Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions apps/cli/src/commands/results/remote.ts
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,12 @@ export interface ResultsPublishOverrides {
readonly push_conflict_policy?: 'block';
}

type RuntimeResultsConfig = Omit<ResultsConfig, 'sync'> & {
readonly sync?: ResultsConfig['sync'] & {
readonly require_push?: boolean;
};
};

const REMOTE_RUN_PREFIX = 'remote::';
const SIZE_WARNING_BYTES = 10 * 1024 * 1024;

Expand Down Expand Up @@ -226,15 +232,11 @@ export async function loadNormalizedResultsConfig(
...(project.results.remote !== undefined && { remote: project.results.remote }),
...(project.results.path !== undefined && { path: project.results.path }),
...((project.results.sync?.autoPush !== undefined ||
project.results.sync?.requirePush !== undefined ||
project.results.sync?.pushConflictPolicy !== undefined) && {
sync: {
...(project.results.sync?.autoPush !== undefined && {
auto_push: project.results.sync.autoPush,
}),
...(project.results.sync?.requirePush !== undefined && {
require_push: project.results.sync.requirePush,
}),
...(project.results.sync?.pushConflictPolicy !== undefined && {
push_conflict_policy: project.results.sync.pushConflictPolicy,
}),
Expand All @@ -260,7 +262,7 @@ export async function loadNormalizedResultsConfig(
return baseConfig;
}

const merged: ResultsConfig = {
const merged: RuntimeResultsConfig = {
mode: 'github',
...(overrides.repo !== undefined
? { repo: overrides.repo }
Expand Down
30 changes: 30 additions & 0 deletions apps/cli/test/commands/results/remote-auto-export.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ describe('maybeAutoExportRunArtifacts', () => {
let cloneDir: string;
let previousHome: string | undefined;
let previousXdgConfigHome: string | undefined;
let previousAgentvHome: string | undefined;

beforeEach(() => {
rootDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-remote-export-test-'));
Expand All @@ -157,10 +158,13 @@ describe('maybeAutoExportRunArtifacts', () => {
// identity setup rather than the developer machine's ~/.gitconfig.
previousHome = process.env.HOME;
previousXdgConfigHome = process.env.XDG_CONFIG_HOME;
previousAgentvHome = process.env.AGENTV_HOME;
process.env.HOME = path.join(rootDir, 'empty-home');
process.env.XDG_CONFIG_HOME = path.join(rootDir, 'empty-xdg-config');
process.env.AGENTV_HOME = path.join(rootDir, 'agentv-home');
mkdirSync(process.env.HOME, { recursive: true });
mkdirSync(process.env.XDG_CONFIG_HOME, { recursive: true });
mkdirSync(process.env.AGENTV_HOME, { recursive: true });
});

afterEach(() => {
Expand All @@ -174,6 +178,11 @@ describe('maybeAutoExportRunArtifacts', () => {
} else {
process.env.XDG_CONFIG_HOME = previousXdgConfigHome;
}
if (previousAgentvHome === undefined) {
process.env.AGENTV_HOME = undefined;
} else {
process.env.AGENTV_HOME = previousAgentvHome;
}
rmSync(rootDir, { recursive: true, force: true });
});

Expand Down Expand Up @@ -272,6 +281,27 @@ describe('maybeAutoExportRunArtifacts', () => {
}
}, 20_000);

it('throws when a CLI-only require-push override cannot push results', async () => {
const runDir = writeRunArtifacts(projectDir);
writeProjectConfig(projectDir, {
repo: `file://${path.join(rootDir, 'missing-remote.git')}`,
path: cloneDir,
autoPush: false,
});
const warnSpy = spyOn(console, 'warn').mockImplementation(() => {});

try {
await expect(
maybeAutoExportRunArtifacts({
...payload(projectDir, runDir),
results_overrides: { require_push: true },
}),
).rejects.toThrow();
} finally {
warnSpy.mockRestore();
}
}, 20_000);

it('publishes locally without pushing when auto-push is disabled', async () => {
const remoteDir = initializeRemoteRepo(rootDir);
const runDir = writeRunArtifacts(projectDir);
Expand Down
4 changes: 1 addition & 3 deletions apps/web/src/content/docs/docs/tools/dashboard.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -291,11 +291,10 @@ projects:
branch: agentv/results/v1
sync:
auto_push: false
require_push: false
push_conflict_policy: block
```

`results.repo.remote` is the Git remote URL used when AgentV creates a fresh results checkout, and the intended remote URL for portable project config. `results.repo.path: .` stores completed run artifacts on a dedicated branch of the source repository without checking out that branch in the source worktree. AgentV does not add or rewrite remotes inside an existing checkout; the checkout's existing `origin` must already point at the repository you want to fetch and push. When `results.repo.remote` is omitted, `results.repo.path` means an existing local Git checkout whose object database and refs AgentV should write to, and the branch defaults to `agentv/results/v1`. AgentV creates the branch automatically on first publish and commits only AgentV result paths into it. `sync.auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. `sync.require_push: true` is for CI workflows where a push failure should fail the command after local artifacts are written. `sync.push_conflict_policy` defaults to `block`; the removed `backup_and_force_push` value is rejected with migration guidance because AgentV never force-pushes result branches. Non-fast-forward result branch pushes are auto-merged with artifact-aware Git merge drivers and pushed as a fast-forward, so the canonical results branch is never force-pushed or rewritten. Genuine overlay conflicts route to a timestamped temp branch plus a GitHub compare link for a human merge instead.
`results.repo.remote` is the Git remote URL used when AgentV creates a fresh results checkout, and the intended remote URL for portable project config. `results.repo.path: .` stores completed run artifacts on a dedicated branch of the source repository without checking out that branch in the source worktree. AgentV does not add or rewrite remotes inside an existing checkout; the checkout's existing `origin` must already point at the repository you want to fetch and push. When `results.repo.remote` is omitted, `results.repo.path` means an existing local Git checkout whose object database and refs AgentV should write to, and the branch defaults to `agentv/results/v1`. AgentV creates the branch automatically on first publish and commits only AgentV result paths into it. `sync.auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. For CI workflows where a push failure should fail the command after local artifacts are written, invoke the run with `agentv eval run --results-require-push`. `sync.push_conflict_policy` defaults to `block`; the removed `backup_and_force_push` value is rejected with migration guidance because AgentV never force-pushes result branches. Non-fast-forward result branch pushes are auto-merged with artifact-aware Git merge drivers and pushed as a fast-forward, so the canonical results branch is never force-pushed or rewritten. Genuine overlay conflicts route to a timestamped temp branch plus a GitHub compare link for a human merge instead.

For a separate results repository, use `results.repo.remote` and an optional managed clone `results.repo.path`:

Expand Down Expand Up @@ -327,7 +326,6 @@ results:
branch: agentv/results/v1
sync:
auto_push: false
require_push: false
push_conflict_policy: block
```

Expand Down
2 changes: 1 addition & 1 deletion apps/web/src/content/docs/docs/tools/results.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ The CLI contract is deliberately narrow: `agentv results` manages local result a

Use these supported remote workflows instead:

- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo.remote` with `repo.path: .` and `repo.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV never adds or rewrites remotes in an existing checkout; that checkout's `origin` must already point at the repository you want to fetch and push. AgentV reserves `agentv/results/v1` for primary results and `agentv/artifacts/v1` for heavy artifact payloads. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs/<run-path>/<pointer.path>`. The configured results branch remains the metadata/control plane (`index.jsonl`, `summary.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. Mutable run tags are stored as `tags.json` with a `tag_revision`; there is no tag event log in the normal results layout. `results.repo.path` without `results.repo.remote` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `sync.auto_push: true` to push after publish, or `sync.require_push: true` in CI to fail when that push fails. Non-fast-forward result branch pushes never force-push: AgentV auto-merges concurrent remote writes with artifact-aware Git merge drivers (a union driver for the append-only `index.jsonl`, a JSON-union driver for tag and feedback overlays) and pushes the merge as a fast-forward, and routes a genuine overlay conflict to a timestamped `agentv/results-sync/...` branch plus a GitHub compare/PR link for a human merge. The removed `sync.push_conflict_policy: backup_and_force_push` value is rejected with migration guidance; remove the field or set it to `block`. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled.
- **Automatic publishing:** configure `projects[].results` or top-level `results`; new `agentv eval` and `agentv pipeline bench` runs publish completed artifacts after the run completes. Use `repo.remote` with `repo.path: .` and `repo.branch: agentv/results/v1` to store primary result records on a dedicated branch of the source repo. AgentV never adds or rewrites remotes in an existing checkout; that checkout's `origin` must already point at the repository you want to fetch and push. AgentV reserves `agentv/results/v1` for primary results and `agentv/artifacts/v1` for heavy artifact payloads. When `index.jsonl` rows point trace or transcript payloads at `agentv/artifacts/v1`, automatic publishing stores those bytes on that artifact branch in the same remote and publishes pointer keys such as `runs/<run-path>/<pointer.path>`. The configured results branch remains the metadata/control plane (`index.jsonl`, `summary.json`, tags, and pointers) instead of duplicating canonical trace/transcript payload bodies. Local pre-publish run workspaces can still contain those files beside the manifest so local tools keep working. Mutable run tags are stored as `tags.json` with a `tag_revision`; there is no tag event log in the normal results layout. `results.repo.path` without `results.repo.remote` means an existing local Git checkout, distinct from `workspace.repos[].repo`, which is a portable repository identity. Set `sync.auto_push: true` to push after publish. In CI, use `agentv eval run --results-require-push` when push failures should fail that invocation after local artifacts are written. Non-fast-forward result branch pushes never force-push: AgentV auto-merges concurrent remote writes with artifact-aware Git merge drivers (a union driver for the append-only `index.jsonl`, a JSON-union driver for tag and feedback overlays) and pushes the merge as a fast-forward, and routes a genuine overlay conflict to a timestamped `agentv/results-sync/...` branch plus a GitHub compare/PR link for a human merge. The removed `sync.push_conflict_policy: backup_and_force_push` value is rejected with migration guidance; remove the field or set it to `block`. While an eval is still running, [WIP checkpoints](/docs/tools/wip-checkpoints/) can keep partial run output durable on `agentv/wip/...` branches when auto-push is enabled.
- **Manual Dashboard sync:** run `agentv dashboard`, open the project, and use **Sync Project**.
- **Manual API sync:** while Dashboard is running, call `GET /api/projects/:projectId/remote/status` or `POST /api/projects/:projectId/remote/sync` for project-scoped automation. Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`.
- **Git escape hatch:** for advanced recovery, inspect or repair the configured `projects[].results.repo.path` clone with `git` directly, then sync again.
8 changes: 4 additions & 4 deletions docs/plans/git-native-results.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ Every completed `agentv eval` publish is one atomic operation:
1. Write artifacts into the normal local run workspace at `.agentv/results/runs/<experiment>/<timestamp>/`.
2. Resolve the results store: either the `repo_path` checkout or the managed clone for `repo_url`.
3. Build a commit for `runs/**` (and `metadata/**`) on the storage branch using git plumbing and a temporary index, so `repo_path: .` never has to check out `agentv/results/v1`.
4. If `sync.auto_push` or `sync.require_push` is enabled, push the storage branch. Non-fast-forward conflicts fetch the remote branch, rebuild the single run commit on the remote base when safe, and retry.
4. If `sync.auto_push` is enabled or the run was invoked with `--results-require-push`, push the storage branch. Non-fast-forward conflicts fetch the remote branch, rebuild the single run commit on the remote base when safe, and retry.

Each run is one commit. Files are unique to that run, so rebases never content-conflict.

Expand Down Expand Up @@ -89,7 +89,7 @@ Each run is one commit. Files are unique to that run, so rebases never content-c
## Implementation notes

- `normalizeResultsConfig()` accepts `repo_url`/legacy `repo` or `repo_path`, but prerelease docs and config examples use `repo_url` or `repo_path`.
- `directPushResults()` resolves the results store, builds one storage-branch commit for the completed run, and pushes when `sync.auto_push` or `sync.require_push` is enabled.
- `directPushResults()` resolves the results store, builds one storage-branch commit for the completed run, and pushes when `sync.auto_push` or the runtime `--results-require-push` override is enabled.
- `commitResultsRunWithTemporaryIndex()` writes blobs into the repo object database and updates the storage branch via a temporary index. This is the normal `repo_path: .` path and avoids copying files into a checked-out results branch.
- `listGitRuns()` uses `git ls-tree` plus `git cat-file --batch` against `runs/**/summary.json`. A not-yet-created storage branch (ref does not exist) returns `[]` rather than throwing, so the Dashboard's remote-results poll stays quiet before the first push.
- `setupWipWorktree()` and `pushWipCheckpoint()` maintain recoverable in-progress branches under `agentv/wip/...`.
Expand All @@ -99,7 +99,7 @@ Each run is one commit. Files are unique to that run, so rebases never content-c
| Change | Impact |
|--------|--------|
| `results.repo` is legacy | Use `results.repo_url` for a remote clone or `results.repo_path` for an existing local checkout |
| `results.auto_push` moved | Use `results.sync.auto_push`; `results.sync.require_push` is the CI fail-on-push-failure knob |
| `results.auto_push` moved | Use `results.sync.auto_push`; use `agentv eval run --results-require-push` as the per-run CI fail-on-push-failure knob |
| `repo_path` configs default to `agentv/results/v1` | Same-repo storage no longer needs an explicit branch in the common case |
| WIP branch namespace is `agentv/wip/...` | Interrupted runs are recoverable, but successful runs delete their WIP branch after final publish |

Expand All @@ -122,7 +122,7 @@ Breaking changes accepted because no production users yet. Document in release n
## Current answers

1. **Branch model**: completed runs use the configured storage branch, defaulting to `agentv/results/v1` for `repo_path`; WIP checkpoints use `agentv/wip/<hostname>/<run-dir-basename>`.
2. **What to do on `git fetch` failures during `agentv eval`**: warn unless `sync.require_push` is true; local eval artifacts are still written first.
2. **What to do on `git fetch` failures during `agentv eval`**: warn unless `--results-require-push` was passed; local eval artifacts are still written first.
3. **`gh` CLI dependency**: the git-native flow uses raw `git`; GitHub-specific tooling stays outside result publishing.

## What this PR does NOT do
Expand Down
8 changes: 4 additions & 4 deletions packages/core/src/evaluation/loaders/config-loader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ export type ResultsConfig = {
readonly auto_push?: boolean;
readonly sync?: {
readonly auto_push?: boolean;
readonly require_push?: boolean;
readonly push_conflict_policy?: ResultPushConflictPolicy;
};
readonly branch_prefix?: string;
Expand Down Expand Up @@ -857,8 +856,10 @@ export function parseResultsConfig(raw: unknown, configPath: string): ResultsCon
logWarning(`Invalid results.sync.auto_push in ${configPath}, expected boolean`);
return undefined;
}
if (syncObj.require_push !== undefined && typeof syncObj.require_push !== 'boolean') {
logWarning(`Invalid results.sync.require_push in ${configPath}, expected boolean`);
if (syncObj.require_push !== undefined) {
logWarning(
`results.sync.require_push in ${configPath} is no longer supported in persistent config. Use the per-run --results-require-push CLI flag instead.`,
);
return undefined;
}
if (syncObj.push_conflict_policy === 'backup_and_force_push') {
Expand All @@ -873,7 +874,6 @@ export function parseResultsConfig(raw: unknown, configPath: string): ResultsCon
}
sync = {
...(typeof syncObj.auto_push === 'boolean' && { auto_push: syncObj.auto_push }),
...(typeof syncObj.require_push === 'boolean' && { require_push: syncObj.require_push }),
...(syncObj.push_conflict_policy === 'block' && {
push_conflict_policy: syncObj.push_conflict_policy,
}),
Expand Down
18 changes: 17 additions & 1 deletion packages/core/src/evaluation/results-repo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,13 @@ export interface NormalizedResultsConfig {
readonly storageBranchWorktree: boolean;
}

export type RuntimeResultsConfig = Omit<ResultsConfig, 'sync'> & {
readonly sync?: ResultsConfig['sync'] & {
/** Runtime-only override, set by --results-require-push. Not YAML config. */
readonly require_push?: boolean;
};
};

type StorageBranchResultsConfig = NormalizedResultsConfig & { readonly branch: string };

export interface DirectPushResultsResult {
Expand Down Expand Up @@ -371,10 +378,19 @@ function resolveLocalPath(p: string, baseDir: string): string {
return path.isAbsolute(expanded) ? expanded : path.resolve(baseDir, expanded);
}

function isNormalizedResultsConfig(
config: RuntimeResultsConfig | NormalizedResultsConfig,
): config is NormalizedResultsConfig {
return typeof (config as NormalizedResultsConfig).storageBranchWorktree === 'boolean';
}

export function normalizeResultsConfig(
config: ResultsConfig,
config: RuntimeResultsConfig | NormalizedResultsConfig,
options?: { baseDir?: string },
): NormalizedResultsConfig {
if (isNormalizedResultsConfig(config)) {
return config;
}
const baseDir = options?.baseDir ?? process.cwd();
const repoUrl = (config.repo_url ?? config.repo)?.trim();
const repoPath = config.repo_path?.trim();
Expand Down
4 changes: 2 additions & 2 deletions packages/core/src/evaluation/validation/config-validator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -443,12 +443,12 @@ function validateResultsSyncAndBranchPrefix(
`Field '${location}.sync.auto_push' must be a boolean`,
);
}
if (syncRecord.require_push !== undefined && typeof syncRecord.require_push !== 'boolean') {
if (syncRecord.require_push !== undefined) {
addError(
errors,
filePath,
`${location}.sync.require_push`,
`Field '${location}.sync.require_push' must be a boolean`,
`Field '${location}.sync.require_push' was removed from persistent config. Use the per-run --results-require-push CLI flag instead.`,
);
}
if (syncRecord.push_conflict_policy === 'backup_and_force_push') {
Expand Down
Loading
Loading