From 2bc52fc21b45daab6b85db19bb2be06394cad737 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Mon, 8 Jun 2026 15:29:01 -0600 Subject: [PATCH 1/4] Skip bundle-backed code functions during migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A code function (function_data.type == "code") can be backed by either inline source (function_data.data.type == "inline") or a compiled bundle/sandbox artifact. The inline variant carries its full code and migrates fine, but the bundle variant only references an artifact produced by the push/eval build pipeline — that artifact isn't exposed by the API and can't be recreated in the destination, so migrating it yields a function with a dangling bundle_id that can't be invoked. FunctionMigrator now detects bundle-backed code functions and skips them, recording a MigrationResult(skipped=True, skip_reason="code_bundle_not_migratable") so they appear in the skip summary (not as failures) and logging each one's name/slug so it can be re-pushed to the destination manually. Inline code functions and all other function types are unaffected. Adds tests for the detection and the migrate_batch skip/keep behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 +- braintrust_migrate/resources/functions.py | 66 ++++++++- tests/unit/test_function_code_bundle_skip.py | 136 +++++++++++++++++++ 3 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_function_code_bundle_skip.py diff --git a/CHANGELOG.md b/CHANGELOG.md index af3fc40..0218f15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ The format is based on Keep a Changelog and this project follows Semantic Versio ### Changed -- N/A +- Skip bundle-backed code functions during migration. A code function's compiled bundle is produced by the push/eval build pipeline and is not exposed by the API, so it can't be recreated in the destination — migrating it produces a broken function. These are now skipped (recorded with `skip_reason="code_bundle_not_migratable"` and logged with name/slug so they can be re-pushed manually). Inline code functions (which carry their source) and all other function types continue to migrate normally. ### Fixed diff --git a/braintrust_migrate/resources/functions.py b/braintrust_migrate/resources/functions.py index d9554fc..cc84eb0 100644 --- a/braintrust_migrate/resources/functions.py +++ b/braintrust_migrate/resources/functions.py @@ -1,6 +1,9 @@ """Function migrator for Braintrust migration tool.""" -from braintrust_migrate.resources.base import ResourceMigrator +from braintrust_migrate.resources.base import MigrationResult, ResourceMigrator + +# skip_reason recorded for bundle-backed code functions we cannot migrate. +CODE_BUNDLE_SKIP_REASON = "code_bundle_not_migratable" class FunctionMigrator(ResourceMigrator[dict]): @@ -14,6 +17,67 @@ def resource_name(self) -> str: """Human-readable name for this resource type.""" return "Functions" + @staticmethod + def _is_code_bundle_function(resource: dict) -> bool: + """Whether a function is code backed by a build artifact (bundle/sandbox). + + Code functions come in two flavors. ``inline`` carries its full source + (``function_data.data.code``) and migrates fine. Anything else + (``bundle``/``sandbox``) only references a compiled artifact produced by + the push/eval build pipeline — that artifact isn't exposed by the API + and can't be recreated in the destination, so such functions would land + broken. We skip those. + """ + function_data = resource.get("function_data") + if not isinstance(function_data, dict) or function_data.get("type") != "code": + return False + data = function_data.get("data") + return not (isinstance(data, dict) and data.get("type") == "inline") + + async def migrate_batch( + self, resources: list[dict], max_concurrent: int | None = None + ) -> list[MigrationResult]: + """Skip bundle-backed code functions; migrate everything else normally.""" + to_migrate: list[dict] = [] + skip_results: list[MigrationResult] = [] + + for resource in resources: + if not self._is_code_bundle_function(resource): + to_migrate.append(resource) + continue + + source_id = self.get_resource_id(resource) + name = resource.get("name") + slug = resource.get("slug") + self._logger.warning( + "⏭️ Skipping bundled code function (its code bundle cannot be " + "migrated; re-push it to the destination)", + source_id=source_id, + name=name, + slug=slug, + function_type=resource.get("function_type"), + ) + metadata: dict = {"skip_reason": CODE_BUNDLE_SKIP_REASON} + if name: + metadata["name"] = name + if slug: + metadata["slug"] = slug + skip_results.append( + MigrationResult( + success=True, + source_id=source_id, + skipped=True, + metadata=metadata, + ) + ) + + migrated_results = ( + await super().migrate_batch(to_migrate, max_concurrent) + if to_migrate + else [] + ) + return skip_results + migrated_results + async def get_dependencies(self, resource: dict) -> list[str]: """Get list of resource IDs that this function depends on. diff --git a/tests/unit/test_function_code_bundle_skip.py b/tests/unit/test_function_code_bundle_skip.py new file mode 100644 index 0000000..e20cd81 --- /dev/null +++ b/tests/unit/test_function_code_bundle_skip.py @@ -0,0 +1,136 @@ +"""FunctionMigrator skips bundle-backed code functions (not migratable). + +A code function's bundle is built by the push/eval pipeline and isn't +retrievable via the API, so it can't be recreated in the destination. Inline +code functions carry their source and migrate fine. +""" + +from __future__ import annotations + +import pytest + +from braintrust_migrate.resources.functions import ( + CODE_BUNDLE_SKIP_REASON, + FunctionMigrator, +) + + +def _bundle_fn(fid: str = "f-bundle") -> dict: + return { + "id": fid, + "name": "bundle scorer", + "slug": "bundle-scorer", + "project_id": "p", + "function_type": "scorer", + "function_data": { + "type": "code", + "data": { + "type": "bundle", + "bundle_id": "b1", + "runtime_context": {"runtime": "node", "version": "20"}, + "location": { + "type": "experiment", + "eval_name": "e", + "position": {"type": "scorer", "index": 0}, + }, + }, + }, + } + + +def _inline_fn(fid: str = "f-inline") -> dict: + return { + "id": fid, + "name": "inline scorer", + "slug": "inline-scorer", + "project_id": "p", + "function_type": "scorer", + "function_data": { + "type": "code", + "data": { + "type": "inline", + "code": "def handler(): return 1", + "runtime_context": {"runtime": "python", "version": "3.11"}, + }, + }, + } + + +def _prompt_fn(fid: str = "f-prompt") -> dict: + return { + "id": fid, + "name": "prompt fn", + "slug": "prompt-fn", + "project_id": "p", + "function_type": "llm", + "function_data": {"type": "prompt"}, + } + + +def test_is_code_bundle_function_detection(): + assert FunctionMigrator._is_code_bundle_function(_bundle_fn()) is True + assert FunctionMigrator._is_code_bundle_function(_inline_fn()) is False + assert FunctionMigrator._is_code_bundle_function(_prompt_fn()) is False + # sandbox / missing data on a code function -> not inline -> skip. + assert ( + FunctionMigrator._is_code_bundle_function( + {"function_data": {"type": "code", "data": {"type": "sandbox"}}} + ) + is True + ) + assert ( + FunctionMigrator._is_code_bundle_function({"function_data": {"type": "code"}}) + is True + ) + # non-code / no function_data -> never skipped here. + assert FunctionMigrator._is_code_bundle_function({}) is False + assert ( + FunctionMigrator._is_code_bundle_function( + {"function_data": {"type": "global"}} + ) + is False + ) + + +@pytest.mark.asyncio +async def test_migrate_batch_skips_bundle_keeps_inline_and_prompt( + mock_source_client, mock_dest_client, temp_checkpoint_dir +): + migrator = FunctionMigrator( + mock_source_client, mock_dest_client, temp_checkpoint_dir + ) + migrator.dest_project_id = "dest-project" + + async def mock_with_retry(operation_name, coro_func, **kwargs): + result = coro_func() + return await result if hasattr(result, "__await__") else result + + mock_dest_client.with_retry.side_effect = mock_with_retry + + created: list[dict] = [] + + async def mock_raw_request(method, path, *, json=None, **kwargs): + created.append(json) + return {"id": f"dest-{json.get('slug')}", "name": json.get("name")} + + mock_dest_client.raw_request.side_effect = mock_raw_request + + results = await migrator.migrate_batch([_bundle_fn(), _inline_fn(), _prompt_fn()]) + by_source = {r.source_id: r for r in results} + + # Bundle function: skipped with the documented reason; never sent to dest. + assert by_source["f-bundle"].skipped is True + assert by_source["f-bundle"].metadata["skip_reason"] == CODE_BUNDLE_SKIP_REASON + + # Inline + prompt: migrated normally. + assert by_source["f-inline"].skipped is False + assert by_source["f-inline"].success is True + assert by_source["f-prompt"].skipped is False + assert by_source["f-prompt"].success is True + + # Exactly the two migratable functions hit the create endpoint; no bundle + # function_data was ever sent. + assert len(created) == 2 + assert {c.get("slug") for c in created} == {"inline-scorer", "prompt-fn"} + for c in created: + assert c.get("function_data", {}).get("data", {}).get("type") != "bundle" From abef859c81cd5cef4dfd584fe9b8101ca080148a Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Mon, 8 Jun 2026 15:38:41 -0600 Subject: [PATCH 2/4] Point console summary at the report for per-item skip detail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failures are printed inline in the console summary, but skips were only shown as counts — so users couldn't tell *what* was skipped without knowing to open migration_report.json. Rather than duplicate the (potentially large) per-item list in the console, print a pointer when anything was skipped: the count plus the report path and the JSON location (detailed_breakdown.skipped) where each skipped item is recorded with its name and skip_reason. Applies to all skips (already-migrated, unresolved-dependency, and the new code_bundle_not_migratable). Adds tests for the pointer shown/hidden cases. Co-Authored-By: Claude Opus 4.8 (1M context) --- braintrust_migrate/cli.py | 14 ++++++ tests/unit/test_cli_skip_report_pointer.py | 51 ++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 tests/unit/test_cli_skip_report_pointer.py diff --git a/braintrust_migrate/cli.py b/braintrust_migrate/cli.py index 9141b94..44d66e1 100644 --- a/braintrust_migrate/cli.py +++ b/braintrust_migrate/cli.py @@ -914,6 +914,20 @@ def _display_results(results: dict) -> None: f" ... and {len(summary['errors']) - MAX_ERRORS_TO_DISPLAY} more errors" ) + # Skips are counted above but the per-item detail (what was skipped and why) + # can be large, so point to the JSON report rather than printing it inline. + report_path = results.get("report_path") + if summary["skipped_resources"]: + msg = ( + f"\n[yellow]{summary['skipped_resources']} resource(s) skipped[/yellow]" + ) + if report_path: + msg += ( + f" — per-item detail (name + reason) in:\n [dim]{report_path}[/dim]" + ' → "detailed_breakdown.skipped"' + ) + console.print(msg) + @app.command() def validate( diff --git a/tests/unit/test_cli_skip_report_pointer.py b/tests/unit/test_cli_skip_report_pointer.py new file mode 100644 index 0000000..8a90e8a --- /dev/null +++ b/tests/unit/test_cli_skip_report_pointer.py @@ -0,0 +1,51 @@ +"""The console summary points to the JSON report for per-item skip detail.""" + +from __future__ import annotations + +from rich.console import Console + +import braintrust_migrate.cli as cli_module + + +def _results(*, skipped: int, report_path: str | None) -> dict: + res: dict = { + "summary": { + "total_projects": 1, + "total_resources": 3, + "migrated_resources": 3 - skipped, + "skipped_resources": skipped, + "failed_resources": 0, + "errors": [], + }, + "projects": {}, + } + if report_path is not None: + res["report_path"] = report_path + return res + + +def test_skip_pointer_shown_when_resources_skipped(monkeypatch): + rec = Console(record=True, width=200) + monkeypatch.setattr(cli_module, "console", rec) + + cli_module._display_results( + _results(skipped=2, report_path="/tmp/checkpoints/migration_report.json") + ) + + out = rec.export_text() + assert "2 resource(s) skipped" in out + assert "/tmp/checkpoints/migration_report.json" in out + assert "detailed_breakdown.skipped" in out + + +def test_no_skip_pointer_when_nothing_skipped(monkeypatch): + rec = Console(record=True, width=200) + monkeypatch.setattr(cli_module, "console", rec) + + cli_module._display_results( + _results(skipped=0, report_path="/tmp/checkpoints/migration_report.json") + ) + + out = rec.export_text() + assert "resource(s) skipped" not in out + assert "detailed_breakdown.skipped" not in out From 90e396e1440aa7cc09a0badbe3e67599ed158b74 Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Mon, 8 Jun 2026 15:54:32 -0600 Subject: [PATCH 3/4] Document that bundled code functions are skipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an explicit README callout (Resource Types) explaining that code functions backed by a compiled bundle — e.g. code-based scorers/tools/tasks pushed via `braintrust push` or an eval — cannot be migrated (the API only exposes a bundle reference, not the code, and there's no re-upload path), so they are skipped and recorded in migration_report.json with skip_reason="code_bundle_not_migratable". Users should re-push them manually. Inline code functions and all other function types migrate normally. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 23d45f9..8f31f97 100644 --- a/README.md +++ b/README.md @@ -504,7 +504,7 @@ The following resource types are supported: - **Datasets** - **Project Tags** - **Span Iframes** -- **Functions** +- **Functions** (⚠️ bundled code functions are **skipped** — see note below) - **Prompts** - **Project Scores** - **Experiments** @@ -514,6 +514,24 @@ The following resource types are supported: > **Note:** Agents and users are not supported for migration. +> **⚠️ Code functions with bundled code are skipped (re-push them manually).** +> A code function created by *pushing* code — e.g. a code-based **scorer**, tool, +> or task deployed via `braintrust push` or an eval — ships a compiled **bundle** +> that is produced by the push/eval build pipeline and stored separately. The API +> only exposes a *reference* to that bundle (a `bundle_id` and a short `preview`), +> not the code itself, and there is no way to re-upload it into the destination +> org — so these functions **cannot be recreated by the migration** and would +> otherwise land broken (a dangling `bundle_id` that can't be invoked). +> +> The migration therefore **skips** them. Each skipped function is logged with its +> name/slug and recorded in `migration_report.json` under +> `detailed_breakdown.skipped` with `skip_reason="code_bundle_not_migratable"`, so +> you have an exact list. **After migrating, re-push these functions to the +> destination manually** (e.g. `braintrust push`). +> +> *Inline* code functions (whose source is stored directly on the function) and +> all other function types migrate normally. + ## Troubleshooting ### Common Issues From d10be39b455fad4d1f424b3ebe3b824cac834dee Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Mon, 8 Jun 2026 15:56:49 -0600 Subject: [PATCH 4/4] Always surface the report path in the console summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Print the full per-item report location regardless of whether anything was skipped — it's general drill-in info (migrated / skipped / failed detail), not specific to skips. The Skipped count is already shown in the summary table, so the pointer no longer repeats it; it just points to migration_report.json. Co-Authored-By: Claude Opus 4.8 (1M context) --- braintrust_migrate/cli.py | 17 ++++------ tests/unit/test_cli_skip_report_pointer.py | 38 ++++++++++++---------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/braintrust_migrate/cli.py b/braintrust_migrate/cli.py index 44d66e1..d3e08a6 100644 --- a/braintrust_migrate/cli.py +++ b/braintrust_migrate/cli.py @@ -914,19 +914,14 @@ def _display_results(results: dict) -> None: f" ... and {len(summary['errors']) - MAX_ERRORS_TO_DISPLAY} more errors" ) - # Skips are counted above but the per-item detail (what was skipped and why) - # can be large, so point to the JSON report rather than printing it inline. + # Always point to the full per-item report so users can drill into exactly + # what was migrated / skipped / failed (and why) — counts above, detail here. report_path = results.get("report_path") - if summary["skipped_resources"]: - msg = ( - f"\n[yellow]{summary['skipped_resources']} resource(s) skipped[/yellow]" + if report_path: + console.print( + f"\n[dim]Full per-item report (migrated / skipped / failed): " + f"{report_path}[/dim]" ) - if report_path: - msg += ( - f" — per-item detail (name + reason) in:\n [dim]{report_path}[/dim]" - ' → "detailed_breakdown.skipped"' - ) - console.print(msg) @app.command() diff --git a/tests/unit/test_cli_skip_report_pointer.py b/tests/unit/test_cli_skip_report_pointer.py index 8a90e8a..32577bf 100644 --- a/tests/unit/test_cli_skip_report_pointer.py +++ b/tests/unit/test_cli_skip_report_pointer.py @@ -1,4 +1,4 @@ -"""The console summary points to the JSON report for per-item skip detail.""" +"""The console summary always points to the JSON report to drill into detail.""" from __future__ import annotations @@ -24,28 +24,32 @@ def _results(*, skipped: int, report_path: str | None) -> dict: return res -def test_skip_pointer_shown_when_resources_skipped(monkeypatch): +def _render(monkeypatch, results: dict) -> str: rec = Console(record=True, width=200) monkeypatch.setattr(cli_module, "console", rec) + cli_module._display_results(results) + return rec.export_text() - cli_module._display_results( - _results(skipped=2, report_path="/tmp/checkpoints/migration_report.json") - ) - out = rec.export_text() - assert "2 resource(s) skipped" in out +def test_report_path_shown_when_resources_skipped(monkeypatch): + out = _render( + monkeypatch, + _results(skipped=2, report_path="/tmp/checkpoints/migration_report.json"), + ) assert "/tmp/checkpoints/migration_report.json" in out - assert "detailed_breakdown.skipped" in out + assert "Full per-item report" in out -def test_no_skip_pointer_when_nothing_skipped(monkeypatch): - rec = Console(record=True, width=200) - monkeypatch.setattr(cli_module, "console", rec) - - cli_module._display_results( - _results(skipped=0, report_path="/tmp/checkpoints/migration_report.json") +def test_report_path_shown_even_when_nothing_skipped(monkeypatch): + # The report pointer is general drill-in info, not gated on skips. + out = _render( + monkeypatch, + _results(skipped=0, report_path="/tmp/checkpoints/migration_report.json"), ) + assert "/tmp/checkpoints/migration_report.json" in out + assert "Full per-item report" in out + - out = rec.export_text() - assert "resource(s) skipped" not in out - assert "detailed_breakdown.skipped" not in out +def test_no_report_line_when_report_path_absent(monkeypatch): + out = _render(monkeypatch, _results(skipped=1, report_path=None)) + assert "Full per-item report" not in out