diff --git a/CHANGELOG.md b/CHANGELOG.md index af3fc40..0218f15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ The format is based on Keep a Changelog and this project follows Semantic Versio ### Changed -- N/A +- Skip bundle-backed code functions during migration. A code function's compiled bundle is produced by the push/eval build pipeline and is not exposed by the API, so it can't be recreated in the destination — migrating it produces a broken function. These are now skipped (recorded with `skip_reason="code_bundle_not_migratable"` and logged with name/slug so they can be re-pushed manually). Inline code functions (which carry their source) and all other function types continue to migrate normally. ### Fixed diff --git a/README.md b/README.md index 23d45f9..8f31f97 100644 --- a/README.md +++ b/README.md @@ -504,7 +504,7 @@ The following resource types are supported: - **Datasets** - **Project Tags** - **Span Iframes** -- **Functions** +- **Functions** (⚠️ bundled code functions are **skipped** — see note below) - **Prompts** - **Project Scores** - **Experiments** @@ -514,6 +514,24 @@ The following resource types are supported: > **Note:** Agents and users are not supported for migration. +> **⚠️ Code functions with bundled code are skipped (re-push them manually).** +> A code function created by *pushing* code — e.g. a code-based **scorer**, tool, +> or task deployed via `braintrust push` or an eval — ships a compiled **bundle** +> that is produced by the push/eval build pipeline and stored separately. The API +> only exposes a *reference* to that bundle (a `bundle_id` and a short `preview`), +> not the code itself, and there is no way to re-upload it into the destination +> org — so these functions **cannot be recreated by the migration** and would +> otherwise land broken (a dangling `bundle_id` that can't be invoked). +> +> The migration therefore **skips** them. Each skipped function is logged with its +> name/slug and recorded in `migration_report.json` under +> `detailed_breakdown.skipped` with `skip_reason="code_bundle_not_migratable"`, so +> you have an exact list. **After migrating, re-push these functions to the +> destination manually** (e.g. `braintrust push`). +> +> *Inline* code functions (whose source is stored directly on the function) and +> all other function types migrate normally. + ## Troubleshooting ### Common Issues diff --git a/braintrust_migrate/cli.py b/braintrust_migrate/cli.py index 9141b94..d3e08a6 100644 --- a/braintrust_migrate/cli.py +++ b/braintrust_migrate/cli.py @@ -914,6 +914,15 @@ def _display_results(results: dict) -> None: f" ... and {len(summary['errors']) - MAX_ERRORS_TO_DISPLAY} more errors" ) + # Always point to the full per-item report so users can drill into exactly + # what was migrated / skipped / failed (and why) — counts above, detail here. + report_path = results.get("report_path") + if report_path: + console.print( + f"\n[dim]Full per-item report (migrated / skipped / failed): " + f"{report_path}[/dim]" + ) + @app.command() def validate( diff --git a/braintrust_migrate/resources/functions.py b/braintrust_migrate/resources/functions.py index d9554fc..cc84eb0 100644 --- a/braintrust_migrate/resources/functions.py +++ b/braintrust_migrate/resources/functions.py @@ -1,6 +1,9 @@ """Function migrator for Braintrust migration tool.""" -from braintrust_migrate.resources.base import ResourceMigrator +from braintrust_migrate.resources.base import MigrationResult, ResourceMigrator + +# skip_reason recorded for bundle-backed code functions we cannot migrate. +CODE_BUNDLE_SKIP_REASON = "code_bundle_not_migratable" class FunctionMigrator(ResourceMigrator[dict]): @@ -14,6 +17,67 @@ def resource_name(self) -> str: """Human-readable name for this resource type.""" return "Functions" + @staticmethod + def _is_code_bundle_function(resource: dict) -> bool: + """Whether a function is code backed by a build artifact (bundle/sandbox). + + Code functions come in two flavors. ``inline`` carries its full source + (``function_data.data.code``) and migrates fine. Anything else + (``bundle``/``sandbox``) only references a compiled artifact produced by + the push/eval build pipeline — that artifact isn't exposed by the API + and can't be recreated in the destination, so such functions would land + broken. We skip those. + """ + function_data = resource.get("function_data") + if not isinstance(function_data, dict) or function_data.get("type") != "code": + return False + data = function_data.get("data") + return not (isinstance(data, dict) and data.get("type") == "inline") + + async def migrate_batch( + self, resources: list[dict], max_concurrent: int | None = None + ) -> list[MigrationResult]: + """Skip bundle-backed code functions; migrate everything else normally.""" + to_migrate: list[dict] = [] + skip_results: list[MigrationResult] = [] + + for resource in resources: + if not self._is_code_bundle_function(resource): + to_migrate.append(resource) + continue + + source_id = self.get_resource_id(resource) + name = resource.get("name") + slug = resource.get("slug") + self._logger.warning( + "⏭️ Skipping bundled code function (its code bundle cannot be " + "migrated; re-push it to the destination)", + source_id=source_id, + name=name, + slug=slug, + function_type=resource.get("function_type"), + ) + metadata: dict = {"skip_reason": CODE_BUNDLE_SKIP_REASON} + if name: + metadata["name"] = name + if slug: + metadata["slug"] = slug + skip_results.append( + MigrationResult( + success=True, + source_id=source_id, + skipped=True, + metadata=metadata, + ) + ) + + migrated_results = ( + await super().migrate_batch(to_migrate, max_concurrent) + if to_migrate + else [] + ) + return skip_results + migrated_results + async def get_dependencies(self, resource: dict) -> list[str]: """Get list of resource IDs that this function depends on. diff --git a/tests/unit/test_cli_skip_report_pointer.py b/tests/unit/test_cli_skip_report_pointer.py new file mode 100644 index 0000000..32577bf --- /dev/null +++ b/tests/unit/test_cli_skip_report_pointer.py @@ -0,0 +1,55 @@ +"""The console summary always points to the JSON report to drill into detail.""" + +from __future__ import annotations + +from rich.console import Console + +import braintrust_migrate.cli as cli_module + + +def _results(*, skipped: int, report_path: str | None) -> dict: + res: dict = { + "summary": { + "total_projects": 1, + "total_resources": 3, + "migrated_resources": 3 - skipped, + "skipped_resources": skipped, + "failed_resources": 0, + "errors": [], + }, + "projects": {}, + } + if report_path is not None: + res["report_path"] = report_path + return res + + +def _render(monkeypatch, results: dict) -> str: + rec = Console(record=True, width=200) + monkeypatch.setattr(cli_module, "console", rec) + cli_module._display_results(results) + return rec.export_text() + + +def test_report_path_shown_when_resources_skipped(monkeypatch): + out = _render( + monkeypatch, + _results(skipped=2, report_path="/tmp/checkpoints/migration_report.json"), + ) + assert "/tmp/checkpoints/migration_report.json" in out + assert "Full per-item report" in out + + +def test_report_path_shown_even_when_nothing_skipped(monkeypatch): + # The report pointer is general drill-in info, not gated on skips. + out = _render( + monkeypatch, + _results(skipped=0, report_path="/tmp/checkpoints/migration_report.json"), + ) + assert "/tmp/checkpoints/migration_report.json" in out + assert "Full per-item report" in out + + +def test_no_report_line_when_report_path_absent(monkeypatch): + out = _render(monkeypatch, _results(skipped=1, report_path=None)) + assert "Full per-item report" not in out diff --git a/tests/unit/test_function_code_bundle_skip.py b/tests/unit/test_function_code_bundle_skip.py new file mode 100644 index 0000000..e20cd81 --- /dev/null +++ b/tests/unit/test_function_code_bundle_skip.py @@ -0,0 +1,136 @@ +"""FunctionMigrator skips bundle-backed code functions (not migratable). + +A code function's bundle is built by the push/eval pipeline and isn't +retrievable via the API, so it can't be recreated in the destination. Inline +code functions carry their source and migrate fine. +""" + +from __future__ import annotations + +import pytest + +from braintrust_migrate.resources.functions import ( + CODE_BUNDLE_SKIP_REASON, + FunctionMigrator, +) + + +def _bundle_fn(fid: str = "f-bundle") -> dict: + return { + "id": fid, + "name": "bundle scorer", + "slug": "bundle-scorer", + "project_id": "p", + "function_type": "scorer", + "function_data": { + "type": "code", + "data": { + "type": "bundle", + "bundle_id": "b1", + "runtime_context": {"runtime": "node", "version": "20"}, + "location": { + "type": "experiment", + "eval_name": "e", + "position": {"type": "scorer", "index": 0}, + }, + }, + }, + } + + +def _inline_fn(fid: str = "f-inline") -> dict: + return { + "id": fid, + "name": "inline scorer", + "slug": "inline-scorer", + "project_id": "p", + "function_type": "scorer", + "function_data": { + "type": "code", + "data": { + "type": "inline", + "code": "def handler(): return 1", + "runtime_context": {"runtime": "python", "version": "3.11"}, + }, + }, + } + + +def _prompt_fn(fid: str = "f-prompt") -> dict: + return { + "id": fid, + "name": "prompt fn", + "slug": "prompt-fn", + "project_id": "p", + "function_type": "llm", + "function_data": {"type": "prompt"}, + } + + +def test_is_code_bundle_function_detection(): + assert FunctionMigrator._is_code_bundle_function(_bundle_fn()) is True + assert FunctionMigrator._is_code_bundle_function(_inline_fn()) is False + assert FunctionMigrator._is_code_bundle_function(_prompt_fn()) is False + # sandbox / missing data on a code function -> not inline -> skip. + assert ( + FunctionMigrator._is_code_bundle_function( + {"function_data": {"type": "code", "data": {"type": "sandbox"}}} + ) + is True + ) + assert ( + FunctionMigrator._is_code_bundle_function({"function_data": {"type": "code"}}) + is True + ) + # non-code / no function_data -> never skipped here. + assert FunctionMigrator._is_code_bundle_function({}) is False + assert ( + FunctionMigrator._is_code_bundle_function( + {"function_data": {"type": "global"}} + ) + is False + ) + + +@pytest.mark.asyncio +async def test_migrate_batch_skips_bundle_keeps_inline_and_prompt( + mock_source_client, mock_dest_client, temp_checkpoint_dir +): + migrator = FunctionMigrator( + mock_source_client, mock_dest_client, temp_checkpoint_dir + ) + migrator.dest_project_id = "dest-project" + + async def mock_with_retry(operation_name, coro_func, **kwargs): + result = coro_func() + return await result if hasattr(result, "__await__") else result + + mock_dest_client.with_retry.side_effect = mock_with_retry + + created: list[dict] = [] + + async def mock_raw_request(method, path, *, json=None, **kwargs): + created.append(json) + return {"id": f"dest-{json.get('slug')}", "name": json.get("name")} + + mock_dest_client.raw_request.side_effect = mock_raw_request + + results = await migrator.migrate_batch([_bundle_fn(), _inline_fn(), _prompt_fn()]) + by_source = {r.source_id: r for r in results} + + # Bundle function: skipped with the documented reason; never sent to dest. + assert by_source["f-bundle"].skipped is True + assert by_source["f-bundle"].metadata["skip_reason"] == CODE_BUNDLE_SKIP_REASON + + # Inline + prompt: migrated normally. + assert by_source["f-inline"].skipped is False + assert by_source["f-inline"].success is True + assert by_source["f-prompt"].skipped is False + assert by_source["f-prompt"].success is True + + # Exactly the two migratable functions hit the create endpoint; no bundle + # function_data was ever sent. + assert len(created) == 2 + assert {c.get("slug") for c in created} == {"inline-scorer", "prompt-fn"} + for c in created: + assert c.get("function_data", {}).get("data", {}).get("type") != "bundle"