From b3246b2c59287a413201b0b865bf9b9ec98a0381 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 23 Aug 2026 14:15:14 -0700 Subject: [PATCH 01/18] feat(relay): add dynamic plugin on runner Signed-off-by: Bryan Bednarski --- CHANGELOG.md | 15 + Cargo.lock | 509 ++++++---- Cargo.toml | 3 + README.md | 2 + .../switchyard-nemo-relay-plugin/Cargo.toml | 30 + crates/switchyard-nemo-relay-plugin/README.md | 382 ++++++++ .../config.schema.json | 210 +++++ .../relay-plugin.toml | 31 + .../scripts/package_bundle.py | 99 ++ .../src/client.rs | 296 ++++++ .../src/config.rs | 606 ++++++++++++ .../src/config/tests.rs | 566 +++++++++++ .../switchyard-nemo-relay-plugin/src/lib.rs | 273 ++++++ .../src/runtime.rs | 552 +++++++++++ .../src/runtime/tests.rs | 878 ++++++++++++++++++ .../src/translation.rs | 116 +++ .../tests/test_package_bundle.py | 76 ++ docs/index.md | 2 + 18 files changed, 4464 insertions(+), 182 deletions(-) create mode 100644 crates/switchyard-nemo-relay-plugin/Cargo.toml create mode 100644 crates/switchyard-nemo-relay-plugin/README.md create mode 100644 crates/switchyard-nemo-relay-plugin/config.schema.json create mode 100644 crates/switchyard-nemo-relay-plugin/relay-plugin.toml create mode 100644 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py create mode 100644 crates/switchyard-nemo-relay-plugin/src/client.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/config.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/config/tests.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/lib.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/runtime.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/translation.rs create mode 100644 crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 47e1a6b41..91b3192e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,21 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **NeMo Relay native plugin** — a dynamically loaded integration that runs + libsy's weighted-random, LLM-classifier, escalation, and stage-router + algorithms in process while Switchyard owns provider HTTP dispatch, + credentials, translation, retries, and fallback. Managed calls require NeMo + Relay 0.7 or newer and do not depend on `switchyard-server`. Target bindings + accept non-secret `extra_body` provider defaults, preserved requests are + re-encoded after routing mutations, and synthetic Relay gateway identities do + not become shared router session state. + +- **NeMo Relay routing-model usage marks** — classifier judges, escalation + judges and discarded weak candidates, and failed routing candidates now emit + `switchyard.routing.llm_call` ATOF marks with normalized token usage and + latency. The final serving call remains represented only by Relay's outer LLM + lifecycle event to prevent double-counting. + - **Advisor-gate routing** — new `advisor` route type pairing the serving executor with a stronger judge-only advisor that reviews terminal turns: APPROVE releases the buffered turn, REDO discards it and feeds the advisor's diff --git a/Cargo.lock b/Cargo.lock index c3029243f..e046d1ded 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,9 +18,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -106,6 +106,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -130,13 +142,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -153,9 +165,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ "aws-lc-sys", "zeroize", @@ -163,9 +175,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" dependencies = [ "cc", "cmake", @@ -274,6 +286,9 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] [[package]] name = "borrow-or-share" @@ -301,9 +316,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.2.67" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" dependencies = [ "find-msvc-tools", "jobserver", @@ -334,11 +349,21 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "clap" -version = "4.6.2" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -346,9 +371,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.2" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -358,14 +383,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -399,6 +424,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -424,6 +458,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "data-encoding" version = "2.11.1" @@ -456,13 +496,13 @@ checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -473,9 +513,9 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "either" -version = "1.16.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "email_address" @@ -502,11 +542,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fancy-regex" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +checksum = "476de73bddf2ef8490aa4ee8f1cf40b430bf1d56c48c22080e5186952cd580e6" dependencies = [ "bit-set", "regex-automata", @@ -521,9 +581,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "fluent-uri" @@ -585,9 +645,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -600,9 +660,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -610,15 +670,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -627,38 +687,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -714,9 +774,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228" dependencies = [ "atomic-waker", "bytes", @@ -756,9 +816,9 @@ checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -776,9 +836,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -807,9 +867,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -867,9 +927,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -881,9 +941,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -894,9 +954,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -908,16 +968,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -928,15 +989,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -980,9 +1041,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "is_terminal_polyfill" @@ -1017,7 +1078,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.18", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -1066,9 +1127,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -1077,9 +1138,9 @@ dependencies = [ [[package]] name = "jsonptr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85019623956752c8dd1f04a7b05d066187e0c9b217454246d8397d2a4893cc83" +checksum = "72c22d99362c7aac7d77efab6b49c2181674b165d7e6e7c755b3432ded64bf51" dependencies = [ "serde", "serde_json", @@ -1087,9 +1148,9 @@ dependencies = [ [[package]] name = "jsonschema" -version = "0.49.4" +version = "0.49.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257549c093d8f3d043337ba0d507e8eeace2447dfae3f0ee37eb0f57d3df7655" +checksum = "59ec8a241beed129f06114aa68007e905ca350e7baeb6e17a7631bb7978d91b2" dependencies = [ "ahash", "bytecount", @@ -1116,18 +1177,18 @@ dependencies = [ [[package]] name = "jsonschema-regex" -version = "0.49.4" +version = "0.49.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "474790e948498099d61ec85c10dc72d459d61298b7975eda7fb163af1934b134" +checksum = "91994f45017ed5e66aa8e59b8415f4cb033a6380d7200387b7cf117595fbdf85" dependencies = [ "regex-syntax", ] [[package]] name = "jsonschema-value" -version = "0.49.4" +version = "0.49.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8212315eb8e0bc44959d1af653cd5ec515771a3cdca966cfc0a10999bff144b9" +checksum = "7ec7637f83e510868ae6ed625f7ebfbbde4554ee8ce49854caa5126a8b9b9ecb" dependencies = [ "ahash", "bytecount", @@ -1145,9 +1206,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "linux-raw-sys" @@ -1157,9 +1218,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -1172,9 +1233,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru-slab" @@ -1226,6 +1287,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nemo-relay-plugin" +version = "0.8.0" +source = "git+https://github.com/NVIDIA/NeMo-Relay.git?rev=ca08901629e6058c2d5d65cc7708ec5264073d7b#ca08901629e6058c2d5d65cc7708ec5264073d7b" +dependencies = [ + "futures", + "nemo-relay-types", + "serde", + "serde_json", + "tokio", + "tokio-util", +] + +[[package]] +name = "nemo-relay-types" +version = "0.8.0" +source = "git+https://github.com/NVIDIA/NeMo-Relay.git?rev=ca08901629e6058c2d5d65cc7708ec5264073d7b#ca08901629e6058c2d5d65cc7708ec5264073d7b" +dependencies = [ + "bitflags", + "chrono", + "serde", + "serde_json", + "typed-builder", + "uuid", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1276,9 +1363,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -1351,7 +1438,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -1381,7 +1468,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1421,7 +1508,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", ] @@ -1431,6 +1518,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1468,21 +1561,21 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -1519,9 +1612,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -1538,7 +1631,7 @@ dependencies = [ "memchr", "parking_lot", "protobuf", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1680,7 +1773,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -1688,9 +1781,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", @@ -1703,7 +1796,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -1725,9 +1818,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -1810,18 +1903,18 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", @@ -1830,9 +1923,9 @@ dependencies = [ [[package]] name = "referencing" -version = "0.49.4" +version = "0.49.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bf1a74e036be2546c0c81cdfad6b2def6a25bf31c4e34a468037058d3bd05c6" +checksum = "6efa2154ea6f5ce0fdecdd2a8d18f2fa1a39a8fbba91564f555a592e4dce8278" dependencies = [ "ahash", "fluent-uri", @@ -1859,9 +1952,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1959,9 +2052,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "once_cell", @@ -1985,9 +2078,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -2022,9 +2115,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -2099,9 +2192,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -2109,29 +2202,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", @@ -2292,7 +2385,7 @@ dependencies = [ "serde", "serde_json", "switchyard-protocol", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", "tracing", @@ -2317,7 +2410,7 @@ dependencies = [ "switchyard-libsy", "switchyard-protocol", "switchyard-translation", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "tracing-opentelemetry", @@ -2325,6 +2418,24 @@ dependencies = [ "wiremock", ] +[[package]] +name = "switchyard-nemo-relay-plugin" +version = "0.2.0" +dependencies = [ + "async-channel", + "async-trait", + "futures-util", + "http", + "nemo-relay-plugin", + "serde", + "serde_json", + "switchyard-libsy", + "switchyard-llm-client", + "switchyard-protocol", + "switchyard-translation", + "tokio", +] + [[package]] name = "switchyard-protocol" version = "0.2.0" @@ -2334,7 +2445,7 @@ dependencies = [ "http", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -2367,7 +2478,7 @@ dependencies = [ "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "toml", "tracing", @@ -2415,7 +2526,7 @@ dependencies = [ "async-trait", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", ] @@ -2446,7 +2557,7 @@ dependencies = [ "serde", "serde_json", "switchyard-protocol", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -2521,11 +2632,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.20", ] [[package]] @@ -2541,13 +2652,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -2561,9 +2672,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -2586,9 +2697,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.4" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317fafbbe3f02fc663dad00ea6186197de963cd4190e86a26d8d0fae095539af" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -2603,13 +2714,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -2624,9 +2735,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -2635,22 +2746,24 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", + "libc", "pin-project-lite", "tokio", ] [[package]] name = "toml" -version = "1.1.3+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap", "serde_core", @@ -2672,9 +2785,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] @@ -2815,6 +2928,26 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -2857,6 +2990,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "uuid-simd" version = "0.8.0" @@ -2921,9 +3066,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -2934,9 +3079,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ "js-sys", "wasm-bindgen", @@ -2944,9 +3089,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2954,9 +3099,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -2967,9 +3112,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] @@ -2989,9 +3134,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -3150,9 +3295,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "yansi" @@ -3185,18 +3330,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", @@ -3232,9 +3377,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -3243,9 +3388,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -3254,13 +3399,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4a46fe6ff..90eed1722 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/prefill-router", "crates/switchyard-py", "crates/protocol", + "crates/switchyard-nemo-relay-plugin", "crates/switchyard-runner", "crates/switchyard-server", "crates/switchyard-skill-distillation", @@ -25,6 +26,7 @@ repository = "https://github.com/NVIDIA-NeMo/Switchyard" rust-version = "1.96.1" [workspace.dependencies] +async-channel = "2" async-stream = "0.3" async-trait = "0.1" futures = "0.3" @@ -33,6 +35,7 @@ http = "1" httpdate = "1" jsonschema = { version = "0.49.4", default-features = false } jsonptr = { version = "0.8.1", default-features = false, features = ["std", "json", "resolve"] } +nemo-relay-plugin = { git = "https://github.com/NVIDIA/NeMo-Relay.git", rev = "ca08901629e6058c2d5d65cc7708ec5264073d7b" } parking_lot = "0.12" rand = "0.10" regex = "1" diff --git a/README.md b/README.md index 142f80ee7..bc83ad51e 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ algorithm you write yourself. - **Protocol Translation**: convert between OpenAI Chat, Anthropic Messages, and OpenAI Responses formats - **Multi-Backend Routing**: random routing, LLM-as-classifier routing, signal-driven stage-router, or your own algorithm - **Operational Metrics**: Prometheus metrics cover requests, errors, latency, tokens, and routing overhead +- **NeMo Relay Plugin**: run random, classifier, escalation, or stage routing in Relay while Switchyard owns provider HTTP dispatch ## Maturity @@ -123,6 +124,7 @@ configured LLM client selects one upstream format. - **[`switchyard-libsy`](crates/libsy/README.md)**: embed routing algorithms in a Rust application - **[`switchyard-protocol`](crates/protocol/README.md)**: provider-neutral request, response, and streaming types - **[`switchyard-translation`](crates/switchyard-translation/README.md)**: request, response, and stream translation +- **[`switchyard-nemo-relay-plugin`](crates/switchyard-nemo-relay-plugin/README.md)**: install Switchyard as a native NeMo Relay plugin ## Community diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml new file mode 100644 index 000000000..243e66cc0 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-nemo-relay-plugin" +version.workspace = true +description = "Switchyard-owned HTTP routing plugin for NeMo Relay" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +async-channel.workspace = true +async-trait.workspace = true +futures-util.workspace = true +http.workspace = true +nemo-relay-plugin.workspace = true +serde.workspace = true +serde_json.workspace = true +switchyard-libsy.workspace = true +switchyard-llm-client.workspace = true +switchyard-protocol.workspace = true +switchyard-translation.workspace = true +tokio.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md new file mode 100644 index 000000000..20abab1d7 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -0,0 +1,382 @@ + + +# Switchyard NeMo Relay Dynamic Plugin + +This crate builds the external `nvidia.switchyard` native plugin. It embeds +`switchyard-libsy`, drives it through `switchyard-llm-client::run`, and uses +`switchyard-llm-client` for provider HTTP calls. Managed calls use Relay's +typed asynchronous middleware API and do not require a targeted provider +continuation from Relay. + +The plugin uses NeMo Relay native API v1. It depends on the small +`nemo-relay-plugin` authoring SDK, not the Relay runtime, and does not start +`switchyard-server`. Managed provider calls do not use Relay's provider +continuation. + +## Ownership boundary + +For a managed LLM call: + +1. Relay invokes the native LLM execution intercept. +2. The plugin decodes the caller JSON through `switchyard-translation`. +3. The plugin passes the configured algorithm and its target-to-client map to + `switchyard-llm-client::run`, using the library's public execution and + observation boundary. +4. For every routed call, the selected target client translates the neutral + request, applies its URL and credentials, and performs the HTTP request. +5. `switchyard-llm-client` drives libsy to its final response while the plugin + records decisions and routing-only model usage. +6. The plugin encodes the final neutral response into the caller's protocol. + +Relay still owns the outer LLM lifecycle, dynamic-plugin loading, plugin +configuration, and event substrate. Relay's downstream LLM continuation is +used only for calls whose inbound protocol is not managed by this plugin. + +```mermaid +flowchart LR + A["Caller JSON"] --> B["Relay LLM execution intercept"] + B --> C["Switchyard decode"] + C --> D["switchyard-llm-client run"] + D --> E["libsy algorithm"] + E --> F["target ClientRouter"] + F --> G["Provider HTTP endpoint"] + G --> H["Switchyard response or event decode"] + H --> I["libsy final response"] + I --> J["routing observations"] + J --> K["Switchyard encode"] + K --> A + + U["Unmanaged profile"] -.-> V["Relay v1 continuation"] +``` + +This boundary has two important consequences: + +- Managed provider calls do not traverse Relay middleware registered after the + Switchyard intercept and do not use the host's provider callback. Provider + transport activity is therefore not represented as nested Relay LLM + lifecycle events. Relay records the outer managed call and the plugin emits + Switchyard routing marks; bridging Switchyard transport spans into Relay is + future work. Relay's typed middleware adapter propagates the active scope so + asynchronous routing marks retain their event parent. +- Switchyard owns provider URLs, credentials, HTTP retry behavior, and + translation for managed calls. Relay neither validates nor transports those + target details. + +## Native API v1 and typed asynchronous execution + +The manifest remains `compat.native_api = "1"`, and the plugin requires Relay +`>=0.8.0,<1.0` with native host ABI v4. It registers typed buffered and +incremental streaming intercepts through `nemo-relay-plugin`. Relay owns the plugin executor, +continuation and output-stream lifecycle, cancellation, backpressure, and +scope propagation; Relay workers therefore do not wait synchronously for +provider I/O. + +The stream adapter preserves the plugin's bounded 32-message routing channel. +Relay's typed stream adapter forwards response events to its bounded output +queue and handles cancellation and backpressure. Cancelling a buffered or +streaming call drops its in-flight provider future. + +Unmanaged profiles use Relay's typed continuations for pass-through. The HTTP, +routing, and translation behavior remains in Switchyard; Relay owns the native +dynamic-library boundary. + +## Supported routers + +The plugin supports four libsy routing modes: + +- seeded, weighted `random` routing; and +- capability-based `llm_classifier` routing, where a judge selects the weak or + strong target before the final provider call; +- escalation-mode `llm_classifier` routing, where a judge evaluates the weak + model's completed turn and latches a session to the strong target after a + configured confirmation streak; and +- signal-driven `stage_router` routing, with optional handoff notes, tier + prompts, and a capability-classifier fallback for ambiguous turns. + +Unsupported algorithm kinds are rejected instead of being approximated. + +## Compatibility Matrix + +The following matrix describes the algorithm behavior implemented by the +plugin. `Conditional` means that the feature is implemented with the constraint +shown in the table; it does not mean that the feature falls back to a different +algorithm. + +| Compatibility Area | `random` | `llm_classifier` (`capability`) | `llm_classifier` (`escalation`) | `stage_router` | +|---|---|---|---|---| +| Version-2 configuration and static validation | Supported | Supported | Supported | Supported | +| Caller protocols | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | +| Serving-target protocols | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | +| Structured-output judge protocols | Not applicable | OpenAI Chat or OpenAI Responses | OpenAI Chat or OpenAI Responses | OpenAI Chat or OpenAI Responses for the optional classifier | +| Buffered responses | Supported | Supported | Supported | Supported | +| Streaming responses | Supported | Supported after the judge selects a target | Conditional: an unlatched weak stream is aggregated before the judge runs | Supported after the signal cascade selects a target | +| Retained routing state | No selection affinity; context-overflow eviction can use session identity | Optional session affinity and message-hash fallback | Confirmation streak and strong latch require stable session identity | No classifier affinity; context-overflow eviction can use session identity | +| Router-specific prompts | Not applicable | Optional judge prompt | Optional escalation-judge prompt | Optional tier prompts, handoff notes, and classifier prompt | +| Relay decision marks | Algorithm, attempt, selected target, and identity | Algorithm, attempt, selected target, and identity | Algorithm, attempt, selected target, and identity | Algorithm, attempt, selected target, and identity | +| ATOF routing-LLM usage | Not applicable unless a failed candidate is replaced | Judge calls, plus failed candidates | Judge calls and discarded weak candidates | Optional classifier judge calls, plus failed candidates | + +Anthropic Messages is supported for callers and serving targets, but not for a +structured-output judge. That restriction is intentional and fails during +static configuration loading. Same-protocol streaming preserves parsed provider +events when the router does not aggregate or replace them; raw SSE bytes and +framing are not part of the compatibility contract. + +### Known issue: OpenAI Responses structured-output judges + +OpenAI Responses targets are accepted for structured-output judges, but the +shared Responses request encoder currently emits the Chat-compatible JSON +Schema object directly under `text.format`. This places `name`, `schema`, and +`strict` under `text.format.json_schema`; conforming Responses endpoints expect +those fields directly under `text.format`. InferenceHub therefore returns HTTP +400 with `Missing required parameter: 'text.format.name'`, and the affected +router follows its existing judge-failure or fall-open path. + +OpenAI Responses remains supported as a caller and ordinary serving-target +protocol. Until the shared `switchyard-translation` encoder is corrected, +configure structured-output judges with `protocol = "openai_chat"`. Follow-up +work must add the inverse of the existing Responses-to-neutral schema conversion +plus core and process-level regression coverage for all three affected router +paths. + +Managed inner provider calls also do not re-enter Relay's downstream provider +middleware. This behavior is part of the current ownership boundary, not an +automatic compatibility fallback. + +Each completed routing-only model call emits a +`switchyard.routing.llm_call` ATOF mark. Its data identifies the algorithm, +attempt, call order, target, routing role, outcome, +latency, and normalized provider token `usage`. The +successful call that serves the caller is deliberately excluded because +Relay's outer LLM end event already records that usage. A failed call, or a +provider response that omits usage, has `usage = null`. Consumers can therefore +add these marks to the outer LLM usage to measure total request compute without +double-counting the serving model. + +`switchyard-llm-client` owns provider retries. Every target, including the +trusted fallback target, uses its default of two additional attempts for +transient provider failures and honors capped `Retry-After` delays. A retry +stays on the selected target and does not rerun the routing algorithm. A random +target with `weight = 0` is fallback-only and is not considered by the +algorithm. Trusted fallback is attempted at most once and, for streaming +responses, only before the first caller event is emitted. + +## Translation and stream fidelity + +`switchyard-translation` is the only request, response, and event translation +layer. It decodes caller JSON into Switchyard's neutral protocol, encodes each +selected call for the target protocol, decodes provider results, and encodes +`ReturnToAgent` back to the caller protocol. Relay codecs are not used. + +The streaming contract carries each parsed provider JSON event in a preservation +envelope alongside its normalized `LlmResponseChunk` representation. +Same-protocol routes replay the preserved JSON unchanged, including +provider-specific fields; this preserves parsed events, not raw SSE bytes or +framing. Cross-protocol routes encode only normalized chunks, and the streaming +helpers still do not expose the buffered translation engine's reject-lossy +diagnostics, so unsupported fields may be normalized or omitted. Replacing +normalized stream content or folding a stream into an aggregate drops the +per-event preservation envelope. + +## Configuration + +The manifest declares `compat.native_api = "1"` and Relay `>=0.8.0,<1.0`. +The manifest API value selects Relay's native plugin contract; the binary uses +Relay 0.8's typed middleware API and native host ABI v4. Rebuild the bundle +when changing SDK versions rather than assuming Rust dynamic-library +compatibility from the manifest value alone. + +A Relay project can configure a seeded weighted-random router as follows: + +```toml +version = 1 + +[[plugins.dynamic]] +manifest = "/opt/switchyard-relay-plugin/relay-plugin.toml" + +[plugins.dynamic.config] +version = 2 +priority = 0 +[plugins.dynamic.config.algorithm] +kind = "random" +seed = 42 + +[plugins.dynamic.config.default_targets] +openai_chat = "fast" + +[plugins.dynamic.config.targets.fast] +model = "provider/model" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" +weight = 1 +drop_caller_extra_body = true + +[plugins.dynamic.config.targets.fast.header_env] +authorization = "PROVIDER_AUTHORIZATION" +``` + +Target map keys such as `fast` are stable semantic names visible to libsy. The +target binding is authoritative for the provider model, protocol, endpoint, +base URL, weight, and environment-backed headers. Each `default_targets` key +both enables that inbound protocol and names its trusted fallback. + +`header_env` is the only custom provider-header source. It resolves values in +the plugin process at registration time so literal header values never appear +in configuration. Environment values must not appear in errors, routing marks, +spans, or debug output. The plugin does not inherit caller credentials for +managed calls. Each variable supplies the complete header value, so an +`authorization` value must include its scheme, such as `Bearer`. Literal +`headers` configuration is rejected; non-secret routing or tenancy headers must +also use `header_env`. + +Relay may intercept an OpenAI SDK call before the SDK materializes its +`extra_body` option into a provider request. Targets that reject this +caller-specific wrapper can set `drop_caller_extra_body = true`. The plugin +then drops the wrapper and its contents; it does not promote those values to +top-level provider fields. The default is `false` so lossless same-format +forwarding remains unchanged for targets that consume the extension. + +`extra_body` supplies non-secret provider defaults for a target. It is useful +for provider-specific controls such as disabling reasoning on a dedicated +judge model. Fields already present on the caller's request take precedence. +Do not put credentials in `extra_body`; use `header_env` for secrets. + +For `kind = "llm_classifier"`, the classifier target must use `openai_chat` or +`openai_responses`; libsy's judge request uses a JSON-schema response format +that cannot be represented losslessly by Anthropic Messages. Omitting `mode` +selects `capability`, preserving the original version-2 configuration shape. + +Escalation mode evaluates the weak model's completed response before returning +it or replacing it with a strong-model response: + +```toml +[plugins.dynamic.config.algorithm] +kind = "llm_classifier" +mode = "escalation" +classifier_target = "judge" +weak_target = "weak" +strong_target = "strong" +prompt = "Judge whether the weak model is stuck." +max_output_tokens = 512 + +[plugins.dynamic.config.algorithm.escalation] +confirmations = 2 +recent_turn_window = 28 +window_message_chars = 500 +``` + +`judge`, `weak`, and `strong` are keys in +`plugins.dynamic.config.targets`, configured with the same model, protocol, +URL, and `header_env` fields shown above. The judge must use `openai_chat` or +`openai_responses`; the serving targets may use any supported protocol. + +Use a dedicated, non-reasoning model for the judge when possible. Providers +that expose a reasoning switch can configure it on that target, for example: + +```toml +[plugins.dynamic.config.targets.judge] +model = "provider/non-reasoning-judge" +protocol = "openai_chat" +base_url = "https://provider.example.com" +extra_body = { think = false } +``` + +The packaged escalation rubric is intentionally detailed and can consume +roughly two thousand or more input tokens depending on the tokenizer. Every +unlatched request also pays for a complete judge call. A custom `prompt` can +reduce that cost, but should be evaluated against representative trajectories +before deployment. Reasoning models may spend `max_output_tokens` on hidden or +visible reasoning before returning the structured verdict; disable reasoning +with provider-supported `extra_body` controls or raise the cap after measuring. + +An unlatched streaming escalation request is intentionally buffered. Libsy must +read the complete weak response before asking the judge, so caller first-token +delivery waits for the weak call and judge verdict. A declined escalation is +reconstructed as a stream from the aggregate response, which drops the +provider-event preservation envelope. A confirmed escalation discards that +weak response and serves the strong target. + +The default `confirmations = 2` retains a streak per Switchyard session. Callers +must send a stable `x-switchyard-session-id` header for the streak and strong +latch to survive across turns. Without session identity each request has +isolated state and a multi-confirmation escalation cannot latch. + +A full stage router can combine tool-result signals, model-specific prompts, +handoff notes, and an optional judge for ambiguous turns: + +```toml +[plugins.dynamic.config.algorithm] +kind = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 0.5 +recent_turn_window = 3 +capable_system_prompt = "Diagnose before editing." +efficient_system_prompt = "Follow the settled plan." + +[plugins.dynamic.config.algorithm.handoff_notes] +escalation_note = "The previous model was stalling; pick up the diagnosis." +deescalation_note = "The task is settled; continue with the mechanical work." +only_on_wrong_signal_escalation = true + +[plugins.dynamic.config.algorithm.classifier] +target = "judge" +base_threshold = 0.5 +threshold_step = 0.1 +recent_turn_window = 3 +prompt = "Estimate whether the efficient target can finish this turn." +max_output_tokens = 512 +``` + +Stage routing reads normalized tool calls and tool results from OpenAI Chat, +OpenAI Responses, and Anthropic Messages traffic. When the signals do not cross +`confidence_threshold`, the optional classifier decides; if it is absent or +cannot decide, the configured picker's default tier serves the turn. The +classifier target has the same structured-output protocol restriction as the +standalone classifier. + +Ambiguous turns that reach the optional classifier add one judge call; +decisive tool signals do not. Decision marks report the selected model from +libsy's `RoutingOutcome`. + +Version-1 service configuration, decision-only execution, and observe-only +mode are rejected. + +## Build and bundle + +The crate is a non-publishable member of the Switchyard Cargo workspace. +Operators install a binary bundle rather than a Rust crate: + +```bash +cargo build --release \ + --manifest-path crates/switchyard-nemo-relay-plugin/Cargo.toml +python3 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py \ + --library target/release/libswitchyard_nemo_relay_plugin.so \ + --output build/switchyard-nemo-relay-plugin-linux-x86_64 \ + --archive dist/switchyard-nemo-relay-plugin-0.2.0-linux-x86_64.tar.gz +``` + +On macOS the library suffix is `.dylib`; Windows builds use `.dll`. The bundle +builder creates the Relay package: the shared library, a materialized manifest +with Relay's inline SHA-256 integrity digest, the JSON schema, and the project +license files. Use `.tar.gz` archives on Linux and macOS and `.zip` on Windows. +The archive's top-level directory is always `switchyard-nemo-relay-plugin`. + +The release archive convention is +`switchyard-nemo-relay-plugin--.`. A future Actions +matrix should upload each archive under the artifact name +`switchyard-nemo-relay-plugin-`, matching Switchyard's existing +platform-qualified artifact convention. + +Install the materialized bundle with Relay's normal lifecycle commands: + +```bash +nemo-relay plugins validate /opt/switchyard-relay-plugin/relay-plugin.toml +nemo-relay plugins add /opt/switchyard-relay-plugin/relay-plugin.toml +nemo-relay plugins enable nvidia.switchyard +nemo-relay plugins inspect nvidia.switchyard +``` diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json new file mode 100644 index 000000000..8ea12cece --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -0,0 +1,210 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Switchyard NeMo Relay Plugin", + "description": "In-process Switchyard routing with Switchyard-owned provider HTTP dispatch.", + "type": "object", + "additionalProperties": false, + "required": ["version", "algorithm", "targets", "default_targets"], + "properties": { + "version": { + "const": 2, + "description": "Library-only Switchyard configuration version." + }, + "priority": { + "type": "integer", + "default": 0 + }, + "algorithm": { + "description": "In-process random, capability, escalation, or stage-router configuration.", + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { "const": "random" }, + "seed": { "type": ["integer", "null"], "minimum": 0 } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "classifier_target", + "weak_target", + "strong_target", + "base_threshold" + ], + "properties": { + "kind": { "const": "llm_classifier" }, + "mode": { "const": "capability", "default": "capability" }, + "classifier_target": { + "type": "string", + "minLength": 1, + "description": "Semantic target name for the judge. The target must use openai_chat or openai_responses because the judge requires a JSON-schema response format." + }, + "weak_target": { "type": "string", "minLength": 1 }, + "strong_target": { "type": "string", "minLength": 1 }, + "base_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "threshold_step": { "type": "number", "minimum": 0, "default": 0 }, + "recent_turn_window": { + "type": ["integer", "null"], + "minimum": 0 + }, + "max_output_tokens": { + "type": "integer", + "minimum": 1, + "default": 4096 + }, + "prompt": { "type": "string" }, + "session_affinity": { "type": "boolean", "default": false }, + "message_hash_fallback": { "type": "boolean", "default": false } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "mode", + "classifier_target", + "weak_target", + "strong_target", + "escalation" + ], + "properties": { + "kind": { "const": "llm_classifier" }, + "mode": { "const": "escalation" }, + "classifier_target": { + "type": "string", + "minLength": 1, + "description": "Semantic target name for the trajectory judge. The target must use openai_chat or openai_responses." + }, + "weak_target": { "type": "string", "minLength": 1 }, + "strong_target": { "type": "string", "minLength": 1 }, + "prompt": { "type": "string" }, + "max_output_tokens": { + "type": "integer", + "minimum": 1, + "default": 4096 + }, + "escalation": { + "type": "object", + "additionalProperties": false, + "properties": { + "confirmations": { "type": "integer", "minimum": 1, "default": 2 }, + "recent_turn_window": { "type": "integer", "minimum": 1, "default": 28 }, + "window_message_chars": { "type": "integer", "minimum": 50, "default": 500 } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "capable_target", + "efficient_target", + "picker", + "confidence_threshold" + ], + "properties": { + "kind": { "const": "stage_router" }, + "capable_target": { "type": "string", "minLength": 1 }, + "efficient_target": { "type": "string", "minLength": 1 }, + "picker": { "enum": ["capable_first", "efficient_first"] }, + "confidence_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "recent_turn_window": { + "type": ["integer", "null"], + "minimum": 0, + "description": "Trailing tool results scored for the current turn. Null uses libsy's default window." + }, + "capable_system_prompt": { "type": "string" }, + "efficient_system_prompt": { "type": "string" }, + "handoff_notes": { + "type": "object", + "additionalProperties": false, + "required": ["escalation_note"], + "properties": { + "escalation_note": { "type": "string", "minLength": 1 }, + "deescalation_note": { "type": ["string", "null"], "minLength": 1 }, + "only_on_wrong_signal_escalation": { "type": "boolean", "default": true } + } + }, + "classifier": { + "type": "object", + "additionalProperties": false, + "required": ["target", "base_threshold"], + "properties": { + "target": { + "type": "string", + "minLength": 1, + "description": "Judge target used only when stage signals are ambiguous. It must use openai_chat or openai_responses." + }, + "base_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "threshold_step": { "type": "number", "minimum": 0, "default": 0 }, + "recent_turn_window": { "type": ["integer", "null"], "minimum": 0 }, + "prompt": { "type": "string" }, + "max_output_tokens": { "type": "integer", "minimum": 1, "default": 4096 } + } + } + } + } + ] + }, + "targets": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["model", "protocol", "base_url"], + "properties": { + "model": { "type": "string", "minLength": 1 }, + "protocol": { + "enum": ["openai_chat", "openai_responses", "anthropic_messages"] + }, + "endpoint": { + "type": "string", + "pattern": "^$|^/", + "description": "Optional provider endpoint override. The resolved URL must end in the canonical route for the selected protocol." + }, + "base_url": { + "type": "string", + "pattern": "^https?://" + }, + "weight": { "type": "number", "minimum": 0, "default": 1 }, + "drop_caller_extra_body": { + "type": "boolean", + "default": false, + "description": "Drop an intercepted OpenAI SDK extra_body wrapper instead of forwarding it to targets that reject caller-specific extensions." + }, + "extra_body": { + "type": "object", + "default": {}, + "description": "Non-secret provider request defaults, such as judge reasoning controls. Caller-provided fields take precedence.", + "additionalProperties": true + }, + "header_env": { + "type": "object", + "description": "Sole custom provider-header source. Maps header names to environment-variable names resolved by the plugin process so literal values are never stored in configuration.", + "additionalProperties": { "type": "string", "minLength": 1 } + } + } + } + }, + "default_targets": { + "type": "object", + "description": "Maps each managed inbound protocol to its trusted fallback target.", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "openai_chat": { "type": "string", "minLength": 1 }, + "openai_responses": { "type": "string", "minLength": 1 }, + "anthropic_messages": { "type": "string", "minLength": 1 } + } + } + } +} diff --git a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml new file mode 100644 index 000000000..ba5992bd6 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +manifest_version = 1 + +[plugin] +id = "nvidia.switchyard" +kind = "rust_dynamic" + +[compat] +relay = ">=0.8.0,<1.0" +native_api = "1" + +[defaults] +enabled = false + +[capabilities] +items = ["plugin_native", "config_schema"] + +[config_schema] +path = "config.schema.json" + +[source] +artifact = "" + +[integrity] +sha256 = "sha256:" + +[load] +library = "" +symbol = "nemo_relay_register_plugin" diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py new file mode 100644 index 000000000..c5ea9e946 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Materialize the minimal Relay plugin bundle from a compiled cdylib.""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +import tarfile +import zipfile +from pathlib import Path + +CRATE_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = CRATE_ROOT.parents[1] +PACKAGE_NAME = "switchyard-nemo-relay-plugin" + + +def digest(path: Path) -> str: + """Return the lowercase SHA-256 digest for a file.""" + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def archive_bundle(bundle: Path, archive: Path) -> None: + """Archive a materialized bundle under the stable package directory name.""" + archive.parent.mkdir(parents=True, exist_ok=True) + if archive.exists(): + raise ValueError(f"bundle archive already exists: {archive}") + + if archive.name.endswith(".tar.gz"): + with tarfile.open(archive, "w:gz") as stream: + stream.add(bundle, arcname=PACKAGE_NAME) + return + + if archive.suffix == ".zip": + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as stream: + for path in sorted(bundle.rglob("*")): + if path.is_file(): + stream.write(path, Path(PACKAGE_NAME) / path.relative_to(bundle)) + return + + raise ValueError("bundle archive must end in .tar.gz or .zip") + + +def main() -> None: + """Materialize a Relay-loadable plugin bundle in an empty directory.""" + parser = argparse.ArgumentParser() + parser.add_argument("--library", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--archive", type=Path) + args = parser.parse_args() + + library = args.library.resolve() + if not library.is_file(): + parser.error(f"compiled plugin library does not exist: {library}") + + manifest = (CRATE_ROOT / "relay-plugin.toml").read_text(encoding="utf-8") + placeholders = ("", "") + missing = [placeholder for placeholder in placeholders if placeholder not in manifest] + if missing: + parser.error(f"plugin manifest is missing placeholders: {', '.join(missing)}") + + output = args.output.resolve() + if output.exists() and not output.is_dir(): + parser.error(f"bundle output exists and is not a directory: {output}") + if output.is_dir() and any(output.iterdir()): + parser.error(f"bundle output directory must be empty: {output}") + output.mkdir(parents=True, exist_ok=True) + + artifact = output / library.name + shutil.copy2(library, artifact) + shutil.copy2(CRATE_ROOT / "config.schema.json", output / "config.schema.json") + for filename in ("LICENSE", "NOTICE"): + shutil.copy2(REPOSITORY_ROOT / filename, output / filename) + + artifact_digest = digest(artifact) + manifest = manifest.replace("", artifact.name) + manifest = manifest.replace("", artifact_digest) + (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8") + + if args.archive is None: + print(output) + return + + archive = args.archive.resolve() + try: + archive_bundle(output, archive) + except ValueError as error: + parser.error(str(error)) + print(archive) + + +if __name__ == "__main__": + main() diff --git a/crates/switchyard-nemo-relay-plugin/src/client.rs b/crates/switchyard-nemo-relay-plugin/src/client.rs new file mode 100644 index 000000000..95d7a7a14 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/client.rs @@ -0,0 +1,296 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Switchyard-owned target adapters over a shared HTTP client. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::Value as Json; +use switchyard_llm_client::{ + Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, TranslatingLlmClient, +}; +use switchyard_protocol::{ + LlmClientError, ModelId, Request, Response, RoutedLlmClient, WireFormat, +}; +use switchyard_translation::TranslationEngine; + +use crate::translation; + +/// A provider target adapter over a shared HTTP client. +/// +/// libsy routes with a stable semantic name (for example `fast`). The provider +/// still expects its own model id (for example `meta/llama-3.1-8b-instruct`). +/// Keeping that mapping here prevents an algorithm's semantic labels from +/// leaking into provider requests. +pub(crate) struct TargetClient { + provider_model: ModelId, + target_format: WireFormat, + drop_caller_extra_body: bool, + inner: Arc, + translation: TranslationEngine, +} + +impl TargetClient { + pub(crate) fn model_config( + provider_model: String, + target_format: WireFormat, + dispatch_url: String, + headers: BTreeMap, + extra_body: BTreeMap, + ) -> ModelConfig { + let backend_config = HttpBackendConfig { + // `dispatch_url` is already resolved by configuration. Backend URL + // joining accepts a complete canonical endpoint as well as a base + // URL/prefix. + base_url: dispatch_url, + api_key: None, + forward_auth: false, + extra_headers: headers, + extra_body, + max_retries: DEFAULT_MAX_RETRIES, + }; + let backend = match target_format { + WireFormat::OpenAiChat => Backend::OpenAiChat(backend_config), + WireFormat::OpenAiResponses => Backend::OpenAiResponses(backend_config), + WireFormat::AnthropicMessages => Backend::Anthropic(backend_config), + }; + ModelConfig::new(provider_model, backend, None) + } + + pub(crate) fn new( + provider_model: ModelId, + target_format: WireFormat, + drop_caller_extra_body: bool, + inner: Arc, + ) -> Self { + Self { + provider_model, + target_format, + drop_caller_extra_body, + inner, + translation: TranslationEngine::default(), + } + } + + /// Retargets only the provider-facing transport metadata. + /// + /// Correlation and agent identity remain available to libsy, while inbound + /// HTTP headers are deliberately removed. Provider credentials come solely + /// from this target's `header_env` configuration. + fn prepare_request(&self, mut request: Request) -> Request { + let metadata = request.metadata.get_or_insert_default(); + metadata.wire_format = Some(self.target_format); + metadata.http_headers = None; + if self.drop_caller_extra_body { + request.llm_request.extensions.fields.remove("extra_body"); + for preserved in request.llm_request.preservation.requests.values_mut() { + if let Some(body) = preserved.as_object_mut() { + body.remove("extra_body"); + } + } + } + request + } +} + +#[async_trait] +impl RoutedLlmClient for TargetClient { + async fn call(&self, request: Request) -> Result { + let request = self.prepare_request(request); + translation::validate_target_request( + &self.translation, + self.target_format, + &request.llm_request, + ) + .map_err(LlmClientError::RequestEncoding)?; + self.inner + .call_rewrite_model(request, Some(&self.provider_model)) + .await + } +} + +#[cfg(test)] +mod tests { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::thread; + + use super::*; + use serde_json::json; + use switchyard_protocol::{ + LlmRequest, LlmResponse, Metadata, PreservationMetadata, ProviderExtensions, text_request, + }; + + fn client(format: WireFormat) -> TargetClient { + client_with_options( + format, + match format { + WireFormat::OpenAiChat => "https://provider.example/v1/chat/completions".into(), + WireFormat::OpenAiResponses => "https://provider.example/v1/responses".into(), + WireFormat::AnthropicMessages => "https://provider.example/v1/messages".into(), + }, + false, + ) + } + + fn client_with_options( + format: WireFormat, + dispatch_url: String, + drop_caller_extra_body: bool, + ) -> TargetClient { + let model = TargetClient::model_config( + "provider/model".into(), + format, + dispatch_url, + BTreeMap::new(), + BTreeMap::new(), + ); + let inner = Arc::new(TranslatingLlmClient::new(&[model]).unwrap()); + TargetClient::new( + ModelId::from("provider/model"), + format, + drop_caller_extra_body, + inner, + ) + } + + #[test] + fn target_preparation_forces_format_and_removes_inbound_headers() { + let client = client(WireFormat::AnthropicMessages); + let request = Request { + metadata: Some(Metadata { + correlation_id: Some("request-123".into()), + wire_format: Some(WireFormat::OpenAiChat), + http_headers: Some(http::HeaderMap::from_iter([ + ( + http::HeaderName::from_static("authorization"), + http::HeaderValue::from_static("Bearer caller-secret"), + ), + ( + http::HeaderName::from_static("x-caller-only"), + http::HeaderValue::from_static("must-not-forward"), + ), + ])), + ..Metadata::default() + }), + ..Request::default() + }; + + let prepared = client.prepare_request(request); + let metadata = prepared.metadata.unwrap(); + assert_eq!(metadata.wire_format, Some(WireFormat::AnthropicMessages)); + assert_eq!(metadata.correlation_id.as_deref(), Some("request-123")); + assert!(metadata.http_headers.is_none()); + } + + #[test] + fn missing_metadata_is_created_for_the_target_format() { + let client = client(WireFormat::OpenAiResponses); + let prepared = client.prepare_request(Request::default()); + assert_eq!( + prepared.metadata.and_then(|metadata| metadata.wire_format), + Some(WireFormat::OpenAiResponses) + ); + } + + #[test] + fn configured_target_drops_intercepted_caller_extra_body() { + let client = client_with_options( + WireFormat::OpenAiChat, + "https://provider.example/v1/chat/completions".into(), + true, + ); + let request = Request { + llm_request: LlmRequest { + extensions: ProviderExtensions { + fields: serde_json::Map::from_iter([( + "extra_body".into(), + json!({"reasoning": {"effort": "medium"}}), + )]), + }, + preservation: PreservationMetadata { + requests: BTreeMap::from([( + WireFormat::OpenAiChat.into(), + json!({ + "model": "route", + "messages": [{"role": "user", "content": "hello"}], + "extra_body": { + "reasoning": {"effort": "medium"}, + "session_id": "hermes-session" + } + }), + )]), + ..PreservationMetadata::default() + }, + ..LlmRequest::default() + }, + ..Request::default() + }; + + let prepared = client.prepare_request(request); + assert!( + !prepared + .llm_request + .extensions + .fields + .contains_key("extra_body") + ); + assert!( + prepared + .llm_request + .preservation + .requests + .values() + .all(|body| body.get("extra_body").is_none()) + ); + } + + #[tokio::test] + async fn target_client_retries_transient_provider_failures() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind provider stub"); + let address = listener.local_addr().expect("read provider stub address"); + let server = thread::spawn(move || -> std::io::Result<()> { + let response_body = "{\"id\":\"chatcmpl-1\",\"model\":\"provider/model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"recovered\"},\"finish_reason\":\"stop\"}],\"usage\":{}}"; + let responses = [ + "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 11\r\nConnection: close\r\n\r\nunavailable".to_string(), + "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 11\r\nConnection: close\r\n\r\nunavailable".to_string(), + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response_body}", + response_body.len() + ), + ]; + for response in responses { + let (mut stream, _) = listener.accept()?; + let mut request = [0_u8; 1024]; + if stream.read(&mut request)? == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "client closed before sending a request", + )); + } + stream.write_all(response.as_bytes())?; + } + Ok(()) + }); + let client = client_with_options( + WireFormat::OpenAiChat, + format!("http://{address}/v1"), + false, + ); + let response = client + .call(Request { + llm_request: text_request(Some("route".into()), "hello"), + ..Request::default() + }) + .await + .expect("third provider attempt succeeds"); + + assert!(matches!(response.llm_response, LlmResponse::Agg(_))); + server + .join() + .expect("provider stub panicked") + .expect("provider stub failed"); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs new file mode 100644 index 000000000..e280de103 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -0,0 +1,606 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use http::Uri; +use http::header::{HeaderName, HeaderValue}; +use serde::Deserialize; +use serde_json::Value as Json; +use switchyard_libsy::{ + Algorithm, ClassifierContractConfig, EscalationJudgeConfig, HandoffNoteConfig, + LlmClassifierConfig, LlmFallback, LlmTaskClassifier, PickerMode, Random, StageRouter, + StageRouterConfig, TargetPrompts, TaskClassifierConfig, +}; +use switchyard_llm_client::{ModelConfig, TranslatingLlmClient}; +use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat}; + +use crate::client::TargetClient; + +pub(crate) fn protocol_from_call(name: &str) -> Option { + match name { + "openai.chat_completions" => Some(WireFormat::OpenAiChat), + "openai.responses" => Some(WireFormat::OpenAiResponses), + "anthropic.messages" => Some(WireFormat::AnthropicMessages), + _ => None, + } +} + +const fn default_endpoint(protocol: WireFormat) -> &'static str { + match protocol { + WireFormat::OpenAiChat => "/v1/chat/completions", + WireFormat::OpenAiResponses => "/v1/responses", + WireFormat::AnthropicMessages => "/v1/messages", + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TargetBinding { + model: String, + protocol: WireFormat, + #[serde(default)] + endpoint: String, + base_url: String, + #[serde(default = "default_weight")] + weight: f64, + #[serde(default)] + drop_caller_extra_body: bool, + #[serde(default)] + header_env: BTreeMap, + #[serde(default)] + extra_body: BTreeMap, +} + +impl TargetBinding { + fn dispatch_url(&self) -> String { + let base = self.base_url.trim_end_matches('/'); + let default = default_endpoint(self.protocol); + if self.endpoint.is_empty() && base.ends_with(default) { + return base.to_string(); + } + let endpoint = if self.endpoint.is_empty() { + default + } else { + &self.endpoint + }; + let endpoint = if base.ends_with("/v1") && endpoint.starts_with("/v1/") { + &endpoint[3..] + } else { + endpoint + }; + format!("{base}{endpoint}") + } + + fn validate(&self, name: &str) -> Result<(), String> { + if self.model.trim().is_empty() { + return Err(format!("target {name:?} model must be non-empty")); + } + if !self.endpoint.is_empty() && !self.endpoint.starts_with('/') { + return Err(format!( + "target {name:?} endpoint must be empty or begin with '/'" + )); + } + if !self.weight.is_finite() || self.weight < 0.0 { + return Err(format!( + "target {name:?} weight must be finite and nonnegative" + )); + } + validate_dispatch_url(name, self.protocol, &self.dispatch_url())?; + self.validate_headers(name) + } + + fn validate_headers(&self, target_name: &str) -> Result<(), String> { + let mut normalized = BTreeSet::new(); + for (name, variable) in &self.header_env { + let canonical = validate_header_name(name)?; + if !normalized.insert(canonical) { + return Err(format!( + "target {target_name:?} configures header {name:?} more than once (header names are case-insensitive)" + )); + } + if variable.trim().is_empty() { + return Err(format!( + "environment variable name for target header {name:?} must not be empty" + )); + } + if variable.as_bytes().contains(&b'=') || variable.as_bytes().contains(&b'\0') { + return Err(format!( + "environment variable name for target header {name:?} must not contain '=' or NUL" + )); + } + } + Ok(()) + } + + fn prepare(&self) -> Result { + let mut headers = BTreeMap::new(); + for (name, variable) in &self.header_env { + let value = std::env::var(variable) + .map_err(|_| format!("environment variable {variable:?} is not set"))?; + validate_header(name, &value)?; + headers.insert(name.clone(), value); + } + let model_config = TargetClient::model_config( + self.model.clone(), + self.protocol, + self.dispatch_url(), + headers, + self.extra_body.clone(), + ); + Ok(PreparedTargetTransport { + provider_model: ModelId::from(self.model.clone()), + protocol: self.protocol, + drop_caller_extra_body: self.drop_caller_extra_body, + model_config, + }) + } +} + +struct PreparedTargetTransport { + provider_model: ModelId, + protocol: WireFormat, + drop_caller_extra_body: bool, + model_config: ModelConfig, +} + +pub(crate) struct PreparedTargetBinding { + pub(crate) client: Arc, +} + +#[derive(Clone, Copy, Default, Deserialize)] +#[serde(rename_all = "snake_case")] +enum LlmClassifierMode { + #[default] + Capability, + Escalation, +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct LlmClassifierAlgorithmConfig { + #[serde(default)] + mode: LlmClassifierMode, + classifier_target: String, + weak_target: String, + strong_target: String, + #[serde(default)] + base_threshold: Option, + #[serde(default)] + threshold_step: Option, + #[serde(default)] + session_affinity: Option, + #[serde(default)] + message_hash_fallback: Option, + #[serde(default)] + recent_turn_window: Option, + #[serde(default)] + prompt: Option, + #[serde(default = "default_classifier_max_output_tokens")] + max_output_tokens: u64, + #[serde(default)] + escalation: Option, +} + +impl LlmClassifierAlgorithmConfig { + fn capability_config(&self) -> Result { + if self.escalation.is_some() { + return Err( + "llm_classifier capability mode does not accept escalation settings".into(), + ); + } + let base_threshold = self + .base_threshold + .ok_or_else(|| "llm_classifier capability mode requires base_threshold".to_string())?; + let mut contract = ClassifierContractConfig::default(); + if let Some(prompt) = &self.prompt { + contract = contract.with_prompt(prompt.clone()); + } + Ok(TaskClassifierConfig { + base_threshold, + threshold_step: self.threshold_step.unwrap_or_default(), + session_affinity: self.session_affinity.unwrap_or_default(), + message_hash_fallback: self.message_hash_fallback.unwrap_or_default(), + recent_turn_window: self.recent_turn_window, + contract, + max_output_tokens: self.max_output_tokens, + }) + } + + fn escalation_config( + &self, + ) -> Result<(ClassifierContractConfig, EscalationJudgeConfig), String> { + if self.base_threshold.is_some() + || self.threshold_step.is_some() + || self.session_affinity.is_some() + || self.message_hash_fallback.is_some() + || self.recent_turn_window.is_some() + { + return Err( + "llm_classifier escalation mode does not accept capability settings".into(), + ); + } + let config = self.escalation.clone().ok_or_else(|| { + "llm_classifier escalation mode requires escalation settings".to_string() + })?; + let mut contract = ClassifierContractConfig::default(); + if let Some(prompt) = &self.prompt { + contract = contract.with_prompt(prompt.clone()); + } + Ok((contract, config)) + } +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct StageFallbackConfig { + target: String, + base_threshold: f64, + #[serde(default)] + threshold_step: f64, + #[serde(default)] + recent_turn_window: Option, + #[serde(default)] + prompt: Option, + #[serde(default = "default_classifier_max_output_tokens")] + max_output_tokens: u64, +} + +impl StageFallbackConfig { + fn classifier_config(&self) -> TaskClassifierConfig { + let mut contract = ClassifierContractConfig::default(); + if let Some(prompt) = &self.prompt { + contract = contract.with_prompt(prompt.clone()); + } + TaskClassifierConfig { + base_threshold: self.base_threshold, + threshold_step: self.threshold_step, + session_affinity: false, + message_hash_fallback: false, + recent_turn_window: self.recent_turn_window, + contract, + max_output_tokens: self.max_output_tokens, + } + } +} + +#[derive(Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum AlgorithmConfig { + Random { + #[serde(default)] + seed: Option, + }, + LlmClassifier { + #[serde(flatten)] + config: LlmClassifierAlgorithmConfig, + }, + StageRouter { + capable_target: String, + efficient_target: String, + picker: PickerMode, + confidence_threshold: f64, + #[serde(default)] + recent_turn_window: Option, + #[serde(default)] + capable_system_prompt: Option, + #[serde(default)] + efficient_system_prompt: Option, + #[serde(default)] + handoff_notes: Option, + #[serde(default)] + classifier: Option, + }, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SwitchyardConfig { + version: u32, + #[serde(default)] + pub(crate) priority: i32, + algorithm: AlgorithmConfig, + targets: BTreeMap, + default_targets: BTreeMap, +} + +pub(crate) struct PreparedConfig { + pub(crate) algorithm: Arc, + pub(crate) targets: BTreeMap, + pub(crate) default_targets: BTreeMap, +} + +impl SwitchyardConfig { + pub(crate) fn validate(&self) -> Result<(), String> { + self.validate_structure()?; + self.build_algorithm(None).map(drop) + } + + fn validate_structure(&self) -> Result<(), String> { + if self.version != 2 { + return Err(format!( + "unsupported Switchyard config version {}; version 1 used switchyard-server; migrate to version = 2", + self.version + )); + } + if self.targets.is_empty() { + return Err("targets must not be empty".into()); + } + if self.default_targets.is_empty() { + return Err("default_targets must not be empty".into()); + } + for (name, target) in &self.targets { + if name.trim().is_empty() { + return Err("target names must be non-empty".into()); + } + target.validate(name)?; + } + for (protocol, fallback) in &self.default_targets { + let target = self + .targets + .get(fallback) + .ok_or_else(|| format!("default target {fallback:?} is not configured"))?; + if target.protocol != *protocol { + return Err(format!( + "default target {fallback:?} must use protocol {}", + protocol.as_str() + )); + } + } + Ok(()) + } + + pub(crate) fn prepare(self) -> Result { + self.validate_structure()?; + let transports = self + .targets + .iter() + .map(|(name, target)| target.prepare().map(|prepared| (name.clone(), prepared))) + .collect::, _>>()?; + let models = transports + .values() + .map(|transport| transport.model_config.clone()) + .collect::>(); + // One multi-model client shares reqwest connection pools across targets. + let client = Arc::new( + TranslatingLlmClient::new(&models) + .map_err(|error| format!("failed to create target HTTP client: {error}"))?, + ); + let targets = transports + .into_iter() + .map(|(name, transport)| { + let target = TargetClient::new( + transport.provider_model, + transport.protocol, + transport.drop_caller_extra_body, + client.clone(), + ); + ( + name, + PreparedTargetBinding { + client: Arc::new(target), + }, + ) + }) + .collect(); + let algorithm = self.build_algorithm(Some(&targets))?; + Ok(PreparedConfig { + algorithm, + targets, + default_targets: self.default_targets, + }) + } + + fn build_algorithm( + &self, + prepared: Option<&BTreeMap>, + ) -> Result, String> { + let target = |name: &str| { + if !self.targets.contains_key(name) { + return Err(format!("algorithm target {name:?} is not configured")); + } + Ok(match prepared { + Some(targets) => { + targets + .get(name) + .ok_or_else(|| format!("algorithm target {name:?} was not prepared"))?; + ModelId::from(name) + } + None => ModelId::from(name), + }) + }; + + match &self.algorithm { + AlgorithmConfig::Random { seed } => { + let routable = self + .targets + .iter() + .filter(|(_, binding)| binding.weight > 0.0) + .collect::>(); + if routable.is_empty() { + return Err( + "random routing requires at least one positive target weight".into(), + ); + } + let targets = routable + .iter() + .map(|(name, _)| target(name)) + .collect::, _>>()?; + let weights = routable + .iter() + .map(|(_, binding)| binding.weight) + .collect::>(); + Random::new(targets, Some(weights), *seed) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } + AlgorithmConfig::LlmClassifier { config } => { + self.validate_judge_target(&config.classifier_target)?; + let algorithm = match config.mode { + LlmClassifierMode::Capability => LlmClassifierConfig::Capability { + judge_target: target(&config.classifier_target)?, + efficient_target: target(&config.weak_target)?, + capable_target: target(&config.strong_target)?, + config: config.capability_config()?, + }, + LlmClassifierMode::Escalation => { + let (contract, escalation) = config.escalation_config()?; + LlmClassifierConfig::Escalation { + judge_target: target(&config.classifier_target)?, + efficient_target: target(&config.weak_target)?, + capable_target: target(&config.strong_target)?, + contract, + config: escalation, + max_output_tokens: config.max_output_tokens, + } + } + }; + LlmTaskClassifier::new(algorithm) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } + AlgorithmConfig::StageRouter { + capable_target, + efficient_target, + picker, + confidence_threshold, + recent_turn_window, + capable_system_prompt, + efficient_system_prompt, + handoff_notes, + classifier, + } => { + let capable = target(capable_target)?; + let efficient = target(efficient_target)?; + let mut config = StageRouterConfig::new(*picker, *confidence_threshold); + config.recent_window = *recent_turn_window; + config.handoff_notes = handoff_notes.clone(); + let mut prompts = TargetPrompts::default(); + if let Some(prompt) = capable_system_prompt { + prompts = prompts.with(capable_target.as_str(), prompt); + } + if let Some(prompt) = efficient_system_prompt { + prompts = prompts.with(efficient_target.as_str(), prompt); + } + config.tier_prompts = prompts; + if let Some(classifier) = classifier { + self.validate_judge_target(&classifier.target)?; + config.llm_fallback = Some(LlmFallback { + judge_target: target(&classifier.target)?, + config: classifier.classifier_config(), + }); + } + StageRouter::new(capable, efficient, config) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } + } + } + + fn validate_judge_target(&self, name: &str) -> Result<(), String> { + let binding = self + .targets + .get(name) + .ok_or_else(|| format!("algorithm target {name:?} is not configured"))?; + if binding.protocol == WireFormat::AnthropicMessages { + return Err(format!( + "classifier target {name:?} uses anthropic_messages, which cannot encode the required JSON-schema response format without loss; use an openai_chat or openai_responses target" + )); + } + Ok(()) + } +} + +fn validate_dispatch_url( + target_name: &str, + protocol: WireFormat, + dispatch_url: &str, +) -> Result<(), String> { + let uri = dispatch_url + .parse::() + .map_err(|error| format!("target {target_name:?} has invalid URL: {error}"))?; + if !matches!(uri.scheme_str(), Some("http" | "https")) { + return Err(format!( + "target {target_name:?} base_url must use http or https" + )); + } + let authority = uri + .authority() + .ok_or_else(|| format!("target {target_name:?} URL must include a host"))?; + if authority.host().is_empty() { + return Err(format!("target {target_name:?} URL must include a host")); + } + if authority.as_str().contains('@') { + return Err(format!( + "target {target_name:?} URL must not contain embedded credentials" + )); + } + if uri.query().is_some() { + return Err(format!( + "target {target_name:?} URL query parameters are not supported" + )); + } + + // The current switchyard-llm-client accepts provider base URLs and complete + // canonical endpoints. Reject a custom terminal route to avoid + // allowing Backend::url() to append another provider suffix silently. + let expected_suffix = match protocol { + WireFormat::OpenAiChat => "/chat/completions", + WireFormat::OpenAiResponses => "/responses", + WireFormat::AnthropicMessages => "/v1/messages", + }; + if !uri.path().ends_with(expected_suffix) { + return Err(format!( + "target {target_name:?} endpoint must resolve to a canonical {protocol} route ending in {expected_suffix:?}" + )); + } + Ok(()) +} + +fn validate_header_name(name: &str) -> Result { + let parsed = HeaderName::from_bytes(name.as_bytes()) + .map_err(|error| format!("invalid target header name {name:?}: {error}"))?; + let canonical = parsed.as_str().to_ascii_lowercase(); + if is_forbidden_target_header(&canonical) { + return Err(format!( + "target header {name:?} is controlled by the HTTP transport and cannot be configured" + )); + } + Ok(canonical) +} + +fn validate_header(name: &str, value: &str) -> Result { + let canonical = validate_header_name(name)?; + HeaderValue::from_str(value) + .map_err(|error| format!("invalid target header value for {name:?}: {error}"))?; + Ok(canonical) +} + +fn is_forbidden_target_header(name: &str) -> bool { + matches!( + name, + "connection" + | "content-length" + | "host" + | "keep-alive" + | "proxy-connection" + | "proxy-authenticate" + | "proxy-authorization" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + ) || name.starts_with("x-nemo-relay-internal-") +} + +const fn default_weight() -> f64 { + 1.0 +} + +fn default_classifier_max_output_tokens() -> u64 { + TaskClassifierConfig::default().max_output_tokens +} + +#[cfg(test)] +mod tests; diff --git a/crates/switchyard-nemo-relay-plugin/src/config/tests.rs b/crates/switchyard-nemo-relay-plugin/src/config/tests.rs new file mode 100644 index 000000000..06b5d6d31 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/config/tests.rs @@ -0,0 +1,566 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use serde_json::{Value, json}; + +fn binding(protocol: WireFormat, model: &str) -> TargetBinding { + TargetBinding { + model: model.into(), + protocol, + endpoint: String::new(), + base_url: "https://provider.example/v1".into(), + weight: 1.0, + drop_caller_extra_body: false, + header_env: BTreeMap::new(), + extra_body: BTreeMap::new(), + } +} + +fn config() -> SwitchyardConfig { + SwitchyardConfig { + version: 2, + priority: 0, + algorithm: AlgorithmConfig::Random { seed: Some(42) }, + targets: BTreeMap::from([ + ( + "chat".into(), + binding(WireFormat::OpenAiChat, "provider/chat"), + ), + ( + "responses".into(), + binding(WireFormat::OpenAiResponses, "provider/responses"), + ), + ( + "anthropic".into(), + binding(WireFormat::AnthropicMessages, "provider/anthropic"), + ), + ]), + default_targets: BTreeMap::from([ + (WireFormat::OpenAiChat, "chat".into()), + (WireFormat::OpenAiResponses, "responses".into()), + (WireFormat::AnthropicMessages, "anthropic".into()), + ]), + } +} + +#[test] +fn version_two_random_configuration_builds_clients_without_a_service() { + let config = config(); + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "random"); + assert_eq!(prepared.targets.len(), 3); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); +} + +#[test] +fn target_endpoints_must_be_canonical_for_the_current_http_client() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat".into(); + let error = config.validate().unwrap_err(); + assert!(error.contains("ending in \"/chat/completions\"")); + + config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat/completions".into(); + config.validate().unwrap(); + assert_eq!( + config.targets["chat"].dispatch_url(), + "https://provider.example/v1/custom/chat/completions" + ); +} + +#[test] +fn complete_provider_endpoint_is_not_appended_twice() { + let mut config = config(); + let chat = config.targets.get_mut("chat").unwrap(); + chat.base_url = "https://provider.example/v1/chat/completions/".into(); + assert_eq!( + chat.dispatch_url(), + "https://provider.example/v1/chat/completions" + ); + config.validate().unwrap(); +} + +#[test] +fn absolute_urls_cannot_embed_credentials_or_query_parameters() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().base_url = + "https://user:password@provider.example/v1".into(); + assert!( + config + .validate() + .unwrap_err() + .contains("embedded credentials") + ); + + config.targets.get_mut("chat").unwrap().base_url = + "https://provider.example/v1?api-version=1".into(); + assert!(config.validate().unwrap_err().contains("query parameters")); +} + +#[test] +fn transport_owned_and_case_duplicate_environment_headers_are_rejected() { + let mut host_header_config = config(); + let chat = host_header_config.targets.get_mut("chat").unwrap(); + chat.header_env.insert("Host".into(), "TARGET_HOST".into()); + assert!( + host_header_config + .validate() + .unwrap_err() + .contains("HTTP transport") + ); + + let mut duplicate_config = config(); + let chat = duplicate_config.targets.get_mut("chat").unwrap(); + chat.header_env + .insert("X-Tenant".into(), "TARGET_TENANT_A".into()); + chat.header_env + .insert("x-tenant".into(), "TARGET_TENANT_B".into()); + assert!( + duplicate_config + .validate() + .unwrap_err() + .contains("more than once") + ); +} + +#[test] +fn only_canonical_relay_execution_names_resolve_protocols() { + assert_eq!( + protocol_from_call("openai.chat_completions"), + Some(WireFormat::OpenAiChat) + ); + assert_eq!( + protocol_from_call("openai.responses"), + Some(WireFormat::OpenAiResponses) + ); + assert_eq!( + protocol_from_call("anthropic.messages"), + Some(WireFormat::AnthropicMessages) + ); + assert_eq!(protocol_from_call("openai_chat"), None); +} + +#[test] +fn schema_required_contract_fields_do_not_default_during_deserialization() { + let base = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1" + } + }, + "default_targets": {"openai_chat": "chat"} + }); + for field in ["version", "algorithm", "default_targets"] { + let mut value = base.clone(); + value.as_object_mut().unwrap().remove(field); + let error = serde_json::from_value::(value) + .err() + .expect("required field must not default"); + assert!(error.to_string().contains(field), "field={field}: {error}"); + } +} + +#[test] +fn plugin_retry_budget_is_not_configurable() { + let value = json!({ + "version": 2, + "max_retries": 3, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1" + } + }, + "default_targets": {"openai_chat": "chat"} + }); + + let error = serde_json::from_value::(value) + .err() + .expect("plugin retry budget must be rejected"); + assert!(error.to_string().contains("max_retries")); +} + +#[test] +fn unknown_target_fields_are_rejected() { + let value = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1", + "unexpected_setting": true + } + }, + "default_targets": {"openai_chat": "chat"} + }); + let error = serde_json::from_value::(value) + .err() + .expect("unknown target field must be rejected"); + assert!(error.to_string().contains("unexpected_setting")); +} + +#[test] +fn literal_target_headers_are_rejected() { + let value = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1", + "headers": {"x-provider-token": "plaintext-secret"} + } + }, + "default_targets": {"openai_chat": "chat"} + }); + let error = serde_json::from_value::(value) + .err() + .expect("literal target headers must be rejected") + .to_string(); + assert!(error.contains("unknown field `headers`")); + assert!(!error.contains("plaintext-secret")); +} + +#[test] +fn unknown_algorithm_fields_are_rejected() { + let error = serde_json::from_value::(json!({ + "kind": "random", + "seed": 42, + "unexpected_setting": true + })) + .err() + .expect("unknown algorithm field must be rejected"); + assert!(error.to_string().contains("unexpected_setting")); +} + +#[test] +fn classifier_prepares_clients_for_judge_and_routed_targets() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "recent_turn_window": 4, + "max_output_tokens": 512 + })) + .unwrap(); + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); +} + +#[test] +fn target_provider_defaults_are_accepted_for_judge_controls() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().extra_body = + BTreeMap::from([("think".into(), json!(false))]); + + config.validate().unwrap(); + config.prepare().unwrap(); +} + +#[test] +fn classifier_rejects_anthropic_judge_targets_before_dispatch() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "anthropic", + "weak_target": "responses", + "strong_target": "chat", + "base_threshold": 0.5 + })) + .unwrap(); + + let error = config.validate().unwrap_err(); + assert!(error.contains("classifier target \"anthropic\" uses anthropic_messages")); +} + +#[test] +fn validation_does_not_resolve_environment_backed_headers() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().header_env = BTreeMap::from([( + "authorization".into(), + "SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET".into(), + )]); + + config.validate().unwrap(); + let error = config + .prepare() + .err() + .expect("preparation must resolve headers"); + assert!(error.contains("SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET")); +} + +#[test] +fn invalid_environment_variable_names_are_rejected_before_resolution() { + for variable in ["INVALID=VARIABLE", "INVALID\0VARIABLE"] { + let mut config = config(); + config.targets.get_mut("chat").unwrap().header_env = + BTreeMap::from([("authorization".into(), variable.into())]); + + let error = config.validate().unwrap_err(); + assert!(error.contains("must not contain '=' or NUL")); + } +} + +#[test] +fn static_validation_preserves_algorithm_constructor_checks() { + let mut random = config(); + for target in random.targets.values_mut() { + target.weight = 0.0; + } + assert!( + random + .validate() + .unwrap_err() + .contains("at least one positive target weight") + ); + + let mut classifier = config(); + classifier.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 1.1 + })) + .unwrap(); + assert!( + classifier + .validate() + .unwrap_err() + .contains("base_threshold must be between 0 and 1") + ); +} + +#[test] +fn escalation_classifier_builds_with_defaulted_policy_settings() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "prompt": "Judge the completed trajectory.", + "max_output_tokens": 256, + "escalation": {} + })) + .unwrap(); + + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); +} + +#[test] +fn classifier_modes_reject_mixed_or_missing_settings() { + let mut capability = config(); + capability.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "escalation": {} + })) + .unwrap(); + assert!( + capability + .validate() + .unwrap_err() + .contains("capability mode does not accept escalation") + ); + + let mut escalation = config(); + escalation.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "escalation": {} + })) + .unwrap(); + assert!( + escalation + .validate() + .unwrap_err() + .contains("escalation mode does not accept capability") + ); + + let mut missing = config(); + missing.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic" + })) + .unwrap(); + assert!( + missing + .validate() + .unwrap_err() + .contains("requires escalation settings") + ); +} + +#[test] +fn escalation_settings_are_validated_by_the_libsy_constructor() { + for (settings, expected) in [ + ( + json!({"confirmations": 0}), + "confirmations must be at least 1", + ), + ( + json!({"recent_turn_window": 0}), + "recent_turn_window must be at least 1", + ), + ( + json!({"window_message_chars": 49}), + "window_message_chars must be at least 50", + ), + ] { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "escalation": settings + })) + .unwrap(); + assert!(config.validate().unwrap_err().contains(expected)); + } +} + +#[test] +fn full_stage_router_configuration_builds_all_clients() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "stage_router", + "capable_target": "anthropic", + "efficient_target": "responses", + "picker": "efficient_first", + "confidence_threshold": 0.5, + "recent_turn_window": 3, + "capable_system_prompt": "Diagnose before editing.", + "efficient_system_prompt": "Follow the settled plan.", + "handoff_notes": { + "escalation_note": "The previous model was stalling.", + "deescalation_note": "The task is settled.", + "only_on_wrong_signal_escalation": true + }, + "classifier": { + "target": "chat", + "base_threshold": 0.5, + "threshold_step": 0.1, + "recent_turn_window": 3, + "prompt": "Can the efficient tier finish this turn?", + "max_output_tokens": 256 + } + })) + .unwrap(); + + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "stage_router"); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); +} + +#[test] +fn stage_router_validates_threshold_targets_and_judge_protocol() { + let stage = |classifier: Value, threshold: f64| { + serde_json::from_value(json!({ + "kind": "stage_router", + "capable_target": "anthropic", + "efficient_target": "responses", + "picker": "capable_first", + "confidence_threshold": threshold, + "classifier": classifier + })) + .unwrap() + }; + + let mut invalid_threshold = config(); + invalid_threshold.algorithm = stage(Value::Null, 1.1); + assert!( + invalid_threshold + .validate() + .unwrap_err() + .contains("confidence_threshold must be between 0 and 1") + ); + + let mut missing_target = config(); + missing_target.algorithm = serde_json::from_value(json!({ + "kind": "stage_router", + "capable_target": "missing", + "efficient_target": "responses", + "picker": "capable_first", + "confidence_threshold": 0.5 + })) + .unwrap(); + assert!( + missing_target + .validate() + .unwrap_err() + .contains("algorithm target \"missing\" is not configured") + ); + + let mut anthropic_judge = config(); + anthropic_judge.algorithm = stage(json!({"target": "anthropic", "base_threshold": 0.5}), 0.5); + assert!( + anthropic_judge + .validate() + .unwrap_err() + .contains("classifier target \"anthropic\" uses anthropic_messages") + ); +} + +#[test] +fn zero_weight_random_targets_are_fallback_only() { + let mut config = config(); + config.targets.get_mut("anthropic").unwrap().weight = 0.0; + let prepared = config.prepare().unwrap(); + + assert_eq!(Arc::strong_count(&prepared.targets["anthropic"].client), 1); + assert_eq!(Arc::strong_count(&prepared.targets["chat"].client), 1); + assert_eq!(Arc::strong_count(&prepared.targets["responses"].client), 1); +} diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs new file mode 100644 index 000000000..0f9446c61 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -0,0 +1,273 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod client; +mod config; +mod runtime; +mod translation; + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use nemo_relay_plugin::{ + ConfigDiagnostic, DiagnosticLevel, Json, LlmJsonAsyncStream, NativePlugin, PluginContext, + PluginRuntime, +}; +use serde_json::Map; + +use crate::config::SwitchyardConfig; +use crate::runtime::{RoutingMark, StreamMessage, SwitchyardRuntime}; + +#[derive(Default)] +struct SwitchyardPlugin; + +impl NativePlugin for SwitchyardPlugin { + fn plugin_kind(&self) -> &str { + "nvidia.switchyard" + } + + fn allows_multiple_components(&self) -> bool { + false + } + + fn validate(&self, plugin_config: &Map) -> Vec { + match parse_config(plugin_config).and_then(|config| config.validate()) { + Ok(()) => Vec::new(), + Err(message) => vec![ConfigDiagnostic { + level: DiagnosticLevel::Error, + code: "switchyard.invalid_config".into(), + component: Some("nvidia.switchyard".into()), + field: Some("config".into()), + message, + }], + } + } + + fn register( + &mut self, + plugin_config: &Map, + ctx: &mut PluginContext<'_>, + ) -> nemo_relay_plugin::Result<()> { + let config = parse_config(plugin_config)?; + let priority = config.priority; + let runtime = Arc::new(SwitchyardRuntime::new(config)?); + let plugin_runtime = ctx.runtime(); + + register_buffered(ctx, priority, Arc::clone(&runtime), plugin_runtime.clone())?; + register_stream(ctx, priority, runtime, plugin_runtime)?; + Ok(()) + } +} + +fn register_buffered( + ctx: &mut PluginContext<'_>, + priority: i32, + runtime: Arc, + plugin_runtime: PluginRuntime, +) -> Result<(), String> { + ctx.register_llm_execution_intercept( + "switchyard.run_stream.buffered", + priority, + move |name, request, next| { + let runtime = Arc::clone(&runtime); + let plugin_runtime = plugin_runtime.clone(); + async move { + let Some(inbound) = runtime.managed_protocol(&name) else { + return next.call(request).await; + }; + let request = runtime.decode_request(inbound, &request, false)?; + let mut marks = Vec::new(); + let response = runtime.execute_buffered(inbound, request, &mut marks).await; + emit_marks(&plugin_runtime, marks); + response + } + }, + ) +} + +fn register_stream( + ctx: &mut PluginContext<'_>, + priority: i32, + runtime: Arc, + plugin_runtime: PluginRuntime, +) -> Result<(), String> { + ctx.register_llm_stream_execution_intercept( + "switchyard.run_stream.streaming", + priority, + move |name, request, next| { + let runtime = Arc::clone(&runtime); + let plugin_runtime = plugin_runtime.clone(); + async move { + let Some(inbound) = runtime.managed_protocol(&name) else { + return next.call(request).await; + }; + let request = runtime.decode_request(inbound, &request, true)?; + Ok(Box::pin(ManagedStream::new( + runtime, + plugin_runtime, + inbound, + request, + )) as LlmJsonAsyncStream) + } + }, + ) +} + +fn parse_config(plugin_config: &Map) -> Result { + match plugin_config.get("version").and_then(Json::as_u64) { + Some(2) => {} + Some(version) => { + return Err(format!( + "unsupported Switchyard config version {version}; version 1 used switchyard-server; migrate to version = 2" + )); + } + None => { + return Err("invalid Switchyard configuration: version must be the integer 2".into()); + } + } + serde_json::from_value(Json::Object(plugin_config.clone())) + .map_err(|error| format!("invalid Switchyard configuration: {error}")) +} + +fn emit_marks(runtime: &PluginRuntime, marks: Vec) { + for mark in marks { + emit_mark(runtime, mark); + } +} + +fn emit_mark(runtime: &PluginRuntime, mark: RoutingMark) { + if let Err(error) = runtime.emit_mark(&mark.name, Some(&mark.data), Some(&mark.metadata)) { + eprintln!( + "Switchyard could not emit routing mark {:?}: {error}", + mark.name + ); + } +} + +type StreamExecution = Pin> + Send>>; + +struct ManagedStream { + execution: Option, + messages: Pin>>, + emit_mark: Arc, + terminal_error: Option, +} + +impl ManagedStream { + fn new( + runtime: Arc, + plugin_runtime: PluginRuntime, + inbound: switchyard_protocol::WireFormat, + request: switchyard_protocol::Request, + ) -> Self { + let (sender, messages) = async_channel::bounded(32); + let execution = async move { runtime.execute_stream(inbound, request, &sender).await }; + let emit_mark = Arc::new(move |mark| emit_mark(&plugin_runtime, mark)); + Self { + execution: Some(Box::pin(execution)), + messages: Box::pin(messages), + emit_mark, + terminal_error: None, + } + } +} + +impl futures_util::Stream for ManagedStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if let Some(execution) = self.execution.as_mut() { + match execution.as_mut().poll(cx) { + Poll::Ready(Ok(())) => self.execution = None, + Poll::Ready(Err(error)) => { + self.execution = None; + self.terminal_error = Some(error); + } + Poll::Pending => {} + } + } + + loop { + match self.messages.as_mut().poll_next(cx) { + Poll::Ready(Some(StreamMessage::Mark(mark))) => (self.emit_mark)(mark), + Poll::Ready(Some(StreamMessage::Event(event))) => { + return Poll::Ready(Some(Ok(event))); + } + Poll::Ready(None) => { + return Poll::Ready(self.terminal_error.take().map(Err)); + } + Poll::Pending => return Poll::Pending, + } + } + } +} + +nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlugin::default); + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use futures_util::StreamExt; + use serde_json::json; + + use super::*; + + #[tokio::test] + async fn managed_stream_delivers_queued_events_before_terminal_error() { + let (sender, messages) = async_channel::bounded(32); + let execution = async move { + sender + .send(StreamMessage::Event(json!({"id": "committed"}))) + .await + .expect("queue committed event"); + Err("stream failed after commitment".into()) + }; + let mut stream = ManagedStream { + execution: Some(Box::pin(execution)), + messages: Box::pin(messages), + emit_mark: Arc::new(|_| {}), + terminal_error: None, + }; + + assert_eq!(stream.next().await, Some(Ok(json!({"id": "committed"})))); + assert_eq!( + stream.next().await, + Some(Err("stream failed after commitment".into())) + ); + assert_eq!(stream.next().await, None); + } + + #[test] + fn version_one_service_config_gets_a_migration_error_before_v2_deserialization() { + let value = json!({ + "version": 1, + "service_url": "http://127.0.0.1:8080", + "health_endpoint": "/healthz" + }); + let plugin_config = value.as_object().unwrap(); + + let error = parse_config(plugin_config) + .err() + .expect("version one must be rejected"); + assert!(error.contains("version 1 used switchyard-server")); + assert!(error.contains("migrate to version = 2")); + assert!(!error.contains("unknown field")); + } + + #[test] + fn version_must_be_an_integer() { + let value = json!({"version": "2"}); + let plugin_config = value.as_object().unwrap(); + + let error = parse_config(plugin_config) + .err() + .expect("non-integer versions must be rejected"); + assert_eq!( + error, + "invalid Switchyard configuration: version must be the integer 2" + ); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs new file mode 100644 index 000000000..48914eab3 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -0,0 +1,552 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::{BTreeMap, HashMap}; +use std::sync::{Arc, Mutex}; + +use futures_util::{StreamExt, stream}; +use nemo_relay_plugin::{Json, LlmRequest as RelayRequest}; +use serde_json::{Map, json}; +use switchyard_libsy::{Algorithm, LibsyError}; +use switchyard_llm_client::{ClientRouter, LlmCallObservation, RunObservation, RunObserver, run}; +use switchyard_protocol::{ + LlmClientError, LlmResponse, Metadata, ModelId, Request, Response, WireFormat, +}; +use switchyard_translation::{TranslationEngine, encode_stream}; + +use crate::config::{PreparedTargetBinding, SwitchyardConfig, protocol_from_call}; +use crate::translation; + +#[derive(Debug)] +pub(crate) struct RoutingMark { + pub(crate) name: String, + pub(crate) data: Json, + pub(crate) metadata: Json, +} + +#[derive(Debug)] +pub(crate) enum StreamMessage { + Mark(RoutingMark), + Event(Json), +} + +pub(crate) struct SwitchyardRuntime { + algorithm: Arc, + targets: BTreeMap, + default_targets: BTreeMap, + translation: TranslationEngine, +} + +impl SwitchyardRuntime { + pub(crate) fn new(config: SwitchyardConfig) -> Result { + let prepared = config.prepare()?; + Ok(Self { + algorithm: prepared.algorithm, + targets: prepared.targets, + default_targets: prepared.default_targets, + translation: TranslationEngine::default(), + }) + } + + pub(crate) fn managed_protocol(&self, name: &str) -> Option { + protocol_from_call(name).filter(|protocol| self.default_targets.contains_key(protocol)) + } + + pub(crate) fn decode_request( + &self, + inbound: WireFormat, + request: &RelayRequest, + streaming: bool, + ) -> Result { + let mut llm_request = translation::decode_request(&self.translation, inbound, request)?; + llm_request.stream = streaming; + let headers = string_headers(&request.headers); + let mut metadata = Metadata::from_headers(&headers); + let relay_gateway_placeholder = !headers.contains_key("x-switchyard-session-id") + && headers + .get("x-nemo-relay-source") + .and_then(|value| value.to_str().ok()) + == Some("gateway") + && metadata.session_id.as_deref() == Some("gateway-gateway"); + if relay_gateway_placeholder { + metadata.session_id = None; + } + // Keep identity/routing metadata, but target clients deliberately clear + // these caller headers before HTTP dispatch. + metadata.http_headers = Some(headers); + metadata.wire_format = Some(inbound); + Ok(Request { + llm_request, + raw_request: Some(request.content.clone()), + metadata: Some(metadata), + }) + } + + pub(crate) async fn execute_buffered( + &self, + inbound: WireFormat, + request: Request, + marks: &mut Vec, + ) -> Result { + let metadata = identity_metadata(request.metadata.as_ref()); + self.mark( + marks, + "switchyard.routing.requested", + json!({"algorithm": self.algorithm.name(), "attempt": 1}), + &metadata, + ); + let result = self + .drive(request.clone(), 1, marks, &metadata) + .await + .and_then(|response| { + finalize_buffered_response(&self.translation, inbound, response) + .map_err(|source| LibsyError::client_call("return_to_agent", source)) + }); + match result { + Ok(response) => Ok(response), + Err(failure) => { + self.mark( + marks, + "switchyard.routing.error", + failure_mark_data(1, &failure), + &metadata, + ); + let response = self + .fallback_response(inbound, request, marks, &metadata) + .await?; + finalize_buffered_response(&self.translation, inbound, response) + .map_err(|error| public_response_failure("trusted fallback response", &error)) + } + } + } + + pub(crate) async fn execute_stream( + &self, + inbound: WireFormat, + request: Request, + output: &async_channel::Sender, + ) -> Result<(), String> { + let metadata = identity_metadata(request.metadata.as_ref()); + let mut marks = Vec::new(); + self.mark( + &mut marks, + "switchyard.routing.requested", + json!({"algorithm": self.algorithm.name(), "attempt": 1}), + &metadata, + ); + let (response, mut fallback_used) = + match self.drive(request.clone(), 1, &mut marks, &metadata).await { + Ok(response) => (response, false), + Err(failure) => { + self.mark( + &mut marks, + "switchyard.routing.error", + failure_mark_data(1, &failure), + &metadata, + ); + let fallback = self + .fallback_response(inbound, request.clone(), &mut marks, &metadata) + .await; + send_marks(output, &mut marks).await?; + (fallback?, true) + } + }; + send_marks(output, &mut marks).await?; + + let mut events = match returned_events(response, inbound).await { + Ok(events) => events, + Err(failure) if !fallback_used => { + self.mark( + &mut marks, + "switchyard.routing.error", + failure_mark_data(1, &failure), + &metadata, + ); + fallback_used = true; + let fallback = self + .fallback_response(inbound, request.clone(), &mut marks, &metadata) + .await; + send_marks(output, &mut marks).await?; + let fallback = fallback?; + returned_events(fallback, inbound) + .await + .map_err(|error| public_libsy_failure("trusted fallback stream", &error))? + } + Err(failure) => { + return Err(public_libsy_failure("trusted fallback stream", &failure)); + } + }; + + let mut committed = false; + while let Some(item) = events.next().await { + match item { + Ok(event) => { + send_event(output, event).await?; + committed = true; + } + Err(failure) if !fallback_used && !committed => { + self.mark( + &mut marks, + "switchyard.routing.error", + failure_mark_data(1, &failure), + &metadata, + ); + let fallback = self + .fallback_response(inbound, request.clone(), &mut marks, &metadata) + .await; + send_marks(output, &mut marks).await?; + let fallback = fallback?; + let mut fallback = returned_events(fallback, inbound) + .await + .map_err(|error| public_libsy_failure("trusted fallback stream", &error))?; + while let Some(item) = fallback.next().await { + let event = item.map_err(|error| { + public_libsy_failure("trusted fallback stream", &error) + })?; + send_event(output, event).await?; + } + return Ok(()); + } + Err(failure) if !committed => { + return Err(public_libsy_failure("trusted fallback stream", &failure)); + } + Err(failure) => { + self.mark( + &mut marks, + "switchyard.routing.error", + failure_mark_data(1, &failure), + &metadata, + ); + send_marks(output, &mut marks).await?; + return Err(public_libsy_failure( + "Switchyard stream failed after response commitment", + &failure, + )); + } + } + } + if committed { + Ok(()) + } else { + Err("Switchyard response stream produced no caller events".into()) + } + } + + async fn drive( + &self, + request: Request, + attempt: u32, + marks: &mut Vec, + mark_metadata: &Json, + ) -> Result { + let observations = Arc::new(Mutex::new(Vec::new())); + let observed_calls = observations.clone(); + let observer: RunObserver = Arc::new(move |observation| { + if let RunObservation::LlmCall(call) = observation { + observed_calls + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(call); + } + }); + let clients = ClientRouter::new( + self.targets + .iter() + .map(|(name, target)| (ModelId::from(name.as_str()), target.client.clone())) + .collect::>(), + ); + match run(self.algorithm.clone(), clients, request, Some(observer)).await { + Ok((selected_model_id, response)) => { + self.emit_decision(marks, &selected_model_id, attempt, mark_metadata); + self.emit_routing_llm_calls( + marks, + take_observed_calls(&observations), + attempt, + mark_metadata, + ); + Ok(response) + } + Err(error) => { + self.emit_routing_llm_calls( + marks, + take_observed_calls(&observations), + attempt, + mark_metadata, + ); + Err(error) + } + } + } + + async fn fallback_response( + &self, + inbound: WireFormat, + request: Request, + marks: &mut Vec, + metadata: &Json, + ) -> Result { + let target_name = self.default_target(inbound)?; + let target = self.target(target_name)?; + self.mark( + marks, + "switchyard.routing.fallback", + json!({"selected_target": target_name}), + metadata, + ); + target + .client + .call(request) + .await + .map_err(|error| public_client_failure("trusted fallback", &error)) + } + + fn target(&self, name: &str) -> Result<&PreparedTargetBinding, String> { + self.targets + .get(name) + .ok_or_else(|| format!("libsy selected unknown target {name:?}")) + } + + fn default_target(&self, protocol: WireFormat) -> Result<&str, String> { + self.default_targets + .get(&protocol) + .map(String::as_str) + .ok_or_else(|| format!("managed protocol {protocol} has no default target")) + } + + fn mark(&self, marks: &mut Vec, name: &str, data: Json, metadata: &Json) { + marks.push(RoutingMark { + name: name.to_string(), + data, + metadata: metadata.clone(), + }); + } + + fn emit_decision( + &self, + marks: &mut Vec, + selected_model_id: &ModelId, + attempt: u32, + metadata: &Json, + ) { + self.mark( + marks, + "switchyard.routing.decision", + json!({ + "algorithm": self.algorithm.name(), + "attempt": attempt, + "selected_target": selected_model_id, + }), + metadata, + ); + } + + fn emit_routing_llm_calls( + &self, + marks: &mut Vec, + calls: Vec, + attempt: u32, + metadata: &Json, + ) { + for (index, call) in calls.into_iter().enumerate() { + self.mark( + marks, + "switchyard.routing.llm_call", + json!({ + "algorithm": self.algorithm.name(), + "attempt": attempt, + "call_index": index + 1, + "selected_target": call.selected_model, + "call_role": "routing", + "outcome": if call.is_success { "ok" } else { "error" }, + "latency_ms": call.duration.as_secs_f64() * 1_000.0, + "usage": call.usage, + "contributes_to_routing_overhead": true, + }), + metadata, + ); + } + } +} + +fn take_observed_calls(observations: &Mutex>) -> Vec { + std::mem::take( + &mut *observations + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()), + ) +} + +async fn send_marks( + output: &async_channel::Sender, + marks: &mut Vec, +) -> Result<(), String> { + for mark in marks.drain(..) { + output + .send(StreamMessage::Mark(mark)) + .await + .map_err(|_| "Relay cancelled the Switchyard response stream".to_string())?; + } + Ok(()) +} + +async fn send_event( + output: &async_channel::Sender, + event: Json, +) -> Result<(), String> { + output + .send(StreamMessage::Event(event)) + .await + .map_err(|_| "Relay cancelled the Switchyard response stream".to_string()) +} + +type ReturnedEventStream = + std::pin::Pin> + Send>>; + +fn finalize_buffered_response( + translation_engine: &TranslationEngine, + inbound: WireFormat, + response: Response, +) -> Result { + let LlmResponse::Agg(response) = response.llm_response else { + return Err(LlmClientError::InvalidResponse { + source: Box::new(std::io::Error::other( + "libsy returned a stream for a buffered request", + )), + }); + }; + translation::encode_response(translation_engine, inbound, &response) + .map_err(LlmClientError::ResponseTranslation) +} + +async fn returned_events( + response: Response, + inbound: WireFormat, +) -> Result { + let chunks = match response.llm_response { + LlmResponse::Agg(response) => response.into_stream(), + LlmResponse::Stream(mut chunks) => { + let Some(first) = chunks.next().await else { + return Err(LibsyError::client_call( + "return_to_agent", + LlmClientError::InvalidResponse { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "provider returned an empty stream", + )), + }, + )); + }; + Box::pin(stream::once(async move { first }).chain(chunks)) + } + }; + let events = encode_stream(chunks, inbound, None) + .map_err(|error| LibsyError::client_call("return_to_agent", error))?; + Ok(Box::pin(events.map(|item| { + item.map_err(|source| match source.downcast::() { + Ok(source) => LibsyError::client_call("return_to_agent", *source), + Err(source) => LibsyError::client_call( + "return_to_agent", + LlmClientError::ResponseTranslation(source.to_string()), + ), + }) + }))) +} + +fn failure_mark_data(attempt: u32, failure: &LibsyError) -> Json { + let mut data = Map::from_iter([("attempt".into(), Json::from(attempt))]); + match failure { + LibsyError::ClientCall { + source: LlmClientError::UpstreamHttp { status, .. }, + .. + } => { + data.insert("failure_kind".into(), Json::from("http")); + data.insert("http_status".into(), Json::from(status.as_u16())); + } + LibsyError::ClientCall { source, .. } => { + data.insert("failure_kind".into(), Json::from("non_http")); + data.insert( + "non_http_kind".into(), + Json::from(client_error_label(source)), + ); + } + _ => { + data.insert("failure_kind".into(), Json::from("algorithm")); + } + } + Json::Object(data) +} + +fn client_error_label(error: &LlmClientError) -> &'static str { + match error { + LlmClientError::InvalidRequest { .. } => "invalid_request", + LlmClientError::RequestTranslation(_) => "request_translation", + LlmClientError::RequestEncoding(_) => "request_encoding", + LlmClientError::ResponseTranslation(_) => "response_translation", + LlmClientError::Configuration { .. } => "configuration", + LlmClientError::Transport { .. } => "transport", + LlmClientError::Timeout { .. } => "timeout", + LlmClientError::ContextWindowExceeded { .. } => "context_window_exceeded", + LlmClientError::UpstreamHttp { .. } => "http", + LlmClientError::InvalidResponse { .. } => "invalid_response", + LlmClientError::Ffi { .. } => "ffi", + LlmClientError::General(_) => "general", + _ => "unknown", + } +} + +fn public_libsy_failure(prefix: &str, error: &LibsyError) -> String { + match error { + LibsyError::ClientCall { source, .. } => public_client_failure(prefix, source), + _ => format!("{prefix}: Switchyard algorithm failure"), + } +} + +fn public_response_failure(prefix: &str, error: &LlmClientError) -> String { + match error { + LlmClientError::InvalidResponse { .. } => format!("{prefix}: invalid response"), + LlmClientError::ResponseTranslation(_) => { + format!("{prefix}: response translation failure") + } + _ => format!("{prefix}: response finalization failure"), + } +} + +fn public_client_failure(prefix: &str, error: &LlmClientError) -> String { + match error { + LlmClientError::UpstreamHttp { status, .. } => { + format!("{prefix}: provider returned HTTP {status}") + } + _ => format!("{prefix}: provider {} failure", client_error_label(error)), + } +} + +fn string_headers(headers: &Map) -> http::HeaderMap { + let mut parsed = http::HeaderMap::with_capacity(headers.len()); + for (name, value) in headers { + let Some(value) = value.as_str() else { + continue; + }; + let (Ok(name), Ok(value)) = ( + http::HeaderName::from_bytes(name.as_bytes()), + http::HeaderValue::from_str(value), + ) else { + continue; + }; + parsed.insert(name, value); + } + parsed +} + +fn identity_metadata(metadata: Option<&Metadata>) -> Json { + json!({ + "session_id": metadata.and_then(|value| value.session_id.as_deref()), + "agent_id": metadata.and_then(|value| value.agent_id.as_deref()), + "parent_agent_id": metadata.and_then(|value| value.parent_agent_id.as_deref()), + "task_id": metadata.and_then(|value| value.task_id.as_deref()), + "turn_id": metadata.and_then(|value| value.turn_id.as_deref()), + "correlation_id": metadata.and_then(|value| value.correlation_id.as_deref()), + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs b/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs new file mode 100644 index 000000000..b0dcaa39c --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs @@ -0,0 +1,878 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use switchyard_libsy::{ + ClassifierContractConfig, EscalationJudgeConfig, LlmClassifierConfig, LlmFallback, + LlmTaskClassifier, Passthrough, PickerMode, StageRouter, StageRouterConfig, + TaskClassifierConfig, +}; +use switchyard_protocol::{ + LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, ModelId, RoutedLlmClient, Usage, + text_request, text_response, +}; + +use super::*; + +enum ScriptedBehavior { + Text(&'static str), + EmptyBuffered, + EmptyStream, + FailingStream, + PartialThenFailure, + TransportFailure(&'static str), +} + +struct ScriptedClient { + behavior: ScriptedBehavior, + calls: AtomicUsize, +} + +fn scripted(behavior: ScriptedBehavior) -> Arc { + Arc::new(ScriptedClient { + behavior, + calls: AtomicUsize::new(0), + }) +} + +#[async_trait::async_trait] +impl RoutedLlmClient for ScriptedClient { + async fn call(&self, request: Request) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + match self.behavior { + ScriptedBehavior::Text(text) => { + let mut response = text_response(None, text); + response.usage = Usage { + input_tokens: Some(11), + output_tokens: Some(7), + total_tokens: Some(18), + ..Usage::default() + }; + Ok(Response { + llm_response: LlmResponse::Agg(response), + metadata: request.metadata, + }) + } + ScriptedBehavior::EmptyBuffered => Ok(Response { + llm_response: LlmResponse::Agg(Default::default()), + metadata: None, + }), + ScriptedBehavior::EmptyStream => Ok(Response { + llm_response: LlmResponse::Stream(Box::pin(stream::empty())), + metadata: None, + }), + ScriptedBehavior::FailingStream => { + let stream: LlmResponseStream = Box::pin(stream::once(async { + Err(LlmClientError::Transport { + source: Box::new(std::io::Error::other("fallback stream failed")), + }) + })); + Ok(Response { + llm_response: LlmResponse::Stream(stream), + metadata: None, + }) + } + ScriptedBehavior::PartialThenFailure => { + let stream: LlmResponseStream = Box::pin(stream::iter(vec![ + Ok(LlmResponseStreamEvent::from(LlmResponseChunk::TextDelta { + index: 0, + text: "partial".into(), + })), + Err(LlmClientError::Transport { + source: Box::new(std::io::Error::other("stream failed after a chunk")), + }), + ])); + Ok(Response { + llm_response: LlmResponse::Stream(stream), + metadata: None, + }) + } + ScriptedBehavior::TransportFailure(message) => Err(LlmClientError::Transport { + source: Box::new(std::io::Error::other(message)), + }), + } + } +} + +fn fixed_target(name: &str) -> ModelId { + ModelId::from(name) +} + +fn runtime_with_algorithm( + algorithm: Arc, + fallback: Arc, + protocol: WireFormat, +) -> SwitchyardRuntime { + runtime_with_algorithm_clients(algorithm, fallback, protocol, Vec::new()) +} + +fn runtime_with_algorithm_clients( + algorithm: Arc, + fallback: Arc, + protocol: WireFormat, + clients: Vec<(&str, Arc)>, +) -> SwitchyardRuntime { + let mut targets = BTreeMap::from([( + "fallback".into(), + PreparedTargetBinding { + client: fallback as Arc, + }, + )]); + for (name, client) in clients { + targets.insert( + name.to_string(), + PreparedTargetBinding { + client: client as Arc, + }, + ); + } + SwitchyardRuntime { + algorithm, + targets, + default_targets: BTreeMap::from([(protocol, "fallback".into())]), + translation: TranslationEngine::default(), + } +} + +fn request_with_session(protocol: WireFormat, session: Option<&str>) -> Request { + Request { + llm_request: text_request(Some("auto".into()), "fix the build"), + raw_request: None, + metadata: Some(Metadata { + wire_format: Some(protocol), + session_id: session.map(str::to_string), + ..Metadata::default() + }), + } +} + +fn stage_signal_relay_request(protocol: WireFormat) -> RelayRequest { + let content = match protocol { + WireFormat::OpenAiChat => json!({ + "model": "auto", + "messages": [ + {"role": "user", "content": "fix the build"}, + { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "call-1", + "type": "function", + "function": { + "name": "bash", + "arguments": "{\"cmd\":\"cargo test\"}" + } + }] + }, + { + "role": "tool", + "tool_call_id": "call-1", + "content": "fatal runtime error: out of memory" + } + ] + }), + WireFormat::OpenAiResponses => json!({ + "model": "auto", + "input": [ + {"type": "message", "role": "user", "content": "fix the build"}, + { + "type": "function_call", + "call_id": "call-1", + "name": "bash", + "arguments": "{\"cmd\":\"cargo test\"}" + }, + { + "type": "function_call_output", + "call_id": "call-1", + "output": "fatal runtime error: out of memory" + } + ] + }), + WireFormat::AnthropicMessages => json!({ + "model": "auto", + "max_tokens": 128, + "messages": [ + {"role": "user", "content": "fix the build"}, + { + "role": "assistant", + "content": [{ + "type": "tool_use", + "id": "call-1", + "name": "bash", + "input": {"cmd": "cargo test"} + }] + }, + { + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "call-1", + "content": "fatal runtime error: out of memory", + "is_error": true + }] + } + ] + }), + }; + + RelayRequest { + headers: Map::from_iter([( + "x-switchyard-session-id".into(), + json!(format!("stage-{}", protocol.as_str())), + )]), + content, + } +} + +#[test] +fn relay_gateway_placeholder_session_is_not_retained() { + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let runtime = runtime_with_algorithm( + Arc::new(Passthrough::new(ModelId::from("selected"))), + fallback, + WireFormat::OpenAiChat, + ); + let request = RelayRequest { + headers: Map::from_iter([ + ("x-nemo-relay-source".into(), json!("gateway")), + ("x-nemo-relay-session-id".into(), json!("gateway-gateway")), + ("x-dynamo-session-id".into(), json!("gateway-gateway")), + ]), + content: json!({ + "model": "router", + "messages": [{"role": "user", "content": "hello"}] + }), + }; + + let decoded = runtime + .decode_request(WireFormat::OpenAiChat, &request, false) + .unwrap(); + + assert_eq!(decoded.metadata.unwrap().session_id, None); +} + +#[test] +fn explicit_switchyard_session_overrides_relay_gateway_placeholder() { + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let runtime = runtime_with_algorithm( + Arc::new(Passthrough::new(ModelId::from("selected"))), + fallback, + WireFormat::OpenAiChat, + ); + let request = RelayRequest { + headers: Map::from_iter([ + ("x-switchyard-session-id".into(), json!("caller-session")), + ("x-nemo-relay-source".into(), json!("gateway")), + ("x-nemo-relay-session-id".into(), json!("gateway-gateway")), + ]), + content: json!({ + "model": "router", + "messages": [{"role": "user", "content": "hello"}] + }), + }; + + let decoded = runtime + .decode_request(WireFormat::OpenAiChat, &request, false) + .unwrap(); + + assert_eq!( + decoded.metadata.unwrap().session_id.as_deref(), + Some("caller-session") + ); +} + +#[tokio::test] +async fn buffered_finalization_failure_uses_fallback_once() { + let selected = scripted(ScriptedBehavior::EmptyStream); + let fallback = scripted(ScriptedBehavior::EmptyBuffered); + let runtime = SwitchyardRuntime { + algorithm: Arc::new(Passthrough::new(ModelId::from("selected"))), + targets: BTreeMap::from([ + ( + "selected".into(), + PreparedTargetBinding { + client: selected.clone(), + }, + ), + ( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + ), + ]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + translation: TranslationEngine::default(), + }; + let mut marks = Vec::new(); + + let response = runtime + .execute_buffered(WireFormat::OpenAiChat, Request::default(), &mut marks) + .await + .expect("the buffered fallback response should be encoded"); + + assert!(response.is_object()); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); + let error = marks + .iter() + .find(|mark| mark.name == "switchyard.routing.error") + .expect("finalization failure should emit an error mark"); + assert_eq!(error.data["non_http_kind"], "invalid_response"); + assert_eq!( + marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.fallback") + .count(), + 1 + ); +} + +#[tokio::test] +async fn returned_events_replays_preserved_openai_chat_without_duplicate_terminal() { + let content = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "system_fingerprint": "fp_provider_specific", + "choices": [{ + "index": 0, + "delta": {"content": "Hi"}, + "finish_reason": null + }] + }); + let terminal = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "choices": [{ + "index": 0, + "delta": {}, + "finish_reason": "stop" + }] + }); + let body = format!("data: {content}\n\ndata: {terminal}\n\ndata: [DONE]\n\n").into_bytes(); + let stream = switchyard_translation::decode_stream( + stream::once(async move { Ok::<_, LlmClientError>(body) }), + WireFormat::OpenAiChat, + ) + .expect("provider SSE should decode"); + let response = Response { + llm_response: LlmResponse::Stream(stream), + metadata: None, + }; + + let replayed = returned_events(response, WireFormat::OpenAiChat) + .await + .expect("return stream should encode") + .collect::>() + .await + .into_iter() + .collect::, _>>() + .expect("return stream should not fail"); + + assert_eq!(replayed, vec![content, terminal]); +} + +#[tokio::test] +async fn invalid_selected_stream_does_not_invoke_failing_fallback_twice() { + let selected = scripted(ScriptedBehavior::EmptyStream); + let fallback = scripted(ScriptedBehavior::FailingStream); + let runtime = SwitchyardRuntime { + algorithm: Arc::new(Passthrough::new(ModelId::from("selected"))), + targets: BTreeMap::from([ + ( + "selected".into(), + PreparedTargetBinding { + client: selected.clone(), + }, + ), + ( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + ), + ]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + translation: TranslationEngine::default(), + }; + let (output, _messages) = async_channel::bounded(32); + + let error = runtime + .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) + .await + .expect_err("the failing fallback stream must fail the request"); + + assert_eq!(error, "trusted fallback stream: provider transport failure"); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn failing_fallback_call_flushes_error_and_fallback_marks() { + let selected = scripted(ScriptedBehavior::EmptyStream); + let fallback = scripted(ScriptedBehavior::TransportFailure("fallback call failed")); + let runtime = SwitchyardRuntime { + algorithm: Arc::new(Passthrough::new(ModelId::from("selected"))), + targets: BTreeMap::from([ + ( + "selected".into(), + PreparedTargetBinding { + client: selected.clone(), + }, + ), + ( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + ), + ]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + translation: TranslationEngine::default(), + }; + let (output, messages) = async_channel::bounded(32); + + let error = runtime + .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) + .await + .expect_err("the failing fallback call must fail the request"); + + assert_eq!(error, "trusted fallback: provider transport failure"); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); + let mut terminal_marks = Vec::new(); + while let Ok(message) = messages.try_recv() { + if let StreamMessage::Mark(mark) = message + && matches!( + mark.name.as_str(), + "switchyard.routing.error" | "switchyard.routing.fallback" + ) + { + terminal_marks.push(mark.name); + } + } + assert_eq!( + terminal_marks, + ["switchyard.routing.error", "switchyard.routing.fallback"] + ); +} + +#[tokio::test] +async fn committed_stream_failure_does_not_fallback() { + let selected = scripted(ScriptedBehavior::PartialThenFailure); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let runtime = runtime_with_algorithm_clients( + Arc::new(Passthrough::new(ModelId::from("selected"))), + fallback.clone(), + WireFormat::OpenAiChat, + vec![("selected", selected.clone())], + ); + let (output, messages) = async_channel::bounded(32); + + let error = runtime + .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) + .await + .expect_err("committed stream failures must reject the stream"); + + assert_eq!( + error, + "Switchyard stream failed after response commitment: provider transport failure" + ); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + let mut emitted_event = false; + while let Ok(message) = messages.try_recv() { + emitted_event |= matches!(message, StreamMessage::Event(_)); + } + assert!(emitted_event); +} + +#[tokio::test] +async fn capability_classifier_emits_judge_usage_without_serving_usage() { + let weak = scripted(ScriptedBehavior::Text("weak answer")); + let strong = scripted(ScriptedBehavior::Text("strong answer")); + let judge = scripted(ScriptedBehavior::Text( + r#"{"crux":"bounded","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, + )); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Capability { + judge_target: fixed_target("judge"), + efficient_target: fixed_target("weak"), + capable_target: fixed_target("strong"), + config: TaskClassifierConfig { + base_threshold: 0.5, + ..TaskClassifierConfig::default() + }, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback, + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + let mut marks = Vec::new(); + + runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("capability")), + &mut marks, + ) + .await + .unwrap(); + + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(weak.calls.load(Ordering::Relaxed), 1); + assert_eq!(strong.calls.load(Ordering::Relaxed), 0); + let routing_calls = marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.llm_call") + .collect::>(); + assert_eq!(routing_calls.len(), 1); + assert_eq!(routing_calls[0].data["selected_target"], "judge"); + assert_eq!(routing_calls[0].data["usage"]["total_tokens"], 18); +} + +#[tokio::test] +async fn escalation_buffers_weak_stream_then_latches_the_session_to_strong() { + let weak = scripted(ScriptedBehavior::Text("weak draft")); + let strong = scripted(ScriptedBehavior::Text("strong answer")); + let judge = scripted(ScriptedBehavior::Text( + r#"{"escalate":true,"reason":"stuck"}"#, + )); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: fixed_target("judge"), + efficient_target: fixed_target("weak"), + capable_target: fixed_target("strong"), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 1, + ..EscalationJudgeConfig::default() + }, + max_output_tokens: 128, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + + let mut first = request_with_session(WireFormat::OpenAiChat, Some("session-1")); + first.llm_request.stream = true; + let (output, messages) = async_channel::bounded(32); + runtime + .execute_stream(WireFormat::OpenAiChat, first, &output) + .await + .unwrap(); + let mut streamed = Vec::new(); + let mut routing_calls = Vec::new(); + while let Ok(message) = messages.try_recv() { + match message { + StreamMessage::Event(event) => streamed.push(event), + StreamMessage::Mark(mark) if mark.name == "switchyard.routing.llm_call" => { + routing_calls.push(mark.data) + } + StreamMessage::Mark(_) => {} + } + } + assert!(!streamed.is_empty()); + assert!( + streamed + .iter() + .any(|event| event.to_string().contains("strong answer")) + ); + assert_eq!(routing_calls.len(), 2); + assert_eq!(routing_calls[0]["selected_target"], "weak"); + assert_eq!(routing_calls[0]["call_role"], "routing"); + assert_eq!(routing_calls[0]["usage"]["total_tokens"], 18); + assert_eq!(routing_calls[1]["selected_target"], "judge"); + assert_eq!(routing_calls[1]["call_role"], "routing"); + assert_eq!(routing_calls[1]["usage"]["total_tokens"], 18); + assert!( + routing_calls + .iter() + .all(|call| call["selected_target"] != "strong") + ); + + let mut marks = Vec::new(); + let response = runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("session-1")), + &mut marks, + ) + .await + .unwrap(); + assert!(response.to_string().contains("strong answer")); + assert_eq!(weak.calls.load(Ordering::Relaxed), 1); + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(strong.calls.load(Ordering::Relaxed), 2); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + assert!( + !marks + .iter() + .any(|mark| mark.name == "switchyard.routing.llm_call") + ); + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" + && mark.data["selected_target"] == "strong" + && mark.metadata["session_id"] == "session-1" + })); +} + +#[tokio::test] +async fn escalation_judge_failure_falls_open_to_the_buffered_weak_response() { + let weak = scripted(ScriptedBehavior::Text("weak answer")); + let strong = scripted(ScriptedBehavior::Text("strong answer")); + let judge = scripted(ScriptedBehavior::TransportFailure("scripted failure")); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: fixed_target("judge"), + efficient_target: fixed_target("weak"), + capable_target: fixed_target("strong"), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig::default(), + max_output_tokens: 128, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + let mut marks = Vec::new(); + + let response = runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("session-1")), + &mut marks, + ) + .await + .unwrap(); + + assert!(response.to_string().contains("weak answer")); + assert_eq!(weak.calls.load(Ordering::Relaxed), 1); + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(strong.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + let routing_calls = marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.llm_call") + .collect::>(); + assert_eq!(routing_calls.len(), 1); + assert_eq!(routing_calls[0].data["selected_target"], "judge"); + assert_eq!(routing_calls[0].data["call_role"], "routing"); + assert_eq!(routing_calls[0].data["outcome"], "error"); + assert!(routing_calls[0].data["usage"].is_null()); +} + +#[tokio::test] +async fn escalation_without_session_identity_cannot_accumulate_confirmations() { + let weak = scripted(ScriptedBehavior::Text("weak answer")); + let strong = scripted(ScriptedBehavior::Text("strong answer")); + let judge = scripted(ScriptedBehavior::Text( + r#"{"escalate":true,"reason":"stuck"}"#, + )); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: fixed_target("judge"), + efficient_target: fixed_target("weak"), + capable_target: fixed_target("strong"), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 2, + ..EscalationJudgeConfig::default() + }, + max_output_tokens: 128, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + + for _ in 0..2 { + let mut marks = Vec::new(); + let response = runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, None), + &mut marks, + ) + .await + .unwrap(); + assert!(response.to_string().contains("weak answer")); + } + assert_eq!(weak.calls.load(Ordering::Relaxed), 2); + assert_eq!(judge.calls.load(Ordering::Relaxed), 2); + assert_eq!(strong.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn stage_router_uses_tool_signals_for_every_managed_protocol() { + for protocol in [ + WireFormat::OpenAiChat, + WireFormat::OpenAiResponses, + WireFormat::AnthropicMessages, + ] { + let capable = scripted(ScriptedBehavior::Text("capable answer")); + let efficient = scripted(ScriptedBehavior::Text("efficient answer")); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = StageRouter::new( + fixed_target("strong"), + fixed_target("weak"), + StageRouterConfig::new(PickerMode::EfficientFirst, 0.5), + ) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + protocol, + vec![("strong", capable.clone()), ("weak", efficient.clone())], + ); + let mut marks = Vec::new(); + let relay_request = stage_signal_relay_request(protocol); + let request = runtime + .decode_request(protocol, &relay_request, false) + .unwrap(); + + let response = runtime + .execute_buffered(protocol, request, &mut marks) + .await + .unwrap(); + + assert!(response.to_string().contains("capable answer")); + assert_eq!(capable.calls.load(Ordering::Relaxed), 1); + assert_eq!(efficient.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" + && mark.data["algorithm"] == "stage_router" + && mark.data["attempt"] == 1 + && mark.data["selected_target"] == "strong" + && mark.metadata["session_id"] == format!("stage-{}", protocol.as_str()) + })); + } +} + +#[tokio::test] +async fn stage_router_falls_open_to_each_picker_default_without_tool_history() { + for (picker, expected) in [ + (PickerMode::CapableFirst, "strong"), + (PickerMode::EfficientFirst, "weak"), + ] { + let capable = scripted(ScriptedBehavior::Text("strong")); + let efficient = scripted(ScriptedBehavior::Text("weak")); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = StageRouter::new( + fixed_target("strong"), + fixed_target("weak"), + StageRouterConfig::new(picker, 0.5), + ) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback, + WireFormat::OpenAiChat, + vec![("strong", capable), ("weak", efficient)], + ); + let mut marks = Vec::new(); + + runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, None), + &mut marks, + ) + .await + .unwrap(); + + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" && mark.data["selected_target"] == expected + })); + } +} + +#[tokio::test] +async fn stage_router_classifier_resolves_an_ambiguous_turn() { + let capable = scripted(ScriptedBehavior::Text("strong")); + let efficient = scripted(ScriptedBehavior::Text("weak")); + let judge = scripted(ScriptedBehavior::Text( + r#"{"crux":"bounded","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, + )); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let mut config = StageRouterConfig::new(PickerMode::CapableFirst, 0.5); + config.llm_fallback = Some(LlmFallback { + judge_target: fixed_target("judge"), + config: TaskClassifierConfig { + base_threshold: 0.5, + ..TaskClassifierConfig::default() + }, + }); + let algorithm = StageRouter::new(fixed_target("strong"), fixed_target("weak"), config).unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("strong", capable.clone()), + ("weak", efficient.clone()), + ("judge", judge.clone()), + ], + ); + let mut marks = Vec::new(); + + runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("stage-classifier")), + &mut marks, + ) + .await + .unwrap(); + + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(efficient.calls.load(Ordering::Relaxed), 1); + assert_eq!(capable.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + let routing_calls = marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.llm_call") + .collect::>(); + assert_eq!(routing_calls.len(), 1); + assert_eq!(routing_calls[0].data["selected_target"], "judge"); + assert_eq!(routing_calls[0].data["call_role"], "routing"); + assert_eq!(routing_calls[0].data["outcome"], "ok"); + assert_eq!(routing_calls[0].data["usage"]["total_tokens"], 18); + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" && mark.data["selected_target"] == "weak" + })); +} diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs new file mode 100644 index 000000000..6f6f965fe --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use nemo_relay_plugin::LlmRequest as RelayRequest; +use serde_json::Value as Json; +use switchyard_protocol::{AggLlmResponse, LlmRequest, WireFormat}; +use switchyard_translation::{ + DeterministicIdPolicy, DiagnosticSeverity, LossyConversionPolicy, PreservationPolicy, + TargetCapabilities, TranslationDiagnostic, TranslationEngine, TranslationPolicy, + UnknownFieldPolicy, +}; + +pub(crate) fn decode_request( + engine: &TranslationEngine, + protocol: WireFormat, + request: &RelayRequest, +) -> Result { + let output = engine + .decode_request(protocol, &request.content, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.request) +} + +pub(crate) fn validate_target_request( + engine: &TranslationEngine, + protocol: WireFormat, + request: &LlmRequest, +) -> Result<(), String> { + let output = engine + .encode_request(protocol, request, &request_policy(protocol)) + .map_err(error)?; + safe(&output.diagnostics) +} + +pub(crate) fn encode_response( + engine: &TranslationEngine, + protocol: WireFormat, + response: &AggLlmResponse, +) -> Result { + let output = engine + .encode_response(protocol, response, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.body) +} + +fn policy() -> TranslationPolicy { + TranslationPolicy { + unknown_field_policy: UnknownFieldPolicy::Preserve, + lossy_conversion_policy: LossyConversionPolicy::Reject, + deterministic_ids: DeterministicIdPolicy::GenerateStable { + prefix: "relay".into(), + }, + preservation: PreservationPolicy::InMemory, + target_capabilities: TargetCapabilities::default(), + } +} + +fn request_policy(protocol: WireFormat) -> TranslationPolicy { + let mut policy = policy(); + if protocol == WireFormat::AnthropicMessages { + policy + .target_capabilities + .supports_json_schema_response_format = Some(false); + } + policy +} + +fn safe(diagnostics: &[TranslationDiagnostic]) -> Result<(), String> { + let unsafe_diagnostics = diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity != DiagnosticSeverity::Info) + .collect::>(); + if unsafe_diagnostics.is_empty() { + Ok(()) + } else { + Err(format!( + "Switchyard translation was not lossless: {unsafe_diagnostics:?}" + )) + } +} + +fn error(error: switchyard_translation::TranslationError) -> String { + format!("Switchyard translation failed: {error}") +} + +#[cfg(test)] +mod tests { + use serde_json::{Map, json}; + + use super::*; + + #[test] + fn same_protocol_request_preserves_unknown_fields() { + let request = RelayRequest { + headers: Map::new(), + content: json!({ + "model": "route", + "messages": [{"role": "user", "content": "hello"}], + "provider_extension": {"exact": true} + }), + }; + let engine = TranslationEngine::default(); + let decoded = decode_request(&engine, WireFormat::OpenAiChat, &request).unwrap(); + validate_target_request(&engine, WireFormat::OpenAiChat, &decoded).unwrap(); + assert_eq!( + decoded + .preservation + .requests + .get(&WireFormat::OpenAiChat.into()) + .and_then(|body| body.get("provider_extension")), + Some(&json!({"exact": true})) + ); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py b/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py new file mode 100644 index 000000000..296cbc4b8 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the native Relay plugin bundle packager.""" + +from __future__ import annotations + +import hashlib +import subprocess +import sys +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path + +CRATE_ROOT = Path(__file__).resolve().parents[1] +PACKAGER = CRATE_ROOT / "scripts" / "package_bundle.py" +PACKAGE_NAME = "switchyard-nemo-relay-plugin" + + +class PackageBundleTest(unittest.TestCase): + """Verify materialized and archived plugin bundle contents.""" + + def test_materializes_and_archives_supported_formats(self) -> None: + for archive_suffix in (".tar.gz", ".zip"): + with self.subTest(archive_suffix=archive_suffix), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + library = root / "libswitchyard_nemo_relay_plugin.so" + library.write_bytes(b"compiled plugin") + output = root / "bundle" + archive = root / f"{PACKAGE_NAME}-0.2.0-linux-x86_64{archive_suffix}" + + subprocess.run( + [ + sys.executable, + str(PACKAGER), + "--library", + str(library), + "--output", + str(output), + "--archive", + str(archive), + ], + check=True, + capture_output=True, + text=True, + ) + + expected = { + "LICENSE", + "NOTICE", + "config.schema.json", + library.name, + "relay-plugin.toml", + } + self.assertEqual({path.name for path in output.iterdir()}, expected) + manifest = (output / "relay-plugin.toml").read_text(encoding="utf-8") + self.assertIn(f'artifact = "{library.name}"', manifest) + self.assertIn(hashlib.sha256(library.read_bytes()).hexdigest(), manifest) + self.assertNotIn("", manifest) + self.assertNotIn("", manifest) + self.assertEqual(self.archive_members(archive), {f"{PACKAGE_NAME}/{name}" for name in expected}) + + @staticmethod + def archive_members(archive: Path) -> set[str]: + """Return regular-file paths from a supported bundle archive.""" + if archive.name.endswith(".tar.gz"): + with tarfile.open(archive) as stream: + return {member.name for member in stream.getmembers() if member.isfile()} + with zipfile.ZipFile(archive) as stream: + return {member.filename for member in stream.infolist() if not member.is_dir()} + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/index.md b/docs/index.md index e68ed477b..9b8fb17f9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,6 +9,7 @@ It supports OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages. |---|---|---| | Run Switchyard as a standalone proxy for API clients | Server Path | [Build and run the Rust server](getting_started.md#server-path) | | Add Switchyard routing to a Rust application | Library Path | [`switchyard-libsy`](../crates/libsy/README.md) | +| Add Switchyard routing to NeMo Relay | Native Plugin Path | [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md) | The Server Path builds and runs the standalone `switchyard-server` binary. @@ -27,3 +28,4 @@ The Server Path builds and runs the standalone `switchyard-server` binary. - [`switchyard-libsy`](reference/rust_api.md#switchyard-libsy): embeddable routing algorithms - [`switchyard-protocol`](reference/rust_api.md#switchyard-protocol): provider-neutral API types - [`switchyard-translation`](../crates/switchyard-translation/README.md): protocol translation +- [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md): native NeMo Relay integration From 52129a311019911a64c9b443e6259db98c3e9bc7 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Sun, 23 Aug 2026 14:33:20 -0700 Subject: [PATCH 02/18] refactor(relay): use shared configured runner Signed-off-by: Bryan Bednarski --- CHANGELOG.md | 23 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 4 +- crates/switchyard-nemo-relay-plugin/README.md | 398 +------- .../config.schema.json | 206 +--- .../src/client.rs | 296 ------ .../src/config.rs | 587 +----------- .../src/config/tests.rs | 566 ----------- .../switchyard-nemo-relay-plugin/src/lib.rs | 189 +--- .../src/runtime.rs | 581 ++++-------- .../src/runtime/tests.rs | 878 ------------------ 10 files changed, 276 insertions(+), 3452 deletions(-) delete mode 100644 crates/switchyard-nemo-relay-plugin/src/client.rs delete mode 100644 crates/switchyard-nemo-relay-plugin/src/config/tests.rs delete mode 100644 crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 91b3192e9..e949c4f98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,20 +8,15 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added -- **NeMo Relay native plugin** — a dynamically loaded integration that runs - libsy's weighted-random, LLM-classifier, escalation, and stage-router - algorithms in process while Switchyard owns provider HTTP dispatch, - credentials, translation, retries, and fallback. Managed calls require NeMo - Relay 0.7 or newer and do not depend on `switchyard-server`. Target bindings - accept non-secret `extra_body` provider defaults, preserved requests are - re-encoded after routing mutations, and synthetic Relay gateway identities do - not become shared router session state. - -- **NeMo Relay routing-model usage marks** — classifier judges, escalation - judges and discarded weak candidates, and failed routing candidates now emit - `switchyard.routing.llm_call` ATOF marks with normalized token usage and - latency. The final serving call remains represented only by Relay's outer LLM - lifecycle event to prevent double-counting. +- **NeMo Relay native plugin** — a dynamically loaded integration that loads + Switchyard's standard TOML deployment and executes its named routes in + process. Managed calls require NeMo Relay 0.8.0 or newer; unknown models use + Relay's continuation unchanged. + +- **NeMo Relay routing marks** — routing-model usage, measured routing + overhead, and selected-model decisions are emitted as ATOF marks. The final + serving call remains represented only by Relay's outer LLM lifecycle event to + prevent double-counting. - **Advisor-gate routing** — new `advisor` route type pairing the serving executor with a stronger judge-only advisor that reviews terminal turns: diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 243e66cc0..f6a4b1d64 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -16,15 +16,13 @@ publish = false crate-type = ["cdylib"] [dependencies] -async-channel.workspace = true -async-trait.workspace = true futures-util.workspace = true http.workspace = true nemo-relay-plugin.workspace = true serde.workspace = true serde_json.workspace = true -switchyard-libsy.workspace = true switchyard-llm-client.workspace = true switchyard-protocol.workspace = true +switchyard-runner.workspace = true switchyard-translation.workspace = true tokio.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 20abab1d7..8ce7093f1 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -1,382 +1,56 @@ - +# Switchyard NeMo Relay Plugin -# Switchyard NeMo Relay Dynamic Plugin +`switchyard-nemo-relay-plugin` is a native NeMo Relay dynamic plugin. It loads +a standard Switchyard TOML deployment and executes its configured routes in +Relay through `switchyard-runner`. -This crate builds the external `nvidia.switchyard` native plugin. It embeds -`switchyard-libsy`, drives it through `switchyard-llm-client::run`, and uses -`switchyard-llm-client` for provider HTTP calls. Managed calls use Relay's -typed asynchronous middleware API and do not require a targeted provider -continuation from Relay. +The plugin does not define a second routing or target configuration language. +`switchyard-server` and Relay therefore use the same targets, client pooling, +algorithm construction, retry policy, and route validation. -The plugin uses NeMo Relay native API v1. It depends on the small -`nemo-relay-plugin` authoring SDK, not the Relay runtime, and does not start -`switchyard-server`. Managed provider calls do not use Relay's provider -continuation. +## Install -## Ownership boundary +Build the platform bundle with the package script, then configure Relay to load +the generated `relay-plugin.toml` manifest. The plugin requires NeMo Relay +`>=0.8.0,<1.0`. -For a managed LLM call: +## Configure Relay -1. Relay invokes the native LLM execution intercept. -2. The plugin decodes the caller JSON through `switchyard-translation`. -3. The plugin passes the configured algorithm and its target-to-client map to - `switchyard-llm-client::run`, using the library's public execution and - observation boundary. -4. For every routed call, the selected target client translates the neutral - request, applies its URL and credentials, and performs the HTTP request. -5. `switchyard-llm-client` drives libsy to its final response while the plugin - records decisions and routing-only model usage. -6. The plugin encodes the final neutral response into the caller's protocol. - -Relay still owns the outer LLM lifecycle, dynamic-plugin loading, plugin -configuration, and event substrate. Relay's downstream LLM continuation is -used only for calls whose inbound protocol is not managed by this plugin. - -```mermaid -flowchart LR - A["Caller JSON"] --> B["Relay LLM execution intercept"] - B --> C["Switchyard decode"] - C --> D["switchyard-llm-client run"] - D --> E["libsy algorithm"] - E --> F["target ClientRouter"] - F --> G["Provider HTTP endpoint"] - G --> H["Switchyard response or event decode"] - H --> I["libsy final response"] - I --> J["routing observations"] - J --> K["Switchyard encode"] - K --> A - - U["Unmanaged profile"] -.-> V["Relay v1 continuation"] -``` - -This boundary has two important consequences: - -- Managed provider calls do not traverse Relay middleware registered after the - Switchyard intercept and do not use the host's provider callback. Provider - transport activity is therefore not represented as nested Relay LLM - lifecycle events. Relay records the outer managed call and the plugin emits - Switchyard routing marks; bridging Switchyard transport spans into Relay is - future work. Relay's typed middleware adapter propagates the active scope so - asynchronous routing marks retain their event parent. -- Switchyard owns provider URLs, credentials, HTTP retry behavior, and - translation for managed calls. Relay neither validates nor transports those - target details. - -## Native API v1 and typed asynchronous execution - -The manifest remains `compat.native_api = "1"`, and the plugin requires Relay -`>=0.8.0,<1.0` with native host ABI v4. It registers typed buffered and -incremental streaming intercepts through `nemo-relay-plugin`. Relay owns the plugin executor, -continuation and output-stream lifecycle, cancellation, backpressure, and -scope propagation; Relay workers therefore do not wait synchronously for -provider I/O. - -The stream adapter preserves the plugin's bounded 32-message routing channel. -Relay's typed stream adapter forwards response events to its bounded output -queue and handles cancellation and backpressure. Cancelling a buffered or -streaming call drops its in-flight provider future. - -Unmanaged profiles use Relay's typed continuations for pass-through. The HTTP, -routing, and translation behavior remains in Switchyard; Relay owns the native -dynamic-library boundary. - -## Supported routers - -The plugin supports four libsy routing modes: - -- seeded, weighted `random` routing; and -- capability-based `llm_classifier` routing, where a judge selects the weak or - strong target before the final provider call; -- escalation-mode `llm_classifier` routing, where a judge evaluates the weak - model's completed turn and latches a session to the strong target after a - configured confirmation streak; and -- signal-driven `stage_router` routing, with optional handoff notes, tier - prompts, and a capability-classifier fallback for ambiguous turns. - -Unsupported algorithm kinds are rejected instead of being approximated. - -## Compatibility Matrix - -The following matrix describes the algorithm behavior implemented by the -plugin. `Conditional` means that the feature is implemented with the constraint -shown in the table; it does not mean that the feature falls back to a different -algorithm. - -| Compatibility Area | `random` | `llm_classifier` (`capability`) | `llm_classifier` (`escalation`) | `stage_router` | -|---|---|---|---|---| -| Version-2 configuration and static validation | Supported | Supported | Supported | Supported | -| Caller protocols | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | -| Serving-target protocols | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | -| Structured-output judge protocols | Not applicable | OpenAI Chat or OpenAI Responses | OpenAI Chat or OpenAI Responses | OpenAI Chat or OpenAI Responses for the optional classifier | -| Buffered responses | Supported | Supported | Supported | Supported | -| Streaming responses | Supported | Supported after the judge selects a target | Conditional: an unlatched weak stream is aggregated before the judge runs | Supported after the signal cascade selects a target | -| Retained routing state | No selection affinity; context-overflow eviction can use session identity | Optional session affinity and message-hash fallback | Confirmation streak and strong latch require stable session identity | No classifier affinity; context-overflow eviction can use session identity | -| Router-specific prompts | Not applicable | Optional judge prompt | Optional escalation-judge prompt | Optional tier prompts, handoff notes, and classifier prompt | -| Relay decision marks | Algorithm, attempt, selected target, and identity | Algorithm, attempt, selected target, and identity | Algorithm, attempt, selected target, and identity | Algorithm, attempt, selected target, and identity | -| ATOF routing-LLM usage | Not applicable unless a failed candidate is replaced | Judge calls, plus failed candidates | Judge calls and discarded weak candidates | Optional classifier judge calls, plus failed candidates | - -Anthropic Messages is supported for callers and serving targets, but not for a -structured-output judge. That restriction is intentional and fails during -static configuration loading. Same-protocol streaming preserves parsed provider -events when the router does not aggregate or replace them; raw SSE bytes and -framing are not part of the compatibility contract. - -### Known issue: OpenAI Responses structured-output judges - -OpenAI Responses targets are accepted for structured-output judges, but the -shared Responses request encoder currently emits the Chat-compatible JSON -Schema object directly under `text.format`. This places `name`, `schema`, and -`strict` under `text.format.json_schema`; conforming Responses endpoints expect -those fields directly under `text.format`. InferenceHub therefore returns HTTP -400 with `Missing required parameter: 'text.format.name'`, and the affected -router follows its existing judge-failure or fall-open path. - -OpenAI Responses remains supported as a caller and ordinary serving-target -protocol. Until the shared `switchyard-translation` encoder is corrected, -configure structured-output judges with `protocol = "openai_chat"`. Follow-up -work must add the inverse of the existing Responses-to-neutral schema conversion -plus core and process-level regression coverage for all three affected router -paths. - -Managed inner provider calls also do not re-enter Relay's downstream provider -middleware. This behavior is part of the current ownership boundary, not an -automatic compatibility fallback. - -Each completed routing-only model call emits a -`switchyard.routing.llm_call` ATOF mark. Its data identifies the algorithm, -attempt, call order, target, routing role, outcome, -latency, and normalized provider token `usage`. The -successful call that serves the caller is deliberately excluded because -Relay's outer LLM end event already records that usage. A failed call, or a -provider response that omits usage, has `usage = null`. Consumers can therefore -add these marks to the outer LLM usage to measure total request compute without -double-counting the serving model. - -`switchyard-llm-client` owns provider retries. Every target, including the -trusted fallback target, uses its default of two additional attempts for -transient provider failures and honors capped `Retry-After` delays. A retry -stays on the selected target and does not rerun the routing algorithm. A random -target with `weight = 0` is fallback-only and is not considered by the -algorithm. Trusted fallback is attempted at most once and, for streaming -responses, only before the first caller event is emitted. - -## Translation and stream fidelity - -`switchyard-translation` is the only request, response, and event translation -layer. It decodes caller JSON into Switchyard's neutral protocol, encodes each -selected call for the target protocol, decodes provider results, and encodes -`ReturnToAgent` back to the caller protocol. Relay codecs are not used. - -The streaming contract carries each parsed provider JSON event in a preservation -envelope alongside its normalized `LlmResponseChunk` representation. -Same-protocol routes replay the preserved JSON unchanged, including -provider-specific fields; this preserves parsed events, not raw SSE bytes or -framing. Cross-protocol routes encode only normalized chunks, and the streaming -helpers still do not expose the buffered translation engine's reject-lossy -diagnostics, so unsupported fields may be normalized or omitted. Replacing -normalized stream content or folding a stream into an aggregate drops the -per-event preservation envelope. - -## Configuration - -The manifest declares `compat.native_api = "1"` and Relay `>=0.8.0,<1.0`. -The manifest API value selects Relay's native plugin contract; the binary uses -Relay 0.8's typed middleware API and native host ABI v4. Rebuild the bundle -when changing SDK versions rather than assuming Rust dynamic-library -compatibility from the manifest value alone. - -A Relay project can configure a seeded weighted-random router as follows: +Point the dynamic plugin configuration at an existing Switchyard deployment: ```toml -version = 1 - [[plugins.dynamic]] -manifest = "/opt/switchyard-relay-plugin/relay-plugin.toml" +plugin_id = "nvidia.switchyard" [plugins.dynamic.config] -version = 2 priority = 0 -[plugins.dynamic.config.algorithm] -kind = "random" -seed = 42 - -[plugins.dynamic.config.default_targets] -openai_chat = "fast" - -[plugins.dynamic.config.targets.fast] -model = "provider/model" -protocol = "openai_chat" -endpoint = "/v1/chat/completions" -base_url = "https://provider.example.com" -weight = 1 -drop_caller_extra_body = true - -[plugins.dynamic.config.targets.fast.header_env] -authorization = "PROVIDER_AUTHORIZATION" -``` - -Target map keys such as `fast` are stable semantic names visible to libsy. The -target binding is authoritative for the provider model, protocol, endpoint, -base URL, weight, and environment-backed headers. Each `default_targets` key -both enables that inbound protocol and names its trusted fallback. - -`header_env` is the only custom provider-header source. It resolves values in -the plugin process at registration time so literal header values never appear -in configuration. Environment values must not appear in errors, routing marks, -spans, or debug output. The plugin does not inherit caller credentials for -managed calls. Each variable supplies the complete header value, so an -`authorization` value must include its scheme, such as `Bearer`. Literal -`headers` configuration is rejected; non-secret routing or tenancy headers must -also use `header_env`. - -Relay may intercept an OpenAI SDK call before the SDK materializes its -`extra_body` option into a provider request. Targets that reject this -caller-specific wrapper can set `drop_caller_extra_body = true`. The plugin -then drops the wrapper and its contents; it does not promote those values to -top-level provider fields. The default is `false` so lossless same-format -forwarding remains unchanged for targets that consume the extension. - -`extra_body` supplies non-secret provider defaults for a target. It is useful -for provider-specific controls such as disabling reasoning on a dedicated -judge model. Fields already present on the caller's request take precedence. -Do not put credentials in `extra_body`; use `header_env` for secrets. - -For `kind = "llm_classifier"`, the classifier target must use `openai_chat` or -`openai_responses`; libsy's judge request uses a JSON-schema response format -that cannot be represented losslessly by Anthropic Messages. Omitting `mode` -selects `capability`, preserving the original version-2 configuration shape. - -Escalation mode evaluates the weak model's completed response before returning -it or replacing it with a strong-model response: - -```toml -[plugins.dynamic.config.algorithm] -kind = "llm_classifier" -mode = "escalation" -classifier_target = "judge" -weak_target = "weak" -strong_target = "strong" -prompt = "Judge whether the weak model is stuck." -max_output_tokens = 512 - -[plugins.dynamic.config.algorithm.escalation] -confirmations = 2 -recent_turn_window = 28 -window_message_chars = 500 +deployment_path = "/etc/switchyard/routes.toml" ``` -`judge`, `weak`, and `strong` are keys in -`plugins.dynamic.config.targets`, configured with the same model, protocol, -URL, and `header_env` fields shown above. The judge must use `openai_chat` or -`openai_responses`; the serving targets may use any supported protocol. +`deployment_path` is a Switchyard version-1 TOML deployment, accepted by both +`switchyard-server` and `switchyard-runner`. See the +[server configuration guide](../switchyard-server/CONFIGURATION.md) for the +deployment schema and routing algorithms. -Use a dedicated, non-reasoning model for the judge when possible. Providers -that expose a reasoning switch can configure it on that target, for example: +## Request handling -```toml -[plugins.dynamic.config.targets.judge] -model = "provider/non-reasoning-judge" -protocol = "openai_chat" -base_url = "https://provider.example.com" -extra_body = { think = false } -``` +For OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages calls, +the plugin decodes the Relay request and checks the requested model against the +deployment's route IDs. -The packaged escalation rubric is intentionally detailed and can consume -roughly two thousand or more input tokens depending on the tokenizer. Every -unlatched request also pays for a complete judge call. A custom `prompt` can -reduce that cost, but should be evaluated against representative trajectories -before deployment. Reasoning models may spend `max_output_tokens` on hidden or -visible reasoning before returning the structured verdict; disable reasoning -with provider-supported `extra_body` controls or raise the cap after measuring. +- A configured route is executed by `switchyard-runner`. +- An unknown model calls Relay's continuation unchanged. +- The returned provider response is encoded back into the caller's wire format. +- Streaming responses are returned as unpolled translated streams; Relay owns + cancellation and the outer serving-call lifecycle. -An unlatched streaming escalation request is intentionally buffered. Libsy must -read the complete weak response before asking the judge, so caller first-token -delivery waits for the weak call and judge verdict. A declined escalation is -reconstructed as a stream from the aggregate response, which drops the -provider-event preservation envelope. A confirmed escalation discards that -weak response and serves the strong target. +The plugin emits a routing request mark, routing-model usage marks, measured +routing-overhead marks, and a selected-model decision mark. Answer-call usage +continues to belong to Relay's outer LLM lifecycle. -The default `confirmations = 2` retains a streak per Switchyard session. Callers -must send a stable `x-switchyard-session-id` header for the streak and strong -latch to survive across turns. Without session identity each request has -isolated state and a multi-confirmation escalation cannot latch. +## Failure policy -A full stage router can combine tool-result signals, model-specific prompts, -handoff notes, and an optional judge for ambiguous turns: - -```toml -[plugins.dynamic.config.algorithm] -kind = "stage_router" -capable_target = "strong" -efficient_target = "weak" -picker = "efficient_first" -confidence_threshold = 0.5 -recent_turn_window = 3 -capable_system_prompt = "Diagnose before editing." -efficient_system_prompt = "Follow the settled plan." - -[plugins.dynamic.config.algorithm.handoff_notes] -escalation_note = "The previous model was stalling; pick up the diagnosis." -deescalation_note = "The task is settled; continue with the mechanical work." -only_on_wrong_signal_escalation = true - -[plugins.dynamic.config.algorithm.classifier] -target = "judge" -base_threshold = 0.5 -threshold_step = 0.1 -recent_turn_window = 3 -prompt = "Estimate whether the efficient target can finish this turn." -max_output_tokens = 512 -``` - -Stage routing reads normalized tool calls and tool results from OpenAI Chat, -OpenAI Responses, and Anthropic Messages traffic. When the signals do not cross -`confidence_threshold`, the optional classifier decides; if it is absent or -cannot decide, the configured picker's default tier serves the turn. The -classifier target has the same structured-output protocol restriction as the -standalone classifier. - -Ambiguous turns that reach the optional classifier add one judge call; -decisive tool signals do not. Decision marks report the selected model from -libsy's `RoutingOutcome`. - -Version-1 service configuration, decision-only execution, and observe-only -mode are rejected. - -## Build and bundle - -The crate is a non-publishable member of the Switchyard Cargo workspace. -Operators install a binary bundle rather than a Rust crate: - -```bash -cargo build --release \ - --manifest-path crates/switchyard-nemo-relay-plugin/Cargo.toml -python3 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py \ - --library target/release/libswitchyard_nemo_relay_plugin.so \ - --output build/switchyard-nemo-relay-plugin-linux-x86_64 \ - --archive dist/switchyard-nemo-relay-plugin-0.2.0-linux-x86_64.tar.gz -``` - -On macOS the library suffix is `.dylib`; Windows builds use `.dll`. The bundle -builder creates the Relay package: the shared library, a materialized manifest -with Relay's inline SHA-256 integrity digest, the JSON schema, and the project -license files. Use `.tar.gz` archives on Linux and macOS and `.zip` on Windows. -The archive's top-level directory is always `switchyard-nemo-relay-plugin`. - -The release archive convention is -`switchyard-nemo-relay-plugin--.`. A future Actions -matrix should upload each archive under the artifact name -`switchyard-nemo-relay-plugin-`, matching Switchyard's existing -platform-qualified artifact convention. - -Install the materialized bundle with Relay's normal lifecycle commands: - -```bash -nemo-relay plugins validate /opt/switchyard-relay-plugin/relay-plugin.toml -nemo-relay plugins add /opt/switchyard-relay-plugin/relay-plugin.toml -nemo-relay plugins enable nvidia.switchyard -nemo-relay plugins inspect nvidia.switchyard -``` +`switchyard-llm-client` owns provider retry and route-candidate fallback +behavior. The plugin does not maintain a separate trusted-default target or +rerun routing after an execution failure. Failures outside the shared runner, +including response translation failures, are returned to Relay. diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json index 8ea12cece..474f212f7 100644 --- a/crates/switchyard-nemo-relay-plugin/config.schema.json +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -1,210 +1,20 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "Switchyard NeMo Relay Plugin", - "description": "In-process Switchyard routing with Switchyard-owned provider HTTP dispatch.", + "description": "Runs a Switchyard deployment through its shared in-process runner.", "type": "object", "additionalProperties": false, - "required": ["version", "algorithm", "targets", "default_targets"], + "required": ["deployment_path"], "properties": { - "version": { - "const": 2, - "description": "Library-only Switchyard configuration version." - }, "priority": { "type": "integer", - "default": 0 - }, - "algorithm": { - "description": "In-process random, capability, escalation, or stage-router configuration.", - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "required": ["kind"], - "properties": { - "kind": { "const": "random" }, - "seed": { "type": ["integer", "null"], "minimum": 0 } - } - }, - { - "type": "object", - "additionalProperties": false, - "required": [ - "kind", - "classifier_target", - "weak_target", - "strong_target", - "base_threshold" - ], - "properties": { - "kind": { "const": "llm_classifier" }, - "mode": { "const": "capability", "default": "capability" }, - "classifier_target": { - "type": "string", - "minLength": 1, - "description": "Semantic target name for the judge. The target must use openai_chat or openai_responses because the judge requires a JSON-schema response format." - }, - "weak_target": { "type": "string", "minLength": 1 }, - "strong_target": { "type": "string", "minLength": 1 }, - "base_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, - "threshold_step": { "type": "number", "minimum": 0, "default": 0 }, - "recent_turn_window": { - "type": ["integer", "null"], - "minimum": 0 - }, - "max_output_tokens": { - "type": "integer", - "minimum": 1, - "default": 4096 - }, - "prompt": { "type": "string" }, - "session_affinity": { "type": "boolean", "default": false }, - "message_hash_fallback": { "type": "boolean", "default": false } - } - }, - { - "type": "object", - "additionalProperties": false, - "required": [ - "kind", - "mode", - "classifier_target", - "weak_target", - "strong_target", - "escalation" - ], - "properties": { - "kind": { "const": "llm_classifier" }, - "mode": { "const": "escalation" }, - "classifier_target": { - "type": "string", - "minLength": 1, - "description": "Semantic target name for the trajectory judge. The target must use openai_chat or openai_responses." - }, - "weak_target": { "type": "string", "minLength": 1 }, - "strong_target": { "type": "string", "minLength": 1 }, - "prompt": { "type": "string" }, - "max_output_tokens": { - "type": "integer", - "minimum": 1, - "default": 4096 - }, - "escalation": { - "type": "object", - "additionalProperties": false, - "properties": { - "confirmations": { "type": "integer", "minimum": 1, "default": 2 }, - "recent_turn_window": { "type": "integer", "minimum": 1, "default": 28 }, - "window_message_chars": { "type": "integer", "minimum": 50, "default": 500 } - } - } - } - }, - { - "type": "object", - "additionalProperties": false, - "required": [ - "kind", - "capable_target", - "efficient_target", - "picker", - "confidence_threshold" - ], - "properties": { - "kind": { "const": "stage_router" }, - "capable_target": { "type": "string", "minLength": 1 }, - "efficient_target": { "type": "string", "minLength": 1 }, - "picker": { "enum": ["capable_first", "efficient_first"] }, - "confidence_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, - "recent_turn_window": { - "type": ["integer", "null"], - "minimum": 0, - "description": "Trailing tool results scored for the current turn. Null uses libsy's default window." - }, - "capable_system_prompt": { "type": "string" }, - "efficient_system_prompt": { "type": "string" }, - "handoff_notes": { - "type": "object", - "additionalProperties": false, - "required": ["escalation_note"], - "properties": { - "escalation_note": { "type": "string", "minLength": 1 }, - "deescalation_note": { "type": ["string", "null"], "minLength": 1 }, - "only_on_wrong_signal_escalation": { "type": "boolean", "default": true } - } - }, - "classifier": { - "type": "object", - "additionalProperties": false, - "required": ["target", "base_threshold"], - "properties": { - "target": { - "type": "string", - "minLength": 1, - "description": "Judge target used only when stage signals are ambiguous. It must use openai_chat or openai_responses." - }, - "base_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, - "threshold_step": { "type": "number", "minimum": 0, "default": 0 }, - "recent_turn_window": { "type": ["integer", "null"], "minimum": 0 }, - "prompt": { "type": "string" }, - "max_output_tokens": { "type": "integer", "minimum": 1, "default": 4096 } - } - } - } - } - ] - }, - "targets": { - "type": "object", - "minProperties": 1, - "additionalProperties": { - "type": "object", - "additionalProperties": false, - "required": ["model", "protocol", "base_url"], - "properties": { - "model": { "type": "string", "minLength": 1 }, - "protocol": { - "enum": ["openai_chat", "openai_responses", "anthropic_messages"] - }, - "endpoint": { - "type": "string", - "pattern": "^$|^/", - "description": "Optional provider endpoint override. The resolved URL must end in the canonical route for the selected protocol." - }, - "base_url": { - "type": "string", - "pattern": "^https?://" - }, - "weight": { "type": "number", "minimum": 0, "default": 1 }, - "drop_caller_extra_body": { - "type": "boolean", - "default": false, - "description": "Drop an intercepted OpenAI SDK extra_body wrapper instead of forwarding it to targets that reject caller-specific extensions." - }, - "extra_body": { - "type": "object", - "default": {}, - "description": "Non-secret provider request defaults, such as judge reasoning controls. Caller-provided fields take precedence.", - "additionalProperties": true - }, - "header_env": { - "type": "object", - "description": "Sole custom provider-header source. Maps header names to environment-variable names resolved by the plugin process so literal values are never stored in configuration.", - "additionalProperties": { "type": "string", "minLength": 1 } - } - } - } + "default": 0, + "description": "NeMo Relay execution-intercept priority." }, - "default_targets": { - "type": "object", - "description": "Maps each managed inbound protocol to its trusted fallback target.", - "minProperties": 1, - "additionalProperties": false, - "properties": { - "openai_chat": { "type": "string", "minLength": 1 }, - "openai_responses": { "type": "string", "minLength": 1 }, - "anthropic_messages": { "type": "string", "minLength": 1 } - } + "deployment_path": { + "type": "string", + "minLength": 1, + "description": "Path to a Switchyard version-1 TOML deployment shared with switchyard-server." } } } diff --git a/crates/switchyard-nemo-relay-plugin/src/client.rs b/crates/switchyard-nemo-relay-plugin/src/client.rs deleted file mode 100644 index 95d7a7a14..000000000 --- a/crates/switchyard-nemo-relay-plugin/src/client.rs +++ /dev/null @@ -1,296 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Switchyard-owned target adapters over a shared HTTP client. - -use std::collections::BTreeMap; -use std::sync::Arc; - -use async_trait::async_trait; -use serde_json::Value as Json; -use switchyard_llm_client::{ - Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, TranslatingLlmClient, -}; -use switchyard_protocol::{ - LlmClientError, ModelId, Request, Response, RoutedLlmClient, WireFormat, -}; -use switchyard_translation::TranslationEngine; - -use crate::translation; - -/// A provider target adapter over a shared HTTP client. -/// -/// libsy routes with a stable semantic name (for example `fast`). The provider -/// still expects its own model id (for example `meta/llama-3.1-8b-instruct`). -/// Keeping that mapping here prevents an algorithm's semantic labels from -/// leaking into provider requests. -pub(crate) struct TargetClient { - provider_model: ModelId, - target_format: WireFormat, - drop_caller_extra_body: bool, - inner: Arc, - translation: TranslationEngine, -} - -impl TargetClient { - pub(crate) fn model_config( - provider_model: String, - target_format: WireFormat, - dispatch_url: String, - headers: BTreeMap, - extra_body: BTreeMap, - ) -> ModelConfig { - let backend_config = HttpBackendConfig { - // `dispatch_url` is already resolved by configuration. Backend URL - // joining accepts a complete canonical endpoint as well as a base - // URL/prefix. - base_url: dispatch_url, - api_key: None, - forward_auth: false, - extra_headers: headers, - extra_body, - max_retries: DEFAULT_MAX_RETRIES, - }; - let backend = match target_format { - WireFormat::OpenAiChat => Backend::OpenAiChat(backend_config), - WireFormat::OpenAiResponses => Backend::OpenAiResponses(backend_config), - WireFormat::AnthropicMessages => Backend::Anthropic(backend_config), - }; - ModelConfig::new(provider_model, backend, None) - } - - pub(crate) fn new( - provider_model: ModelId, - target_format: WireFormat, - drop_caller_extra_body: bool, - inner: Arc, - ) -> Self { - Self { - provider_model, - target_format, - drop_caller_extra_body, - inner, - translation: TranslationEngine::default(), - } - } - - /// Retargets only the provider-facing transport metadata. - /// - /// Correlation and agent identity remain available to libsy, while inbound - /// HTTP headers are deliberately removed. Provider credentials come solely - /// from this target's `header_env` configuration. - fn prepare_request(&self, mut request: Request) -> Request { - let metadata = request.metadata.get_or_insert_default(); - metadata.wire_format = Some(self.target_format); - metadata.http_headers = None; - if self.drop_caller_extra_body { - request.llm_request.extensions.fields.remove("extra_body"); - for preserved in request.llm_request.preservation.requests.values_mut() { - if let Some(body) = preserved.as_object_mut() { - body.remove("extra_body"); - } - } - } - request - } -} - -#[async_trait] -impl RoutedLlmClient for TargetClient { - async fn call(&self, request: Request) -> Result { - let request = self.prepare_request(request); - translation::validate_target_request( - &self.translation, - self.target_format, - &request.llm_request, - ) - .map_err(LlmClientError::RequestEncoding)?; - self.inner - .call_rewrite_model(request, Some(&self.provider_model)) - .await - } -} - -#[cfg(test)] -mod tests { - use std::io::{Read, Write}; - use std::net::TcpListener; - use std::thread; - - use super::*; - use serde_json::json; - use switchyard_protocol::{ - LlmRequest, LlmResponse, Metadata, PreservationMetadata, ProviderExtensions, text_request, - }; - - fn client(format: WireFormat) -> TargetClient { - client_with_options( - format, - match format { - WireFormat::OpenAiChat => "https://provider.example/v1/chat/completions".into(), - WireFormat::OpenAiResponses => "https://provider.example/v1/responses".into(), - WireFormat::AnthropicMessages => "https://provider.example/v1/messages".into(), - }, - false, - ) - } - - fn client_with_options( - format: WireFormat, - dispatch_url: String, - drop_caller_extra_body: bool, - ) -> TargetClient { - let model = TargetClient::model_config( - "provider/model".into(), - format, - dispatch_url, - BTreeMap::new(), - BTreeMap::new(), - ); - let inner = Arc::new(TranslatingLlmClient::new(&[model]).unwrap()); - TargetClient::new( - ModelId::from("provider/model"), - format, - drop_caller_extra_body, - inner, - ) - } - - #[test] - fn target_preparation_forces_format_and_removes_inbound_headers() { - let client = client(WireFormat::AnthropicMessages); - let request = Request { - metadata: Some(Metadata { - correlation_id: Some("request-123".into()), - wire_format: Some(WireFormat::OpenAiChat), - http_headers: Some(http::HeaderMap::from_iter([ - ( - http::HeaderName::from_static("authorization"), - http::HeaderValue::from_static("Bearer caller-secret"), - ), - ( - http::HeaderName::from_static("x-caller-only"), - http::HeaderValue::from_static("must-not-forward"), - ), - ])), - ..Metadata::default() - }), - ..Request::default() - }; - - let prepared = client.prepare_request(request); - let metadata = prepared.metadata.unwrap(); - assert_eq!(metadata.wire_format, Some(WireFormat::AnthropicMessages)); - assert_eq!(metadata.correlation_id.as_deref(), Some("request-123")); - assert!(metadata.http_headers.is_none()); - } - - #[test] - fn missing_metadata_is_created_for_the_target_format() { - let client = client(WireFormat::OpenAiResponses); - let prepared = client.prepare_request(Request::default()); - assert_eq!( - prepared.metadata.and_then(|metadata| metadata.wire_format), - Some(WireFormat::OpenAiResponses) - ); - } - - #[test] - fn configured_target_drops_intercepted_caller_extra_body() { - let client = client_with_options( - WireFormat::OpenAiChat, - "https://provider.example/v1/chat/completions".into(), - true, - ); - let request = Request { - llm_request: LlmRequest { - extensions: ProviderExtensions { - fields: serde_json::Map::from_iter([( - "extra_body".into(), - json!({"reasoning": {"effort": "medium"}}), - )]), - }, - preservation: PreservationMetadata { - requests: BTreeMap::from([( - WireFormat::OpenAiChat.into(), - json!({ - "model": "route", - "messages": [{"role": "user", "content": "hello"}], - "extra_body": { - "reasoning": {"effort": "medium"}, - "session_id": "hermes-session" - } - }), - )]), - ..PreservationMetadata::default() - }, - ..LlmRequest::default() - }, - ..Request::default() - }; - - let prepared = client.prepare_request(request); - assert!( - !prepared - .llm_request - .extensions - .fields - .contains_key("extra_body") - ); - assert!( - prepared - .llm_request - .preservation - .requests - .values() - .all(|body| body.get("extra_body").is_none()) - ); - } - - #[tokio::test] - async fn target_client_retries_transient_provider_failures() { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind provider stub"); - let address = listener.local_addr().expect("read provider stub address"); - let server = thread::spawn(move || -> std::io::Result<()> { - let response_body = "{\"id\":\"chatcmpl-1\",\"model\":\"provider/model\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"recovered\"},\"finish_reason\":\"stop\"}],\"usage\":{}}"; - let responses = [ - "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 11\r\nConnection: close\r\n\r\nunavailable".to_string(), - "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 11\r\nConnection: close\r\n\r\nunavailable".to_string(), - format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response_body}", - response_body.len() - ), - ]; - for response in responses { - let (mut stream, _) = listener.accept()?; - let mut request = [0_u8; 1024]; - if stream.read(&mut request)? == 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "client closed before sending a request", - )); - } - stream.write_all(response.as_bytes())?; - } - Ok(()) - }); - let client = client_with_options( - WireFormat::OpenAiChat, - format!("http://{address}/v1"), - false, - ); - let response = client - .call(Request { - llm_request: text_request(Some("route".into()), "hello"), - ..Request::default() - }) - .await - .expect("third provider attempt succeeds"); - - assert!(matches!(response.llm_response, LlmResponse::Agg(_))); - server - .join() - .expect("provider stub panicked") - .expect("provider stub failed"); - } -} diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index e280de103..9b235a62d 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -1,22 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Arc; +use std::path::PathBuf; -use http::Uri; -use http::header::{HeaderName, HeaderValue}; use serde::Deserialize; -use serde_json::Value as Json; -use switchyard_libsy::{ - Algorithm, ClassifierContractConfig, EscalationJudgeConfig, HandoffNoteConfig, - LlmClassifierConfig, LlmFallback, LlmTaskClassifier, PickerMode, Random, StageRouter, - StageRouterConfig, TargetPrompts, TaskClassifierConfig, -}; -use switchyard_llm_client::{ModelConfig, TranslatingLlmClient}; -use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat}; - -use crate::client::TargetClient; +use switchyard_protocol::WireFormat; +use switchyard_runner::Runner; pub(crate) fn protocol_from_call(name: &str) -> Option { match name { @@ -27,580 +16,16 @@ pub(crate) fn protocol_from_call(name: &str) -> Option { } } -const fn default_endpoint(protocol: WireFormat) -> &'static str { - match protocol { - WireFormat::OpenAiChat => "/v1/chat/completions", - WireFormat::OpenAiResponses => "/v1/responses", - WireFormat::AnthropicMessages => "/v1/messages", - } -} - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct TargetBinding { - model: String, - protocol: WireFormat, - #[serde(default)] - endpoint: String, - base_url: String, - #[serde(default = "default_weight")] - weight: f64, - #[serde(default)] - drop_caller_extra_body: bool, - #[serde(default)] - header_env: BTreeMap, - #[serde(default)] - extra_body: BTreeMap, -} - -impl TargetBinding { - fn dispatch_url(&self) -> String { - let base = self.base_url.trim_end_matches('/'); - let default = default_endpoint(self.protocol); - if self.endpoint.is_empty() && base.ends_with(default) { - return base.to_string(); - } - let endpoint = if self.endpoint.is_empty() { - default - } else { - &self.endpoint - }; - let endpoint = if base.ends_with("/v1") && endpoint.starts_with("/v1/") { - &endpoint[3..] - } else { - endpoint - }; - format!("{base}{endpoint}") - } - - fn validate(&self, name: &str) -> Result<(), String> { - if self.model.trim().is_empty() { - return Err(format!("target {name:?} model must be non-empty")); - } - if !self.endpoint.is_empty() && !self.endpoint.starts_with('/') { - return Err(format!( - "target {name:?} endpoint must be empty or begin with '/'" - )); - } - if !self.weight.is_finite() || self.weight < 0.0 { - return Err(format!( - "target {name:?} weight must be finite and nonnegative" - )); - } - validate_dispatch_url(name, self.protocol, &self.dispatch_url())?; - self.validate_headers(name) - } - - fn validate_headers(&self, target_name: &str) -> Result<(), String> { - let mut normalized = BTreeSet::new(); - for (name, variable) in &self.header_env { - let canonical = validate_header_name(name)?; - if !normalized.insert(canonical) { - return Err(format!( - "target {target_name:?} configures header {name:?} more than once (header names are case-insensitive)" - )); - } - if variable.trim().is_empty() { - return Err(format!( - "environment variable name for target header {name:?} must not be empty" - )); - } - if variable.as_bytes().contains(&b'=') || variable.as_bytes().contains(&b'\0') { - return Err(format!( - "environment variable name for target header {name:?} must not contain '=' or NUL" - )); - } - } - Ok(()) - } - - fn prepare(&self) -> Result { - let mut headers = BTreeMap::new(); - for (name, variable) in &self.header_env { - let value = std::env::var(variable) - .map_err(|_| format!("environment variable {variable:?} is not set"))?; - validate_header(name, &value)?; - headers.insert(name.clone(), value); - } - let model_config = TargetClient::model_config( - self.model.clone(), - self.protocol, - self.dispatch_url(), - headers, - self.extra_body.clone(), - ); - Ok(PreparedTargetTransport { - provider_model: ModelId::from(self.model.clone()), - protocol: self.protocol, - drop_caller_extra_body: self.drop_caller_extra_body, - model_config, - }) - } -} - -struct PreparedTargetTransport { - provider_model: ModelId, - protocol: WireFormat, - drop_caller_extra_body: bool, - model_config: ModelConfig, -} - -pub(crate) struct PreparedTargetBinding { - pub(crate) client: Arc, -} - -#[derive(Clone, Copy, Default, Deserialize)] -#[serde(rename_all = "snake_case")] -enum LlmClassifierMode { - #[default] - Capability, - Escalation, -} - -#[derive(Clone, Deserialize)] -#[serde(deny_unknown_fields)] -struct LlmClassifierAlgorithmConfig { - #[serde(default)] - mode: LlmClassifierMode, - classifier_target: String, - weak_target: String, - strong_target: String, - #[serde(default)] - base_threshold: Option, - #[serde(default)] - threshold_step: Option, - #[serde(default)] - session_affinity: Option, - #[serde(default)] - message_hash_fallback: Option, - #[serde(default)] - recent_turn_window: Option, - #[serde(default)] - prompt: Option, - #[serde(default = "default_classifier_max_output_tokens")] - max_output_tokens: u64, - #[serde(default)] - escalation: Option, -} - -impl LlmClassifierAlgorithmConfig { - fn capability_config(&self) -> Result { - if self.escalation.is_some() { - return Err( - "llm_classifier capability mode does not accept escalation settings".into(), - ); - } - let base_threshold = self - .base_threshold - .ok_or_else(|| "llm_classifier capability mode requires base_threshold".to_string())?; - let mut contract = ClassifierContractConfig::default(); - if let Some(prompt) = &self.prompt { - contract = contract.with_prompt(prompt.clone()); - } - Ok(TaskClassifierConfig { - base_threshold, - threshold_step: self.threshold_step.unwrap_or_default(), - session_affinity: self.session_affinity.unwrap_or_default(), - message_hash_fallback: self.message_hash_fallback.unwrap_or_default(), - recent_turn_window: self.recent_turn_window, - contract, - max_output_tokens: self.max_output_tokens, - }) - } - - fn escalation_config( - &self, - ) -> Result<(ClassifierContractConfig, EscalationJudgeConfig), String> { - if self.base_threshold.is_some() - || self.threshold_step.is_some() - || self.session_affinity.is_some() - || self.message_hash_fallback.is_some() - || self.recent_turn_window.is_some() - { - return Err( - "llm_classifier escalation mode does not accept capability settings".into(), - ); - } - let config = self.escalation.clone().ok_or_else(|| { - "llm_classifier escalation mode requires escalation settings".to_string() - })?; - let mut contract = ClassifierContractConfig::default(); - if let Some(prompt) = &self.prompt { - contract = contract.with_prompt(prompt.clone()); - } - Ok((contract, config)) - } -} - -#[derive(Clone, Deserialize)] -#[serde(deny_unknown_fields)] -struct StageFallbackConfig { - target: String, - base_threshold: f64, - #[serde(default)] - threshold_step: f64, - #[serde(default)] - recent_turn_window: Option, - #[serde(default)] - prompt: Option, - #[serde(default = "default_classifier_max_output_tokens")] - max_output_tokens: u64, -} - -impl StageFallbackConfig { - fn classifier_config(&self) -> TaskClassifierConfig { - let mut contract = ClassifierContractConfig::default(); - if let Some(prompt) = &self.prompt { - contract = contract.with_prompt(prompt.clone()); - } - TaskClassifierConfig { - base_threshold: self.base_threshold, - threshold_step: self.threshold_step, - session_affinity: false, - message_hash_fallback: false, - recent_turn_window: self.recent_turn_window, - contract, - max_output_tokens: self.max_output_tokens, - } - } -} - -#[derive(Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -enum AlgorithmConfig { - Random { - #[serde(default)] - seed: Option, - }, - LlmClassifier { - #[serde(flatten)] - config: LlmClassifierAlgorithmConfig, - }, - StageRouter { - capable_target: String, - efficient_target: String, - picker: PickerMode, - confidence_threshold: f64, - #[serde(default)] - recent_turn_window: Option, - #[serde(default)] - capable_system_prompt: Option, - #[serde(default)] - efficient_system_prompt: Option, - #[serde(default)] - handoff_notes: Option, - #[serde(default)] - classifier: Option, - }, -} - #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct SwitchyardConfig { - version: u32, #[serde(default)] pub(crate) priority: i32, - algorithm: AlgorithmConfig, - targets: BTreeMap, - default_targets: BTreeMap, -} - -pub(crate) struct PreparedConfig { - pub(crate) algorithm: Arc, - pub(crate) targets: BTreeMap, - pub(crate) default_targets: BTreeMap, + pub(crate) deployment_path: PathBuf, } impl SwitchyardConfig { - pub(crate) fn validate(&self) -> Result<(), String> { - self.validate_structure()?; - self.build_algorithm(None).map(drop) - } - - fn validate_structure(&self) -> Result<(), String> { - if self.version != 2 { - return Err(format!( - "unsupported Switchyard config version {}; version 1 used switchyard-server; migrate to version = 2", - self.version - )); - } - if self.targets.is_empty() { - return Err("targets must not be empty".into()); - } - if self.default_targets.is_empty() { - return Err("default_targets must not be empty".into()); - } - for (name, target) in &self.targets { - if name.trim().is_empty() { - return Err("target names must be non-empty".into()); - } - target.validate(name)?; - } - for (protocol, fallback) in &self.default_targets { - let target = self - .targets - .get(fallback) - .ok_or_else(|| format!("default target {fallback:?} is not configured"))?; - if target.protocol != *protocol { - return Err(format!( - "default target {fallback:?} must use protocol {}", - protocol.as_str() - )); - } - } - Ok(()) - } - - pub(crate) fn prepare(self) -> Result { - self.validate_structure()?; - let transports = self - .targets - .iter() - .map(|(name, target)| target.prepare().map(|prepared| (name.clone(), prepared))) - .collect::, _>>()?; - let models = transports - .values() - .map(|transport| transport.model_config.clone()) - .collect::>(); - // One multi-model client shares reqwest connection pools across targets. - let client = Arc::new( - TranslatingLlmClient::new(&models) - .map_err(|error| format!("failed to create target HTTP client: {error}"))?, - ); - let targets = transports - .into_iter() - .map(|(name, transport)| { - let target = TargetClient::new( - transport.provider_model, - transport.protocol, - transport.drop_caller_extra_body, - client.clone(), - ); - ( - name, - PreparedTargetBinding { - client: Arc::new(target), - }, - ) - }) - .collect(); - let algorithm = self.build_algorithm(Some(&targets))?; - Ok(PreparedConfig { - algorithm, - targets, - default_targets: self.default_targets, - }) - } - - fn build_algorithm( - &self, - prepared: Option<&BTreeMap>, - ) -> Result, String> { - let target = |name: &str| { - if !self.targets.contains_key(name) { - return Err(format!("algorithm target {name:?} is not configured")); - } - Ok(match prepared { - Some(targets) => { - targets - .get(name) - .ok_or_else(|| format!("algorithm target {name:?} was not prepared"))?; - ModelId::from(name) - } - None => ModelId::from(name), - }) - }; - - match &self.algorithm { - AlgorithmConfig::Random { seed } => { - let routable = self - .targets - .iter() - .filter(|(_, binding)| binding.weight > 0.0) - .collect::>(); - if routable.is_empty() { - return Err( - "random routing requires at least one positive target weight".into(), - ); - } - let targets = routable - .iter() - .map(|(name, _)| target(name)) - .collect::, _>>()?; - let weights = routable - .iter() - .map(|(_, binding)| binding.weight) - .collect::>(); - Random::new(targets, Some(weights), *seed) - .map(|algorithm| Arc::new(algorithm) as Arc) - .map_err(|error| error.to_string()) - } - AlgorithmConfig::LlmClassifier { config } => { - self.validate_judge_target(&config.classifier_target)?; - let algorithm = match config.mode { - LlmClassifierMode::Capability => LlmClassifierConfig::Capability { - judge_target: target(&config.classifier_target)?, - efficient_target: target(&config.weak_target)?, - capable_target: target(&config.strong_target)?, - config: config.capability_config()?, - }, - LlmClassifierMode::Escalation => { - let (contract, escalation) = config.escalation_config()?; - LlmClassifierConfig::Escalation { - judge_target: target(&config.classifier_target)?, - efficient_target: target(&config.weak_target)?, - capable_target: target(&config.strong_target)?, - contract, - config: escalation, - max_output_tokens: config.max_output_tokens, - } - } - }; - LlmTaskClassifier::new(algorithm) - .map(|algorithm| Arc::new(algorithm) as Arc) - .map_err(|error| error.to_string()) - } - AlgorithmConfig::StageRouter { - capable_target, - efficient_target, - picker, - confidence_threshold, - recent_turn_window, - capable_system_prompt, - efficient_system_prompt, - handoff_notes, - classifier, - } => { - let capable = target(capable_target)?; - let efficient = target(efficient_target)?; - let mut config = StageRouterConfig::new(*picker, *confidence_threshold); - config.recent_window = *recent_turn_window; - config.handoff_notes = handoff_notes.clone(); - let mut prompts = TargetPrompts::default(); - if let Some(prompt) = capable_system_prompt { - prompts = prompts.with(capable_target.as_str(), prompt); - } - if let Some(prompt) = efficient_system_prompt { - prompts = prompts.with(efficient_target.as_str(), prompt); - } - config.tier_prompts = prompts; - if let Some(classifier) = classifier { - self.validate_judge_target(&classifier.target)?; - config.llm_fallback = Some(LlmFallback { - judge_target: target(&classifier.target)?, - config: classifier.classifier_config(), - }); - } - StageRouter::new(capable, efficient, config) - .map(|algorithm| Arc::new(algorithm) as Arc) - .map_err(|error| error.to_string()) - } - } - } - - fn validate_judge_target(&self, name: &str) -> Result<(), String> { - let binding = self - .targets - .get(name) - .ok_or_else(|| format!("algorithm target {name:?} is not configured"))?; - if binding.protocol == WireFormat::AnthropicMessages { - return Err(format!( - "classifier target {name:?} uses anthropic_messages, which cannot encode the required JSON-schema response format without loss; use an openai_chat or openai_responses target" - )); - } - Ok(()) + pub(crate) fn load_runner(&self) -> Result { + Runner::load(&self.deployment_path).map_err(|error| error.to_string()) } } - -fn validate_dispatch_url( - target_name: &str, - protocol: WireFormat, - dispatch_url: &str, -) -> Result<(), String> { - let uri = dispatch_url - .parse::() - .map_err(|error| format!("target {target_name:?} has invalid URL: {error}"))?; - if !matches!(uri.scheme_str(), Some("http" | "https")) { - return Err(format!( - "target {target_name:?} base_url must use http or https" - )); - } - let authority = uri - .authority() - .ok_or_else(|| format!("target {target_name:?} URL must include a host"))?; - if authority.host().is_empty() { - return Err(format!("target {target_name:?} URL must include a host")); - } - if authority.as_str().contains('@') { - return Err(format!( - "target {target_name:?} URL must not contain embedded credentials" - )); - } - if uri.query().is_some() { - return Err(format!( - "target {target_name:?} URL query parameters are not supported" - )); - } - - // The current switchyard-llm-client accepts provider base URLs and complete - // canonical endpoints. Reject a custom terminal route to avoid - // allowing Backend::url() to append another provider suffix silently. - let expected_suffix = match protocol { - WireFormat::OpenAiChat => "/chat/completions", - WireFormat::OpenAiResponses => "/responses", - WireFormat::AnthropicMessages => "/v1/messages", - }; - if !uri.path().ends_with(expected_suffix) { - return Err(format!( - "target {target_name:?} endpoint must resolve to a canonical {protocol} route ending in {expected_suffix:?}" - )); - } - Ok(()) -} - -fn validate_header_name(name: &str) -> Result { - let parsed = HeaderName::from_bytes(name.as_bytes()) - .map_err(|error| format!("invalid target header name {name:?}: {error}"))?; - let canonical = parsed.as_str().to_ascii_lowercase(); - if is_forbidden_target_header(&canonical) { - return Err(format!( - "target header {name:?} is controlled by the HTTP transport and cannot be configured" - )); - } - Ok(canonical) -} - -fn validate_header(name: &str, value: &str) -> Result { - let canonical = validate_header_name(name)?; - HeaderValue::from_str(value) - .map_err(|error| format!("invalid target header value for {name:?}: {error}"))?; - Ok(canonical) -} - -fn is_forbidden_target_header(name: &str) -> bool { - matches!( - name, - "connection" - | "content-length" - | "host" - | "keep-alive" - | "proxy-connection" - | "proxy-authenticate" - | "proxy-authorization" - | "te" - | "trailer" - | "transfer-encoding" - | "upgrade" - ) || name.starts_with("x-nemo-relay-internal-") -} - -const fn default_weight() -> f64 { - 1.0 -} - -fn default_classifier_max_output_tokens() -> u64 { - TaskClassifierConfig::default().max_output_tokens -} - -#[cfg(test)] -mod tests; diff --git a/crates/switchyard-nemo-relay-plugin/src/config/tests.rs b/crates/switchyard-nemo-relay-plugin/src/config/tests.rs deleted file mode 100644 index 06b5d6d31..000000000 --- a/crates/switchyard-nemo-relay-plugin/src/config/tests.rs +++ /dev/null @@ -1,566 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use super::*; -use serde_json::{Value, json}; - -fn binding(protocol: WireFormat, model: &str) -> TargetBinding { - TargetBinding { - model: model.into(), - protocol, - endpoint: String::new(), - base_url: "https://provider.example/v1".into(), - weight: 1.0, - drop_caller_extra_body: false, - header_env: BTreeMap::new(), - extra_body: BTreeMap::new(), - } -} - -fn config() -> SwitchyardConfig { - SwitchyardConfig { - version: 2, - priority: 0, - algorithm: AlgorithmConfig::Random { seed: Some(42) }, - targets: BTreeMap::from([ - ( - "chat".into(), - binding(WireFormat::OpenAiChat, "provider/chat"), - ), - ( - "responses".into(), - binding(WireFormat::OpenAiResponses, "provider/responses"), - ), - ( - "anthropic".into(), - binding(WireFormat::AnthropicMessages, "provider/anthropic"), - ), - ]), - default_targets: BTreeMap::from([ - (WireFormat::OpenAiChat, "chat".into()), - (WireFormat::OpenAiResponses, "responses".into()), - (WireFormat::AnthropicMessages, "anthropic".into()), - ]), - } -} - -#[test] -fn version_two_random_configuration_builds_clients_without_a_service() { - let config = config(); - config.validate().unwrap(); - let prepared = config.prepare().unwrap(); - assert_eq!(prepared.algorithm.name(), "random"); - assert_eq!(prepared.targets.len(), 3); - assert!( - prepared - .targets - .values() - .all(|target| Arc::strong_count(&target.client) == 1) - ); -} - -#[test] -fn target_endpoints_must_be_canonical_for_the_current_http_client() { - let mut config = config(); - config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat".into(); - let error = config.validate().unwrap_err(); - assert!(error.contains("ending in \"/chat/completions\"")); - - config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat/completions".into(); - config.validate().unwrap(); - assert_eq!( - config.targets["chat"].dispatch_url(), - "https://provider.example/v1/custom/chat/completions" - ); -} - -#[test] -fn complete_provider_endpoint_is_not_appended_twice() { - let mut config = config(); - let chat = config.targets.get_mut("chat").unwrap(); - chat.base_url = "https://provider.example/v1/chat/completions/".into(); - assert_eq!( - chat.dispatch_url(), - "https://provider.example/v1/chat/completions" - ); - config.validate().unwrap(); -} - -#[test] -fn absolute_urls_cannot_embed_credentials_or_query_parameters() { - let mut config = config(); - config.targets.get_mut("chat").unwrap().base_url = - "https://user:password@provider.example/v1".into(); - assert!( - config - .validate() - .unwrap_err() - .contains("embedded credentials") - ); - - config.targets.get_mut("chat").unwrap().base_url = - "https://provider.example/v1?api-version=1".into(); - assert!(config.validate().unwrap_err().contains("query parameters")); -} - -#[test] -fn transport_owned_and_case_duplicate_environment_headers_are_rejected() { - let mut host_header_config = config(); - let chat = host_header_config.targets.get_mut("chat").unwrap(); - chat.header_env.insert("Host".into(), "TARGET_HOST".into()); - assert!( - host_header_config - .validate() - .unwrap_err() - .contains("HTTP transport") - ); - - let mut duplicate_config = config(); - let chat = duplicate_config.targets.get_mut("chat").unwrap(); - chat.header_env - .insert("X-Tenant".into(), "TARGET_TENANT_A".into()); - chat.header_env - .insert("x-tenant".into(), "TARGET_TENANT_B".into()); - assert!( - duplicate_config - .validate() - .unwrap_err() - .contains("more than once") - ); -} - -#[test] -fn only_canonical_relay_execution_names_resolve_protocols() { - assert_eq!( - protocol_from_call("openai.chat_completions"), - Some(WireFormat::OpenAiChat) - ); - assert_eq!( - protocol_from_call("openai.responses"), - Some(WireFormat::OpenAiResponses) - ); - assert_eq!( - protocol_from_call("anthropic.messages"), - Some(WireFormat::AnthropicMessages) - ); - assert_eq!(protocol_from_call("openai_chat"), None); -} - -#[test] -fn schema_required_contract_fields_do_not_default_during_deserialization() { - let base = json!({ - "version": 2, - "algorithm": {"kind": "random"}, - "targets": { - "chat": { - "model": "provider/chat", - "protocol": "openai_chat", - "base_url": "https://provider.example/v1" - } - }, - "default_targets": {"openai_chat": "chat"} - }); - for field in ["version", "algorithm", "default_targets"] { - let mut value = base.clone(); - value.as_object_mut().unwrap().remove(field); - let error = serde_json::from_value::(value) - .err() - .expect("required field must not default"); - assert!(error.to_string().contains(field), "field={field}: {error}"); - } -} - -#[test] -fn plugin_retry_budget_is_not_configurable() { - let value = json!({ - "version": 2, - "max_retries": 3, - "algorithm": {"kind": "random"}, - "targets": { - "chat": { - "model": "provider/chat", - "protocol": "openai_chat", - "base_url": "https://provider.example/v1" - } - }, - "default_targets": {"openai_chat": "chat"} - }); - - let error = serde_json::from_value::(value) - .err() - .expect("plugin retry budget must be rejected"); - assert!(error.to_string().contains("max_retries")); -} - -#[test] -fn unknown_target_fields_are_rejected() { - let value = json!({ - "version": 2, - "algorithm": {"kind": "random"}, - "targets": { - "chat": { - "model": "provider/chat", - "protocol": "openai_chat", - "base_url": "https://provider.example/v1", - "unexpected_setting": true - } - }, - "default_targets": {"openai_chat": "chat"} - }); - let error = serde_json::from_value::(value) - .err() - .expect("unknown target field must be rejected"); - assert!(error.to_string().contains("unexpected_setting")); -} - -#[test] -fn literal_target_headers_are_rejected() { - let value = json!({ - "version": 2, - "algorithm": {"kind": "random"}, - "targets": { - "chat": { - "model": "provider/chat", - "protocol": "openai_chat", - "base_url": "https://provider.example/v1", - "headers": {"x-provider-token": "plaintext-secret"} - } - }, - "default_targets": {"openai_chat": "chat"} - }); - let error = serde_json::from_value::(value) - .err() - .expect("literal target headers must be rejected") - .to_string(); - assert!(error.contains("unknown field `headers`")); - assert!(!error.contains("plaintext-secret")); -} - -#[test] -fn unknown_algorithm_fields_are_rejected() { - let error = serde_json::from_value::(json!({ - "kind": "random", - "seed": 42, - "unexpected_setting": true - })) - .err() - .expect("unknown algorithm field must be rejected"); - assert!(error.to_string().contains("unexpected_setting")); -} - -#[test] -fn classifier_prepares_clients_for_judge_and_routed_targets() { - let mut config = config(); - config.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "base_threshold": 0.5, - "recent_turn_window": 4, - "max_output_tokens": 512 - })) - .unwrap(); - config.validate().unwrap(); - let prepared = config.prepare().unwrap(); - assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); - assert!( - prepared - .targets - .values() - .all(|target| Arc::strong_count(&target.client) == 1) - ); -} - -#[test] -fn target_provider_defaults_are_accepted_for_judge_controls() { - let mut config = config(); - config.targets.get_mut("chat").unwrap().extra_body = - BTreeMap::from([("think".into(), json!(false))]); - - config.validate().unwrap(); - config.prepare().unwrap(); -} - -#[test] -fn classifier_rejects_anthropic_judge_targets_before_dispatch() { - let mut config = config(); - config.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "classifier_target": "anthropic", - "weak_target": "responses", - "strong_target": "chat", - "base_threshold": 0.5 - })) - .unwrap(); - - let error = config.validate().unwrap_err(); - assert!(error.contains("classifier target \"anthropic\" uses anthropic_messages")); -} - -#[test] -fn validation_does_not_resolve_environment_backed_headers() { - let mut config = config(); - config.targets.get_mut("chat").unwrap().header_env = BTreeMap::from([( - "authorization".into(), - "SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET".into(), - )]); - - config.validate().unwrap(); - let error = config - .prepare() - .err() - .expect("preparation must resolve headers"); - assert!(error.contains("SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET")); -} - -#[test] -fn invalid_environment_variable_names_are_rejected_before_resolution() { - for variable in ["INVALID=VARIABLE", "INVALID\0VARIABLE"] { - let mut config = config(); - config.targets.get_mut("chat").unwrap().header_env = - BTreeMap::from([("authorization".into(), variable.into())]); - - let error = config.validate().unwrap_err(); - assert!(error.contains("must not contain '=' or NUL")); - } -} - -#[test] -fn static_validation_preserves_algorithm_constructor_checks() { - let mut random = config(); - for target in random.targets.values_mut() { - target.weight = 0.0; - } - assert!( - random - .validate() - .unwrap_err() - .contains("at least one positive target weight") - ); - - let mut classifier = config(); - classifier.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "base_threshold": 1.1 - })) - .unwrap(); - assert!( - classifier - .validate() - .unwrap_err() - .contains("base_threshold must be between 0 and 1") - ); -} - -#[test] -fn escalation_classifier_builds_with_defaulted_policy_settings() { - let mut config = config(); - config.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "mode": "escalation", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "prompt": "Judge the completed trajectory.", - "max_output_tokens": 256, - "escalation": {} - })) - .unwrap(); - - config.validate().unwrap(); - let prepared = config.prepare().unwrap(); - assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); - assert!( - prepared - .targets - .values() - .all(|target| Arc::strong_count(&target.client) == 1) - ); -} - -#[test] -fn classifier_modes_reject_mixed_or_missing_settings() { - let mut capability = config(); - capability.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "base_threshold": 0.5, - "escalation": {} - })) - .unwrap(); - assert!( - capability - .validate() - .unwrap_err() - .contains("capability mode does not accept escalation") - ); - - let mut escalation = config(); - escalation.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "mode": "escalation", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "base_threshold": 0.5, - "escalation": {} - })) - .unwrap(); - assert!( - escalation - .validate() - .unwrap_err() - .contains("escalation mode does not accept capability") - ); - - let mut missing = config(); - missing.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "mode": "escalation", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic" - })) - .unwrap(); - assert!( - missing - .validate() - .unwrap_err() - .contains("requires escalation settings") - ); -} - -#[test] -fn escalation_settings_are_validated_by_the_libsy_constructor() { - for (settings, expected) in [ - ( - json!({"confirmations": 0}), - "confirmations must be at least 1", - ), - ( - json!({"recent_turn_window": 0}), - "recent_turn_window must be at least 1", - ), - ( - json!({"window_message_chars": 49}), - "window_message_chars must be at least 50", - ), - ] { - let mut config = config(); - config.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "mode": "escalation", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "escalation": settings - })) - .unwrap(); - assert!(config.validate().unwrap_err().contains(expected)); - } -} - -#[test] -fn full_stage_router_configuration_builds_all_clients() { - let mut config = config(); - config.algorithm = serde_json::from_value(json!({ - "kind": "stage_router", - "capable_target": "anthropic", - "efficient_target": "responses", - "picker": "efficient_first", - "confidence_threshold": 0.5, - "recent_turn_window": 3, - "capable_system_prompt": "Diagnose before editing.", - "efficient_system_prompt": "Follow the settled plan.", - "handoff_notes": { - "escalation_note": "The previous model was stalling.", - "deescalation_note": "The task is settled.", - "only_on_wrong_signal_escalation": true - }, - "classifier": { - "target": "chat", - "base_threshold": 0.5, - "threshold_step": 0.1, - "recent_turn_window": 3, - "prompt": "Can the efficient tier finish this turn?", - "max_output_tokens": 256 - } - })) - .unwrap(); - - config.validate().unwrap(); - let prepared = config.prepare().unwrap(); - assert_eq!(prepared.algorithm.name(), "stage_router"); - assert!( - prepared - .targets - .values() - .all(|target| Arc::strong_count(&target.client) == 1) - ); -} - -#[test] -fn stage_router_validates_threshold_targets_and_judge_protocol() { - let stage = |classifier: Value, threshold: f64| { - serde_json::from_value(json!({ - "kind": "stage_router", - "capable_target": "anthropic", - "efficient_target": "responses", - "picker": "capable_first", - "confidence_threshold": threshold, - "classifier": classifier - })) - .unwrap() - }; - - let mut invalid_threshold = config(); - invalid_threshold.algorithm = stage(Value::Null, 1.1); - assert!( - invalid_threshold - .validate() - .unwrap_err() - .contains("confidence_threshold must be between 0 and 1") - ); - - let mut missing_target = config(); - missing_target.algorithm = serde_json::from_value(json!({ - "kind": "stage_router", - "capable_target": "missing", - "efficient_target": "responses", - "picker": "capable_first", - "confidence_threshold": 0.5 - })) - .unwrap(); - assert!( - missing_target - .validate() - .unwrap_err() - .contains("algorithm target \"missing\" is not configured") - ); - - let mut anthropic_judge = config(); - anthropic_judge.algorithm = stage(json!({"target": "anthropic", "base_threshold": 0.5}), 0.5); - assert!( - anthropic_judge - .validate() - .unwrap_err() - .contains("classifier target \"anthropic\" uses anthropic_messages") - ); -} - -#[test] -fn zero_weight_random_targets_are_fallback_only() { - let mut config = config(); - config.targets.get_mut("anthropic").unwrap().weight = 0.0; - let prepared = config.prepare().unwrap(); - - assert_eq!(Arc::strong_count(&prepared.targets["anthropic"].client), 1); - assert_eq!(Arc::strong_count(&prepared.targets["chat"].client), 1); - assert_eq!(Arc::strong_count(&prepared.targets["responses"].client), 1); -} diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs index 0f9446c61..f115923b3 100644 --- a/crates/switchyard-nemo-relay-plugin/src/lib.rs +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -1,15 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -mod client; mod config; mod runtime; mod translation; -use std::future::Future; -use std::pin::Pin; use std::sync::Arc; -use std::task::{Context, Poll}; use nemo_relay_plugin::{ ConfigDiagnostic, DiagnosticLevel, Json, LlmJsonAsyncStream, NativePlugin, PluginContext, @@ -17,8 +13,8 @@ use nemo_relay_plugin::{ }; use serde_json::Map; -use crate::config::SwitchyardConfig; -use crate::runtime::{RoutingMark, StreamMessage, SwitchyardRuntime}; +use crate::config::{SwitchyardConfig, protocol_from_call}; +use crate::runtime::{RoutingMark, SwitchyardRuntime}; #[derive(Default)] struct SwitchyardPlugin; @@ -33,8 +29,8 @@ impl NativePlugin for SwitchyardPlugin { } fn validate(&self, plugin_config: &Map) -> Vec { - match parse_config(plugin_config).and_then(|config| config.validate()) { - Ok(()) => Vec::new(), + match parse_config(plugin_config).and_then(SwitchyardRuntime::new) { + Ok(_) => Vec::new(), Err(message) => vec![ConfigDiagnostic { level: DiagnosticLevel::Error, code: "switchyard.invalid_config".into(), @@ -54,7 +50,6 @@ impl NativePlugin for SwitchyardPlugin { let priority = config.priority; let runtime = Arc::new(SwitchyardRuntime::new(config)?); let plugin_runtime = ctx.runtime(); - register_buffered(ctx, priority, Arc::clone(&runtime), plugin_runtime.clone())?; register_stream(ctx, priority, runtime, plugin_runtime)?; Ok(()) @@ -68,20 +63,22 @@ fn register_buffered( plugin_runtime: PluginRuntime, ) -> Result<(), String> { ctx.register_llm_execution_intercept( - "switchyard.run_stream.buffered", + "switchyard.runner.buffered", priority, move |name, request, next| { let runtime = Arc::clone(&runtime); let plugin_runtime = plugin_runtime.clone(); async move { - let Some(inbound) = runtime.managed_protocol(&name) else { + let Some(inbound) = protocol_from_call(&name) else { return next.call(request).await; }; - let request = runtime.decode_request(inbound, &request, false)?; - let mut marks = Vec::new(); - let response = runtime.execute_buffered(inbound, request, &mut marks).await; - emit_marks(&plugin_runtime, marks); - response + let decoded = runtime.decode_request(inbound, &request, false)?; + if !runtime.manages(&decoded) { + return next.call(request).await; + } + let execution = runtime.execute_buffered(inbound, decoded).await; + emit_marks(&plugin_runtime, execution.marks); + execution.result } }, ) @@ -94,112 +91,38 @@ fn register_stream( plugin_runtime: PluginRuntime, ) -> Result<(), String> { ctx.register_llm_stream_execution_intercept( - "switchyard.run_stream.streaming", + "switchyard.runner.streaming", priority, move |name, request, next| { let runtime = Arc::clone(&runtime); let plugin_runtime = plugin_runtime.clone(); async move { - let Some(inbound) = runtime.managed_protocol(&name) else { + let Some(inbound) = protocol_from_call(&name) else { return next.call(request).await; }; - let request = runtime.decode_request(inbound, &request, true)?; - Ok(Box::pin(ManagedStream::new( - runtime, - plugin_runtime, - inbound, - request, - )) as LlmJsonAsyncStream) + let decoded = runtime.decode_request(inbound, &request, true)?; + if !runtime.manages(&decoded) { + return next.call(request).await; + } + let execution = runtime.execute_stream(inbound, decoded).await; + emit_marks(&plugin_runtime, execution.marks); + execution + .result + .map(|stream| Box::pin(stream) as LlmJsonAsyncStream) } }, ) } fn parse_config(plugin_config: &Map) -> Result { - match plugin_config.get("version").and_then(Json::as_u64) { - Some(2) => {} - Some(version) => { - return Err(format!( - "unsupported Switchyard config version {version}; version 1 used switchyard-server; migrate to version = 2" - )); - } - None => { - return Err("invalid Switchyard configuration: version must be the integer 2".into()); - } - } serde_json::from_value(Json::Object(plugin_config.clone())) .map_err(|error| format!("invalid Switchyard configuration: {error}")) } fn emit_marks(runtime: &PluginRuntime, marks: Vec) { for mark in marks { - emit_mark(runtime, mark); - } -} - -fn emit_mark(runtime: &PluginRuntime, mark: RoutingMark) { - if let Err(error) = runtime.emit_mark(&mark.name, Some(&mark.data), Some(&mark.metadata)) { - eprintln!( - "Switchyard could not emit routing mark {:?}: {error}", - mark.name - ); - } -} - -type StreamExecution = Pin> + Send>>; - -struct ManagedStream { - execution: Option, - messages: Pin>>, - emit_mark: Arc, - terminal_error: Option, -} - -impl ManagedStream { - fn new( - runtime: Arc, - plugin_runtime: PluginRuntime, - inbound: switchyard_protocol::WireFormat, - request: switchyard_protocol::Request, - ) -> Self { - let (sender, messages) = async_channel::bounded(32); - let execution = async move { runtime.execute_stream(inbound, request, &sender).await }; - let emit_mark = Arc::new(move |mark| emit_mark(&plugin_runtime, mark)); - Self { - execution: Some(Box::pin(execution)), - messages: Box::pin(messages), - emit_mark, - terminal_error: None, - } - } -} - -impl futures_util::Stream for ManagedStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - if let Some(execution) = self.execution.as_mut() { - match execution.as_mut().poll(cx) { - Poll::Ready(Ok(())) => self.execution = None, - Poll::Ready(Err(error)) => { - self.execution = None; - self.terminal_error = Some(error); - } - Poll::Pending => {} - } - } - - loop { - match self.messages.as_mut().poll_next(cx) { - Poll::Ready(Some(StreamMessage::Mark(mark))) => (self.emit_mark)(mark), - Poll::Ready(Some(StreamMessage::Event(event))) => { - return Poll::Ready(Some(Ok(event))); - } - Poll::Ready(None) => { - return Poll::Ready(self.terminal_error.take().map(Err)); - } - Poll::Pending => return Poll::Pending, - } + if let Err(error) = runtime.emit_mark(&mark.name, Some(&mark.data), Some(&mark.metadata)) { + eprintln!("Switchyard could not emit routing mark {:?}: {error}", mark.name); } } } @@ -208,66 +131,14 @@ nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlug #[cfg(test)] mod tests { - use std::sync::Arc; - - use futures_util::StreamExt; use serde_json::json; use super::*; - #[tokio::test] - async fn managed_stream_delivers_queued_events_before_terminal_error() { - let (sender, messages) = async_channel::bounded(32); - let execution = async move { - sender - .send(StreamMessage::Event(json!({"id": "committed"}))) - .await - .expect("queue committed event"); - Err("stream failed after commitment".into()) - }; - let mut stream = ManagedStream { - execution: Some(Box::pin(execution)), - messages: Box::pin(messages), - emit_mark: Arc::new(|_| {}), - terminal_error: None, - }; - - assert_eq!(stream.next().await, Some(Ok(json!({"id": "committed"})))); - assert_eq!( - stream.next().await, - Some(Err("stream failed after commitment".into())) - ); - assert_eq!(stream.next().await, None); - } - #[test] - fn version_one_service_config_gets_a_migration_error_before_v2_deserialization() { - let value = json!({ - "version": 1, - "service_url": "http://127.0.0.1:8080", - "health_endpoint": "/healthz" - }); - let plugin_config = value.as_object().unwrap(); - - let error = parse_config(plugin_config) - .err() - .expect("version one must be rejected"); - assert!(error.contains("version 1 used switchyard-server")); - assert!(error.contains("migrate to version = 2")); - assert!(!error.contains("unknown field")); - } - - #[test] - fn version_must_be_an_integer() { - let value = json!({"version": "2"}); - let plugin_config = value.as_object().unwrap(); - - let error = parse_config(plugin_config) - .err() - .expect("non-integer versions must be rejected"); - assert_eq!( - error, - "invalid Switchyard configuration: version must be the integer 2" - ); + fn plugin_configuration_requires_a_deployment_path() { + let config = json!({"priority": 0}); + let error = parse_config(config.as_object().unwrap()).unwrap_err(); + assert!(error.contains("deployment_path")); } } diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 48914eab3..06b507fe9 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -1,20 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::{BTreeMap, HashMap}; +use std::pin::Pin; use std::sync::{Arc, Mutex}; -use futures_util::{StreamExt, stream}; +use futures_util::{Stream, StreamExt}; use nemo_relay_plugin::{Json, LlmRequest as RelayRequest}; use serde_json::{Map, json}; -use switchyard_libsy::{Algorithm, LibsyError}; -use switchyard_llm_client::{ClientRouter, LlmCallObservation, RunObservation, RunObserver, run}; -use switchyard_protocol::{ - LlmClientError, LlmResponse, Metadata, ModelId, Request, Response, WireFormat, -}; +use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver}; +use switchyard_protocol::{LlmResponse, Metadata, Request, Response, WireFormat}; +use switchyard_runner::{Route, Runner}; use switchyard_translation::{TranslationEngine, encode_stream}; -use crate::config::{PreparedTargetBinding, SwitchyardConfig, protocol_from_call}; +use crate::config::SwitchyardConfig; use crate::translation; #[derive(Debug)] @@ -24,32 +22,32 @@ pub(crate) struct RoutingMark { pub(crate) metadata: Json, } -#[derive(Debug)] -pub(crate) enum StreamMessage { - Mark(RoutingMark), - Event(Json), +pub(crate) type ReturnedEventStream = Pin> + Send>>; + +pub(crate) struct Execution { + pub(crate) result: Result, + pub(crate) marks: Vec, } pub(crate) struct SwitchyardRuntime { - algorithm: Arc, - targets: BTreeMap, - default_targets: BTreeMap, + runner: Runner, translation: TranslationEngine, } impl SwitchyardRuntime { pub(crate) fn new(config: SwitchyardConfig) -> Result { - let prepared = config.prepare()?; Ok(Self { - algorithm: prepared.algorithm, - targets: prepared.targets, - default_targets: prepared.default_targets, + runner: config.load_runner()?, translation: TranslationEngine::default(), }) } - pub(crate) fn managed_protocol(&self, name: &str) -> Option { - protocol_from_call(name).filter(|protocol| self.default_targets.contains_key(protocol)) + pub(crate) fn manages(&self, request: &Request) -> bool { + request + .llm_request + .model + .as_deref() + .is_some_and(|model| self.runner.route(model).is_some()) } pub(crate) fn decode_request( @@ -71,8 +69,6 @@ impl SwitchyardRuntime { if relay_gateway_placeholder { metadata.session_id = None; } - // Keep identity/routing metadata, but target clients deliberately clear - // these caller headers before HTTP dispatch. metadata.http_headers = Some(headers); metadata.wire_format = Some(inbound); Ok(Request { @@ -86,289 +82,155 @@ impl SwitchyardRuntime { &self, inbound: WireFormat, request: Request, - marks: &mut Vec, - ) -> Result { - let metadata = identity_metadata(request.metadata.as_ref()); - self.mark( - marks, - "switchyard.routing.requested", - json!({"algorithm": self.algorithm.name(), "attempt": 1}), - &metadata, - ); - let result = self - .drive(request.clone(), 1, marks, &metadata) - .await - .and_then(|response| { - finalize_buffered_response(&self.translation, inbound, response) - .map_err(|source| LibsyError::client_call("return_to_agent", source)) - }); - match result { - Ok(response) => Ok(response), - Err(failure) => { - self.mark( - marks, - "switchyard.routing.error", - failure_mark_data(1, &failure), - &metadata, - ); - let response = self - .fallback_response(inbound, request, marks, &metadata) - .await?; - finalize_buffered_response(&self.translation, inbound, response) - .map_err(|error| public_response_failure("trusted fallback response", &error)) + ) -> Execution { + let mut execution = self.execute(request).await; + if let Ok(response) = execution.result { + execution.result = finalize_buffered_response(&self.translation, inbound, response); + if execution.result.is_err() { + self.error_mark(&mut execution.marks, "response_finalization", None); } } + execution } pub(crate) async fn execute_stream( &self, inbound: WireFormat, request: Request, - output: &async_channel::Sender, - ) -> Result<(), String> { - let metadata = identity_metadata(request.metadata.as_ref()); - let mut marks = Vec::new(); - self.mark( - &mut marks, - "switchyard.routing.requested", - json!({"algorithm": self.algorithm.name(), "attempt": 1}), - &metadata, - ); - let (response, mut fallback_used) = - match self.drive(request.clone(), 1, &mut marks, &metadata).await { - Ok(response) => (response, false), - Err(failure) => { - self.mark( - &mut marks, - "switchyard.routing.error", - failure_mark_data(1, &failure), - &metadata, - ); - let fallback = self - .fallback_response(inbound, request.clone(), &mut marks, &metadata) - .await; - send_marks(output, &mut marks).await?; - (fallback?, true) - } - }; - send_marks(output, &mut marks).await?; - - let mut events = match returned_events(response, inbound).await { - Ok(events) => events, - Err(failure) if !fallback_used => { - self.mark( - &mut marks, - "switchyard.routing.error", - failure_mark_data(1, &failure), - &metadata, - ); - fallback_used = true; - let fallback = self - .fallback_response(inbound, request.clone(), &mut marks, &metadata) - .await; - send_marks(output, &mut marks).await?; - let fallback = fallback?; - returned_events(fallback, inbound) - .await - .map_err(|error| public_libsy_failure("trusted fallback stream", &error))? - } - Err(failure) => { - return Err(public_libsy_failure("trusted fallback stream", &failure)); - } - }; - - let mut committed = false; - while let Some(item) = events.next().await { - match item { - Ok(event) => { - send_event(output, event).await?; - committed = true; - } - Err(failure) if !fallback_used && !committed => { - self.mark( - &mut marks, - "switchyard.routing.error", - failure_mark_data(1, &failure), - &metadata, - ); - let fallback = self - .fallback_response(inbound, request.clone(), &mut marks, &metadata) - .await; - send_marks(output, &mut marks).await?; - let fallback = fallback?; - let mut fallback = returned_events(fallback, inbound) - .await - .map_err(|error| public_libsy_failure("trusted fallback stream", &error))?; - while let Some(item) = fallback.next().await { - let event = item.map_err(|error| { - public_libsy_failure("trusted fallback stream", &error) - })?; - send_event(output, event).await?; - } - return Ok(()); - } - Err(failure) if !committed => { - return Err(public_libsy_failure("trusted fallback stream", &failure)); - } - Err(failure) => { - self.mark( - &mut marks, - "switchyard.routing.error", - failure_mark_data(1, &failure), - &metadata, - ); - send_marks(output, &mut marks).await?; - return Err(public_libsy_failure( - "Switchyard stream failed after response commitment", - &failure, - )); - } + ) -> Execution { + let mut execution = self.execute(request).await; + if let Ok(response) = execution.result { + execution.result = returned_events(response, inbound); + if execution.result.is_err() { + self.error_mark(&mut execution.marks, "response_finalization", None); } } - if committed { - Ok(()) - } else { - Err("Switchyard response stream produced no caller events".into()) - } + execution } - async fn drive( - &self, - request: Request, - attempt: u32, - marks: &mut Vec, - mark_metadata: &Json, - ) -> Result { + async fn execute(&self, request: Request) -> Execution { + let Some(route) = self.route(&request) else { + return Execution { + result: Err("Switchyard has no route for this request model".into()), + marks: Vec::new(), + }; + }; + let metadata = identity_metadata(request.metadata.as_ref()); + let mut marks = vec![RoutingMark { + name: "switchyard.routing.requested".into(), + data: json!({"algorithm": route.algorithm_name()}), + metadata: metadata.clone(), + }]; + if let Err(error) = route.check_caller_format(metadata_wire_format(&request)) { + self.error_mark(&mut marks, "caller_format", None); + return Execution { + result: Err(format!("Switchyard caller format is incompatible: {error}")), + marks, + }; + } let observations = Arc::new(Mutex::new(Vec::new())); - let observed_calls = observations.clone(); + let observed = Arc::clone(&observations); let observer: RunObserver = Arc::new(move |observation| { - if let RunObservation::LlmCall(call) = observation { - observed_calls - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .push(call); - } + observed + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(observation); }); - let clients = ClientRouter::new( - self.targets - .iter() - .map(|(name, target)| (ModelId::from(name.as_str()), target.client.clone())) - .collect::>(), - ); - match run(self.algorithm.clone(), clients, request, Some(observer)).await { - Ok((selected_model_id, response)) => { - self.emit_decision(marks, &selected_model_id, attempt, mark_metadata); - self.emit_routing_llm_calls( + match route.execute(request, Some(observer)).await { + Ok(output) => { + self.emit_observations(&mut marks, take_observations(&observations), &metadata); + marks.push(RoutingMark { + name: "switchyard.routing.decision".into(), + data: json!({ + "algorithm": route.algorithm_name(), + "selected_model": output.selected_model, + }), + metadata, + }); + Execution { + result: Ok(output.response), marks, - take_observed_calls(&observations), - attempt, - mark_metadata, - ); - Ok(response) + } } - Err(error) => { - self.emit_routing_llm_calls( + Err(_) => { + self.emit_observations(&mut marks, take_observations(&observations), &metadata); + self.error_mark(&mut marks, "route_execution", None); + Execution { + result: Err("Switchyard route execution failed".into()), marks, - take_observed_calls(&observations), - attempt, - mark_metadata, - ); - Err(error) + } } } } - async fn fallback_response( - &self, - inbound: WireFormat, - request: Request, - marks: &mut Vec, - metadata: &Json, - ) -> Result { - let target_name = self.default_target(inbound)?; - let target = self.target(target_name)?; - self.mark( - marks, - "switchyard.routing.fallback", - json!({"selected_target": target_name}), - metadata, - ); - target - .client - .call(request) - .await - .map_err(|error| public_client_failure("trusted fallback", &error)) + fn route(&self, request: &Request) -> Option<&Route> { + request + .llm_request + .model + .as_deref() + .and_then(|model| self.runner.route(model)) } - fn target(&self, name: &str) -> Result<&PreparedTargetBinding, String> { - self.targets - .get(name) - .ok_or_else(|| format!("libsy selected unknown target {name:?}")) - } - - fn default_target(&self, protocol: WireFormat) -> Result<&str, String> { - self.default_targets - .get(&protocol) - .map(String::as_str) - .ok_or_else(|| format!("managed protocol {protocol} has no default target")) - } - - fn mark(&self, marks: &mut Vec, name: &str, data: Json, metadata: &Json) { - marks.push(RoutingMark { - name: name.to_string(), - data, - metadata: metadata.clone(), - }); - } - - fn emit_decision( + fn emit_observations( &self, marks: &mut Vec, - selected_model_id: &ModelId, - attempt: u32, + observations: Vec, metadata: &Json, ) { - self.mark( - marks, - "switchyard.routing.decision", - json!({ - "algorithm": self.algorithm.name(), - "attempt": attempt, - "selected_target": selected_model_id, - }), - metadata, - ); + let mut call_index = 0; + for observation in observations { + match observation { + RunObservation::LlmCall(call) => { + call_index += 1; + self.routing_call_mark(marks, call, call_index, metadata); + } + RunObservation::RoutingOverhead(duration) => marks.push(RoutingMark { + name: "switchyard.routing.overhead".into(), + data: json!({"latency_ms": duration.as_secs_f64() * 1_000.0}), + metadata: metadata.clone(), + }), + RunObservation::AnswerCall(_) => {} + } + } } - fn emit_routing_llm_calls( + fn routing_call_mark( &self, marks: &mut Vec, - calls: Vec, - attempt: u32, + call: LlmCallObservation, + call_index: usize, metadata: &Json, ) { - for (index, call) in calls.into_iter().enumerate() { - self.mark( - marks, - "switchyard.routing.llm_call", - json!({ - "algorithm": self.algorithm.name(), - "attempt": attempt, - "call_index": index + 1, - "selected_target": call.selected_model, - "call_role": "routing", - "outcome": if call.is_success { "ok" } else { "error" }, - "latency_ms": call.duration.as_secs_f64() * 1_000.0, - "usage": call.usage, - "contributes_to_routing_overhead": true, - }), - metadata, - ); - } + marks.push(RoutingMark { + name: "switchyard.routing.llm_call".into(), + data: json!({ + "call_index": call_index, + "selected_model": call.selected_model, + "call_role": "routing", + "outcome": if call.is_success { "ok" } else { "error" }, + "latency_ms": call.duration.as_secs_f64() * 1_000.0, + "usage": call.usage, + }), + metadata: metadata.clone(), + }); + } + + fn error_mark(&self, marks: &mut Vec, failure_kind: &str, metadata: Option<&Json>) { + let metadata = metadata.cloned().unwrap_or_else(|| { + marks + .first() + .map(|mark| mark.metadata.clone()) + .unwrap_or_else(|| Json::Object(Map::new())) + }); + marks.push(RoutingMark { + name: "switchyard.routing.error".into(), + data: json!({"failure_kind": failure_kind}), + metadata, + }); } } -fn take_observed_calls(observations: &Mutex>) -> Vec { +fn take_observations(observations: &Mutex>) -> Vec { std::mem::take( &mut *observations .lock() @@ -376,150 +238,37 @@ fn take_observed_calls(observations: &Mutex>) -> Vec, - marks: &mut Vec, -) -> Result<(), String> { - for mark in marks.drain(..) { - output - .send(StreamMessage::Mark(mark)) - .await - .map_err(|_| "Relay cancelled the Switchyard response stream".to_string())?; - } - Ok(()) -} - -async fn send_event( - output: &async_channel::Sender, - event: Json, -) -> Result<(), String> { - output - .send(StreamMessage::Event(event)) - .await - .map_err(|_| "Relay cancelled the Switchyard response stream".to_string()) +fn metadata_wire_format(request: &Request) -> WireFormat { + request + .metadata + .as_ref() + .and_then(|metadata| metadata.wire_format) + .expect("decoded Relay requests always carry a wire format") } -type ReturnedEventStream = - std::pin::Pin> + Send>>; - fn finalize_buffered_response( translation_engine: &TranslationEngine, inbound: WireFormat, response: Response, -) -> Result { +) -> Result { let LlmResponse::Agg(response) = response.llm_response else { - return Err(LlmClientError::InvalidResponse { - source: Box::new(std::io::Error::other( - "libsy returned a stream for a buffered request", - )), - }); + return Err("Switchyard returned a stream for a buffered request".into()); }; translation::encode_response(translation_engine, inbound, &response) - .map_err(LlmClientError::ResponseTranslation) } -async fn returned_events( - response: Response, - inbound: WireFormat, -) -> Result { +fn returned_events(response: Response, inbound: WireFormat) -> Result { let chunks = match response.llm_response { LlmResponse::Agg(response) => response.into_stream(), - LlmResponse::Stream(mut chunks) => { - let Some(first) = chunks.next().await else { - return Err(LibsyError::client_call( - "return_to_agent", - LlmClientError::InvalidResponse { - source: Box::new(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "provider returned an empty stream", - )), - }, - )); - }; - Box::pin(stream::once(async move { first }).chain(chunks)) - } + LlmResponse::Stream(chunks) => chunks, }; let events = encode_stream(chunks, inbound, None) - .map_err(|error| LibsyError::client_call("return_to_agent", error))?; + .map_err(|error| format!("Switchyard response stream setup failed: {error}"))?; Ok(Box::pin(events.map(|item| { - item.map_err(|source| match source.downcast::() { - Ok(source) => LibsyError::client_call("return_to_agent", *source), - Err(source) => LibsyError::client_call( - "return_to_agent", - LlmClientError::ResponseTranslation(source.to_string()), - ), - }) + item.map_err(|error| format!("Switchyard response stream failed: {error}")) }))) } -fn failure_mark_data(attempt: u32, failure: &LibsyError) -> Json { - let mut data = Map::from_iter([("attempt".into(), Json::from(attempt))]); - match failure { - LibsyError::ClientCall { - source: LlmClientError::UpstreamHttp { status, .. }, - .. - } => { - data.insert("failure_kind".into(), Json::from("http")); - data.insert("http_status".into(), Json::from(status.as_u16())); - } - LibsyError::ClientCall { source, .. } => { - data.insert("failure_kind".into(), Json::from("non_http")); - data.insert( - "non_http_kind".into(), - Json::from(client_error_label(source)), - ); - } - _ => { - data.insert("failure_kind".into(), Json::from("algorithm")); - } - } - Json::Object(data) -} - -fn client_error_label(error: &LlmClientError) -> &'static str { - match error { - LlmClientError::InvalidRequest { .. } => "invalid_request", - LlmClientError::RequestTranslation(_) => "request_translation", - LlmClientError::RequestEncoding(_) => "request_encoding", - LlmClientError::ResponseTranslation(_) => "response_translation", - LlmClientError::Configuration { .. } => "configuration", - LlmClientError::Transport { .. } => "transport", - LlmClientError::Timeout { .. } => "timeout", - LlmClientError::ContextWindowExceeded { .. } => "context_window_exceeded", - LlmClientError::UpstreamHttp { .. } => "http", - LlmClientError::InvalidResponse { .. } => "invalid_response", - LlmClientError::Ffi { .. } => "ffi", - LlmClientError::General(_) => "general", - _ => "unknown", - } -} - -fn public_libsy_failure(prefix: &str, error: &LibsyError) -> String { - match error { - LibsyError::ClientCall { source, .. } => public_client_failure(prefix, source), - _ => format!("{prefix}: Switchyard algorithm failure"), - } -} - -fn public_response_failure(prefix: &str, error: &LlmClientError) -> String { - match error { - LlmClientError::InvalidResponse { .. } => format!("{prefix}: invalid response"), - LlmClientError::ResponseTranslation(_) => { - format!("{prefix}: response translation failure") - } - _ => format!("{prefix}: response finalization failure"), - } -} - -fn public_client_failure(prefix: &str, error: &LlmClientError) -> String { - match error { - LlmClientError::UpstreamHttp { status, .. } => { - format!("{prefix}: provider returned HTTP {status}") - } - _ => format!("{prefix}: provider {} failure", client_error_label(error)), - } -} - fn string_headers(headers: &Map) -> http::HeaderMap { let mut parsed = http::HeaderMap::with_capacity(headers.len()); for (name, value) in headers { @@ -549,4 +298,46 @@ fn identity_metadata(metadata: Option<&Metadata>) -> Json { } #[cfg(test)] -mod tests; +mod tests { + use std::collections::{BTreeMap, HashMap}; + + use switchyard_llm_client::ClientRouter; + use switchyard_protocol::{ModelId, text_request}; + use switchyard_runner::{AlgorithmSpec, ModelCapabilities}; + + use super::*; + + fn runtime_for(model: &str) -> SwitchyardRuntime { + let algorithm = AlgorithmSpec::Noop {} + .build("relay", &BTreeMap::new()) + .expect("noop route should build"); + let route = Route::new( + algorithm, + ClientRouter::new(HashMap::new()), + None, + ModelCapabilities::default(), + None, + Vec::new(), + ); + SwitchyardRuntime { + runner: Runner::new(vec![(ModelId::from(model), route)]), + translation: TranslationEngine::default(), + } + } + + #[test] + fn only_configured_route_models_are_managed() { + let runtime = runtime_for("switchyard"); + let configured = Request { + llm_request: text_request(Some("switchyard".into()), "hello"), + ..Request::default() + }; + let other = Request { + llm_request: text_request(Some("other".into()), "hello"), + ..Request::default() + }; + + assert!(runtime.manages(&configured)); + assert!(!runtime.manages(&other)); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs b/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs deleted file mode 100644 index b0dcaa39c..000000000 --- a/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs +++ /dev/null @@ -1,878 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::atomic::{AtomicUsize, Ordering}; - -use switchyard_libsy::{ - ClassifierContractConfig, EscalationJudgeConfig, LlmClassifierConfig, LlmFallback, - LlmTaskClassifier, Passthrough, PickerMode, StageRouter, StageRouterConfig, - TaskClassifierConfig, -}; -use switchyard_protocol::{ - LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, ModelId, RoutedLlmClient, Usage, - text_request, text_response, -}; - -use super::*; - -enum ScriptedBehavior { - Text(&'static str), - EmptyBuffered, - EmptyStream, - FailingStream, - PartialThenFailure, - TransportFailure(&'static str), -} - -struct ScriptedClient { - behavior: ScriptedBehavior, - calls: AtomicUsize, -} - -fn scripted(behavior: ScriptedBehavior) -> Arc { - Arc::new(ScriptedClient { - behavior, - calls: AtomicUsize::new(0), - }) -} - -#[async_trait::async_trait] -impl RoutedLlmClient for ScriptedClient { - async fn call(&self, request: Request) -> Result { - self.calls.fetch_add(1, Ordering::Relaxed); - match self.behavior { - ScriptedBehavior::Text(text) => { - let mut response = text_response(None, text); - response.usage = Usage { - input_tokens: Some(11), - output_tokens: Some(7), - total_tokens: Some(18), - ..Usage::default() - }; - Ok(Response { - llm_response: LlmResponse::Agg(response), - metadata: request.metadata, - }) - } - ScriptedBehavior::EmptyBuffered => Ok(Response { - llm_response: LlmResponse::Agg(Default::default()), - metadata: None, - }), - ScriptedBehavior::EmptyStream => Ok(Response { - llm_response: LlmResponse::Stream(Box::pin(stream::empty())), - metadata: None, - }), - ScriptedBehavior::FailingStream => { - let stream: LlmResponseStream = Box::pin(stream::once(async { - Err(LlmClientError::Transport { - source: Box::new(std::io::Error::other("fallback stream failed")), - }) - })); - Ok(Response { - llm_response: LlmResponse::Stream(stream), - metadata: None, - }) - } - ScriptedBehavior::PartialThenFailure => { - let stream: LlmResponseStream = Box::pin(stream::iter(vec![ - Ok(LlmResponseStreamEvent::from(LlmResponseChunk::TextDelta { - index: 0, - text: "partial".into(), - })), - Err(LlmClientError::Transport { - source: Box::new(std::io::Error::other("stream failed after a chunk")), - }), - ])); - Ok(Response { - llm_response: LlmResponse::Stream(stream), - metadata: None, - }) - } - ScriptedBehavior::TransportFailure(message) => Err(LlmClientError::Transport { - source: Box::new(std::io::Error::other(message)), - }), - } - } -} - -fn fixed_target(name: &str) -> ModelId { - ModelId::from(name) -} - -fn runtime_with_algorithm( - algorithm: Arc, - fallback: Arc, - protocol: WireFormat, -) -> SwitchyardRuntime { - runtime_with_algorithm_clients(algorithm, fallback, protocol, Vec::new()) -} - -fn runtime_with_algorithm_clients( - algorithm: Arc, - fallback: Arc, - protocol: WireFormat, - clients: Vec<(&str, Arc)>, -) -> SwitchyardRuntime { - let mut targets = BTreeMap::from([( - "fallback".into(), - PreparedTargetBinding { - client: fallback as Arc, - }, - )]); - for (name, client) in clients { - targets.insert( - name.to_string(), - PreparedTargetBinding { - client: client as Arc, - }, - ); - } - SwitchyardRuntime { - algorithm, - targets, - default_targets: BTreeMap::from([(protocol, "fallback".into())]), - translation: TranslationEngine::default(), - } -} - -fn request_with_session(protocol: WireFormat, session: Option<&str>) -> Request { - Request { - llm_request: text_request(Some("auto".into()), "fix the build"), - raw_request: None, - metadata: Some(Metadata { - wire_format: Some(protocol), - session_id: session.map(str::to_string), - ..Metadata::default() - }), - } -} - -fn stage_signal_relay_request(protocol: WireFormat) -> RelayRequest { - let content = match protocol { - WireFormat::OpenAiChat => json!({ - "model": "auto", - "messages": [ - {"role": "user", "content": "fix the build"}, - { - "role": "assistant", - "content": null, - "tool_calls": [{ - "id": "call-1", - "type": "function", - "function": { - "name": "bash", - "arguments": "{\"cmd\":\"cargo test\"}" - } - }] - }, - { - "role": "tool", - "tool_call_id": "call-1", - "content": "fatal runtime error: out of memory" - } - ] - }), - WireFormat::OpenAiResponses => json!({ - "model": "auto", - "input": [ - {"type": "message", "role": "user", "content": "fix the build"}, - { - "type": "function_call", - "call_id": "call-1", - "name": "bash", - "arguments": "{\"cmd\":\"cargo test\"}" - }, - { - "type": "function_call_output", - "call_id": "call-1", - "output": "fatal runtime error: out of memory" - } - ] - }), - WireFormat::AnthropicMessages => json!({ - "model": "auto", - "max_tokens": 128, - "messages": [ - {"role": "user", "content": "fix the build"}, - { - "role": "assistant", - "content": [{ - "type": "tool_use", - "id": "call-1", - "name": "bash", - "input": {"cmd": "cargo test"} - }] - }, - { - "role": "user", - "content": [{ - "type": "tool_result", - "tool_use_id": "call-1", - "content": "fatal runtime error: out of memory", - "is_error": true - }] - } - ] - }), - }; - - RelayRequest { - headers: Map::from_iter([( - "x-switchyard-session-id".into(), - json!(format!("stage-{}", protocol.as_str())), - )]), - content, - } -} - -#[test] -fn relay_gateway_placeholder_session_is_not_retained() { - let fallback = scripted(ScriptedBehavior::Text("fallback")); - let runtime = runtime_with_algorithm( - Arc::new(Passthrough::new(ModelId::from("selected"))), - fallback, - WireFormat::OpenAiChat, - ); - let request = RelayRequest { - headers: Map::from_iter([ - ("x-nemo-relay-source".into(), json!("gateway")), - ("x-nemo-relay-session-id".into(), json!("gateway-gateway")), - ("x-dynamo-session-id".into(), json!("gateway-gateway")), - ]), - content: json!({ - "model": "router", - "messages": [{"role": "user", "content": "hello"}] - }), - }; - - let decoded = runtime - .decode_request(WireFormat::OpenAiChat, &request, false) - .unwrap(); - - assert_eq!(decoded.metadata.unwrap().session_id, None); -} - -#[test] -fn explicit_switchyard_session_overrides_relay_gateway_placeholder() { - let fallback = scripted(ScriptedBehavior::Text("fallback")); - let runtime = runtime_with_algorithm( - Arc::new(Passthrough::new(ModelId::from("selected"))), - fallback, - WireFormat::OpenAiChat, - ); - let request = RelayRequest { - headers: Map::from_iter([ - ("x-switchyard-session-id".into(), json!("caller-session")), - ("x-nemo-relay-source".into(), json!("gateway")), - ("x-nemo-relay-session-id".into(), json!("gateway-gateway")), - ]), - content: json!({ - "model": "router", - "messages": [{"role": "user", "content": "hello"}] - }), - }; - - let decoded = runtime - .decode_request(WireFormat::OpenAiChat, &request, false) - .unwrap(); - - assert_eq!( - decoded.metadata.unwrap().session_id.as_deref(), - Some("caller-session") - ); -} - -#[tokio::test] -async fn buffered_finalization_failure_uses_fallback_once() { - let selected = scripted(ScriptedBehavior::EmptyStream); - let fallback = scripted(ScriptedBehavior::EmptyBuffered); - let runtime = SwitchyardRuntime { - algorithm: Arc::new(Passthrough::new(ModelId::from("selected"))), - targets: BTreeMap::from([ - ( - "selected".into(), - PreparedTargetBinding { - client: selected.clone(), - }, - ), - ( - "fallback".into(), - PreparedTargetBinding { - client: fallback.clone(), - }, - ), - ]), - default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), - translation: TranslationEngine::default(), - }; - let mut marks = Vec::new(); - - let response = runtime - .execute_buffered(WireFormat::OpenAiChat, Request::default(), &mut marks) - .await - .expect("the buffered fallback response should be encoded"); - - assert!(response.is_object()); - assert_eq!(selected.calls.load(Ordering::Relaxed), 1); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); - let error = marks - .iter() - .find(|mark| mark.name == "switchyard.routing.error") - .expect("finalization failure should emit an error mark"); - assert_eq!(error.data["non_http_kind"], "invalid_response"); - assert_eq!( - marks - .iter() - .filter(|mark| mark.name == "switchyard.routing.fallback") - .count(), - 1 - ); -} - -#[tokio::test] -async fn returned_events_replays_preserved_openai_chat_without_duplicate_terminal() { - let content = json!({ - "id": "chatcmpl-test", - "object": "chat.completion.chunk", - "model": "gpt-4o", - "system_fingerprint": "fp_provider_specific", - "choices": [{ - "index": 0, - "delta": {"content": "Hi"}, - "finish_reason": null - }] - }); - let terminal = json!({ - "id": "chatcmpl-test", - "object": "chat.completion.chunk", - "model": "gpt-4o", - "choices": [{ - "index": 0, - "delta": {}, - "finish_reason": "stop" - }] - }); - let body = format!("data: {content}\n\ndata: {terminal}\n\ndata: [DONE]\n\n").into_bytes(); - let stream = switchyard_translation::decode_stream( - stream::once(async move { Ok::<_, LlmClientError>(body) }), - WireFormat::OpenAiChat, - ) - .expect("provider SSE should decode"); - let response = Response { - llm_response: LlmResponse::Stream(stream), - metadata: None, - }; - - let replayed = returned_events(response, WireFormat::OpenAiChat) - .await - .expect("return stream should encode") - .collect::>() - .await - .into_iter() - .collect::, _>>() - .expect("return stream should not fail"); - - assert_eq!(replayed, vec![content, terminal]); -} - -#[tokio::test] -async fn invalid_selected_stream_does_not_invoke_failing_fallback_twice() { - let selected = scripted(ScriptedBehavior::EmptyStream); - let fallback = scripted(ScriptedBehavior::FailingStream); - let runtime = SwitchyardRuntime { - algorithm: Arc::new(Passthrough::new(ModelId::from("selected"))), - targets: BTreeMap::from([ - ( - "selected".into(), - PreparedTargetBinding { - client: selected.clone(), - }, - ), - ( - "fallback".into(), - PreparedTargetBinding { - client: fallback.clone(), - }, - ), - ]), - default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), - translation: TranslationEngine::default(), - }; - let (output, _messages) = async_channel::bounded(32); - - let error = runtime - .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) - .await - .expect_err("the failing fallback stream must fail the request"); - - assert_eq!(error, "trusted fallback stream: provider transport failure"); - assert_eq!(selected.calls.load(Ordering::Relaxed), 1); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); -} - -#[tokio::test] -async fn failing_fallback_call_flushes_error_and_fallback_marks() { - let selected = scripted(ScriptedBehavior::EmptyStream); - let fallback = scripted(ScriptedBehavior::TransportFailure("fallback call failed")); - let runtime = SwitchyardRuntime { - algorithm: Arc::new(Passthrough::new(ModelId::from("selected"))), - targets: BTreeMap::from([ - ( - "selected".into(), - PreparedTargetBinding { - client: selected.clone(), - }, - ), - ( - "fallback".into(), - PreparedTargetBinding { - client: fallback.clone(), - }, - ), - ]), - default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), - translation: TranslationEngine::default(), - }; - let (output, messages) = async_channel::bounded(32); - - let error = runtime - .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) - .await - .expect_err("the failing fallback call must fail the request"); - - assert_eq!(error, "trusted fallback: provider transport failure"); - assert_eq!(selected.calls.load(Ordering::Relaxed), 1); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); - let mut terminal_marks = Vec::new(); - while let Ok(message) = messages.try_recv() { - if let StreamMessage::Mark(mark) = message - && matches!( - mark.name.as_str(), - "switchyard.routing.error" | "switchyard.routing.fallback" - ) - { - terminal_marks.push(mark.name); - } - } - assert_eq!( - terminal_marks, - ["switchyard.routing.error", "switchyard.routing.fallback"] - ); -} - -#[tokio::test] -async fn committed_stream_failure_does_not_fallback() { - let selected = scripted(ScriptedBehavior::PartialThenFailure); - let fallback = scripted(ScriptedBehavior::Text("fallback")); - let runtime = runtime_with_algorithm_clients( - Arc::new(Passthrough::new(ModelId::from("selected"))), - fallback.clone(), - WireFormat::OpenAiChat, - vec![("selected", selected.clone())], - ); - let (output, messages) = async_channel::bounded(32); - - let error = runtime - .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) - .await - .expect_err("committed stream failures must reject the stream"); - - assert_eq!( - error, - "Switchyard stream failed after response commitment: provider transport failure" - ); - assert_eq!(selected.calls.load(Ordering::Relaxed), 1); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); - let mut emitted_event = false; - while let Ok(message) = messages.try_recv() { - emitted_event |= matches!(message, StreamMessage::Event(_)); - } - assert!(emitted_event); -} - -#[tokio::test] -async fn capability_classifier_emits_judge_usage_without_serving_usage() { - let weak = scripted(ScriptedBehavior::Text("weak answer")); - let strong = scripted(ScriptedBehavior::Text("strong answer")); - let judge = scripted(ScriptedBehavior::Text( - r#"{"crux":"bounded","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, - )); - let fallback = scripted(ScriptedBehavior::Text("fallback")); - let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Capability { - judge_target: fixed_target("judge"), - efficient_target: fixed_target("weak"), - capable_target: fixed_target("strong"), - config: TaskClassifierConfig { - base_threshold: 0.5, - ..TaskClassifierConfig::default() - }, - }) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback, - WireFormat::OpenAiChat, - vec![ - ("weak", weak.clone()), - ("strong", strong.clone()), - ("judge", judge.clone()), - ], - ); - let mut marks = Vec::new(); - - runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, Some("capability")), - &mut marks, - ) - .await - .unwrap(); - - assert_eq!(judge.calls.load(Ordering::Relaxed), 1); - assert_eq!(weak.calls.load(Ordering::Relaxed), 1); - assert_eq!(strong.calls.load(Ordering::Relaxed), 0); - let routing_calls = marks - .iter() - .filter(|mark| mark.name == "switchyard.routing.llm_call") - .collect::>(); - assert_eq!(routing_calls.len(), 1); - assert_eq!(routing_calls[0].data["selected_target"], "judge"); - assert_eq!(routing_calls[0].data["usage"]["total_tokens"], 18); -} - -#[tokio::test] -async fn escalation_buffers_weak_stream_then_latches_the_session_to_strong() { - let weak = scripted(ScriptedBehavior::Text("weak draft")); - let strong = scripted(ScriptedBehavior::Text("strong answer")); - let judge = scripted(ScriptedBehavior::Text( - r#"{"escalate":true,"reason":"stuck"}"#, - )); - let fallback = scripted(ScriptedBehavior::Text("fallback")); - let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { - judge_target: fixed_target("judge"), - efficient_target: fixed_target("weak"), - capable_target: fixed_target("strong"), - contract: ClassifierContractConfig::default(), - config: EscalationJudgeConfig { - confirmations: 1, - ..EscalationJudgeConfig::default() - }, - max_output_tokens: 128, - }) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback.clone(), - WireFormat::OpenAiChat, - vec![ - ("weak", weak.clone()), - ("strong", strong.clone()), - ("judge", judge.clone()), - ], - ); - - let mut first = request_with_session(WireFormat::OpenAiChat, Some("session-1")); - first.llm_request.stream = true; - let (output, messages) = async_channel::bounded(32); - runtime - .execute_stream(WireFormat::OpenAiChat, first, &output) - .await - .unwrap(); - let mut streamed = Vec::new(); - let mut routing_calls = Vec::new(); - while let Ok(message) = messages.try_recv() { - match message { - StreamMessage::Event(event) => streamed.push(event), - StreamMessage::Mark(mark) if mark.name == "switchyard.routing.llm_call" => { - routing_calls.push(mark.data) - } - StreamMessage::Mark(_) => {} - } - } - assert!(!streamed.is_empty()); - assert!( - streamed - .iter() - .any(|event| event.to_string().contains("strong answer")) - ); - assert_eq!(routing_calls.len(), 2); - assert_eq!(routing_calls[0]["selected_target"], "weak"); - assert_eq!(routing_calls[0]["call_role"], "routing"); - assert_eq!(routing_calls[0]["usage"]["total_tokens"], 18); - assert_eq!(routing_calls[1]["selected_target"], "judge"); - assert_eq!(routing_calls[1]["call_role"], "routing"); - assert_eq!(routing_calls[1]["usage"]["total_tokens"], 18); - assert!( - routing_calls - .iter() - .all(|call| call["selected_target"] != "strong") - ); - - let mut marks = Vec::new(); - let response = runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, Some("session-1")), - &mut marks, - ) - .await - .unwrap(); - assert!(response.to_string().contains("strong answer")); - assert_eq!(weak.calls.load(Ordering::Relaxed), 1); - assert_eq!(judge.calls.load(Ordering::Relaxed), 1); - assert_eq!(strong.calls.load(Ordering::Relaxed), 2); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); - assert!( - !marks - .iter() - .any(|mark| mark.name == "switchyard.routing.llm_call") - ); - assert!(marks.iter().any(|mark| { - mark.name == "switchyard.routing.decision" - && mark.data["selected_target"] == "strong" - && mark.metadata["session_id"] == "session-1" - })); -} - -#[tokio::test] -async fn escalation_judge_failure_falls_open_to_the_buffered_weak_response() { - let weak = scripted(ScriptedBehavior::Text("weak answer")); - let strong = scripted(ScriptedBehavior::Text("strong answer")); - let judge = scripted(ScriptedBehavior::TransportFailure("scripted failure")); - let fallback = scripted(ScriptedBehavior::Text("fallback")); - let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { - judge_target: fixed_target("judge"), - efficient_target: fixed_target("weak"), - capable_target: fixed_target("strong"), - contract: ClassifierContractConfig::default(), - config: EscalationJudgeConfig::default(), - max_output_tokens: 128, - }) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback.clone(), - WireFormat::OpenAiChat, - vec![ - ("weak", weak.clone()), - ("strong", strong.clone()), - ("judge", judge.clone()), - ], - ); - let mut marks = Vec::new(); - - let response = runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, Some("session-1")), - &mut marks, - ) - .await - .unwrap(); - - assert!(response.to_string().contains("weak answer")); - assert_eq!(weak.calls.load(Ordering::Relaxed), 1); - assert_eq!(judge.calls.load(Ordering::Relaxed), 1); - assert_eq!(strong.calls.load(Ordering::Relaxed), 0); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); - let routing_calls = marks - .iter() - .filter(|mark| mark.name == "switchyard.routing.llm_call") - .collect::>(); - assert_eq!(routing_calls.len(), 1); - assert_eq!(routing_calls[0].data["selected_target"], "judge"); - assert_eq!(routing_calls[0].data["call_role"], "routing"); - assert_eq!(routing_calls[0].data["outcome"], "error"); - assert!(routing_calls[0].data["usage"].is_null()); -} - -#[tokio::test] -async fn escalation_without_session_identity_cannot_accumulate_confirmations() { - let weak = scripted(ScriptedBehavior::Text("weak answer")); - let strong = scripted(ScriptedBehavior::Text("strong answer")); - let judge = scripted(ScriptedBehavior::Text( - r#"{"escalate":true,"reason":"stuck"}"#, - )); - let fallback = scripted(ScriptedBehavior::Text("fallback")); - let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { - judge_target: fixed_target("judge"), - efficient_target: fixed_target("weak"), - capable_target: fixed_target("strong"), - contract: ClassifierContractConfig::default(), - config: EscalationJudgeConfig { - confirmations: 2, - ..EscalationJudgeConfig::default() - }, - max_output_tokens: 128, - }) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback.clone(), - WireFormat::OpenAiChat, - vec![ - ("weak", weak.clone()), - ("strong", strong.clone()), - ("judge", judge.clone()), - ], - ); - - for _ in 0..2 { - let mut marks = Vec::new(); - let response = runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, None), - &mut marks, - ) - .await - .unwrap(); - assert!(response.to_string().contains("weak answer")); - } - assert_eq!(weak.calls.load(Ordering::Relaxed), 2); - assert_eq!(judge.calls.load(Ordering::Relaxed), 2); - assert_eq!(strong.calls.load(Ordering::Relaxed), 0); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); -} - -#[tokio::test] -async fn stage_router_uses_tool_signals_for_every_managed_protocol() { - for protocol in [ - WireFormat::OpenAiChat, - WireFormat::OpenAiResponses, - WireFormat::AnthropicMessages, - ] { - let capable = scripted(ScriptedBehavior::Text("capable answer")); - let efficient = scripted(ScriptedBehavior::Text("efficient answer")); - let fallback = scripted(ScriptedBehavior::Text("fallback")); - let algorithm = StageRouter::new( - fixed_target("strong"), - fixed_target("weak"), - StageRouterConfig::new(PickerMode::EfficientFirst, 0.5), - ) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback.clone(), - protocol, - vec![("strong", capable.clone()), ("weak", efficient.clone())], - ); - let mut marks = Vec::new(); - let relay_request = stage_signal_relay_request(protocol); - let request = runtime - .decode_request(protocol, &relay_request, false) - .unwrap(); - - let response = runtime - .execute_buffered(protocol, request, &mut marks) - .await - .unwrap(); - - assert!(response.to_string().contains("capable answer")); - assert_eq!(capable.calls.load(Ordering::Relaxed), 1); - assert_eq!(efficient.calls.load(Ordering::Relaxed), 0); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); - assert!(marks.iter().any(|mark| { - mark.name == "switchyard.routing.decision" - && mark.data["algorithm"] == "stage_router" - && mark.data["attempt"] == 1 - && mark.data["selected_target"] == "strong" - && mark.metadata["session_id"] == format!("stage-{}", protocol.as_str()) - })); - } -} - -#[tokio::test] -async fn stage_router_falls_open_to_each_picker_default_without_tool_history() { - for (picker, expected) in [ - (PickerMode::CapableFirst, "strong"), - (PickerMode::EfficientFirst, "weak"), - ] { - let capable = scripted(ScriptedBehavior::Text("strong")); - let efficient = scripted(ScriptedBehavior::Text("weak")); - let fallback = scripted(ScriptedBehavior::Text("fallback")); - let algorithm = StageRouter::new( - fixed_target("strong"), - fixed_target("weak"), - StageRouterConfig::new(picker, 0.5), - ) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback, - WireFormat::OpenAiChat, - vec![("strong", capable), ("weak", efficient)], - ); - let mut marks = Vec::new(); - - runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, None), - &mut marks, - ) - .await - .unwrap(); - - assert!(marks.iter().any(|mark| { - mark.name == "switchyard.routing.decision" && mark.data["selected_target"] == expected - })); - } -} - -#[tokio::test] -async fn stage_router_classifier_resolves_an_ambiguous_turn() { - let capable = scripted(ScriptedBehavior::Text("strong")); - let efficient = scripted(ScriptedBehavior::Text("weak")); - let judge = scripted(ScriptedBehavior::Text( - r#"{"crux":"bounded","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, - )); - let fallback = scripted(ScriptedBehavior::Text("fallback")); - let mut config = StageRouterConfig::new(PickerMode::CapableFirst, 0.5); - config.llm_fallback = Some(LlmFallback { - judge_target: fixed_target("judge"), - config: TaskClassifierConfig { - base_threshold: 0.5, - ..TaskClassifierConfig::default() - }, - }); - let algorithm = StageRouter::new(fixed_target("strong"), fixed_target("weak"), config).unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback.clone(), - WireFormat::OpenAiChat, - vec![ - ("strong", capable.clone()), - ("weak", efficient.clone()), - ("judge", judge.clone()), - ], - ); - let mut marks = Vec::new(); - - runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, Some("stage-classifier")), - &mut marks, - ) - .await - .unwrap(); - - assert_eq!(judge.calls.load(Ordering::Relaxed), 1); - assert_eq!(efficient.calls.load(Ordering::Relaxed), 1); - assert_eq!(capable.calls.load(Ordering::Relaxed), 0); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); - let routing_calls = marks - .iter() - .filter(|mark| mark.name == "switchyard.routing.llm_call") - .collect::>(); - assert_eq!(routing_calls.len(), 1); - assert_eq!(routing_calls[0].data["selected_target"], "judge"); - assert_eq!(routing_calls[0].data["call_role"], "routing"); - assert_eq!(routing_calls[0].data["outcome"], "ok"); - assert_eq!(routing_calls[0].data["usage"]["total_tokens"], 18); - assert!(marks.iter().any(|mark| { - mark.name == "switchyard.routing.decision" && mark.data["selected_target"] == "weak" - })); -} From 7b42ffca7a842191f40e2857778ac1b5233235db Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 24 Aug 2026 12:35:30 -0700 Subject: [PATCH 03/18] build(relay): update dependency to 0.8.0-rc.2 Signed-off-by: Bryan Bednarski --- Cargo.lock | 73 +++++++----------------------------------------------- Cargo.toml | 2 +- 2 files changed, 10 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e046d1ded..2568a0b03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,18 +106,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "async-channel" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - [[package]] name = "async-stream" version = "0.3.6" @@ -424,15 +412,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -458,12 +437,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - [[package]] name = "data-encoding" version = "2.11.1" @@ -539,27 +512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "event-listener" -version = "5.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" -dependencies = [ - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener", - "pin-project-lite", + "windows-sys 0.52.0", ] [[package]] @@ -1290,7 +1243,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/NVIDIA/NeMo-Relay.git?rev=ca08901629e6058c2d5d65cc7708ec5264073d7b#ca08901629e6058c2d5d65cc7708ec5264073d7b" +source = "git+https://github.com/NVIDIA/NeMo-Relay.git?tag=0.8.0-rc.2#34be6d0850eb71be8d44f738d3b7ce4b1f9d7044" dependencies = [ "futures", "nemo-relay-types", @@ -1303,7 +1256,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/NVIDIA/NeMo-Relay.git?rev=ca08901629e6058c2d5d65cc7708ec5264073d7b#ca08901629e6058c2d5d65cc7708ec5264073d7b" +source = "git+https://github.com/NVIDIA/NeMo-Relay.git?tag=0.8.0-rc.2#34be6d0850eb71be8d44f738d3b7ce4b1f9d7044" dependencies = [ "bitflags", "chrono", @@ -1518,12 +1471,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - [[package]] name = "parking_lot" version = "0.12.5" @@ -1813,7 +1760,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2047,7 +1994,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2104,7 +2051,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2422,16 +2369,14 @@ dependencies = [ name = "switchyard-nemo-relay-plugin" version = "0.2.0" dependencies = [ - "async-channel", - "async-trait", "futures-util", "http", "nemo-relay-plugin", "serde", "serde_json", - "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", + "switchyard-runner", "switchyard-translation", "tokio", ] @@ -2618,7 +2563,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3167,7 +3112,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 90eed1722..42828885a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ http = "1" httpdate = "1" jsonschema = { version = "0.49.4", default-features = false } jsonptr = { version = "0.8.1", default-features = false, features = ["std", "json", "resolve"] } -nemo-relay-plugin = { git = "https://github.com/NVIDIA/NeMo-Relay.git", rev = "ca08901629e6058c2d5d65cc7708ec5264073d7b" } +nemo-relay-plugin = { git = "https://github.com/NVIDIA/NeMo-Relay.git", tag = "0.8.0-rc.2" } parking_lot = "0.12" rand = "0.10" regex = "1" From e3d7f624e56216c2a236bbba50cbfa73c2e32fbd Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 24 Aug 2026 13:29:01 -0700 Subject: [PATCH 04/18] fix(relay): validate plugin bundle output Signed-off-by: Bryan Bednarski --- .../scripts/package_bundle.py | 9 ++++- .../src/config.rs | 2 +- .../src/runtime.rs | 40 +++++++++++++------ .../tests/test_package_bundle.py | 29 ++++++++++++++ 4 files changed, 65 insertions(+), 15 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py index c5ea9e946..3b79af769 100644 --- a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py +++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py @@ -28,6 +28,8 @@ def digest(path: Path) -> str: def archive_bundle(bundle: Path, archive: Path) -> None: """Archive a materialized bundle under the stable package directory name.""" + if archive.parent.is_relative_to(bundle): + raise ValueError(f"bundle archive must be outside output directory: {archive}") archive.parent.mkdir(parents=True, exist_ok=True) if archive.exists(): raise ValueError(f"bundle archive already exists: {archive}") @@ -70,6 +72,10 @@ def main() -> None: parser.error(f"bundle output exists and is not a directory: {output}") if output.is_dir() and any(output.iterdir()): parser.error(f"bundle output directory must be empty: {output}") + + archive = args.archive.resolve() if args.archive is not None else None + if archive is not None and archive.parent.is_relative_to(output): + parser.error(f"bundle archive must be outside output directory: {archive}") output.mkdir(parents=True, exist_ok=True) artifact = output / library.name @@ -83,11 +89,10 @@ def main() -> None: manifest = manifest.replace("", artifact_digest) (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8") - if args.archive is None: + if archive is None: print(output) return - archive = args.archive.resolve() try: archive_bundle(output, archive) except ValueError as error: diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 9b235a62d..b9e22516e 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -16,7 +16,7 @@ pub(crate) fn protocol_from_call(name: &str) -> Option { } } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct SwitchyardConfig { #[serde(default)] diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 06b507fe9..5934d4b05 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -83,14 +83,22 @@ impl SwitchyardRuntime { inbound: WireFormat, request: Request, ) -> Execution { - let mut execution = self.execute(request).await; - if let Ok(response) = execution.result { - execution.result = finalize_buffered_response(&self.translation, inbound, response); - if execution.result.is_err() { - self.error_mark(&mut execution.marks, "response_finalization", None); + let Execution { + result, + mut marks, + } = self.execute(request).await; + let (result, finalization_failed) = match result { + Ok(response) => { + let result = finalize_buffered_response(&self.translation, inbound, response); + let failed = result.is_err(); + (result, failed) } + Err(error) => (Err(error), false), + }; + if finalization_failed { + self.error_mark(&mut marks, "response_finalization", None); } - execution + Execution { result, marks } } pub(crate) async fn execute_stream( @@ -98,14 +106,22 @@ impl SwitchyardRuntime { inbound: WireFormat, request: Request, ) -> Execution { - let mut execution = self.execute(request).await; - if let Ok(response) = execution.result { - execution.result = returned_events(response, inbound); - if execution.result.is_err() { - self.error_mark(&mut execution.marks, "response_finalization", None); + let Execution { + result, + mut marks, + } = self.execute(request).await; + let (result, finalization_failed) = match result { + Ok(response) => { + let result = returned_events(response, inbound); + let failed = result.is_err(); + (result, failed) } + Err(error) => (Err(error), false), + }; + if finalization_failed { + self.error_mark(&mut marks, "response_finalization", None); } - execution + Execution { result, marks } } async fn execute(&self, request: Request) -> Execution { diff --git a/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py b/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py index 296cbc4b8..f1dc6d479 100644 --- a/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py +++ b/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py @@ -62,6 +62,35 @@ def test_materializes_and_archives_supported_formats(self) -> None: self.assertNotIn("", manifest) self.assertEqual(self.archive_members(archive), {f"{PACKAGE_NAME}/{name}" for name in expected}) + def test_rejects_archive_inside_output_before_materializing_bundle(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + library = root / "libswitchyard_nemo_relay_plugin.so" + library.write_bytes(b"compiled plugin") + output = root / "bundle" + archive = output / "plugin.zip" + + result = subprocess.run( + [ + sys.executable, + str(PACKAGER), + "--library", + str(library), + "--output", + str(output), + "--archive", + str(archive), + ], + check=False, + capture_output=True, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("bundle archive must be outside output directory", result.stderr) + self.assertFalse(output.exists()) + self.assertFalse(archive.exists()) + @staticmethod def archive_members(archive: Path) -> set[str]: """Return regular-file paths from a supported bundle archive.""" From 79725c90ab0efb72fe1a53912c84ade08cf38303 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 24 Aug 2026 13:29:38 -0700 Subject: [PATCH 05/18] refactor(relay): remove obsolete translation preflight Signed-off-by: Bryan Bednarski --- .../src/translation.rs | 51 ------------------- 1 file changed, 51 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs index 6f6f965fe..cad0c988e 100644 --- a/crates/switchyard-nemo-relay-plugin/src/translation.rs +++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs @@ -22,17 +22,6 @@ pub(crate) fn decode_request( Ok(output.request) } -pub(crate) fn validate_target_request( - engine: &TranslationEngine, - protocol: WireFormat, - request: &LlmRequest, -) -> Result<(), String> { - let output = engine - .encode_request(protocol, request, &request_policy(protocol)) - .map_err(error)?; - safe(&output.diagnostics) -} - pub(crate) fn encode_response( engine: &TranslationEngine, protocol: WireFormat, @@ -57,16 +46,6 @@ fn policy() -> TranslationPolicy { } } -fn request_policy(protocol: WireFormat) -> TranslationPolicy { - let mut policy = policy(); - if protocol == WireFormat::AnthropicMessages { - policy - .target_capabilities - .supports_json_schema_response_format = Some(false); - } - policy -} - fn safe(diagnostics: &[TranslationDiagnostic]) -> Result<(), String> { let unsafe_diagnostics = diagnostics .iter() @@ -84,33 +63,3 @@ fn safe(diagnostics: &[TranslationDiagnostic]) -> Result<(), String> { fn error(error: switchyard_translation::TranslationError) -> String { format!("Switchyard translation failed: {error}") } - -#[cfg(test)] -mod tests { - use serde_json::{Map, json}; - - use super::*; - - #[test] - fn same_protocol_request_preserves_unknown_fields() { - let request = RelayRequest { - headers: Map::new(), - content: json!({ - "model": "route", - "messages": [{"role": "user", "content": "hello"}], - "provider_extension": {"exact": true} - }), - }; - let engine = TranslationEngine::default(); - let decoded = decode_request(&engine, WireFormat::OpenAiChat, &request).unwrap(); - validate_target_request(&engine, WireFormat::OpenAiChat, &decoded).unwrap(); - assert_eq!( - decoded - .preservation - .requests - .get(&WireFormat::OpenAiChat.into()) - .and_then(|body| body.get("provider_extension")), - Some(&json!({"exact": true})) - ); - } -} From 40df0d6029376bf6cb609527edc18739682ec5cb Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 24 Aug 2026 13:51:14 -0700 Subject: [PATCH 06/18] style(relay): format plugin sources Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/src/lib.rs | 5 ++++- .../switchyard-nemo-relay-plugin/src/runtime.rs | 17 ++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs index f115923b3..b67e312d8 100644 --- a/crates/switchyard-nemo-relay-plugin/src/lib.rs +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -122,7 +122,10 @@ fn parse_config(plugin_config: &Map) -> Result) { for mark in marks { if let Err(error) = runtime.emit_mark(&mark.name, Some(&mark.data), Some(&mark.metadata)) { - eprintln!("Switchyard could not emit routing mark {:?}: {error}", mark.name); + eprintln!( + "Switchyard could not emit routing mark {:?}: {error}", + mark.name + ); } } } diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 5934d4b05..71429e581 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -83,10 +83,7 @@ impl SwitchyardRuntime { inbound: WireFormat, request: Request, ) -> Execution { - let Execution { - result, - mut marks, - } = self.execute(request).await; + let Execution { result, mut marks } = self.execute(request).await; let (result, finalization_failed) = match result { Ok(response) => { let result = finalize_buffered_response(&self.translation, inbound, response); @@ -106,10 +103,7 @@ impl SwitchyardRuntime { inbound: WireFormat, request: Request, ) -> Execution { - let Execution { - result, - mut marks, - } = self.execute(request).await; + let Execution { result, mut marks } = self.execute(request).await; let (result, finalization_failed) = match result { Ok(response) => { let result = returned_events(response, inbound); @@ -231,7 +225,12 @@ impl SwitchyardRuntime { }); } - fn error_mark(&self, marks: &mut Vec, failure_kind: &str, metadata: Option<&Json>) { + fn error_mark( + &self, + marks: &mut Vec, + failure_kind: &str, + metadata: Option<&Json>, + ) { let metadata = metadata.cloned().unwrap_or_else(|| { marks .first() From 193772a27ecfabd53d1d94fea64bcb206cef8e14 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 24 Aug 2026 17:23:24 -0700 Subject: [PATCH 07/18] feat(relay): emit typed route failure marks Signed-off-by: Bryan Bednarski --- .../switchyard-nemo-relay-plugin/src/lib.rs | 22 ++- .../src/runtime.rs | 145 +++++++++++++++++- 2 files changed, 146 insertions(+), 21 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs index b67e312d8..9663538d2 100644 --- a/crates/switchyard-nemo-relay-plugin/src/lib.rs +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -14,7 +14,7 @@ use nemo_relay_plugin::{ use serde_json::Map; use crate::config::{SwitchyardConfig, protocol_from_call}; -use crate::runtime::{RoutingMark, SwitchyardRuntime}; +use crate::runtime::{SwitchyardRuntime, emit_mark, emit_marks}; #[derive(Default)] struct SwitchyardPlugin; @@ -104,7 +104,14 @@ fn register_stream( if !runtime.manages(&decoded) { return next.call(request).await; } - let execution = runtime.execute_stream(inbound, decoded).await; + let stream_plugin_runtime = plugin_runtime.clone(); + let execution = runtime + .execute_stream( + inbound, + decoded, + Arc::new(move |mark| emit_mark(&stream_plugin_runtime, mark)), + ) + .await; emit_marks(&plugin_runtime, execution.marks); execution .result @@ -119,17 +126,6 @@ fn parse_config(plugin_config: &Map) -> Result) { - for mark in marks { - if let Err(error) = runtime.emit_mark(&mark.name, Some(&mark.data), Some(&mark.metadata)) { - eprintln!( - "Switchyard could not emit routing mark {:?}: {error}", - mark.name - ); - } - } -} - nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlugin::default); #[cfg(test)] diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 71429e581..b97782f85 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -5,11 +5,11 @@ use std::pin::Pin; use std::sync::{Arc, Mutex}; use futures_util::{Stream, StreamExt}; -use nemo_relay_plugin::{Json, LlmRequest as RelayRequest}; +use nemo_relay_plugin::{Json, LlmRequest as RelayRequest, PluginRuntime}; use serde_json::{Map, json}; use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver}; use switchyard_protocol::{LlmResponse, Metadata, Request, Response, WireFormat}; -use switchyard_runner::{Route, Runner}; +use switchyard_runner::{Route, RouteFailureSummary, Runner, stream_failure_summary}; use switchyard_translation::{TranslationEngine, encode_stream}; use crate::config::SwitchyardConfig; @@ -23,6 +23,7 @@ pub(crate) struct RoutingMark { } pub(crate) type ReturnedEventStream = Pin> + Send>>; +pub(crate) type RoutingMarkEmitter = Arc; pub(crate) struct Execution { pub(crate) result: Result, @@ -102,11 +103,16 @@ impl SwitchyardRuntime { &self, inbound: WireFormat, request: Request, + emit_mark: RoutingMarkEmitter, ) -> Execution { let Execution { result, mut marks } = self.execute(request).await; let (result, finalization_failed) = match result { Ok(response) => { - let result = returned_events(response, inbound); + let metadata = marks + .first() + .map(|mark| mark.metadata.clone()) + .unwrap_or_else(|| Json::Object(Map::new())); + let result = returned_events(response, inbound, metadata, emit_mark); let failed = result.is_err(); (result, failed) } @@ -162,9 +168,13 @@ impl SwitchyardRuntime { marks, } } - Err(_) => { + Err(error) => { self.emit_observations(&mut marks, take_observations(&observations), &metadata); - self.error_mark(&mut marks, "route_execution", None); + self.route_execution_error_mark( + &mut marks, + &error.execution_failure_summary(), + None, + ); Execution { result: Err("Switchyard route execution failed".into()), marks, @@ -243,6 +253,36 @@ impl SwitchyardRuntime { metadata, }); } + + fn route_execution_error_mark( + &self, + marks: &mut Vec, + summary: &RouteFailureSummary, + metadata: Option<&Json>, + ) { + let metadata = metadata.cloned().unwrap_or_else(|| { + marks + .first() + .map(|mark| mark.metadata.clone()) + .unwrap_or_else(|| Json::Object(Map::new())) + }); + marks.push(route_execution_error_mark(summary, metadata)); + } +} + +pub(crate) fn emit_marks(runtime: &PluginRuntime, marks: Vec) { + for mark in marks { + emit_mark(runtime, mark); + } +} + +pub(crate) fn emit_mark(runtime: &PluginRuntime, mark: RoutingMark) { + if let Err(error) = runtime.emit_mark(&mark.name, Some(&mark.data), Some(&mark.metadata)) { + eprintln!( + "Switchyard could not emit routing mark {:?}: {error}", + mark.name + ); + } } fn take_observations(observations: &Mutex>) -> Vec { @@ -272,11 +312,26 @@ fn finalize_buffered_response( translation::encode_response(translation_engine, inbound, &response) } -fn returned_events(response: Response, inbound: WireFormat) -> Result { +fn returned_events( + response: Response, + inbound: WireFormat, + metadata: Json, + emit_mark: RoutingMarkEmitter, +) -> Result { + let served_model = response.served_model().cloned(); let chunks = match response.llm_response { LlmResponse::Agg(response) => response.into_stream(), LlmResponse::Stream(chunks) => chunks, }; + let chunks = Box::pin(chunks.map(move |item| { + if let Err(error) = &item { + emit_mark(route_execution_error_mark( + &stream_failure_summary(error, served_model.as_ref()), + metadata.clone(), + )); + } + item + })); let events = encode_stream(chunks, inbound, None) .map_err(|error| format!("Switchyard response stream setup failed: {error}"))?; Ok(Box::pin(events.map(|item| { @@ -284,6 +339,20 @@ fn returned_events(response: Response, inbound: WireFormat) -> Result RoutingMark { + RoutingMark { + name: "switchyard.routing.error".into(), + data: json!({ + "failure_kind": "route_execution", + "category": summary.category.as_str(), + "phase": summary.phase.as_str(), + "upstream_status": summary.upstream_status, + "target": summary.target.as_ref().map(|target| target.as_str()), + }), + metadata, + } +} + fn string_headers(headers: &Map) -> http::HeaderMap { let mut parsed = http::HeaderMap::with_capacity(headers.len()); for (name, value) in headers { @@ -317,8 +386,8 @@ mod tests { use std::collections::{BTreeMap, HashMap}; use switchyard_llm_client::ClientRouter; - use switchyard_protocol::{ModelId, text_request}; - use switchyard_runner::{AlgorithmSpec, ModelCapabilities}; + use switchyard_protocol::{LlmClientError, LlmResponseStreamEvent, ModelId, text_request}; + use switchyard_runner::{AlgorithmSpec, ModelCapabilities, RunnerError}; use super::*; @@ -355,4 +424,64 @@ mod tests { assert!(runtime.manages(&configured)); assert!(!runtime.manages(&other)); } + + #[test] + fn execution_failure_mark_uses_the_safe_runner_summary() { + let secret = "provider response body"; + let error = RunnerError::Client(LlmClientError::ContextWindowExceeded { + model: ModelId::from("weak"), + message: secret.into(), + }); + + let mark = route_execution_error_mark( + &error.execution_failure_summary(), + json!({"session_id": "session"}), + ); + + assert_eq!(mark.name, "switchyard.routing.error"); + assert_eq!(mark.data["failure_kind"], "route_execution"); + assert_eq!(mark.data["category"], "context_window_exceeded"); + assert_eq!(mark.data["phase"], "before_response"); + assert_eq!(mark.data["target"], "weak"); + assert_eq!(mark.data["upstream_status"], Json::Null); + assert!(!mark.data.to_string().contains(secret)); + } + + #[tokio::test] + async fn stream_failure_emits_a_safe_failure_mark() { + let secret = "provider response body"; + let response = Response { + llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([Err::< + LlmResponseStreamEvent, + LlmClientError, + >( + LlmClientError::ContextWindowExceeded { + model: ModelId::from("weak"), + message: secret.into(), + }, + )]))), + metadata: Some(Metadata { + served_model: Some(ModelId::from("strong")), + ..Default::default() + }), + }; + let captured = Arc::new(Mutex::new(Vec::new())); + let emitted = Arc::clone(&captured); + let stream = returned_events( + response, + WireFormat::OpenAiChat, + json!({"session_id": "session"}), + Arc::new(move |mark| emitted.lock().unwrap().push(mark)), + ) + .expect("stream setup should succeed"); + + let events = stream.collect::>().await; + assert!(events[0].is_err()); + let marks = captured.lock().unwrap(); + assert_eq!(marks.len(), 1); + assert_eq!(marks[0].data["category"], "context_window_exceeded"); + assert_eq!(marks[0].data["phase"], "during_stream"); + assert_eq!(marks[0].data["target"], "strong"); + assert!(!marks[0].data.to_string().contains(secret)); + } } From e7ca74fbb7fe2454047ef187406add5344b3e8de Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 24 Aug 2026 17:30:11 -0700 Subject: [PATCH 08/18] build(relay): update dependency to 0.8.0-rc.3 Signed-off-by: Bryan Bednarski --- Cargo.lock | 18 +++++++++--------- Cargo.toml | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2568a0b03..a5fa22b28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -512,7 +512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1243,7 +1243,7 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" version = "0.8.0" -source = "git+https://github.com/NVIDIA/NeMo-Relay.git?tag=0.8.0-rc.2#34be6d0850eb71be8d44f738d3b7ce4b1f9d7044" +source = "git+https://github.com/NVIDIA/NeMo-Relay.git?tag=0.8.0-rc.3#4b3e66e8f42b113be2b241914674082fa104cdc4" dependencies = [ "futures", "nemo-relay-types", @@ -1256,7 +1256,7 @@ dependencies = [ [[package]] name = "nemo-relay-types" version = "0.8.0" -source = "git+https://github.com/NVIDIA/NeMo-Relay.git?tag=0.8.0-rc.2#34be6d0850eb71be8d44f738d3b7ce4b1f9d7044" +source = "git+https://github.com/NVIDIA/NeMo-Relay.git?tag=0.8.0-rc.3#4b3e66e8f42b113be2b241914674082fa104cdc4" dependencies = [ "bitflags", "chrono", @@ -1543,7 +1543,7 @@ dependencies = [ "pyo3", "pyo3-build-config", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] @@ -1760,7 +1760,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1994,7 +1994,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2051,7 +2051,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2563,7 +2563,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3112,7 +3112,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 42828885a..bd00434e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ http = "1" httpdate = "1" jsonschema = { version = "0.49.4", default-features = false } jsonptr = { version = "0.8.1", default-features = false, features = ["std", "json", "resolve"] } -nemo-relay-plugin = { git = "https://github.com/NVIDIA/NeMo-Relay.git", tag = "0.8.0-rc.2" } +nemo-relay-plugin = { git = "https://github.com/NVIDIA/NeMo-Relay.git", tag = "0.8.0-rc.3" } parking_lot = "0.12" rand = "0.10" regex = "1" From 71cfe1e6890096b8a7b8342d02aa123cff30cb53 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 24 Aug 2026 17:39:52 -0700 Subject: [PATCH 09/18] build(relay): retain focused RC3 lockfile update Signed-off-by: Bryan Bednarski --- Cargo.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5fa22b28..c33716a51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -512,7 +512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1543,7 +1543,7 @@ dependencies = [ "pyo3", "pyo3-build-config", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.18", "tracing", ] @@ -1760,7 +1760,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1994,7 +1994,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2051,7 +2051,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2563,7 +2563,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3112,7 +3112,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From 7cb6b66e67a8f963dab4f83d9c18237fd57769b1 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 24 Aug 2026 17:45:38 -0700 Subject: [PATCH 10/18] build(relay): resolve focused plugin dependencies Signed-off-by: Bryan Bednarski --- Cargo.lock | 378 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 1 - 2 files changed, 188 insertions(+), 191 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c33716a51..83c2f25df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,9 +18,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.5" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -130,13 +130,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.92" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 2.0.119", ] [[package]] @@ -153,9 +153,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.18.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "zeroize", @@ -163,9 +163,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.44.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", @@ -304,9 +304,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.4.4" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -349,9 +349,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" dependencies = [ "clap_builder", "clap_derive", @@ -359,9 +359,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -371,14 +371,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.3", + "syn 2.0.119", ] [[package]] @@ -469,13 +469,13 @@ checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" [[package]] name = "displaydoc" -version = "0.2.7" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 2.0.119", ] [[package]] @@ -486,9 +486,9 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "either" -version = "1.18.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "email_address" @@ -512,14 +512,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "fancy-regex" -version = "0.19.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "476de73bddf2ef8490aa4ee8f1cf40b430bf1d56c48c22080e5186952cd580e6" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" dependencies = [ "bit-set", "regex-automata", @@ -534,9 +534,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fluent-uri" @@ -598,9 +598,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -613,9 +613,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -623,15 +623,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -640,38 +640,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -727,9 +727,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.18" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", @@ -769,9 +769,9 @@ checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "http" -version = "1.5.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -789,9 +789,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -820,9 +820,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hyper" -version = "1.11.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -880,9 +880,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", @@ -894,9 +894,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -907,9 +907,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -921,17 +921,16 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -942,15 +941,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -994,9 +993,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is_terminal_polyfill" @@ -1031,7 +1030,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.18", "walkdir", "windows-link", ] @@ -1080,9 +1079,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", @@ -1091,9 +1090,9 @@ dependencies = [ [[package]] name = "jsonptr" -version = "0.8.2" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c22d99362c7aac7d77efab6b49c2181674b165d7e6e7c755b3432ded64bf51" +checksum = "85019623956752c8dd1f04a7b05d066187e0c9b217454246d8397d2a4893cc83" dependencies = [ "serde", "serde_json", @@ -1101,9 +1100,9 @@ dependencies = [ [[package]] name = "jsonschema" -version = "0.49.9" +version = "0.49.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ec8a241beed129f06114aa68007e905ca350e7baeb6e17a7631bb7978d91b2" +checksum = "257549c093d8f3d043337ba0d507e8eeace2447dfae3f0ee37eb0f57d3df7655" dependencies = [ "ahash", "bytecount", @@ -1130,18 +1129,18 @@ dependencies = [ [[package]] name = "jsonschema-regex" -version = "0.49.9" +version = "0.49.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91994f45017ed5e66aa8e59b8415f4cb033a6380d7200387b7cf117595fbdf85" +checksum = "474790e948498099d61ec85c10dc72d459d61298b7975eda7fb163af1934b134" dependencies = [ "regex-syntax", ] [[package]] name = "jsonschema-value" -version = "0.49.9" +version = "0.49.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ec7637f83e510868ae6ed625f7ebfbbde4554ee8ce49854caa5126a8b9b9ecb" +checksum = "8212315eb8e0bc44959d1af653cd5ec515771a3cdca966cfc0a10999bff144b9" dependencies = [ "ahash", "bytecount", @@ -1159,9 +1158,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "linux-raw-sys" @@ -1171,9 +1170,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -1186,9 +1185,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.34" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru-slab" @@ -1316,9 +1315,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.47" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ "num-traits", ] @@ -1391,7 +1390,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.20", + "thiserror 2.0.18", "tracing", ] @@ -1421,7 +1420,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 2.0.20", + "thiserror 2.0.18", ] [[package]] @@ -1461,7 +1460,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.20", + "thiserror 2.0.18", "tokio", ] @@ -1508,21 +1507,21 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" -version = "1.15.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "potential_utf" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -1559,9 +1558,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.107" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] @@ -1578,7 +1577,7 @@ dependencies = [ "memchr", "parking_lot", "protobuf", - "thiserror 2.0.20", + "thiserror 2.0.18", ] [[package]] @@ -1720,7 +1719,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.20", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -1728,9 +1727,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", @@ -1743,7 +1742,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.20", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -1760,14 +1759,14 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "quote" -version = "1.0.47" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -1850,18 +1849,18 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.27" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.27" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" dependencies = [ "proc-macro2", "quote", @@ -1870,9 +1869,9 @@ dependencies = [ [[package]] name = "referencing" -version = "0.49.9" +version = "0.49.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6efa2154ea6f5ce0fdecdd2a8d18f2fa1a39a8fbba91564f555a592e4dce8278" +checksum = "2bf1a74e036be2546c0c81cdfad6b2def6a25bf31c4e34a468037058d3bd05c6" dependencies = [ "ahash", "fluent-uri", @@ -1899,9 +1898,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.18" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -1994,14 +1993,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "once_cell", @@ -2025,9 +2024,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -2051,7 +2050,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2062,9 +2061,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -2139,9 +2138,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -2149,29 +2148,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 2.0.119", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "indexmap", "itoa", @@ -2332,7 +2331,7 @@ dependencies = [ "serde", "serde_json", "switchyard-protocol", - "thiserror 2.0.20", + "thiserror 2.0.18", "tokio", "tokio-stream", "tracing", @@ -2357,7 +2356,7 @@ dependencies = [ "switchyard-libsy", "switchyard-protocol", "switchyard-translation", - "thiserror 2.0.20", + "thiserror 2.0.18", "tokio", "tracing", "tracing-opentelemetry", @@ -2390,7 +2389,7 @@ dependencies = [ "http", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.18", ] [[package]] @@ -2423,7 +2422,7 @@ dependencies = [ "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", - "thiserror 2.0.20", + "thiserror 2.0.18", "tokio", "toml", "tracing", @@ -2471,7 +2470,7 @@ dependencies = [ "async-trait", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.18", "tokio", ] @@ -2502,7 +2501,7 @@ dependencies = [ "serde", "serde_json", "switchyard-protocol", - "thiserror 2.0.20", + "thiserror 2.0.18", ] [[package]] @@ -2563,7 +2562,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2577,11 +2576,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.18", ] [[package]] @@ -2597,13 +2596,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 2.0.119", ] [[package]] @@ -2617,9 +2616,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -2642,9 +2641,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.53.1" +version = "1.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "317fafbbe3f02fc663dad00ea6186197de963cd4190e86a26d8d0fae095539af" dependencies = [ "bytes", "libc", @@ -2659,13 +2658,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 2.0.119", ] [[package]] @@ -2680,9 +2679,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.19" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -2691,24 +2690,23 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", "futures-sink", "futures-util", - "libc", "pin-project-lite", "tokio", ] [[package]] name = "toml" -version = "1.1.4+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" dependencies = [ "indexmap", "serde_core", @@ -2730,9 +2728,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] @@ -3011,9 +3009,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -3024,9 +3022,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.77" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -3034,9 +3032,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3044,9 +3042,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", @@ -3057,9 +3055,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] @@ -3079,9 +3077,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.104" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -3112,7 +3110,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3240,9 +3238,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "yansi" @@ -3275,18 +3273,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", @@ -3322,9 +3320,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -3333,9 +3331,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -3344,13 +3342,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 2.0.119", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bd00434e3..1e68dcce6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,6 @@ repository = "https://github.com/NVIDIA-NeMo/Switchyard" rust-version = "1.96.1" [workspace.dependencies] -async-channel = "2" async-stream = "0.3" async-trait = "0.1" futures = "0.3" From 5228aa42e1abec455d9453f072a5c40df36e1f9c Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 24 Aug 2026 18:40:37 -0700 Subject: [PATCH 11/18] test(relay): cover plugin protocol mapping Signed-off-by: Bryan Bednarski --- .../src/config.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index b9e22516e..d42028601 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -29,3 +29,22 @@ impl SwitchyardConfig { Runner::load(&self.deployment_path).map_err(|error| error.to_string()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maps_supported_relay_call_names_to_wire_formats() { + let cases = [ + ("openai.chat_completions", Some(WireFormat::OpenAiChat)), + ("openai.responses", Some(WireFormat::OpenAiResponses)), + ("anthropic.messages", Some(WireFormat::AnthropicMessages)), + ("unsupported.call", None), + ]; + + for (name, expected) in cases { + assert_eq!(protocol_from_call(name), expected, "{name}"); + } + } +} From 235b7f35a9c2d0a1866d7f257d3cfea8214a953c Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 25 Aug 2026 08:26:22 -0700 Subject: [PATCH 12/18] refactor(relay): use route error kind Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/src/runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index b97782f85..467be027c 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -344,7 +344,7 @@ fn route_execution_error_mark(summary: &RouteFailureSummary, metadata: Json) -> name: "switchyard.routing.error".into(), data: json!({ "failure_kind": "route_execution", - "category": summary.category.as_str(), + "category": summary.kind.as_str(), "phase": summary.phase.as_str(), "upstream_status": summary.upstream_status, "target": summary.target.as_ref().map(|target| target.as_str()), From d8591707f825251cc3cc9ca470e813f6a958ac83 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 25 Aug 2026 09:00:21 -0700 Subject: [PATCH 13/18] refactor(relay): use route error summary Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/src/runtime.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 467be027c..cd292e7db 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -9,7 +9,7 @@ use nemo_relay_plugin::{Json, LlmRequest as RelayRequest, PluginRuntime}; use serde_json::{Map, json}; use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver}; use switchyard_protocol::{LlmResponse, Metadata, Request, Response, WireFormat}; -use switchyard_runner::{Route, RouteFailureSummary, Runner, stream_failure_summary}; +use switchyard_runner::{Route, RouteErrorSummary, Runner, stream_failure_summary}; use switchyard_translation::{TranslationEngine, encode_stream}; use crate::config::SwitchyardConfig; @@ -257,7 +257,7 @@ impl SwitchyardRuntime { fn route_execution_error_mark( &self, marks: &mut Vec, - summary: &RouteFailureSummary, + summary: &RouteErrorSummary, metadata: Option<&Json>, ) { let metadata = metadata.cloned().unwrap_or_else(|| { @@ -339,7 +339,7 @@ fn returned_events( }))) } -fn route_execution_error_mark(summary: &RouteFailureSummary, metadata: Json) -> RoutingMark { +fn route_execution_error_mark(summary: &RouteErrorSummary, metadata: Json) -> RoutingMark { RoutingMark { name: "switchyard.routing.error".into(), data: json!({ From b6d61351435b5d898ea969d4f6f0708fe69a16d9 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 25 Aug 2026 09:06:19 -0700 Subject: [PATCH 14/18] refactor(relay): use route error summary methods Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/src/runtime.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index cd292e7db..eba12cb68 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -9,7 +9,7 @@ use nemo_relay_plugin::{Json, LlmRequest as RelayRequest, PluginRuntime}; use serde_json::{Map, json}; use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver}; use switchyard_protocol::{LlmResponse, Metadata, Request, Response, WireFormat}; -use switchyard_runner::{Route, RouteErrorSummary, Runner, stream_failure_summary}; +use switchyard_runner::{Route, RouteErrorSummary, Runner, stream_error_summary}; use switchyard_translation::{TranslationEngine, encode_stream}; use crate::config::SwitchyardConfig; @@ -170,11 +170,7 @@ impl SwitchyardRuntime { } Err(error) => { self.emit_observations(&mut marks, take_observations(&observations), &metadata); - self.route_execution_error_mark( - &mut marks, - &error.execution_failure_summary(), - None, - ); + self.route_execution_error_mark(&mut marks, &error.execution_error_summary(), None); Execution { result: Err("Switchyard route execution failed".into()), marks, @@ -326,7 +322,7 @@ fn returned_events( let chunks = Box::pin(chunks.map(move |item| { if let Err(error) = &item { emit_mark(route_execution_error_mark( - &stream_failure_summary(error, served_model.as_ref()), + &stream_error_summary(error, served_model.as_ref()), metadata.clone(), )); } @@ -434,7 +430,7 @@ mod tests { }); let mark = route_execution_error_mark( - &error.execution_failure_summary(), + &error.execution_error_summary(), json!({"session_id": "session"}), ); From 53a30bb784a96a4e9e38100acc24142ea569a1e8 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 25 Aug 2026 11:59:01 -0700 Subject: [PATCH 15/18] feat(relay): support inline Switchyard configuration Signed-off-by: Bryan Bednarski --- Cargo.lock | 2 + .../switchyard-nemo-relay-plugin/Cargo.toml | 4 + crates/switchyard-nemo-relay-plugin/README.md | 40 ++++++- .../config.schema.json | 24 +++- .../src/config.rs | 106 +++++++++++++++++- .../switchyard-nemo-relay-plugin/src/lib.rs | 21 +++- 6 files changed, 183 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 83c2f25df..f5528189e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2377,7 +2377,9 @@ dependencies = [ "switchyard-protocol", "switchyard-runner", "switchyard-translation", + "tempfile", "tokio", + "toml", ] [[package]] diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index f6a4b1d64..283477c86 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -26,3 +26,7 @@ switchyard-protocol.workspace = true switchyard-runner.workspace = true switchyard-translation.workspace = true tokio.workspace = true +toml = "1.1" + +[dev-dependencies] +tempfile = "3" diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 8ce7093f1..c4da8913a 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -1,8 +1,8 @@ # Switchyard NeMo Relay Plugin `switchyard-nemo-relay-plugin` is a native NeMo Relay dynamic plugin. It loads -a standard Switchyard TOML deployment and executes its configured routes in -Relay through `switchyard-runner`. +a standard Switchyard TOML deployment from a file or Relay's nested plugin +configuration and executes its configured routes through `switchyard-runner`. The plugin does not define a second routing or target configuration language. `switchyard-server` and Relay therefore use the same targets, client pooling, @@ -16,22 +16,50 @@ the generated `relay-plugin.toml` manifest. The plugin requires NeMo Relay ## Configure Relay -Point the dynamic plugin configuration at an existing Switchyard deployment: +Use exactly one Switchyard deployment source. To share an existing deployment +file with `switchyard-server`, configure its path: ```toml [[plugins.dynamic]] -plugin_id = "nvidia.switchyard" +manifest = "./plugins/switchyard/relay-plugin.toml" [plugins.dynamic.config] priority = 0 -deployment_path = "/etc/switchyard/routes.toml" +switchyard_config_path = "/etc/switchyard/routes.toml" ``` -`deployment_path` is a Switchyard version-1 TOML deployment, accepted by both +`switchyard_config_path` is a Switchyard version-1 TOML deployment, accepted by both `switchyard-server` and `switchyard-runner`. See the [server configuration guide](../switchyard-server/CONFIGURATION.md) for the deployment schema and routing algorithms. +To keep the deployment in the Relay configuration, nest the same version-1 +Switchyard configuration under `switchyard_config`: + +```toml +[[plugins.dynamic]] +manifest = "./plugins/switchyard/relay-plugin.toml" + +[plugins.dynamic.config] +priority = 0 + +[plugins.dynamic.config.switchyard_config] +schema_version = 1 + +[plugins.dynamic.config.switchyard_config.llm_clients.primary] +format = "openai_chat" +base_url = "https://example.test/v1" + +[plugins.dynamic.config.switchyard_config.targets.default] +id = "example/model" +llm_client = "primary" + +[plugins.dynamic.config.switchyard_config.routes.default] +id = "switchyard/default" +type = "passthrough" +target = "default" +``` + ## Request handling For OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages calls, diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json index 474f212f7..61f178a12 100644 --- a/crates/switchyard-nemo-relay-plugin/config.schema.json +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -4,17 +4,37 @@ "description": "Runs a Switchyard deployment through its shared in-process runner.", "type": "object", "additionalProperties": false, - "required": ["deployment_path"], + "oneOf": [ + { "required": ["switchyard_config_path"] }, + { "required": ["switchyard_config"] } + ], "properties": { "priority": { "type": "integer", "default": 0, "description": "NeMo Relay execution-intercept priority." }, - "deployment_path": { + "executor": { + "type": "object", + "additionalProperties": false, + "description": "Optional NeMo Relay SDK executor override.", + "properties": { + "worker_threads": { + "type": "integer", + "minimum": 1, + "description": "Number of worker threads for the plugin's SDK-owned executor." + } + } + }, + "switchyard_config_path": { "type": "string", "minLength": 1, "description": "Path to a Switchyard version-1 TOML deployment shared with switchyard-server." + }, + "switchyard_config": { + "type": "object", + "minProperties": 1, + "description": "Inline Switchyard version-1 deployment using the same schema as switchyard-server." } } } diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index d42028601..2df6386b3 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use serde::Deserialize; +use serde_json::{Map, Value}; use switchyard_protocol::WireFormat; use switchyard_runner::Runner; @@ -21,19 +22,66 @@ pub(crate) fn protocol_from_call(name: &str) -> Option { pub(crate) struct SwitchyardConfig { #[serde(default)] pub(crate) priority: i32, - pub(crate) deployment_path: PathBuf, + #[serde(default)] + pub(crate) switchyard_config_path: Option, + #[serde(default)] + pub(crate) switchyard_config: Option>, } impl SwitchyardConfig { pub(crate) fn load_runner(&self) -> Result { - Runner::load(&self.deployment_path).map_err(|error| error.to_string()) + match (&self.switchyard_config_path, &self.switchyard_config) { + (Some(path), None) => Runner::load(path).map_err(|error| error.to_string()), + (None, Some(config)) => toml::to_string(config) + .map_err(|error| format!("failed to serialize Switchyard configuration: {error}")) + .and_then(|source| Runner::from_toml(&source).map_err(|error| error.to_string())), + (Some(_), Some(_)) => Err( + "configure exactly one of switchyard_config_path or switchyard_config".to_string(), + ), + (None, None) => { + Err("configure one of switchyard_config_path or switchyard_config".to_string()) + } + } } } #[cfg(test)] mod tests { + use std::io::Write; + + use serde_json::json; + use tempfile::NamedTempFile; + use super::*; + fn inline_deployment() -> Map { + json!({ + "schema_version": 1, + "llm_clients": { + "primary": { + "format": "openai_chat", + "base_url": "https://example.test/v1" + } + }, + "targets": { + "default": { + "id": "example/model", + "llm_client": "primary" + } + }, + "routes": { + "default": { + "id": "switchyard/default", + "type": "passthrough", + "target": "default" + } + } + }) + .as_object() + .unwrap() + .clone() + } + #[test] fn maps_supported_relay_call_names_to_wire_formats() { let cases = [ @@ -47,4 +95,58 @@ mod tests { assert_eq!(protocol_from_call(name), expected, "{name}"); } } + + #[test] + fn builds_runner_from_inline_deployment() { + let config = SwitchyardConfig { + priority: 0, + switchyard_config_path: None, + switchyard_config: Some(inline_deployment()), + }; + + let runner = config.load_runner().unwrap(); + assert!(runner.route("switchyard/default").is_some()); + } + + #[test] + fn builds_runner_from_deployment_path() { + let mut deployment = NamedTempFile::new().unwrap(); + write!( + deployment, + "{}", + toml::to_string(&inline_deployment()).unwrap() + ) + .unwrap(); + let config = SwitchyardConfig { + priority: 0, + switchyard_config_path: Some(deployment.path().to_path_buf()), + switchyard_config: None, + }; + + let runner = config.load_runner().unwrap(); + assert!(runner.route("switchyard/default").is_some()); + } + + #[test] + fn requires_exactly_one_configuration_source() { + let neither = SwitchyardConfig { + priority: 0, + switchyard_config_path: None, + switchyard_config: None, + }; + assert!(matches!( + neither.load_runner(), + Err(error) if error.contains("configure one of switchyard_config_path or switchyard_config") + )); + + let both = SwitchyardConfig { + priority: 0, + switchyard_config_path: Some(PathBuf::from("routes.toml")), + switchyard_config: Some(inline_deployment()), + }; + assert!(matches!( + both.load_runner(), + Err(error) if error.contains("configure exactly one of switchyard_config_path or switchyard_config") + )); + } } diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs index 9663538d2..82196b065 100644 --- a/crates/switchyard-nemo-relay-plugin/src/lib.rs +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -122,7 +122,9 @@ fn register_stream( } fn parse_config(plugin_config: &Map) -> Result { - serde_json::from_value(Json::Object(plugin_config.clone())) + let mut config = plugin_config.clone(); + config.remove("executor"); + serde_json::from_value(Json::Object(config)) .map_err(|error| format!("invalid Switchyard configuration: {error}")) } @@ -135,9 +137,20 @@ mod tests { use super::*; #[test] - fn plugin_configuration_requires_a_deployment_path() { + fn plugin_configuration_requires_a_switchyard_source() { let config = json!({"priority": 0}); - let error = parse_config(config.as_object().unwrap()).unwrap_err(); - assert!(error.contains("deployment_path")); + assert!(matches!( + parse_config(config.as_object().unwrap()).and_then(SwitchyardRuntime::new), + Err(error) if error.contains("switchyard_config_path or switchyard_config") + )); + } + + #[test] + fn plugin_configuration_ignores_the_sdk_executor_override() { + let config = json!({ + "executor": {"worker_threads": 4}, + "switchyard_config_path": "routes.toml" + }); + assert!(parse_config(config.as_object().unwrap()).is_ok()); } } From 66ab08a1634a867abf441b5675e58a3912b2cb6f Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 25 Aug 2026 14:04:13 -0700 Subject: [PATCH 16/18] feat(relay): emit Switchyard routing telemetry Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 22 + .../switchyard-nemo-relay-plugin/src/lib.rs | 8 +- .../src/runtime.rs | 443 +++++++++++++++--- 3 files changed, 394 insertions(+), 79 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index c4da8913a..cb418daee 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -76,6 +76,28 @@ The plugin emits a routing request mark, routing-model usage marks, measured routing-overhead marks, and a selected-model decision mark. Answer-call usage continues to belong to Relay's outer LLM lifecycle. +## Observability + +When Relay is configured with OTLP logs and metrics exporters, the plugin emits +typed telemetry through Relay's native plugin runtime: + +- Routing request, decision, and overhead marks are Info logs. +- Per-routing-model call marks are Debug logs, including their outcome and + normalized usage when available. +- Terminal routing and response-finalization failures are Error logs. Their + payload contains only the safe Switchyard failure summary; it excludes + provider response bodies and free-form provider messages. +- Metrics use bounded attributes only: algorithm for + `switchyard.routing.requests`; outcome for `switchyard.routing.llm_calls` + and `switchyard.routing.llm_call.duration`; + and safe failure kind, category, and phase for + `switchyard.routing.failures`. The plugin also records + `switchyard.routing.overhead` as a histogram. Durations use milliseconds. + +The plugin deliberately does not attach model IDs, sessions, requests, or +provider messages as metric attributes, preventing unbounded cardinality or +sensitive data from reaching the metrics exporter. + ## Failure policy `switchyard-llm-client` owns provider retry and route-candidate fallback diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs index 82196b065..79ade407a 100644 --- a/crates/switchyard-nemo-relay-plugin/src/lib.rs +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -14,7 +14,7 @@ use nemo_relay_plugin::{ use serde_json::Map; use crate::config::{SwitchyardConfig, protocol_from_call}; -use crate::runtime::{SwitchyardRuntime, emit_mark, emit_marks}; +use crate::runtime::{SwitchyardRuntime, emit_event, emit_events}; #[derive(Default)] struct SwitchyardPlugin; @@ -77,7 +77,7 @@ fn register_buffered( return next.call(request).await; } let execution = runtime.execute_buffered(inbound, decoded).await; - emit_marks(&plugin_runtime, execution.marks); + emit_events(&plugin_runtime, execution.events); execution.result } }, @@ -109,10 +109,10 @@ fn register_stream( .execute_stream( inbound, decoded, - Arc::new(move |mark| emit_mark(&stream_plugin_runtime, mark)), + Arc::new(move |event| emit_event(&stream_plugin_runtime, event)), ) .await; - emit_marks(&plugin_runtime, execution.marks); + emit_events(&plugin_runtime, execution.events); execution .result .map(|stream| Box::pin(stream) as LlmJsonAsyncStream) diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index eba12cb68..ba9efa8fa 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -5,7 +5,10 @@ use std::pin::Pin; use std::sync::{Arc, Mutex}; use futures_util::{Stream, StreamExt}; -use nemo_relay_plugin::{Json, LlmRequest as RelayRequest, PluginRuntime}; +use nemo_relay_plugin::{ + Json, LlmRequest as RelayRequest, LogSeverity, MetricKind, MetricMeasurement, MetricValueType, + PluginRuntime, +}; use serde_json::{Map, json}; use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver}; use switchyard_protocol::{LlmResponse, Metadata, Request, Response, WireFormat}; @@ -20,14 +23,36 @@ pub(crate) struct RoutingMark { pub(crate) name: String, pub(crate) data: Json, pub(crate) metadata: Json, + pub(crate) severity: Option, +} + +#[derive(Debug)] +pub(crate) struct RoutingMetric { + pub(crate) name: String, + pub(crate) measurements: Vec, + pub(crate) metadata: Json, +} + +#[derive(Debug)] +pub(crate) enum RoutingEvent { + Mark(RoutingMark), + Metric(RoutingMetric), +} + +struct MetricDescriptor<'a> { + name: &'a str, + kind: MetricKind, + value_type: MetricValueType, + unit: Option<&'a str>, + description: &'a str, } pub(crate) type ReturnedEventStream = Pin> + Send>>; -pub(crate) type RoutingMarkEmitter = Arc; +pub(crate) type RoutingEventEmitter = Arc; pub(crate) struct Execution { pub(crate) result: Result, - pub(crate) marks: Vec, + pub(crate) events: Vec, } pub(crate) struct SwitchyardRuntime { @@ -84,7 +109,7 @@ impl SwitchyardRuntime { inbound: WireFormat, request: Request, ) -> Execution { - let Execution { result, mut marks } = self.execute(request).await; + let Execution { result, mut events } = self.execute(request).await; let (result, finalization_failed) = match result { Ok(response) => { let result = finalize_buffered_response(&self.translation, inbound, response); @@ -94,54 +119,59 @@ impl SwitchyardRuntime { Err(error) => (Err(error), false), }; if finalization_failed { - self.error_mark(&mut marks, "response_finalization", None); + self.error_mark(&mut events, "response_finalization", None); } - Execution { result, marks } + Execution { result, events } } pub(crate) async fn execute_stream( &self, inbound: WireFormat, request: Request, - emit_mark: RoutingMarkEmitter, + emit_event: RoutingEventEmitter, ) -> Execution { - let Execution { result, mut marks } = self.execute(request).await; + let Execution { result, mut events } = self.execute(request).await; let (result, finalization_failed) = match result { Ok(response) => { - let metadata = marks - .first() - .map(|mark| mark.metadata.clone()) + let metadata = events + .iter() + .find_map(|event| match event { + RoutingEvent::Mark(mark) => Some(mark.metadata.clone()), + RoutingEvent::Metric(_) => None, + }) .unwrap_or_else(|| Json::Object(Map::new())); - let result = returned_events(response, inbound, metadata, emit_mark); + let result = returned_events(response, inbound, metadata, emit_event); let failed = result.is_err(); (result, failed) } Err(error) => (Err(error), false), }; if finalization_failed { - self.error_mark(&mut marks, "response_finalization", None); + self.error_mark(&mut events, "response_finalization", None); } - Execution { result, marks } + Execution { result, events } } async fn execute(&self, request: Request) -> Execution { let Some(route) = self.route(&request) else { return Execution { result: Err("Switchyard has no route for this request model".into()), - marks: Vec::new(), + events: Vec::new(), }; }; let metadata = identity_metadata(request.metadata.as_ref()); - let mut marks = vec![RoutingMark { + let mut events = vec![RoutingEvent::Mark(RoutingMark { name: "switchyard.routing.requested".into(), data: json!({"algorithm": route.algorithm_name()}), metadata: metadata.clone(), - }]; + severity: Some(LogSeverity::Info), + })]; + events.push(request_metric(route.algorithm_name(), metadata.clone())); if let Err(error) = route.check_caller_format(metadata_wire_format(&request)) { - self.error_mark(&mut marks, "caller_format", None); + self.error_mark(&mut events, "caller_format", None); return Execution { result: Err(format!("Switchyard caller format is incompatible: {error}")), - marks, + events, }; } let observations = Arc::new(Mutex::new(Vec::new())); @@ -154,26 +184,31 @@ impl SwitchyardRuntime { }); match route.execute(request, Some(observer)).await { Ok(output) => { - self.emit_observations(&mut marks, take_observations(&observations), &metadata); - marks.push(RoutingMark { + self.emit_observations(&mut events, take_observations(&observations), &metadata); + events.push(RoutingEvent::Mark(RoutingMark { name: "switchyard.routing.decision".into(), data: json!({ "algorithm": route.algorithm_name(), "selected_model": output.selected_model, }), metadata, - }); + severity: Some(LogSeverity::Info), + })); Execution { result: Ok(output.response), - marks, + events, } } Err(error) => { - self.emit_observations(&mut marks, take_observations(&observations), &metadata); - self.route_execution_error_mark(&mut marks, &error.execution_error_summary(), None); + self.emit_observations(&mut events, take_observations(&observations), &metadata); + self.route_execution_error_mark( + &mut events, + &error.execution_error_summary(), + None, + ); Execution { result: Err("Switchyard route execution failed".into()), - marks, + events, } } } @@ -189,7 +224,7 @@ impl SwitchyardRuntime { fn emit_observations( &self, - marks: &mut Vec, + events: &mut Vec, observations: Vec, metadata: &Json, ) { @@ -198,86 +233,102 @@ impl SwitchyardRuntime { match observation { RunObservation::LlmCall(call) => { call_index += 1; - self.routing_call_mark(marks, call, call_index, metadata); + self.routing_call_events(events, call, call_index, metadata); + } + RunObservation::RoutingOverhead(duration) => { + let latency_ms = duration.as_secs_f64() * 1_000.0; + events.push(RoutingEvent::Mark(RoutingMark { + name: "switchyard.routing.overhead".into(), + data: json!({"latency_ms": latency_ms}), + metadata: metadata.clone(), + severity: Some(LogSeverity::Info), + })); + events.push(routing_overhead_metric(latency_ms, metadata.clone())); } - RunObservation::RoutingOverhead(duration) => marks.push(RoutingMark { - name: "switchyard.routing.overhead".into(), - data: json!({"latency_ms": duration.as_secs_f64() * 1_000.0}), - metadata: metadata.clone(), - }), RunObservation::AnswerCall(_) => {} } } } - fn routing_call_mark( + fn routing_call_events( &self, - marks: &mut Vec, + events: &mut Vec, call: LlmCallObservation, call_index: usize, metadata: &Json, ) { - marks.push(RoutingMark { + let outcome = if call.is_success { "ok" } else { "error" }; + let latency_ms = call.duration.as_secs_f64() * 1_000.0; + events.push(RoutingEvent::Mark(RoutingMark { name: "switchyard.routing.llm_call".into(), data: json!({ "call_index": call_index, "selected_model": call.selected_model, "call_role": "routing", - "outcome": if call.is_success { "ok" } else { "error" }, - "latency_ms": call.duration.as_secs_f64() * 1_000.0, + "outcome": outcome, + "latency_ms": latency_ms, "usage": call.usage, }), metadata: metadata.clone(), - }); + severity: Some(LogSeverity::Debug), + })); + events.extend(routing_call_metrics(outcome, latency_ms, metadata.clone())); } fn error_mark( &self, - marks: &mut Vec, + events: &mut Vec, failure_kind: &str, metadata: Option<&Json>, ) { - let metadata = metadata.cloned().unwrap_or_else(|| { - marks - .first() - .map(|mark| mark.metadata.clone()) - .unwrap_or_else(|| Json::Object(Map::new())) - }); - marks.push(RoutingMark { + let metadata = metadata + .cloned() + .unwrap_or_else(|| event_metadata(events).unwrap_or_else(|| Json::Object(Map::new()))); + events.push(RoutingEvent::Mark(RoutingMark { name: "switchyard.routing.error".into(), data: json!({"failure_kind": failure_kind}), - metadata, - }); + metadata: metadata.clone(), + severity: Some(LogSeverity::Error), + })); + events.push(failure_metric(failure_kind, None, None, metadata)); } fn route_execution_error_mark( &self, - marks: &mut Vec, + events: &mut Vec, summary: &RouteErrorSummary, metadata: Option<&Json>, ) { - let metadata = metadata.cloned().unwrap_or_else(|| { - marks - .first() - .map(|mark| mark.metadata.clone()) - .unwrap_or_else(|| Json::Object(Map::new())) - }); - marks.push(route_execution_error_mark(summary, metadata)); + let metadata = metadata + .cloned() + .unwrap_or_else(|| event_metadata(events).unwrap_or_else(|| Json::Object(Map::new()))); + events.extend(route_execution_error_events(summary, metadata)); } } -pub(crate) fn emit_marks(runtime: &PluginRuntime, marks: Vec) { - for mark in marks { - emit_mark(runtime, mark); +pub(crate) fn emit_events(runtime: &PluginRuntime, events: Vec) { + for event in events { + emit_event(runtime, event); } } -pub(crate) fn emit_mark(runtime: &PluginRuntime, mark: RoutingMark) { - if let Err(error) = runtime.emit_mark(&mark.name, Some(&mark.data), Some(&mark.metadata)) { - eprintln!( - "Switchyard could not emit routing mark {:?}: {error}", - mark.name - ); +pub(crate) fn emit_event(runtime: &PluginRuntime, event: RoutingEvent) { + let result = match event { + RoutingEvent::Mark(mark) => runtime + .emit_mark_with_options( + &mark.name, + Some(&mark.data), + Some(&mark.metadata), + None, + mark.severity, + ) + .map_err(|error| ("routing mark", mark.name, error)), + RoutingEvent::Metric(metric) => runtime + .emit_metric(&metric.name, metric.measurements, Some(&metric.metadata)) + .map_err(|error| ("routing metric", metric.name, error)), + }; + if let Err((kind, name, error)) = result { + eprintln!("Switchyard could not emit {kind} {name:?}: {error}"); } } @@ -312,7 +363,7 @@ fn returned_events( response: Response, inbound: WireFormat, metadata: Json, - emit_mark: RoutingMarkEmitter, + emit_event: RoutingEventEmitter, ) -> Result { let served_model = response.served_model().cloned(); let chunks = match response.llm_response { @@ -321,10 +372,12 @@ fn returned_events( }; let chunks = Box::pin(chunks.map(move |item| { if let Err(error) = &item { - emit_mark(route_execution_error_mark( + for event in route_execution_error_events( &stream_error_summary(error, served_model.as_ref()), metadata.clone(), - )); + ) { + emit_event(event); + } } item })); @@ -346,9 +399,147 @@ fn route_execution_error_mark(summary: &RouteErrorSummary, metadata: Json) -> Ro "target": summary.target.as_ref().map(|target| target.as_str()), }), metadata, + severity: Some(LogSeverity::Error), } } +fn route_execution_error_events(summary: &RouteErrorSummary, metadata: Json) -> Vec { + vec![ + RoutingEvent::Mark(route_execution_error_mark(summary, metadata.clone())), + failure_metric( + "route_execution", + Some(summary.kind.as_str()), + Some(summary.phase.as_str()), + metadata, + ), + ] +} + +fn event_metadata(events: &[RoutingEvent]) -> Option { + events.iter().find_map(|event| match event { + RoutingEvent::Mark(mark) => Some(mark.metadata.clone()), + RoutingEvent::Metric(_) => None, + }) +} + +fn request_metric(algorithm: &str, metadata: Json) -> RoutingEvent { + counter_metric( + "switchyard.routing.requests", + "Requests managed by Switchyard routing.", + json!({"algorithm": algorithm}), + metadata, + ) +} + +fn routing_call_metrics(outcome: &str, latency_ms: f64, metadata: Json) -> [RoutingEvent; 2] { + let attributes = json!({"outcome": outcome}); + [ + counter_metric( + "switchyard.routing.llm_calls", + "Switchyard model calls made while routing.", + attributes.clone(), + metadata.clone(), + ), + histogram_metric( + "switchyard.routing.llm_call.duration", + "Duration of Switchyard model calls made while routing.", + latency_ms, + attributes, + metadata, + ), + ] +} + +fn routing_overhead_metric(latency_ms: f64, metadata: Json) -> RoutingEvent { + histogram_metric( + "switchyard.routing.overhead", + "Time spent by Switchyard routing outside model calls.", + latency_ms, + json!({}), + metadata, + ) +} + +fn failure_metric( + failure_kind: &str, + category: Option<&str>, + phase: Option<&str>, + metadata: Json, +) -> RoutingEvent { + let mut attributes = Map::new(); + attributes.insert("failure_kind".into(), Json::String(failure_kind.into())); + if let Some(category) = category { + attributes.insert("category".into(), Json::String(category.into())); + } + if let Some(phase) = phase { + attributes.insert("phase".into(), Json::String(phase.into())); + } + counter_metric( + "switchyard.routing.failures", + "Terminal Switchyard routing failures.", + Json::Object(attributes), + metadata, + ) +} + +fn counter_metric(name: &str, description: &str, attributes: Json, metadata: Json) -> RoutingEvent { + metric( + MetricDescriptor { + name, + kind: MetricKind::Counter, + value_type: MetricValueType::U64, + unit: Some("{event}"), + description, + }, + json!(1), + attributes, + metadata, + ) +} + +fn histogram_metric( + name: &str, + description: &str, + value: f64, + attributes: Json, + metadata: Json, +) -> RoutingEvent { + metric( + MetricDescriptor { + name, + kind: MetricKind::Histogram, + value_type: MetricValueType::F64, + unit: Some("ms"), + description, + }, + json!(value), + attributes, + metadata, + ) +} + +fn metric( + descriptor: MetricDescriptor<'_>, + value: Json, + attributes: Json, + metadata: Json, +) -> RoutingEvent { + RoutingEvent::Metric(RoutingMetric { + name: descriptor.name.into(), + measurements: vec![MetricMeasurement { + name: descriptor.name.into(), + kind: descriptor.kind, + value_type: descriptor.value_type, + value, + unit: descriptor.unit.map(Into::into), + description: Some(descriptor.description.into()), + attributes: Some(attributes), + boundaries: None, + }], + metadata, + }) +} + fn string_headers(headers: &Map) -> http::HeaderMap { let mut parsed = http::HeaderMap::with_capacity(headers.len()); for (name, value) in headers { @@ -440,9 +631,95 @@ mod tests { assert_eq!(mark.data["phase"], "before_response"); assert_eq!(mark.data["target"], "weak"); assert_eq!(mark.data["upstream_status"], Json::Null); + assert_eq!(mark.severity, Some(LogSeverity::Error)); assert!(!mark.data.to_string().contains(secret)); } + #[test] + fn routing_observations_emit_debug_marks_and_metrics() { + let runtime = runtime_for("switchyard"); + let mut events = Vec::new(); + runtime.emit_observations( + &mut events, + vec![ + RunObservation::LlmCall(LlmCallObservation { + selected_model: ModelId::from("routing-model"), + is_success: false, + duration: std::time::Duration::from_millis(12), + usage: None, + }), + RunObservation::RoutingOverhead(std::time::Duration::from_millis(3)), + ], + &json!({"session_id": "session"}), + ); + + assert_eq!(events.len(), 5); + let RoutingEvent::Mark(call_mark) = &events[0] else { + panic!("first event should be the routing call mark"); + }; + assert_eq!(call_mark.name, "switchyard.routing.llm_call"); + assert_eq!(call_mark.severity, Some(LogSeverity::Debug)); + assert_eq!(call_mark.data["outcome"], "error"); + + let RoutingEvent::Metric(call_count) = &events[1] else { + panic!("second event should be the routing call counter"); + }; + assert_eq!(call_count.name, "switchyard.routing.llm_calls"); + assert_eq!(call_count.measurements[0].kind, MetricKind::Counter); + assert_eq!( + call_count.measurements[0].attributes, + Some(json!({"outcome": "error"})) + ); + + let RoutingEvent::Metric(call_duration) = &events[2] else { + panic!("third event should be the routing call histogram"); + }; + assert_eq!(call_duration.name, "switchyard.routing.llm_call.duration"); + assert_eq!(call_duration.measurements[0].kind, MetricKind::Histogram); + assert_eq!(call_duration.measurements[0].value, json!(12.0)); + + let RoutingEvent::Mark(overhead_mark) = &events[3] else { + panic!("fourth event should be the routing overhead mark"); + }; + assert_eq!(overhead_mark.severity, Some(LogSeverity::Info)); + + let RoutingEvent::Metric(overhead) = &events[4] else { + panic!("fifth event should be the routing overhead histogram"); + }; + assert_eq!(overhead.name, "switchyard.routing.overhead"); + assert_eq!(overhead.measurements[0].attributes, Some(json!({}))); + } + + #[test] + fn request_and_failure_metrics_use_bounded_attributes() { + let RoutingEvent::Metric(request) = request_metric("stage_router", json!({})) else { + panic!("request should be a metric"); + }; + assert_eq!(request.name, "switchyard.routing.requests"); + assert_eq!( + request.measurements[0].attributes, + Some(json!({"algorithm": "stage_router"})) + ); + + let RoutingEvent::Metric(failure) = failure_metric( + "route_execution", + Some("upstream_http"), + Some("before_response"), + json!({}), + ) else { + panic!("failure should be a metric"); + }; + assert_eq!(failure.name, "switchyard.routing.failures"); + assert_eq!( + failure.measurements[0].attributes, + Some(json!({ + "failure_kind": "route_execution", + "category": "upstream_http", + "phase": "before_response", + })) + ); + } + #[tokio::test] async fn stream_failure_emits_a_safe_failure_mark() { let secret = "provider response body"; @@ -473,11 +750,27 @@ mod tests { let events = stream.collect::>().await; assert!(events[0].is_err()); - let marks = captured.lock().unwrap(); - assert_eq!(marks.len(), 1); - assert_eq!(marks[0].data["category"], "context_window_exceeded"); - assert_eq!(marks[0].data["phase"], "during_stream"); - assert_eq!(marks[0].data["target"], "strong"); - assert!(!marks[0].data.to_string().contains(secret)); + let events = captured.lock().unwrap(); + assert_eq!(events.len(), 2); + let RoutingEvent::Mark(mark) = &events[0] else { + panic!("first event should be the safe failure mark"); + }; + assert_eq!(mark.data["category"], "context_window_exceeded"); + assert_eq!(mark.data["phase"], "during_stream"); + assert_eq!(mark.data["target"], "strong"); + assert_eq!(mark.severity, Some(LogSeverity::Error)); + assert!(!mark.data.to_string().contains(secret)); + let RoutingEvent::Metric(metric) = &events[1] else { + panic!("second event should be the failure counter"); + }; + assert_eq!(metric.name, "switchyard.routing.failures"); + assert_eq!( + metric.measurements[0].attributes, + Some(json!({ + "failure_kind": "route_execution", + "category": "context_window_exceeded", + "phase": "during_stream", + })) + ); } } From 4db22fb64ef43b676772583007240396cebb22ae Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 25 Aug 2026 14:36:35 -0700 Subject: [PATCH 17/18] feat(relay): emit Switchyard token usage metrics Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 26 +-- .../src/runtime.rs | 166 ++++++++++++++++-- 2 files changed, 171 insertions(+), 21 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index cb418daee..38d6f05d7 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -72,9 +72,10 @@ deployment's route IDs. - Streaming responses are returned as unpolled translated streams; Relay owns cancellation and the outer serving-call lifecycle. -The plugin emits a routing request mark, routing-model usage marks, measured -routing-overhead marks, and a selected-model decision mark. Answer-call usage -continues to belong to Relay's outer LLM lifecycle. +The plugin emits a routing request mark, routing-model call marks, measured +routing-overhead marks, and a selected-model decision mark. Token usage is +emitted as Switchyard metrics for both routing-model and answer-model calls; +Relay retains ownership of the outer LLM lifecycle. ## Observability @@ -83,7 +84,7 @@ typed telemetry through Relay's native plugin runtime: - Routing request, decision, and overhead marks are Info logs. - Per-routing-model call marks are Debug logs, including their outcome and - normalized usage when available. + latency, but not token usage. - Terminal routing and response-finalization failures are Error logs. Their payload contains only the safe Switchyard failure summary; it excludes provider response bodies and free-form provider messages. @@ -91,12 +92,17 @@ typed telemetry through Relay's native plugin runtime: `switchyard.routing.requests`; outcome for `switchyard.routing.llm_calls` and `switchyard.routing.llm_call.duration`; and safe failure kind, category, and phase for - `switchyard.routing.failures`. The plugin also records - `switchyard.routing.overhead` as a histogram. Durations use milliseconds. - -The plugin deliberately does not attach model IDs, sessions, requests, or -provider messages as metric attributes, preventing unbounded cardinality or -sensitive data from reaching the metrics exporter. + `switchyard.routing.failures`. `switchyard.routing.overhead` records total + routing latency, including routing-model calls; durations use milliseconds. +- `switchyard.routing.llm_tokens` records normalized token usage with + `call_role` (`routing` or `answer`), configured `target_model`, and + `token_type` attributes. A provider may omit usage for streaming responses; + the plugin does not synthesize zero-value measurements. + +The plugin does not attach sessions, requests, or provider messages as metric +attributes. `target_model` comes from the configured Switchyard target set, +rather than arbitrary caller input, keeping the metric cardinality bounded by +the deployment. ## Failure policy diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index ba9efa8fa..c8cd18fc5 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -245,7 +245,9 @@ impl SwitchyardRuntime { })); events.push(routing_overhead_metric(latency_ms, metadata.clone())); } - RunObservation::AnswerCall(_) => {} + RunObservation::AnswerCall(call) => { + events.extend(token_usage_metrics("answer", &call, metadata)); + } } } } @@ -259,20 +261,21 @@ impl SwitchyardRuntime { ) { let outcome = if call.is_success { "ok" } else { "error" }; let latency_ms = call.duration.as_secs_f64() * 1_000.0; + let token_metrics = token_usage_metrics("routing", &call, metadata); events.push(RoutingEvent::Mark(RoutingMark { name: "switchyard.routing.llm_call".into(), data: json!({ "call_index": call_index, - "selected_model": call.selected_model, + "selected_model": call.selected_model.as_str(), "call_role": "routing", "outcome": outcome, "latency_ms": latency_ms, - "usage": call.usage, }), metadata: metadata.clone(), severity: Some(LogSeverity::Debug), })); events.extend(routing_call_metrics(outcome, latency_ms, metadata.clone())); + events.extend(token_metrics); } fn error_mark( @@ -453,13 +456,53 @@ fn routing_call_metrics(outcome: &str, latency_ms: f64, metadata: Json) -> [Rout fn routing_overhead_metric(latency_ms: f64, metadata: Json) -> RoutingEvent { histogram_metric( "switchyard.routing.overhead", - "Time spent by Switchyard routing outside model calls.", + "Time needed to produce the Switchyard routing outcome, including routing model calls.", latency_ms, json!({}), metadata, ) } +fn token_usage_metrics( + call_role: &str, + call: &LlmCallObservation, + metadata: &Json, +) -> Vec { + let Some(usage) = call.usage.as_ref() else { + return Vec::new(); + }; + [ + ("input", usage.input_tokens), + ("cached_input", usage.cached_input_tokens()), + ("cache_creation_input", usage.cache_creation_input_tokens()), + ("output", usage.output_tokens), + ("reasoning", usage.reasoning_tokens), + ("total", usage.total_tokens), + ] + .into_iter() + .filter_map(|(token_type, value)| { + value.map(|value| { + metric( + MetricDescriptor { + name: "switchyard.routing.llm_tokens", + kind: MetricKind::Counter, + value_type: MetricValueType::U64, + unit: Some("{token}"), + description: "Normalized tokens used by Switchyard model calls.", + }, + json!(value), + json!({ + "call_role": call_role, + "target_model": call.selected_model.as_str(), + "token_type": token_type, + }), + metadata.clone(), + ) + }) + }) + .collect() +} + fn failure_metric( failure_kind: &str, category: Option<&str>, @@ -573,7 +616,9 @@ mod tests { use std::collections::{BTreeMap, HashMap}; use switchyard_llm_client::ClientRouter; - use switchyard_protocol::{LlmClientError, LlmResponseStreamEvent, ModelId, text_request}; + use switchyard_protocol::{ + LlmClientError, LlmResponseStreamEvent, ModelId, Usage, text_request, + }; use switchyard_runner::{AlgorithmSpec, ModelCapabilities, RunnerError}; use super::*; @@ -646,20 +691,24 @@ mod tests { selected_model: ModelId::from("routing-model"), is_success: false, duration: std::time::Duration::from_millis(12), - usage: None, + usage: Some(Usage { + input_tokens: Some(4), + ..Usage::default() + }), }), RunObservation::RoutingOverhead(std::time::Duration::from_millis(3)), ], &json!({"session_id": "session"}), ); - assert_eq!(events.len(), 5); + assert_eq!(events.len(), 6); let RoutingEvent::Mark(call_mark) = &events[0] else { panic!("first event should be the routing call mark"); }; assert_eq!(call_mark.name, "switchyard.routing.llm_call"); assert_eq!(call_mark.severity, Some(LogSeverity::Debug)); assert_eq!(call_mark.data["outcome"], "error"); + assert!(call_mark.data.get("usage").is_none()); let RoutingEvent::Metric(call_count) = &events[1] else { panic!("second event should be the routing call counter"); @@ -678,13 +727,18 @@ mod tests { assert_eq!(call_duration.measurements[0].kind, MetricKind::Histogram); assert_eq!(call_duration.measurements[0].value, json!(12.0)); - let RoutingEvent::Mark(overhead_mark) = &events[3] else { - panic!("fourth event should be the routing overhead mark"); + let RoutingEvent::Metric(tokens) = &events[3] else { + panic!("fourth event should be the routing token counter"); + }; + assert_eq!(tokens.name, "switchyard.routing.llm_tokens"); + + let RoutingEvent::Mark(overhead_mark) = &events[4] else { + panic!("fifth event should be the routing overhead mark"); }; assert_eq!(overhead_mark.severity, Some(LogSeverity::Info)); - let RoutingEvent::Metric(overhead) = &events[4] else { - panic!("fifth event should be the routing overhead histogram"); + let RoutingEvent::Metric(overhead) = &events[5] else { + panic!("sixth event should be the routing overhead histogram"); }; assert_eq!(overhead.name, "switchyard.routing.overhead"); assert_eq!(overhead.measurements[0].attributes, Some(json!({}))); @@ -720,6 +774,96 @@ mod tests { ); } + #[test] + fn token_usage_metrics_distinguish_routing_and_answer_targets() { + let call = LlmCallObservation { + selected_model: ModelId::from("judge-model"), + is_success: true, + duration: std::time::Duration::from_millis(1), + usage: Some(Usage { + input_tokens: Some(11), + cache: Usage::cache_details(Some(3), Some(2)), + output_tokens: Some(7), + total_tokens: Some(23), + reasoning_tokens: Some(5), + }), + }; + + let routing = token_usage_metrics("routing", &call, &json!({"session_id": "session"})); + assert_eq!(routing.len(), 6); + for event in &routing { + let RoutingEvent::Metric(metric) = event else { + panic!("token usage should be emitted as a metric"); + }; + assert_eq!(metric.name, "switchyard.routing.llm_tokens"); + assert_eq!(metric.measurements[0].kind, MetricKind::Counter); + assert_eq!(metric.measurements[0].unit.as_deref(), Some("{token}")); + assert_eq!( + metric.measurements[0].attributes.as_ref().unwrap()["call_role"], + "routing" + ); + assert_eq!( + metric.measurements[0].attributes.as_ref().unwrap()["target_model"], + "judge-model" + ); + } + let token_values = routing + .iter() + .map(|event| { + let RoutingEvent::Metric(metric) = event else { + panic!("token usage should be emitted as a metric"); + }; + metric.measurements[0].value.clone() + }) + .collect::>(); + assert_eq!( + token_values, + vec![json!(11), json!(3), json!(2), json!(7), json!(5), json!(23)] + ); + + let answer = token_usage_metrics("answer", &call, &json!({})); + let RoutingEvent::Metric(metric) = &answer[0] else { + panic!("answer usage should be emitted as a metric"); + }; + assert_eq!( + metric.measurements[0].attributes.as_ref().unwrap()["call_role"], + "answer" + ); + } + + #[test] + fn answer_observations_emit_token_metrics_without_answer_logs() { + let runtime = runtime_for("switchyard"); + let mut events = Vec::new(); + runtime.emit_observations( + &mut events, + vec![RunObservation::AnswerCall(LlmCallObservation { + selected_model: ModelId::from("selected-target"), + is_success: true, + duration: std::time::Duration::from_millis(2), + usage: Some(Usage { + output_tokens: Some(9), + ..Usage::default() + }), + })], + &json!({}), + ); + + assert_eq!(events.len(), 1); + let RoutingEvent::Metric(metric) = &events[0] else { + panic!("answer observation should only emit a token metric"); + }; + assert_eq!(metric.name, "switchyard.routing.llm_tokens"); + assert_eq!( + metric.measurements[0].attributes, + Some(json!({ + "call_role": "answer", + "target_model": "selected-target", + "token_type": "output", + })) + ); + } + #[tokio::test] async fn stream_failure_emits_a_safe_failure_mark() { let secret = "provider response body"; From ee84cf62f0b5efae5fc1537278cd8e16768c89ff Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 25 Aug 2026 15:19:14 -0700 Subject: [PATCH 18/18] fix(relay): retain HTTP status in failure metrics Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 8 +++++++- crates/switchyard-nemo-relay-plugin/src/runtime.rs | 9 ++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 38d6f05d7..aa9d465ce 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -72,6 +72,12 @@ deployment's route IDs. - Streaming responses are returned as unpolled translated streams; Relay owns cancellation and the outer serving-call lifecycle. +Each route's target client must use the caller's wire format: `openai_chat`, +`openai_responses`, or `anthropic_messages`. The runner selects the upstream +backend from that format rather than translating a route to a different +provider API. When one upstream model must serve multiple caller formats, +declare a target and route for each corresponding client format. + The plugin emits a routing request mark, routing-model call marks, measured routing-overhead marks, and a selected-model decision mark. Token usage is emitted as Switchyard metrics for both routing-model and answer-model calls; @@ -91,7 +97,7 @@ typed telemetry through Relay's native plugin runtime: - Metrics use bounded attributes only: algorithm for `switchyard.routing.requests`; outcome for `switchyard.routing.llm_calls` and `switchyard.routing.llm_call.duration`; - and safe failure kind, category, and phase for + and safe failure kind, category, phase, and optional upstream HTTP status for `switchyard.routing.failures`. `switchyard.routing.overhead` records total routing latency, including routing-model calls; durations use milliseconds. - `switchyard.routing.llm_tokens` records normalized token usage with diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index c8cd18fc5..811ef7811 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -293,7 +293,7 @@ impl SwitchyardRuntime { metadata: metadata.clone(), severity: Some(LogSeverity::Error), })); - events.push(failure_metric(failure_kind, None, None, metadata)); + events.push(failure_metric(failure_kind, None, None, None, metadata)); } fn route_execution_error_mark( @@ -413,6 +413,7 @@ fn route_execution_error_events(summary: &RouteErrorSummary, metadata: Json) -> "route_execution", Some(summary.kind.as_str()), Some(summary.phase.as_str()), + summary.upstream_status, metadata, ), ] @@ -507,6 +508,7 @@ fn failure_metric( failure_kind: &str, category: Option<&str>, phase: Option<&str>, + upstream_status: Option, metadata: Json, ) -> RoutingEvent { let mut attributes = Map::new(); @@ -517,6 +519,9 @@ fn failure_metric( if let Some(phase) = phase { attributes.insert("phase".into(), Json::String(phase.into())); } + if let Some(upstream_status) = upstream_status { + attributes.insert("upstream_status".into(), Json::from(upstream_status)); + } counter_metric( "switchyard.routing.failures", "Terminal Switchyard routing failures.", @@ -759,6 +764,7 @@ mod tests { "route_execution", Some("upstream_http"), Some("before_response"), + Some(503), json!({}), ) else { panic!("failure should be a metric"); @@ -770,6 +776,7 @@ mod tests { "failure_kind": "route_execution", "category": "upstream_http", "phase": "before_response", + "upstream_status": 503, })) ); }