From 4c64a916479c38a7d7a09be60dfcf624b5b3ccaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20Hennh=C3=B6fer?= Date: Mon, 3 Aug 2026 09:38:43 +0200 Subject: [PATCH] fix: resolve runtime evidence from canonical agent paths --- README.md | 26 +- .../scripts/inspect-agent-runtime.sh | 300 ++++++++++++++++-- plugins/sol-advisor/scripts/verify.sh | 198 +++++++++++- .../sol-advisor/skills/orchestration/SKILL.md | 13 +- .../references/role-contracts.md | 6 + 5 files changed, 502 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index f963625..21c5ff6 100644 --- a/README.md +++ b/README.md @@ -114,18 +114,32 @@ thread_id="" sh "$plugin_dir/scripts/inspect-agent-runtime.sh" "$thread_id" ~~~ +For native `spawn_agent` routing, capture a UTC cutoff immediately before the spawn, +then retain the canonical `/root/` path that native spawn returns. This avoids +mistaking an earlier rollout that reused the same task path for the new agent: + +~~~sh +runtime_since="$(date -u +%Y-%m-%dT%H:%M:%SZ)" # immediately before native spawn_agent +agent_path="/root/" +sh "$plugin_dir/scripts/inspect-agent-runtime.sh" --agent-path "$agent_path" --since "$runtime_since" +~~~ + +The positional lowercase UUID interface remains supported for compatibility. + For a disposable fixture or a non-default local session root, pass it explicitly: ~~~sh sh "$plugin_dir/scripts/inspect-agent-runtime.sh" --sessions-dir /absolute/path/to/sessions "$thread_id" ~~~ -The helper searches only rollout filenames ending in that exact thread id, then emits a -single compact JSON object with allowlisted routing fields. It never prints prompts, -messages, environment variables, tokens, configuration contents, or arbitrary rollout -payloads. It refuses invalid ids, zero or multiple matches, and missing or inconsistent -role/model/effort; there is no inferred fallback. If public and local evidence both -exist, they must agree. +The UUID mode searches only rollout filenames ending in that exact thread id. Path mode +uses only exact session-metadata `agent_path` records at or after its inclusive cutoff, +resolves their UUID, and then applies the same strict filename and routing validation. +Both modes emit one compact JSON object with allowlisted routing fields. The helper +never prints prompts, messages, environment variables, tokens, configuration contents, +or arbitrary rollout payloads. It refuses invalid ids, no or ambiguous matches, and +missing or inconsistent role/model/effort; there is no inferred fallback. If public and +local evidence both exist, they must agree. ## How routing works diff --git a/plugins/sol-advisor/scripts/inspect-agent-runtime.sh b/plugins/sol-advisor/scripts/inspect-agent-runtime.sh index 49496fb..8a8f57b 100644 --- a/plugins/sol-advisor/scripts/inspect-agent-runtime.sh +++ b/plugins/sol-advisor/scripts/inspect-agent-runtime.sh @@ -6,10 +6,12 @@ set -eu usage() { cat <<'EOF' Usage: inspect-agent-runtime.sh [--sessions-dir DIR] THREAD_ID + inspect-agent-runtime.sh [--sessions-dir DIR] --agent-path PATH --since RFC3339 -Read the one rollout file whose filename ends with THREAD_ID and emit a compact JSON -object containing only safe routing metadata. Without --sessions-dir, the sessions -root is "$CODEX_HOME/sessions" when CODEX_HOME is already set, otherwise +Read one exact rollout selected by its lowercase UUID, or resolve one canonical native +agent path created at or after an RFC3339 cutoff. Emit a compact JSON object containing +only safe routing metadata. Without --sessions-dir, the sessions root is +"$CODEX_HOME/sessions" when CODEX_HOME is already set, otherwise "$HOME/.codex/sessions". EOF } @@ -19,28 +21,170 @@ fail() { exit 1 } +usage_fail() { + usage >&2 + exit 2 +} + +# jq represents an instant as integer UTC base seconds, a leap phase, and a +# trailing-zero-trimmed fractional string to avoid epoch floating-point rounding. +timestamp_jq=' +def leap_year($year): + (($year % 4 == 0) and (($year % 100 != 0) or ($year % 400 == 0))); + +def days_in_month($year; $month): + if $month == 2 then (if leap_year($year) then 29 else 28 end) + elif ($month == 4 or $month == 6 or $month == 9 or $month == 11) then 30 + else 31 + end; + +# Days since 0000-03-01 in the proleptic Gregorian calendar. Integer arithmetic only. +def days_from_civil($year; $month; $day): + ($year - (if $month <= 2 then 1 else 0 end)) as $y | + ($y / 400 | floor) as $era | + ($y - ($era * 400)) as $yoe | + ($month + (if $month > 2 then -3 else 9 end)) as $mp | + (((153 * $mp + 2) / 5 | floor) + $day - 1) as $doy | + ($yoe * 365 + ($yoe / 4 | floor) - ($yoe / 100 | floor) + $doy) as $doe | + ($era * 146097 + $doe); + +def civil_from_days($days): + ($days / 146097 | floor) as $era | + ($days - ($era * 146097)) as $doe | + (($doe - ($doe / 1460 | floor) + ($doe / 36524 | floor) - ($doe / 146096 | floor)) / 365 | floor) as $yoe | + ($yoe + ($era * 400)) as $year | + ($doe - ($yoe * 365) - ($yoe / 4 | floor) + ($yoe / 100 | floor)) as $doy | + ((5 * $doy + 2) / 153 | floor) as $month_prime | + ($doy - ((153 * $month_prime + 2) / 5 | floor) + 1) as $day | + ($month_prime + (if $month_prime < 10 then 3 else -9 end)) as $month | + {year: ($year + (if $month <= 2 then 1 else 0 end)), month: $month, day: $day}; + +def padded($width): + tostring as $text | ($width - ($text | length)) as $padding | + if $padding > 0 then ("0" * $padding) + $text else $text end; + +def rfc3339_instant($value): + ($value | capture("^(?[0-9]{4})-(?[0-9]{2})-(?[0-9]{2})[Tt](?[0-9]{2}):(?[0-9]{2}):(?[0-9]{2})(?\\.[0-9]+)?(?[Zz]|[+-][0-9]{2}:[0-9]{2})$")) as $parts | + ($parts.year | tonumber) as $year | + ($parts.month | tonumber) as $month | + ($parts.day | tonumber) as $day | + ($parts.hour | tonumber) as $hour | + ($parts.minute | tonumber) as $minute | + ($parts.second | tonumber) as $second | + if $month < 1 or $month > 12 then error("invalid month") + elif $day < 1 or $day > days_in_month($year; $month) then error("invalid day") + elif $hour > 23 or $minute > 59 or $second > 60 then error("invalid time") + else + ($parts.zone) as $zone | + (if $zone == "Z" or $zone == "z" then 0 + elif $zone == "-00:00" then error("unknown offset") + else + ($zone | capture("^(?[+-])(?[0-9]{2}):(?[0-9]{2})$")) as $offset | + ($offset.hour | tonumber) as $offset_hour | + ($offset.minute | tonumber) as $offset_minute | + if $offset_hour > 23 or $offset_minute > 59 then error("invalid offset") + else + (($offset_hour * 3600 + $offset_minute * 60) * + (if $offset.sign == "+" then 1 else -1 end)) + end + end) as $offset_seconds | + ((days_from_civil($year; $month; $day) * 86400) + ($hour * 3600) + + ($minute * 60) + (if $second == 60 then 59 else $second end) - $offset_seconds) as $seconds | + (if $second == 60 and ($seconds - (($seconds / 86400 | floor) * 86400)) != 86399 then + error("invalid leap second") + else + {seconds: $seconds, leap_phase: (if $second == 60 then 1 else 0 end), + fraction: (($parts.fraction // "") | sub("^\\."; "") | sub("0+$"; ""))} + end) + end; + +def timestamp_gte($candidate; $cutoff): + if $candidate.seconds > $cutoff.seconds then true + elif $candidate.seconds < $cutoff.seconds then false + elif $candidate.leap_phase > $cutoff.leap_phase then true + elif $candidate.leap_phase < $cutoff.leap_phase then false + else + ([$candidate.fraction | length, $cutoff.fraction | length] | max) as $width | + ($candidate.fraction + ("0" * ($width - ($candidate.fraction | length)))) as $candidate_fraction | + ($cutoff.fraction + ("0" * ($width - ($cutoff.fraction | length)))) as $cutoff_fraction | + $candidate_fraction >= $cutoff_fraction + end; + +# Session directories are calendar-day partitions. Include the day before the UTC +# cutoff day because session metadata can carry any known RFC3339 offset. +def scan_from_date($value): + rfc3339_instant($value) as $instant | + ((($instant.seconds / 86400) | floor) - 1) as $day_number | + civil_from_days($day_number) as $civil | + if $civil.year < 0 or $civil.year > 9999 then null + else "\($civil.year | padded(4))/\($civil.month | padded(2))/\($civil.day | padded(2))" + end; +' + sessions_dir='' -case "$#" in - 1) - thread_id=$1 - ;; - 3) - [ "$1" = "--sessions-dir" ] || { - usage >&2 - exit 2 - } - [ -n "$2" ] || fail "--sessions-dir requires a non-empty directory." - sessions_dir=$2 - thread_id=$3 - ;; - *) - usage >&2 - exit 2 - ;; -esac +thread_id='' +agent_path='' +since='' +seen_sessions_dir=0 +seen_agent_path=0 +seen_since=0 + +while [ "$#" -gt 0 ]; do + case "$1" in + --sessions-dir) + [ "$seen_sessions_dir" -eq 0 ] || usage_fail + [ "$#" -ge 2 ] || usage_fail + sessions_dir=$2 + seen_sessions_dir=1 + shift 2 + ;; + --agent-path) + [ "$seen_agent_path" -eq 0 ] || usage_fail + [ "$#" -ge 2 ] || usage_fail + agent_path=$2 + seen_agent_path=1 + shift 2 + ;; + --since) + [ "$seen_since" -eq 0 ] || usage_fail + [ "$#" -ge 2 ] || usage_fail + since=$2 + seen_since=1 + shift 2 + ;; + --*) usage_fail ;; + *) + [ -z "$thread_id" ] || usage_fail + thread_id=$1 + shift + ;; + esac +done + +if [ "$seen_agent_path" -eq 1 ]; then + [ -z "$thread_id" ] || usage_fail + [ "$seen_since" -eq 1 ] || usage_fail +elif [ "$seen_since" -eq 1 ]; then + usage_fail +elif [ -z "$thread_id" ]; then + usage_fail +fi + +if [ "$seen_sessions_dir" -eq 1 ] && [ -z "$sessions_dir" ]; then + fail "--sessions-dir requires a non-empty directory." +fi -if ! printf '%s\n' "$thread_id" | LC_ALL=C grep -Eq '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; then - fail "THREAD_ID must be a lowercase UUID." +if [ "$seen_agent_path" -eq 1 ]; then + if ! printf '%s\n' "$agent_path" | LC_ALL=C grep -Eq '^/root/[a-z0-9_]+(/[a-z0-9_]+)*$'; then + fail "--agent-path must be a canonical non-empty /root/ path." + fi + if ! jq -en --arg timestamp "$since" "$timestamp_jq rfc3339_instant(\$timestamp)" >/dev/null 2>&1; then + fail "--since must be a valid known-offset RFC3339 timestamp." + fi +else + if ! printf '%s\n' "$thread_id" | LC_ALL=C grep -Eq '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; then + fail "THREAD_ID must be a lowercase UUID." + fi fi if [ -z "$sessions_dir" ]; then @@ -60,24 +204,111 @@ case "$tmp_base" in *) tmp_base=/tmp ;; esac matches_file='' +candidates_file='' +directories_file='' cleanup() { - if [ -n "$matches_file" ] && [ -f "$matches_file" ]; then - case "$matches_file" in - "$tmp_base"/sol-advisor-runtime.*) - rm -f "$matches_file" - ;; - *) - printf '%s\n' "ERROR: refusing cleanup of unexpected temporary file." >&2 - ;; - esac - fi + for temporary_file in "$matches_file" "$candidates_file" "$directories_file"; do + if [ -n "$temporary_file" ] && [ -f "$temporary_file" ]; then + case "$temporary_file" in + "$tmp_base"/sol-advisor-runtime.*) + rm -f "$temporary_file" + ;; + *) + printf '%s\n' "ERROR: refusing cleanup of unexpected temporary file." >&2 + ;; + esac + fi + done } trap cleanup 0 HUP INT TERM +select_scan_directories() { + cutoff_directory=$1 + if [ -z "$cutoff_directory" ]; then + printf '%s\n' "$sessions_dir" > "$directories_file" + return + fi + if ! find "$sessions_dir" -type d -mindepth 3 -maxdepth 3 -print > "$matches_file"; then + return 1 + fi + canonical_count=$(awk -v root="$sessions_dir" ' + BEGIN { prefix = root "/"; count = 0 } + index($0, prefix) == 1 { + relative = substr($0, length(prefix) + 1) + if (relative ~ /^[0-9][0-9][0-9][0-9]\/[0-9][0-9]\/[0-9][0-9]$/) count++ + } + END { print count + 0 } + ' "$matches_file") + if [ "$canonical_count" -eq 0 ]; then + printf '%s\n' "$sessions_dir" > "$directories_file" + return + fi + awk -v root="$sessions_dir" -v cutoff="$cutoff_directory" ' + BEGIN { prefix = root "/" } + index($0, prefix) == 1 { + relative = substr($0, length(prefix) + 1) + if (relative ~ /^[0-9][0-9][0-9][0-9]\/[0-9][0-9]\/[0-9][0-9]$/ && relative >= cutoff) print + } + ' "$matches_file" > "$directories_file" +} + matches_file=$(mktemp "$tmp_base/sol-advisor-runtime.XXXXXX") || fail "could not create a temporary match list." +if [ "$seen_agent_path" -eq 1 ]; then + # Search only session metadata records. A candidate has the exact returned native + # task path and an inclusive timestamp at or after the pre-spawn cutoff. + candidates_file=$(mktemp "$tmp_base/sol-advisor-runtime.XXXXXX") || fail "could not create a temporary candidate list." + directories_file=$(mktemp "$tmp_base/sol-advisor-runtime.XXXXXX") || fail "could not create a temporary directory list." + scan_from=$(jq -nr --arg timestamp "$since" "$timestamp_jq scan_from_date(\$timestamp)" 2>/dev/null || true) + case "$scan_from" in + [0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]) ;; + *) scan_from='' ;; + esac + select_scan_directories "$scan_from" || fail "could not enumerate session date directories." + if ! while IFS= read -r scan_directory; do + # find passes filenames directly to bounded streaming jq invocations; no xargs, + # filename parsing, or rollout slurping is involved. + find "$scan_directory" -type f -name 'rollout-*.jsonl' -exec jq -cr --arg expected_path "$agent_path" --arg since "$since" "$timestamp_jq + rfc3339_instant(\$since) as \$cutoff | + if type != \"object\" then error(\"malformed JSONL record\") + elif .type != \"session_meta\" then empty + elif ((.payload? | type) != \"object\") then error(\"malformed session metadata\") + elif .payload.agent_path? != \$expected_path then empty + elif ((.payload.id? | type) != \"string\") or (.payload.id == \"\") then error(\"missing session id\") + elif ((.timestamp? | type) != \"string\") or (.timestamp == \"\") then error(\"missing session timestamp\") + else + rfc3339_instant(.timestamp) as \$candidate | + if timestamp_gte(\$candidate; \$cutoff) then + {input_filename: input_filename, thread_id: .payload.id} + else empty end + end" {} + 2>/dev/null >> "$candidates_file" || exit 1 + done < "$directories_file"; then + fail "session metadata is malformed or could not be read." + fi + + match_count=$(awk 'END { print NR + 0 }' "$candidates_file") + case "$match_count" in + 0) fail "no session metadata matched the requested agent path and cutoff." ;; + 1) ;; + *) fail "multiple session metadata records matched the requested agent path and cutoff." ;; + esac + if ! thread_id=$(jq -er ' + if type == "object" and (.input_filename | type) == "string" and .input_filename != "" + and (.thread_id | type) == "string" and .thread_id != "" + then .thread_id else error("invalid candidate") end + ' "$candidates_file" 2>/dev/null); then + fail "session metadata candidate is invalid." + fi + if ! candidate_rollout=$(jq -er '.input_filename' "$candidates_file" 2>/dev/null); then + fail "session metadata candidate is invalid." + fi + if ! printf '%s\n' "$thread_id" | LC_ALL=C grep -Eq '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; then + fail "session metadata contains an invalid thread id." + fi +fi + # Match only the exact rollout filename suffix; do not inspect any rollout contents # until exactly one filename has been found. if ! find "$sessions_dir" -type f -name "rollout-*-$thread_id.jsonl" -print > "$matches_file"; then @@ -93,6 +324,9 @@ esac IFS= read -r rollout_file < "$matches_file" || fail "could not read the matched rollout filename." [ -f "$rollout_file" ] || fail "matched rollout is unavailable." +if [ "$seen_agent_path" -eq 1 ] && [ "$rollout_file" != "$candidate_rollout" ]; then + fail "session metadata does not bind to the matched rollout filename." +fi # The jq program reads only the matched JSONL and constructs a new allowlisted object. # It rejects absent or conflicting required routing values instead of inferring them. diff --git a/plugins/sol-advisor/scripts/verify.sh b/plugins/sol-advisor/scripts/verify.sh index 8654167..0d7a2aa 100755 --- a/plugins/sol-advisor/scripts/verify.sh +++ b/plugins/sol-advisor/scripts/verify.sh @@ -227,7 +227,7 @@ runtime_id=11111111-1111-7111-8111-111111111111 runtime_rollout=$runtime_day/rollout-2026-08-02T00-00-00-$runtime_id.jsonl printf '%s\n' \ '{"type":"response_item","payload":{"prompt":"DO_NOT_LEAK_PROMPT"}}' \ - "{\"type\":\"session_meta\",\"payload\":{\"id\":\"$runtime_id\",\"parent_thread_id\":\"00000000-0000-7000-8000-000000000000\",\"agent_role\":\"sol_advisor_terra_implementer\",\"agent_path\":\"/root/fixture\",\"model_provider\":\"openai\",\"cwd\":\"/fixture\"}}" \ + "{\"type\":\"session_meta\",\"timestamp\":\"2026-08-02T00:00:00Z\",\"payload\":{\"id\":\"$runtime_id\",\"parent_thread_id\":\"00000000-0000-7000-8000-000000000000\",\"agent_role\":\"sol_advisor_terra_implementer\",\"agent_path\":\"/root/fixture\",\"model_provider\":\"openai\",\"cwd\":\"/fixture\"}}" \ '{"type":"turn_context","payload":{"model":"gpt-5.6-terra","effort":"high","sandbox_policy":{"type":"danger-full-access"},"permission_profile":{"type":"disabled"},"cwd":"/fixture"}}' \ > "$runtime_rollout" runtime_output=$(sh "$runtime_inspector" --sessions-dir "$runtime_sessions" "$runtime_id") @@ -243,6 +243,202 @@ zero_id=22222222-2222-7222-8222-222222222222 if sh "$runtime_inspector" --sessions-dir "$runtime_sessions" "$zero_id" >/dev/null 2>&1; then fail "runtime inspector accepted zero matches"; fi pass "runtime inspector Terra/High routing and safe refusal" +write_runtime_rollout() { + target_dir=$1 target_id=$2 target_path=$3 target_timestamp=$4 + mkdir -p "$target_dir" + printf '%s\n' \ + "{\"type\":\"session_meta\",\"timestamp\":\"$target_timestamp\",\"payload\":{\"id\":\"$target_id\",\"parent_thread_id\":\"00000000-0000-7000-8000-000000000000\",\"agent_role\":\"sol_advisor_terra_implementer\",\"agent_path\":\"$target_path\",\"model_provider\":\"openai\",\"cwd\":\"/fixture\"}}" \ + '{"type":"turn_context","payload":{"model":"gpt-5.6-terra","effort":"high","sandbox_policy":{"type":"danger-full-access"},"permission_profile":{"type":"disabled"},"cwd":"/fixture"}}' \ + > "$target_dir/rollout-fixture-$target_id.jsonl" +} + +expect_exit() { + expected_status=$1 + shift + if "$@" >/dev/null 2>&1; then actual_status=0; else actual_status=$?; fi + [ "$actual_status" -eq "$expected_status" ] || fail "expected exit $expected_status, got $actual_status: $*" +} + +path_sessions=$tmp_dir/path-sessions +path_id=33333333-3333-7333-8333-333333333333 +write_runtime_rollout "$path_sessions/2026/08/03" "$path_id" /root/fixture 2026-08-02T00:00:00Z +path_output=$(sh "$runtime_inspector" --since 2026-08-02T00:00:00Z --agent-path /root/fixture --sessions-dir "$path_sessions") +uuid_output=$(sh "$runtime_inspector" --sessions-dir "$path_sessions" "$path_id") +[ "$path_output" = "$uuid_output" ] || fail "path-mode output differs from UUID compatibility output" +pass "runtime inspector UUID compatibility, unique path, and cutoff equality" + +reused_sessions=$tmp_dir/reused-path-sessions +old_id=44444444-4444-7444-8444-444444444444 +new_id=55555555-5555-7555-8555-555555555555 +write_runtime_rollout "$reused_sessions/old" "$old_id" /root/reused 2026-08-03T11:59:59Z +write_runtime_rollout "$reused_sessions/new" "$new_id" /root/reused 2026-08-03T12:00:00Z +reused_output=$(sh "$runtime_inspector" --sessions-dir "$reused_sessions" --agent-path /root/reused --since 2026-08-03T12:00:00Z) +printf '%s\n' "$reused_output" | jq -e --arg id "$new_id" '.thread_id == $id' >/dev/null || fail "cutoff did not filter reused agent path" +pass "runtime inspector reused path filtering" + +offset_sessions=$tmp_dir/offset-sessions +utc_id=66666666-6666-7666-8666-666666666666 +fraction_id=77777777-7777-7777-8777-777777777777 +positive_id=88888888-8888-7888-8888-888888888888 +negative_id=99999999-9999-7999-8999-999999999999 +write_runtime_rollout "$offset_sessions/utc" "$utc_id" /root/utc 2026-08-03T12:00:00Z +write_runtime_rollout "$offset_sessions/fraction" "$fraction_id" /root/fraction 2026-08-03T12:00:00.000000000Z +write_runtime_rollout "$offset_sessions/positive" "$positive_id" /root/positive 2026-08-03T13:00:00+01:00 +write_runtime_rollout "$offset_sessions/negative" "$negative_id" /root/negative 2026-08-03T07:00:00-05:00 +for path_id in "/root/utc:$utc_id" "/root/fraction:$fraction_id" "/root/positive:$positive_id" "/root/negative:$negative_id"; do + runtime_path=${path_id%%:*} expected_id=${path_id#*:} + output=$(sh "$runtime_inspector" --sessions-dir "$offset_sessions" --agent-path "$runtime_path" --since 2026-08-03T12:00:00Z) + printf '%s\n' "$output" | jq -e --arg id "$expected_id" '.thread_id == $id' >/dev/null || fail "equivalent RFC3339 instant did not match: $runtime_path" +done +pass "runtime inspector UTC fractional and signed-offset equivalence" + +leap_utc_id=abababab-abab-7bab-8bab-abababababab +leap_offset_id=acacacac-acac-7cac-8cac-acacacacacac +leap_invalid_id=adadadad-adad-7dad-8dad-adadadadadad +write_runtime_rollout "$offset_sessions/leap-utc" "$leap_utc_id" /root/leap_utc 2026-08-03T23:59:60Z +write_runtime_rollout "$offset_sessions/leap-offset" "$leap_offset_id" /root/leap_offset 2026-08-04T00:29:60+00:30 +write_runtime_rollout "$offset_sessions/leap-invalid" "$leap_invalid_id" /root/leap_invalid 2026-08-03T12:59:60Z +leap_utc_output=$(sh "$runtime_inspector" --sessions-dir "$offset_sessions" --agent-path /root/leap_utc --since 2026-08-03T23:59:59.999Z) +printf '%s\n' "$leap_utc_output" | jq -e --arg id "$leap_utc_id" '.thread_id == $id' >/dev/null || fail "UTC leap did not order after :59" +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$offset_sessions" --agent-path /root/leap_utc --since 2026-08-04T00:00:00Z +leap_offset_output=$(sh "$runtime_inspector" --sessions-dir "$offset_sessions" --agent-path /root/leap_offset --since 2026-08-03T23:59:60Z) +printf '%s\n' "$leap_offset_output" | jq -e --arg id "$leap_offset_id" '.thread_id == $id' >/dev/null || fail "offset-equivalent leap did not match" +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$offset_sessions" --agent-path /root/leap_invalid --since 2026-08-03T12:00:00Z +pass "runtime inspector UTC and offset-equivalent leap ordering" + +fraction_order_sessions=$tmp_dir/fraction-order-sessions +fraction_before_id=16161616-1616-7616-8616-161616161616 +fraction_after_id=17171717-1717-7717-8717-171717171717 +fraction_equal_id=18181818-1818-7818-8818-181818181818 +leap_day_id=19191919-1919-7919-8919-191919191919 +write_runtime_rollout "$fraction_order_sessions/before" "$fraction_before_id" /root/fraction_order 2026-08-03T12:00:00.0999Z +write_runtime_rollout "$fraction_order_sessions/after" "$fraction_after_id" /root/fraction_order 2026-08-03t12:00:00.1001z +write_runtime_rollout "$fraction_order_sessions/equal" "$fraction_equal_id" /root/fraction_equal 2026-08-03T12:00:00.1000Z +write_runtime_rollout "$fraction_order_sessions/leap-day" "$leap_day_id" /root/leap_day 2024-02-29T23:59:59Z +fraction_order_output=$(sh "$runtime_inspector" --sessions-dir "$fraction_order_sessions" --agent-path /root/fraction_order --since 2026-08-03T12:00:00.1Z) +printf '%s\n' "$fraction_order_output" | jq -e --arg id "$fraction_after_id" '.thread_id == $id' >/dev/null || fail "fractional cutoff did not exclude before-cutoff same-path record" +fraction_equal_output=$(sh "$runtime_inspector" --sessions-dir "$fraction_order_sessions" --agent-path /root/fraction_equal --since 2026-08-03T12:00:00.1Z) +printf '%s\n' "$fraction_equal_output" | jq -e --arg id "$fraction_equal_id" '.thread_id == $id' >/dev/null || fail "equivalent fractional values did not compare equal" +leap_day_output=$(sh "$runtime_inspector" --sessions-dir "$fraction_order_sessions" --agent-path /root/leap_day --since 2024-02-29T23:59:59Z) +printf '%s\n' "$leap_day_output" | jq -e --arg id "$leap_day_id" '.thread_id == $id' >/dev/null || fail "valid leap-year calendar boundary did not match" +pass "runtime inspector fractional ordering lowercase t/z and leap-year boundary" + +batched_sessions=$tmp_dir/batched-sessions +batched_unrelated=$batched_sessions/unrelated +mkdir -p "$batched_unrelated" +batched_index=1 +while [ "$batched_index" -le 128 ]; do + printf '%s\n' '{"type":"response_item","payload":{"prompt":"DO_NOT_LEAK_BATCHED_UNRELATED"}}' > "$batched_unrelated/rollout-unrelated-$batched_index.jsonl" + batched_index=$((batched_index + 1)) +done +batched_id=21212121-2121-7121-8121-212121212121 +write_runtime_rollout "$batched_sessions/candidate" "$batched_id" /root/batched_candidate 2026-08-03T12:00:00Z +batched_output=$(sh "$runtime_inspector" --sessions-dir "$batched_sessions" --agent-path /root/batched_candidate --since 2026-08-03T12:00:00Z) +printf '%s\n' "$batched_output" | jq -e --arg id "$batched_id" '.thread_id == $id' >/dev/null || fail "batched discovery selected the wrong rollout" +if printf '%s\n' "$batched_output" | grep -Fq DO_NOT_LEAK_BATCHED_UNRELATED; then fail "batched discovery leaked unrelated payload"; fi +pass "runtime inspector batched discovery with many unrelated rollouts" + +pruned_sessions=$tmp_dir/pruned-sessions +pruned_old=$pruned_sessions/2026/08/01 +mkdir -p "$pruned_old" +printf '%s\n' '{malformed-old-rollout' > "$pruned_old/rollout-malformed-old.jsonl" +pruned_current_id=22222222-2222-7222-8222-222222222222 +pruned_future_id=23232323-2323-7323-8323-232323232323 +write_runtime_rollout "$pruned_sessions/2026/08/02" "$pruned_current_id" /root/pruned_current 2026-08-03T12:00:00Z +write_runtime_rollout "$pruned_sessions/2026/08/04" "$pruned_future_id" /root/pruned_future 2026-08-04T12:00:00Z +for path_id in "/root/pruned_current:$pruned_current_id" "/root/pruned_future:$pruned_future_id"; do + runtime_path=${path_id%%:*} expected_id=${path_id#*:} + output=$(sh "$runtime_inspector" --sessions-dir "$pruned_sessions" --agent-path "$runtime_path" --since 2026-08-03T12:00:00Z) + printf '%s\n' "$output" | jq -e --arg id "$expected_id" '.thread_id == $id' >/dev/null || fail "canonical date pruning selected the wrong rollout" +done +pass "runtime inspector prunes old canonical directories" + +binding_sessions=$tmp_dir/binding-sessions +binding_metadata_id=24242424-2424-7424-8424-242424242424 +binding_filename_id=25252525-2525-7525-8525-252525252525 +mkdir -p "$binding_sessions" +printf '%s\n' \ + "{\"type\":\"session_meta\",\"timestamp\":\"2026-08-03T12:00:00Z\",\"payload\":{\"id\":\"$binding_metadata_id\",\"parent_thread_id\":\"00000000-0000-7000-8000-000000000000\",\"agent_role\":\"sol_advisor_terra_implementer\",\"agent_path\":\"/root/binding\",\"model_provider\":\"openai\",\"cwd\":\"/fixture\"}}" \ + '{"type":"turn_context","payload":{"model":"gpt-5.6-terra","effort":"high","sandbox_policy":{"type":"danger-full-access"},"permission_profile":{"type":"disabled"},"cwd":"/fixture"}}' \ + > "$binding_sessions/rollout-fixture-$binding_filename_id.jsonl" +write_runtime_rollout "$binding_sessions" "$binding_metadata_id" /root/not_binding 2026-08-03T12:00:00Z +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$binding_sessions" --agent-path /root/binding --since 2026-08-03T12:00:00Z +pass "runtime inspector binds path metadata to its rollout filename" + +invalid_timestamp_sessions=$tmp_dir/invalid-timestamp-sessions +mkdir -p "$invalid_timestamp_sessions" +for invalid_timestamp in 2026-02-30T12:00:00Z 2026-08-03T24:00:00Z 2026-08-03T12:60:00Z 2026-08-03T12:00:61Z 2026-08-03T12:00:00+24:00 2026-08-03T12:00:00+01:60 2026-08-03T12:00:00-00:00; do + expect_exit 1 sh "$runtime_inspector" --sessions-dir "$invalid_timestamp_sessions" --agent-path /root/invalid --since "$invalid_timestamp" +done +pass "runtime inspector rejects invalid calendar, offset, and unknown-offset timestamps" + +invalid_candidate_sessions=$tmp_dir/invalid-candidate-sessions +invalid_candidate_id=20202020-2020-7020-8020-202020202020 +write_runtime_rollout "$invalid_candidate_sessions" "$invalid_candidate_id" /root/invalid_candidate 2026-02-29T12:00:00Z +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$invalid_candidate_sessions" --agent-path /root/invalid_candidate --since 2026-02-01T00:00:00Z +pass "runtime inspector rejects invalid candidate timestamps" + +expect_exit 2 sh "$runtime_inspector" --sessions-dir "$path_sessions" --sessions-dir "$path_sessions" "$path_id" +expect_exit 2 sh "$runtime_inspector" --agent-path /root/fixture --agent-path /root/fixture --since 2026-08-03T12:00:00Z +expect_exit 2 sh "$runtime_inspector" --agent-path /root/fixture --since 2026-08-03T12:00:00Z --since 2026-08-03T12:00:00Z +expect_exit 2 sh "$runtime_inspector" --unknown "$path_id" +expect_exit 2 sh "$runtime_inspector" --sessions-dir "$path_sessions" "$path_id" "$path_id" +expect_exit 2 sh "$runtime_inspector" --sessions-dir "$path_sessions" "$path_id" --agent-path /root/fixture --since 2026-08-03T12:00:00Z +expect_exit 2 sh "$runtime_inspector" --sessions-dir "$path_sessions" --since 2026-08-03T12:00:00Z +expect_exit 2 sh "$runtime_inspector" --sessions-dir "$path_sessions" --agent-path /root/fixture +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$path_sessions" --agent-path /root/Invalid --since 2026-08-03T12:00:00Z +pass "runtime inspector strict CLI and canonical-path validation" + +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$path_sessions" --agent-path /root/no_match --since 2026-08-03T12:00:00Z +ambiguous_sessions=$tmp_dir/ambiguous-sessions +write_runtime_rollout "$ambiguous_sessions/one" aaaaaaaa-aaaa-7aaa-8aaa-aaaaaaaaaaaa /root/ambiguous 2026-08-03T12:00:00Z +write_runtime_rollout "$ambiguous_sessions/two" bbbbbbbb-bbbb-7bbb-8bbb-bbbbbbbbbbbb /root/ambiguous 2026-08-03T12:00:01Z +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$ambiguous_sessions" --agent-path /root/ambiguous --since 2026-08-03T12:00:00Z +concurrent_sessions=$tmp_dir/concurrent-sessions +write_runtime_rollout "$concurrent_sessions/one" cccccccc-cccc-7ccc-8ccc-cccccccccccc /root/concurrent/one 2026-08-03T12:00:00Z +write_runtime_rollout "$concurrent_sessions/two" dddddddd-dddd-7ddd-8ddd-dddddddddddd /root/concurrent/two 2026-08-03T12:00:00Z +for path_id in /root/concurrent/one:cccccccc-cccc-7ccc-8ccc-cccccccccccc /root/concurrent/two:dddddddd-dddd-7ddd-8ddd-dddddddddddd; do + runtime_path=${path_id%%:*} expected_id=${path_id#*:} + output=$(sh "$runtime_inspector" --sessions-dir "$concurrent_sessions" --agent-path "$runtime_path" --since 2026-08-03T12:00:00Z) + printf '%s\n' "$output" | jq -e --arg id "$expected_id" '.thread_id == $id' >/dev/null || fail "concurrent path lookup selected wrong agent" +done +pass "runtime inspector no-match, ambiguity, and concurrent-path isolation" + +sentinel=DO_NOT_LEAK_PATH_OR_PROMPT +prompt_sessions=$tmp_dir/prompt-sessions +prompt_id=eeeeeeee-eeee-7eee-8eee-eeeeeeeeeeee +write_runtime_rollout "$prompt_sessions/fixture" "$prompt_id" /root/real 2026-08-03T12:00:00Z +printf '%s\n' "{\"type\":\"response_item\",\"payload\":{\"prompt\":\"$sentinel /root/prompt_only\"}}" >> "$prompt_sessions/fixture/rollout-fixture-$prompt_id.jsonl" +prompt_error=$(sh "$runtime_inspector" --sessions-dir "$prompt_sessions" --agent-path /root/prompt_only --since 2026-08-03T12:00:00Z 2>&1 >/dev/null || true) +if printf '%s\n' "$prompt_error" | grep -Fq "$sentinel"; then fail "runtime inspector leaked prompt sentinel in error"; fi +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$prompt_sessions" --agent-path /root/prompt_only --since 2026-08-03T12:00:00Z + +malformed_sessions=$tmp_dir/malformed-sessions +mkdir -p "$malformed_sessions" +printf '%s\n' '{invalid-json' > "$malformed_sessions/rollout-fixture-ffffffff-ffff-7fff-8fff-ffffffffffff.jsonl" +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$malformed_sessions" --agent-path /root/malformed --since 2026-08-03T12:00:00Z +missing_timestamp_sessions=$tmp_dir/missing-timestamp-sessions +mkdir -p "$missing_timestamp_sessions" +printf '%s\n' '{"type":"session_meta","payload":{"id":"12121212-1212-7212-8212-121212121212","agent_path":"/root/missing_timestamp"}}' > "$missing_timestamp_sessions/rollout-fixture-12121212-1212-7212-8212-121212121212.jsonl" +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$missing_timestamp_sessions" --agent-path /root/missing_timestamp --since 2026-08-03T12:00:00Z +malformed_metadata_sessions=$tmp_dir/malformed-metadata-sessions +mkdir -p "$malformed_metadata_sessions" +printf '%s\n' '{"type":"session_meta","timestamp":"2026-08-03T12:00:00Z","payload":null}' > "$malformed_metadata_sessions/rollout-fixture-13131313-1313-7313-8313-131313131313.jsonl" +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$malformed_metadata_sessions" --agent-path /root/malformed_metadata --since 2026-08-03T12:00:00Z +pass "runtime inspector ignores prompt-only false matches and safely refuses malformed metadata" + +strict_sessions=$tmp_dir/strict-sessions +strict_id=14141414-1414-7414-8414-141414141414 +write_runtime_rollout "$strict_sessions/missing" "$strict_id" /root/strict_missing 2026-08-03T12:00:00Z +strict_rollout=$strict_sessions/missing/rollout-fixture-$strict_id.jsonl +sed '2,$d' "$strict_rollout" > "$strict_rollout.first" && mv "$strict_rollout.first" "$strict_rollout" +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$strict_sessions" --agent-path /root/strict_missing --since 2026-08-03T12:00:00Z +conflicting_id=15151515-1515-7515-8515-151515151515 +write_runtime_rollout "$strict_sessions/conflicting" "$conflicting_id" /root/strict_conflicting 2026-08-03T12:00:00Z +printf '%s\n' '{"type":"turn_context","payload":{"model":"different-model","effort":"high","sandbox_policy":{"type":"danger-full-access"},"permission_profile":{"type":"disabled"},"cwd":"/fixture"}}' >> "$strict_sessions/conflicting/rollout-fixture-$conflicting_id.jsonl" +expect_exit 1 sh "$runtime_inspector" --sessions-dir "$strict_sessions" --agent-path /root/strict_conflicting --since 2026-08-03T12:00:00Z +pass "runtime inspector preserves strict missing and conflicting routing refusal" + for document in "$skill" "$contracts"; do grep -Fq 'agent_type: sol_advisor_terra_implementer' "$document" || fail "missing Terra spawn in $document" grep -Fq 'agent_type: sol_advisor_sol_reviewer' "$document" || fail "missing Sol spawn in $document" diff --git a/plugins/sol-advisor/skills/orchestration/SKILL.md b/plugins/sol-advisor/skills/orchestration/SKILL.md index f1647b6..8ab624b 100644 --- a/plugins/sol-advisor/skills/orchestration/SKILL.md +++ b/plugins/sol-advisor/skills/orchestration/SKILL.md @@ -56,6 +56,13 @@ Before every delegation, complete steps 1-2. After spawning a lane, complete ste fresh task, and update Codex if the name remains unavailable. Do not substitute a built-in or similarly named role. +Immediately after steps 1-2 and immediately before the native `spawn_agent` call, +record the UTC cutoff used to distinguish this new rollout from an older reused path: + +~~~sh +runtime_since="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +~~~ + 3. Treat exact templates plus observed runtime routing as an acceptance gate. Inspect public native spawn/details metadata first. It must identify the selected custom role. When it exposes model or effort, compare them with the role pin. @@ -66,13 +73,17 @@ Before every delegation, complete steps 1-2. After spawning a lane, complete ste ~~~sh skill_dir= runtime_inspector="$skill_dir/../../scripts/inspect-agent-runtime.sh" - sh "$runtime_inspector" + # Capture the canonical /root/ path returned by the completed native spawn. + agent_path= + # Reuse the runtime_since recorded immediately before that spawn. + sh "$runtime_inspector" --agent-path "$agent_path" --since "$runtime_since" ~~~ The helper's allowlisted output is the authoritative local fallback for omitted model and effort. If public and local values both exist, they must agree. Accepted values are Terra / high for implementation and Sol / high for review. Missing, inconsistent, unavailable, or unobservable routing stops that lane. + Its positional lowercase UUID interface remains available for compatibility. 4. For every Sol review, capture the observed sandbox policy type and permission profile type. The shipped reviewer requests read-only sandboxing, but the host may diff --git a/plugins/sol-advisor/skills/orchestration/references/role-contracts.md b/plugins/sol-advisor/skills/orchestration/references/role-contracts.md index 9605861..678a8fe 100644 --- a/plugins/sol-advisor/skills/orchestration/references/role-contracts.md +++ b/plugins/sol-advisor/skills/orchestration/references/role-contracts.md @@ -22,6 +22,12 @@ A missing, stale, unsafe, conflicting, unavailable, inconsistent, or unobservabl role/model/effort stops the lane. Never silently fall back. Model and effort are pinned by custom-agent TOML, so omit per-spawn overrides. +When local runtime evidence is needed, complete steps 1-2, then record a UTC RFC3339 +cutoff immediately before native spawn. After that spawn returns, retain its canonical +`/root/` path and call the inspector with `--agent-path "$agent_path" --since +"$runtime_since"`. The positional lowercase UUID inspector interface remains available +for compatibility. + ## Shared implementation contract Every Terra prompt must contain all five sections: