-
Notifications
You must be signed in to change notification settings - Fork 265
feat(harness): add interactive modes and GPU selection #611
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -63,6 +63,14 @@ LLAMA_CACHE_TYPE_K="${LLAMA_CACHE_TYPE_K:-${CACHE_TYPE_K:-q8_0}}" | |
| LLAMA_CACHE_TYPE_V="${LLAMA_CACHE_TYPE_V:-${CACHE_TYPE_V:-q8_0}}" | ||
| MAX_TOKENS="${MAX_TOKENS:-2048}" | ||
| EXTRA_SERVER_ARGS="${EXTRA_SERVER_ARGS:-}" | ||
| TARGET_DEVICE="${TARGET_DEVICE:-}" | ||
| DRAFT_DEVICE="${DRAFT_DEVICE:-${TARGET_DEVICE:-}}" | ||
| HARNESS_INTERACTIVE="${HARNESS_INTERACTIVE:-0}" | ||
| INTERACTIVE_PROMPT="${INTERACTIVE_PROMPT:-}" | ||
| if [[ "$HARNESS_INTERACTIVE" != "0" && "$HARNESS_INTERACTIVE" != "1" ]]; then | ||
| echo "HARNESS_INTERACTIVE must be 0 or 1" >&2 | ||
| exit 2 | ||
| fi | ||
|
|
||
| MODEL_ID="${MODEL_ID:-luce-dflash}" | ||
| API_KEY="${API_KEY:-sk-lucebox}" | ||
|
|
@@ -81,6 +89,37 @@ SERVER_LOG="$LOG_DIR/server.log" | |
|
|
||
| mkdir -p "$LOG_DIR" | ||
|
|
||
| # Keep progress attached to the original terminal even while one-shot client | ||
| # stdout/stderr is redirected to its result file. | ||
| exec 3>&2 | ||
| HARNESS_PROGRESS="${HARNESS_PROGRESS:-1}" | ||
|
|
||
| progress() { | ||
| if [[ "$HARNESS_PROGRESS" != "0" ]]; then | ||
| printf '[harness] %s\n' "$*" >&3 | ||
| fi | ||
| } | ||
|
|
||
| client_home() { | ||
| local client="$1" | ||
| if [[ "$HARNESS_INTERACTIVE" == "1" ]]; then | ||
| printf '%s/interactive/%s\n' "$CLIENT_WORK_DIR" "$client" | ||
| else | ||
| printf '%s/%s-home\n' "$LOG_DIR" "$client" | ||
| fi | ||
| } | ||
|
|
||
| run_interactive_client() { | ||
| local label="$1" | ||
| local client_out="$2" | ||
| shift 2 | ||
| progress "opening interactive $label; exit the client to stop the server" | ||
| "$@" | ||
| local rc=$? | ||
| printf 'Interactive %s terminal output was not captured.\n' "$label" > "$client_out" | ||
| return "$rc" | ||
| } | ||
|
|
||
| require_client_binary() { | ||
| local label="$1" | ||
| local path="$2" | ||
|
|
@@ -118,18 +157,53 @@ run_with_timeout() { | |
| echo "client timeout must be a non-negative integer (seconds; 0 disables it)" >&2 | ||
| return 2 | ||
| fi | ||
| local command_name | ||
| command_name="$(basename "$1")" | ||
| if [[ "$command_name" == "env" ]]; then | ||
| local arg | ||
| for arg in "${@:2}"; do | ||
| if [[ "$arg" != *=* ]]; then | ||
| command_name="$(basename "$arg")" | ||
| break | ||
| fi | ||
| done | ||
| fi | ||
| local timeout_label="${timeout_seconds}s" | ||
| if [[ "$timeout_seconds" == "0" ]]; then | ||
| timeout_label="disabled" | ||
| fi | ||
| progress "running $command_name (timeout: $timeout_label; logs: $LOG_DIR)" | ||
|
|
||
| local started_at heartbeat_pid rc elapsed | ||
| started_at="$(date +%s)" | ||
| ( | ||
| while sleep 10; do | ||
| elapsed=$(( $(date +%s) - started_at )) | ||
| progress "$command_name still running (${elapsed}s elapsed)" | ||
| done | ||
| ) & | ||
| heartbeat_pid=$! | ||
|
|
||
| if [[ "$timeout_seconds" == "0" ]]; then | ||
| "$@" | ||
| rc=$? | ||
| else | ||
| timeout "${timeout_seconds}s" "$@" | ||
| rc=$? | ||
| fi | ||
| kill "$heartbeat_pid" 2>/dev/null || true | ||
| wait "$heartbeat_pid" 2>/dev/null || true | ||
| elapsed=$(( $(date +%s) - started_at )) | ||
| progress "$command_name finished (rc=$rc, ${elapsed}s elapsed)" | ||
| return "$rc" | ||
| } | ||
|
|
||
| draft_enabled() { | ||
| [[ -n "${DRAFT:-}" && "$DRAFT" != "none" && "$DRAFT" != "off" && "$DRAFT" != "0" ]] | ||
| } | ||
|
|
||
| start_lucebox_server() { | ||
| progress "starting $MODEL_SERVER server (log: $SERVER_LOG)" | ||
| if [[ "$MODEL_SERVER" == "llamacpp" ]]; then | ||
| start_llamacpp_server | ||
| return | ||
|
|
@@ -177,17 +251,26 @@ start_dflash_native_server() { | |
| if [[ -n "$FA_WINDOW" ]] && [[ "$FA_WINDOW" != "0" ]]; then | ||
| fa_args=(--fa-window "$FA_WINDOW") | ||
| fi | ||
| local device_args=() | ||
| if [[ -n "$TARGET_DEVICE" ]]; then | ||
| device_args+=(--target-device "$TARGET_DEVICE") | ||
| fi | ||
| if [[ -n "$DRAFT_DEVICE" ]]; then | ||
| device_args+=(--draft-device "$DRAFT_DEVICE") | ||
| fi | ||
| # Export KV cache type env vars for the C++ server to pick up (only when | ||
| # explicitly requested: the per-axis envs override family defaults). | ||
| if [[ -n "$CACHE_TYPE_K" ]]; then export DFLASH27B_KV_K="$CACHE_TYPE_K"; fi | ||
| if [[ -n "$CACHE_TYPE_V" ]]; then export DFLASH27B_KV_V="$CACHE_TYPE_V"; fi | ||
| progress "server placement: target=${TARGET_DEVICE:-auto:0} draft=${DRAFT_DEVICE:-auto:0} CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-unset} HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-unset}" | ||
| "$DFLASH_SERVER_BIN" "$TARGET" \ | ||
| "${draft_args[@]}" \ | ||
| --host "$HOST" \ | ||
| --port "$PORT" \ | ||
| --max-ctx "$MAX_CTX" \ | ||
| --max-tokens "$MAX_TOKENS" \ | ||
| --model-name "$MODEL_ID" \ | ||
| "${device_args[@]}" \ | ||
| "${ddtree_args[@]}" \ | ||
| "${fa_args[@]}" \ | ||
| "${extra_args[@]}" \ | ||
|
|
@@ -264,10 +347,15 @@ start_llamacpp_server() { | |
| } | ||
|
|
||
| wait_lucebox_server() { | ||
| for _ in $(seq 1 300); do | ||
| progress "waiting for server health at $BASE_URL/health" | ||
| for attempt in $(seq 1 300); do | ||
| if curl -fsS "$BASE_URL/health" >/dev/null 2>&1; then | ||
| progress "server is healthy" | ||
| return 0 | ||
| fi | ||
| if (( attempt % 10 == 0 )); then | ||
| progress "server still starting (${attempt}s elapsed; log: $SERVER_LOG)" | ||
| fi | ||
| sleep 1 | ||
| if ! kill -0 "$SERVER_PID" 2>/dev/null; then | ||
| echo "server exited early; log: $SERVER_LOG" >&2 | ||
|
|
@@ -304,5 +392,24 @@ finish_report() { | |
| echo "--- server tail ---" | ||
| tail -n 120 "$SERVER_LOG" || true | ||
| echo "--- gpu ---" | ||
| nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu --format=csv,noheader || true | ||
| local resolved_backend="${TARGET_DEVICE%%:*}" | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: When the HIP server uses default or Prompt for AI agents |
||
| if [[ -z "$TARGET_DEVICE" ]] && grep -Eq 'target_device[[:space:]]*=[[:space:]]*hip:' "$SERVER_LOG"; then | ||
| resolved_backend="hip" | ||
| elif [[ -z "$TARGET_DEVICE" ]] && grep -Eq 'target_device[[:space:]]*=[[:space:]]*cuda:' "$SERVER_LOG"; then | ||
| resolved_backend="cuda" | ||
| fi | ||
| if [[ "$resolved_backend" == "hip" ]]; then | ||
| if command -v rocm-smi >/dev/null 2>&1; then | ||
| local rocm_device_args=() | ||
| if [[ "${HIP_VISIBLE_DEVICES:-}" =~ ^[0-9]+$ ]]; then | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: The rocm-smi Prompt for AI agents |
||
| rocm_device_args=(--device "$HIP_VISIBLE_DEVICES") | ||
| fi | ||
| rocm-smi "${rocm_device_args[@]}" --showproductname --showuse --showmeminfo vram 2>/dev/null || true | ||
| else | ||
| echo "rocm-smi not found" | ||
| fi | ||
| else | ||
| nvidia-smi --query-gpu=name,memory.used,memory.total,utilization.gpu --format=csv,noheader 2>/dev/null || \ | ||
| echo "nvidia-smi unavailable" | ||
| fi | ||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -16,7 +16,7 @@ CLIENT_OUT="$LOG_DIR/codex.out" | |||||
| LAST_MSG="$LOG_DIR/codex-last-message.txt" | ||||||
| CODEX_BIN="${CODEX_BIN:-$CLIENT_WORK_DIR/clients/codex/npm/bin/codex}" | ||||||
| require_client_binary "Codex" "$CODEX_BIN" "codex" "CODEX_BIN" | ||||||
| CODEX_HOME_DIR="$LOG_DIR/codex-home" | ||||||
| CODEX_HOME_DIR="$(client_home codex)" | ||||||
| CODEX_SANDBOX="${CODEX_SANDBOX:-danger-full-access}" | ||||||
| CODEX_WIRE_API="${CODEX_WIRE_API:-responses}" | ||||||
| mkdir -p "$CODEX_HOME_DIR" | ||||||
|
|
@@ -38,22 +38,29 @@ start_lucebox_server | |||||
| trap stop_lucebox_server EXIT | ||||||
| wait_lucebox_server | ||||||
|
|
||||||
| codex_env=("HOME=$CODEX_HOME_DIR" "CODEX_HOME=$CODEX_HOME_DIR" "OPENAI_API_KEY=$API_KEY") | ||||||
| set +e | ||||||
| run_with_timeout "$CODEX_TIMEOUT" env \ | ||||||
| HOME="$CODEX_HOME_DIR" \ | ||||||
| CODEX_HOME="$CODEX_HOME_DIR" \ | ||||||
| OPENAI_API_KEY="$API_KEY" \ | ||||||
| "$CODEX_BIN" exec \ | ||||||
| --skip-git-repo-check \ | ||||||
| --sandbox "$CODEX_SANDBOX" \ | ||||||
| --model "$MODEL_ID" \ | ||||||
| --json \ | ||||||
| --output-last-message "$LAST_MSG" \ | ||||||
| "$PROMPT" \ | ||||||
| < /dev/null > "$CLIENT_OUT" 2>&1 | ||||||
| RC=$? | ||||||
| if [[ "$HARNESS_INTERACTIVE" == "1" ]]; then | ||||||
| codex_cmd=("$CODEX_BIN" --cd "$REPO_DIR" --sandbox "$CODEX_SANDBOX" --model "$MODEL_ID") | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: The interactive Codex branch drops Prompt for AI agents
Suggested change
|
||||||
| if [[ -n "$INTERACTIVE_PROMPT" ]]; then codex_cmd+=("$INTERACTIVE_PROMPT"); fi | ||||||
| run_interactive_client "Codex" "$CLIENT_OUT" env "${codex_env[@]}" "${codex_cmd[@]}" | ||||||
| RC=$? | ||||||
| else | ||||||
| run_with_timeout "$CODEX_TIMEOUT" env "${codex_env[@]}" \ | ||||||
| "$CODEX_BIN" exec \ | ||||||
| --skip-git-repo-check \ | ||||||
| --sandbox "$CODEX_SANDBOX" \ | ||||||
| --model "$MODEL_ID" \ | ||||||
| --json \ | ||||||
| --output-last-message "$LAST_MSG" \ | ||||||
| "$PROMPT" \ | ||||||
| < /dev/null > "$CLIENT_OUT" 2>&1 | ||||||
| RC=$? | ||||||
| fi | ||||||
| set -e | ||||||
|
|
||||||
| cat "$LAST_MSG" >> "$CLIENT_OUT" 2>/dev/null || true | ||||||
| if [[ "$HARNESS_INTERACTIVE" == "0" ]]; then | ||||||
| cat "$LAST_MSG" >> "$CLIENT_OUT" 2>/dev/null || true | ||||||
| fi | ||||||
| finish_report "$CLIENT_OUT" "$RC" | ||||||
| exit "$RC" | ||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P2: If the OpenClaw config preflight fails or times out,
set -eexitsrun_with_timeoutbefore the heartbeat cleanup runs, leaving its background loop alive and continuing to emit progress. Install heartbeat cleanup in an unconditional trap or run the wrapped command through an errexit-safe conditional.Prompt for AI agents