diff --git a/plugins/wff-code/skills/panel/data/README.md b/plugins/wff-code/skills/panel/data/README.md index fd82ec5..e40c637 100644 --- a/plugins/wff-code/skills/panel/data/README.md +++ b/plugins/wff-code/skills/panel/data/README.md @@ -26,3 +26,25 @@ Updates go through the refresh-data workflow. - roles.json: role_definitions (19 roles) and domain_roles (4 domains, additive levels 1-3). This file is internally consistent; domain_roles references only defined roles. + +## Benchmark score provenance + +`humaneval_score` and `swe_bench_score` are hand-rated, and for models with no +published run they are ESTIMATES from provider positioning, not measured +results. Every `:free` row added in the 2026-08-25 refresh is in that category: +the free tier turned over completely (all 20 previous free rows were retired +upstream), and the replacements are rated from model cards and parameter +counts. Treat their relative ordering as a curation judgement, not evidence. + +This matters because the roster selector sorts each cost tier by +`(-humaneval, -swe_bench)`, so these estimates decide which three models fill a +level-1 panel. Re-rate them when real benchmark numbers land. + +## Known gap: refresh does not detect price drift + +`consensus_cli.py refresh` diffs model IDs only. It cannot see a live price +change on a model that is still alive, and `input_cost` is what assigns a model +to a cost tier band and what feeds the cost cap. The 2026-08-25 refresh found +10 rows whose prices had drifted, one by 7.3x (`openai/o4-mini`, 0.15 -> 1.10). +Re-check prices against `https://openrouter.ai/api/v1/models` during any +refresh, not just liveness. diff --git a/plugins/wff-code/skills/panel/data/models.csv b/plugins/wff-code/skills/panel/data/models.csv index 31d6878..9a617d0 100644 --- a/plugins/wff-code/skills/panel/data/models.csv +++ b/plugins/wff-code/skills/panel/data/models.csv @@ -1,40 +1,29 @@ rank,model,provider,tier,status,context,input_cost,output_cost,org_level,specialization,role,strength,humaneval_score,swe_bench_score,openrouter_url,last_updated -1,openai/gpt-5.1,openai,premium,paid,400K,1.25,10.0,executive,general,lead_architect,next_generation,90.0,80.0,https://openrouter.ai/openai/gpt-5.1,2025-11-15 -2,anthropic/claude-opus-4.6,anthropic,premium,paid,200K,5.0,25.0,executive,agentic,system_architect,agentic_workflows,89.0,82.0,https://openrouter.ai/anthropic/claude-opus-4.6,2026-04-11 -3,openai/gpt-5.1-chat,openai,premium,paid,128K,1.25,10.0,executive,conversation,lead_architect,conversational,89.0,79.0,https://openrouter.ai/openai/gpt-5.1-chat,2025-11-15 -3.5,openai/gpt-5.1-codex,openai,premium,paid,400K,1.25,10.0,executive,coding,lead_architect,coding,92.0,85.0,https://openrouter.ai/openai/gpt-5.1-codex,2025-11-15 -3.6,google/gemini-3-pro-preview,google,premium,paid,1048K,2.0,12.0,executive,reasoning,lead_architect,multimodal,88.0,82.0,https://openrouter.ai/google/gemini-3-pro-preview,2025-11-21 -4,anthropic/claude-sonnet-4.5,anthropic,high_perf,paid,1000K,3.0,15.0,senior,reasoning,senior_developer,agentic,87.0,80.0,https://openrouter.ai/anthropic/claude-sonnet-4.5,2025-11-21 -5,openai/gpt-5-mini,openai,high_perf,paid,400K,0.5,2.0,senior,general,senior_developer,efficient,85.0,75.0,https://openrouter.ai/openai/gpt-5-mini,2025-08-11 -6.5,openai/gpt-5.1-codex-mini,openai,high_perf,paid,400K,0.25,2.0,senior,coding,senior_developer,coding,88.0,78.0,https://openrouter.ai/openai/gpt-5.1-codex-mini,2025-11-15 -7,deepseek/deepseek-r1-0528,deepseek,high_perf,paid,65K,0.40,2.0,senior,reasoning,security_analyst,reasoning,82.0,72.0,https://openrouter.ai/deepseek/deepseek-r1-0528,2025-08-11 -8,google/gemini-2.5-flash,google,open_source,paid,1048K,0.075,0.30,senior,general,senior_developer,speed,78.0,65.0,https://openrouter.ai/google/gemini-2.5-flash,2025-08-11 -9,openai/o4-mini,openai,open_source,paid,200K,0.15,0.6,senior,general,senior_developer,compact,80.0,70.0,https://openrouter.ai/openai/o4-mini,2025-08-11 -10,qwen/qwen3-coder,qwen,open_source,paid,131K,0.20,0.80,senior,coding,senior_developer,coding,85.0,70.0,https://openrouter.ai/qwen/qwen3-coder,2025-08-11 -11,mistralai/mistral-large-2411,mistral,open_source,paid,128K,2.0,6.0,senior,general,senior_developer,european,75.0,65.0,https://openrouter.ai/mistralai/mistral-large-2411,2025-08-11 -12,openai/gpt-5-nano,openai,value_tier,paid,400K,0.1,0.4,senior,general,senior_developer,ultralight,78.0,68.0,https://openrouter.ai/openai/gpt-5-nano,2025-08-11 -13,moonshotai/kimi-k2,moonshotai,value_tier,paid,200K,1.0,3.0,senior,general,senior_developer,chinese,75.0,65.0,https://openrouter.ai/moonshotai/kimi-k2,2025-08-11 -14,microsoft/phi-4,microsoft,value_tier,paid,131K,0.50,1.5,senior,debugging,senior_developer,compact,75.0,62.0,https://openrouter.ai/microsoft/phi-4,2025-08-11 -14.5,x-ai/grok-code-fast-1,x-ai,value_tier,paid,256K,0.20,1.5,senior,coding,senior_developer,fast_coding,82.0,70.0,https://openrouter.ai/x-ai/grok-code-fast-1,2025-11-21 -14.6,google/gemini-3-pro-image-preview,google,value_tier,paid,65K,2.0,12.0,senior,vision,senior_developer,image_generation,70.0,50.0,https://openrouter.ai/google/gemini-3-pro-image-preview,2025-11-21 -10,meta-llama/llama-3.1-405b-instruct:free,meta,free_champion,free,131K,0.0,0.0,junior,general,technical_validator,general,80.5,68.0,https://openrouter.ai/meta-llama/llama-3.1-405b-instruct:free,2025-08-11 -11,deepseek/deepseek-r1-distill-llama-70b:free,deepseek,free_champion,free,131K,0.0,0.0,junior,reasoning,code_reviewer,reasoning,75.0,65.0,https://openrouter.ai/deepseek/deepseek-r1-distill-llama-70b:free,2025-08-11 -15,moonshotai/kimi-k2:free,moonshotai,free_champion,free,200K,0.0,0.0,junior,general,technical_validator,chinese,75.0,65.0,https://openrouter.ai/moonshotai/kimi-k2:free,2025-08-11 -16,qwen/qwen-2.5-coder-32b-instruct:free,qwen,free_champion,free,131K,0.0,0.0,junior,coding,code_reviewer,coding,80.0,65.0,https://openrouter.ai/qwen/qwen-2.5-coder-32b-instruct:free,2025-08-11 -17,microsoft/phi-4-reasoning:free,microsoft,free_champion,free,131K,0.0,0.0,junior,debugging,code_reviewer,reasoning,75.0,62.0,https://openrouter.ai/microsoft/phi-4-reasoning:free,2025-08-11 -18,microsoft/mai-ds-r1:free,microsoft,free_champion,free,131K,0.0,0.0,junior,reasoning,code_reviewer,microsoft,72.0,60.0,https://openrouter.ai/microsoft/mai-ds-r1:free,2025-08-11 -13,meta-llama/llama-3.3-70b-instruct:free,meta,free_tier,free,131K,0.0,0.0,junior,general,technical_validator,efficient,72.0,60.0,https://openrouter.ai/meta-llama/llama-3.3-70b-instruct:free,2025-08-11 -19,qwen/qwq-32b:free,qwen,free_tier,free,131K,0.0,0.0,junior,reasoning,code_reviewer,reasoning,70.0,60.0,https://openrouter.ai/qwen/qwq-32b:free,2025-08-11 -20,qwen/qwen3-14b:free,qwen,free_tier,free,131K,0.0,0.0,junior,general,technical_validator,compact,68.0,58.0,https://openrouter.ai/qwen/qwen3-14b:free,2025-08-11 -21,qwen/qwen3-32b:free,qwen,free_tier,free,131K,0.0,0.0,junior,general,technical_validator,balanced,70.0,60.0,https://openrouter.ai/qwen/qwen3-32b:free,2025-08-11 -22,qwen/qwen3-235b-a22b:free,qwen,free_tier,free,131K,0.0,0.0,junior,general,technical_validator,large,72.0,62.0,https://openrouter.ai/qwen/qwen3-235b-a22b:free,2025-08-11 -16,qwen/qwen2.5-vl-72b-instruct:free,qwen,free_tier,free,131K,0.0,0.0,junior,vision,technical_validator,vision,70.0,58.0,https://openrouter.ai/qwen/qwen2.5-vl-72b-instruct:free,2025-08-11 -17,meta-llama/llama-3.2-11b-vision-instruct:free,meta,free_tier,free,131K,0.0,0.0,junior,vision,technical_validator,vision,68.0,55.0,https://openrouter.ai/meta-llama/llama-3.2-11b-vision-instruct:free,2025-08-11 -18,meta-llama/llama-3.2-3b-instruct:free,meta,free_tier,free,131K,0.0,0.0,junior,general,technical_validator,lightweight,65.0,52.0,https://openrouter.ai/meta-llama/llama-3.2-3b-instruct:free,2025-08-11 -23,meta-llama/llama-4-maverick:free,meta,free_tier,free,131K,0.0,0.0,junior,general,technical_validator,experimental,70.0,58.0,https://openrouter.ai/meta-llama/llama-4-maverick:free,2025-08-11 -24,deepseek/deepseek-chat:free,deepseek,free_tier,free,65K,0.0,0.0,junior,general,technical_validator,chat,68.0,55.0,https://openrouter.ai/deepseek/deepseek-chat:free,2025-08-11 -25,deepseek/deepseek-r1-0528-qwen3-8b:free,deepseek,free_tier,free,65K,0.0,0.0,junior,reasoning,code_reviewer,hybrid,66.0,53.0,https://openrouter.ai/deepseek/deepseek-r1-0528-qwen3-8b:free,2025-08-11 -26,mistralai/mistral-nemo:free,mistral,free_tier,free,128K,0.0,0.0,junior,general,technical_validator,european,64.0,52.0,https://openrouter.ai/mistralai/mistral-nemo:free,2025-08-11 -27,openrouter/cypher-alpha:free,openrouter,free_tier,free,131K,0.0,0.0,junior,experimental,technical_validator,experimental,60.0,48.0,https://openrouter.ai/openrouter/cypher-alpha:free,2025-08-11 -28,tngtech/deepseek-r1t-chimera:free,tngtech,free_tier,free,131K,0.0,0.0,junior,experimental,technical_validator,hybrid,58.0,46.0,https://openrouter.ai/tngtech/deepseek-r1t-chimera:free,2025-08-11 -9,google/gemini-2.5-flash-lite,google,value_tier,paid,1048K,0.10,0.40,senior,general,senior_developer,speed,76.0,62.0,https://openrouter.ai/google/gemini-2.5-flash-lite,2025-01-15 +1,openai/gpt-5.1,openai,premium,paid,400K,1.25,10,executive,general,lead_architect,next_generation,90.0,80.0,https://openrouter.ai/openai/gpt-5.1,2026-08-25 +2,anthropic/claude-opus-4.6,anthropic,premium,paid,1M,5,25,executive,agentic,system_architect,agentic_workflows,89.0,82.0,https://openrouter.ai/anthropic/claude-opus-4.6,2026-08-25 +3.5,openai/gpt-5.1-codex,openai,premium,paid,400K,1.25,10,executive,coding,lead_architect,coding,92.0,85.0,https://openrouter.ai/openai/gpt-5.1-codex,2026-08-25 +3.6,google/gemini-3.1-pro-preview,google,premium,paid,1M,2,12,executive,reasoning,lead_architect,multimodal,88.0,82.0,https://openrouter.ai/google/gemini-3.1-pro-preview,2026-08-25 +4,anthropic/claude-sonnet-4.5,anthropic,high_perf,paid,1M,3,15,senior,reasoning,senior_developer,agentic,87.0,80.0,https://openrouter.ai/anthropic/claude-sonnet-4.5,2026-08-25 +5,openai/gpt-5-mini,openai,high_perf,paid,400K,0.25,2,senior,general,senior_developer,efficient,85.0,75.0,https://openrouter.ai/openai/gpt-5-mini,2026-08-25 +6.5,openai/gpt-5.1-codex-mini,openai,high_perf,paid,400K,0.25,2,senior,coding,senior_developer,coding,88.0,78.0,https://openrouter.ai/openai/gpt-5.1-codex-mini,2026-08-25 +7,deepseek/deepseek-r1-0528,deepseek,high_perf,paid,164K,0.5,2.15,senior,reasoning,security_analyst,reasoning,82.0,72.0,https://openrouter.ai/deepseek/deepseek-r1-0528,2026-08-25 +8,google/gemini-2.5-flash,google,open_source,paid,1M,0.3,2.5,senior,general,senior_developer,speed,78.0,65.0,https://openrouter.ai/google/gemini-2.5-flash,2026-08-25 +9,openai/o4-mini,openai,open_source,paid,200K,1.1,4.4,senior,general,senior_developer,compact,80.0,70.0,https://openrouter.ai/openai/o4-mini,2026-08-25 +10,qwen/qwen3-coder,qwen,open_source,paid,262K,0.3,1,senior,coding,senior_developer,coding,85.0,70.0,https://openrouter.ai/qwen/qwen3-coder,2026-08-25 +11,mistralai/mistral-large-2512,mistral,open_source,paid,262K,0.5,1.5,senior,general,senior_developer,european,75.0,65.0,https://openrouter.ai/mistralai/mistral-large-2512,2026-08-25 +12,openai/gpt-5-nano,openai,value_tier,paid,400K,0.05,0.4,senior,general,senior_developer,ultralight,78.0,68.0,https://openrouter.ai/openai/gpt-5-nano,2026-08-25 +13,moonshotai/kimi-k2,moonshotai,value_tier,paid,131K,0.57,2.3,senior,general,senior_developer,chinese,75.0,65.0,https://openrouter.ai/moonshotai/kimi-k2,2026-08-25 +14,microsoft/phi-4,microsoft,value_tier,paid,16K,0.07,0.14,senior,debugging,senior_developer,compact,75.0,62.0,https://openrouter.ai/microsoft/phi-4,2026-08-25 +14.5,x-ai/grok-4.6,x-ai,value_tier,paid,500K,2,6,senior,coding,senior_developer,fast_coding,82.0,70.0,https://openrouter.ai/x-ai/grok-4.6,2026-08-25 +14.6,google/gemini-3-pro-image-preview,google,value_tier,paid,66K,2,12,senior,vision,senior_developer,image_generation,70.0,50.0,https://openrouter.ai/google/gemini-3-pro-image-preview,2026-08-25 +9,google/gemini-2.5-flash-lite,google,value_tier,paid,1M,0.1,0.4,senior,general,senior_developer,speed,76.0,62.0,https://openrouter.ai/google/gemini-2.5-flash-lite,2026-08-25 +10,z-ai/glm-5.2:free,z-ai,free_champion,free,256K,0.0,0.0,junior,reasoning,technical_validator,long_horizon_agentic,84.0,74.0,https://openrouter.ai/z-ai/glm-5.2:free,2026-08-25 +11,thinkingmachines/inkling:free,thinkingmachines,free_champion,free,1M,0.0,0.0,junior,general,code_reviewer,general_reasoning,83.0,72.0,https://openrouter.ai/thinkingmachines/inkling:free,2026-08-25 +12,poolside/laguna-s-2.1:free,poolside,free_champion,free,262K,0.0,0.0,junior,coding,senior_developer,coding_agent,82.0,71.0,https://openrouter.ai/poolside/laguna-s-2.1:free,2026-08-25 +13,minimax/minimax-m3:free,minimax,free_champion,free,1M,0.0,0.0,junior,general,technical_validator,long_context_multimodal,81.0,70.0,https://openrouter.ai/minimax/minimax-m3:free,2026-08-25 +14,nvidia/nemotron-3-ultra-550b-a55b:free,nvidia,free_champion,free,1M,0.0,0.0,junior,reasoning,system_architect,frontier_reasoning,80.0,69.0,https://openrouter.ai/nvidia/nemotron-3-ultra-550b-a55b:free,2026-08-25 +15,minimax/minimax-m2.7:free,minimax,free_tier,free,197K,0.0,0.0,junior,agentic,senior_developer,agentic_workflows,78.0,67.0,https://openrouter.ai/minimax/minimax-m2.7:free,2026-08-25 +16,thinkingmachines/inkling-small:free,thinkingmachines,free_tier,free,1M,0.0,0.0,junior,general,code_reviewer,efficient_reasoning,78.0,66.0,https://openrouter.ai/thinkingmachines/inkling-small:free,2026-08-25 +17,nvidia/nemotron-3-super-120b-a12b:free,nvidia,free_tier,free,262K,0.0,0.0,junior,general,technical_validator,multi_agent,77.0,65.0,https://openrouter.ai/nvidia/nemotron-3-super-120b-a12b:free,2026-08-25 +18,cohere/north-mini-code:free,cohere,free_tier,free,256K,0.0,0.0,junior,coding,senior_developer,agentic_coding,76.0,64.0,https://openrouter.ai/cohere/north-mini-code:free,2026-08-25 +19,google/gemma-4-31b-it:free,google,free_tier,free,262K,0.0,0.0,junior,general,code_reviewer,multimodal_open,74.0,60.0,https://openrouter.ai/google/gemma-4-31b-it:free,2026-08-25