{"cuts":{"non_us":11.4,"open":11.4,"proprietary":12.8,"us":12.8},"models":[{"bantu_blind_mean":19.0,"bantu_scaf_mean":19.3,"country":"US","english":26.0,"license":"proprietary","model":"OpenAI GPT-5.5","n":9,"org":"OpenAI","pinyin_base":25.56,"pinyin_toned":25.43,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":19.56},"bem:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":25.47},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":20.86},"kin:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":6.63},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":16.61},"lue:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":25.89},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":25.56},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":25.43}}},{"bantu_blind_mean":18.0,"bantu_scaf_mean":26.0,"country":"US","english":26.0,"license":"proprietary","model":"Google Gemini 3","n":9,"org":"Google","pinyin_base":25.69,"pinyin_toned":24.14,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":18.09},"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":20.55},"kin:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":15.35},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":25.69},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":24.14}}},{"bantu_blind_mean":17.5,"bantu_scaf_mean":26.0,"country":"US","english":26.0,"license":"proprietary","model":"Anthropic Claude (Opus 4.8)","n":9,"org":"Anthropic","pinyin_base":23.92,"pinyin_toned":25.43,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":19.62},"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":17.5},"kin:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":15.32},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":23.92},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":25.43}}},{"bantu_blind_mean":16.8,"bantu_scaf_mean":26.0,"country":"China","english":26.0,"license":"open","model":"Qwen 3.7 Plus (Fireworks)","n":9,"org":"Alibaba","pinyin_base":25.56,"pinyin_toned":23.17,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":17.86},"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":18.77},"kin:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":13.75},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":25.56},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":23.17}}},{"bantu_blind_mean":15.5,"bantu_scaf_mean":26.0,"country":"US","english":26.0,"license":"proprietary","model":"Anthropic Claude Sonnet 5","n":9,"org":"Anthropic","pinyin_base":25.56,"pinyin_toned":21.22,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":13.75},"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":19.3},"kin:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":13.34},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":25.56},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":21.22}}},{"bantu_blind_mean":13.7,"bantu_scaf_mean":26.0,"country":"China","english":26.0,"license":"open","model":"DeepSeek V4","n":9,"org":"DeepSeek","pinyin_base":25.43,"pinyin_toned":20.44,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":12.26},"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":14.8},"kin:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":25.95},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":13.97},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":25.43},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":20.44}}},{"bantu_blind_mean":12.3,"bantu_scaf_mean":25.9,"country":"US","english":26.0,"license":"open","model":"Nemotron 3 Ultra (Fireworks)","n":9,"org":"NVIDIA","pinyin_base":24.33,"pinyin_toned":24.29,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":5.53},"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":16.48},"kin:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":25.73},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":14.91},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":24.33},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":24.29}}},{"bantu_blind_mean":12.0,"bantu_scaf_mean":26.0,"country":"China","english":26.0,"license":"open","model":"MiniMax M3 (Fireworks)","n":8,"org":"MiniMax","pinyin_base":25.37,"pinyin_toned":null,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":10.81},"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":9.29},"kin:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":15.8},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":25.37}}},{"bantu_blind_mean":11.3,"bantu_scaf_mean":26.0,"country":"China","english":26.0,"license":"open","model":"GLM-5.2 (Fireworks)","n":7,"org":"Z.ai / Zhipu","pinyin_base":25.24,"pinyin_toned":null,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":5.21},"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":17.36},"kin:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":25.24}}},{"bantu_blind_mean":8.5,"bantu_scaf_mean":24.5,"country":"France","english":26.0,"license":"proprietary","model":"Mistral Large 3","n":9,"org":"Mistral AI","pinyin_base":24.74,"pinyin_toned":23.93,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":7.56},"bem:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":24.92},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":9.02},"kin:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":22.55},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":8.87},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":24.74},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":23.93}}},{"bantu_blind_mean":6.2,"bantu_scaf_mean":25.9,"country":"Canada","english":26.0,"license":"proprietary","model":"Cohere Command A+","n":9,"org":"Cohere","pinyin_base":16.5,"pinyin_toned":8.97,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":8.53},"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":10.12},"kin:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":25.84},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":0.08},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":16.5},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":8.97}}},{"bantu_blind_mean":5.1,"bantu_scaf_mean":26.0,"country":"US","english":26.0,"license":"proprietary","model":"xAI Grok 4.3","n":8,"org":"xAI","pinyin_base":25.37,"pinyin_toned":24.31,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":0.54},"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":14.16},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":0.53},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":25.37},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":24.31}}},{"bantu_blind_mean":2.1,"bantu_scaf_mean":16.2,"country":"US","english":26.0,"license":"open","model":"Groq (GPT-OSS 120B)","n":9,"org":"OpenAI","pinyin_base":24.55,"pinyin_toned":0,"tracks":{"bem:blind":{"failure_mode":null,"grade":"FAIL","l26":6.24},"bem:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":11.14},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:blind":{"failure_mode":null,"grade":"FAIL","l26":0.01},"kin:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":11.89},"lue:blind":{"failure_mode":null,"grade":"FAIL","l26":0.09},"lue:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":25.47},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":24.55},"pinyin_toned:blind":{"failure_mode":null,"grade":"FAIL","l26":0}}},{"bantu_blind_mean":1.6,"bantu_scaf_mean":null,"country":"unknown","english":26.0,"license":"unknown","model":"acme-eval-sim","n":2,"org":"unknown","pinyin_base":null,"pinyin_toned":null,"tracks":{"bem:blind":{"failure_mode":"under_recall","grade":"FAIL","l26":1.62},"english:blind":{"failure_mode":"mastered","grade":"PASS","l26":26.0}}},{"bantu_blind_mean":null,"bantu_scaf_mean":null,"country":"China","english":26.0,"license":"open","model":"Moonshot Kimi K2","n":1,"org":"Moonshot AI","pinyin_base":null,"pinyin_toned":null,"tracks":{"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0}}},{"bantu_blind_mean":null,"bantu_scaf_mean":26.0,"country":"China","english":26.0,"license":"open","model":"Kimi K2.7 Code (Fireworks)","n":5,"org":"Moonshot AI","pinyin_base":25.56,"pinyin_toned":null,"tracks":{"bem:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":25.56}}},{"bantu_blind_mean":null,"bantu_scaf_mean":18.4,"country":"US","english":26.0,"license":"open","model":"Microsoft Phi-4-reasoning","n":2,"org":"Microsoft","pinyin_base":null,"pinyin_toned":null,"tracks":{"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"lue:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":18.41}}},{"bantu_blind_mean":null,"bantu_scaf_mean":25.7,"country":"China","english":26.0,"license":"open","model":"DeepSeek V4 Pro (Fireworks)","n":5,"org":"DeepSeek","pinyin_base":25.31,"pinyin_toned":null,"tracks":{"bem:scaffolded":{"failure_mode":null,"grade":"FAIL","l26":25.24},"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0},"kin:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"lue:scaffolded":{"failure_mode":null,"grade":"PASS","l26":26.0},"pinyin_base:blind":{"failure_mode":null,"grade":"FAIL","l26":25.31}}},{"bantu_blind_mean":null,"bantu_scaf_mean":null,"country":"US","english":26.0,"license":"proprietary","model":"Anthropic Claude Fable 5","n":1,"org":"Anthropic","pinyin_base":null,"pinyin_toned":null,"tracks":{"english:blind":{"failure_mode":null,"grade":"PASS","l26":26.0}}}],"n_runs":129}