{ "schema_version": "1.1", "config_version": "1.3", "environment": { "cpu": "Apple M4 Pro (container)", "logical_cpus": 14, "memory_bytes": 18829856768, "gpu": null, "os": "Linux-7.0.14-linuxkit-aarch64-with-glibc2.41", "python": "3.13.18", "libraries": { "tokenizers": "0.23.3", "tiktoken": "0.14.0", "transformers": "5.22.2", "numpy": "2.5.4" }, "git_commit": "ef2fc972f4e24836ebc9fea119b9338548969a1d", "git_dirty": false }, "provenance": { "measured": "observed in this run", "estimated": "arithmetic on measured values and stated assumptions", "unavailable": "not measured; the note says why" }, "assumptions": { "hidden_sizes": [ 2048, 4096 ], "bytes_per_parameter": { "fp16": 2, "bf16": 2 }, "embedding_rows": "config.json vocab_size of the published model when present (embedding matrix rows), otherwise the tokenizer's vocabulary", "price_usd_per_million_tokens": 1.0, "price_note": "a unit rate, not a real price: cost = tokens x rate, so costs compare tokenizers on the same model. Akshara has no hosted model and no price." }, "phases": { "1_integrity": { "status": "done", "outputs": "tokenizers[].correctness, groups" }, "2_static": { "status": "done", "outputs": "tokenizers[].per_set, parity, vocabulary, splitting, embedding" }, "3_runtime": { "status": "done", "outputs": "tokenizers[].runtime", "environment": { "cpu": "Apple M4 Pro (container)", "logical_cpus": 14, "memory_bytes": 18829856768, "gpu": null, "os": "Linux-7.0.14-linuxkit-aarch64-with-glibc2.41", "python": "3.13.18", "libraries": { "tokenizers": "0.23.3", "tiktoken": "0.14.0", "transformers": "5.22.2", "numpy": "2.5.4" }, "git_commit": "ef2fc972f4e24836ebc9fea119b9338548969a1d", "git_dirty": false } }, "4_workloads": { "status": "done", "outputs": "tokenizers[].workloads" }, "5_model_quality": { "status": "skipped", "reason": "training even one 50-150M-parameter model per tokenizer needs GPU time; this run had a 4-core CPU and no GPU. The plan and a ready-to-run script are in evaluation/model_quality/." }, "6_reporting": { "status": "done", "outputs": "rankings, composite, report.md, demo/data/evaluation.json" } }, "groups": [ { "members": [ "akshara-448k" ], "identical_ids": true, "label": "Akshara 448k", "id": "akshara-448k", "near_duplicates": [] }, { "members": [ "akshara-248k" ], "identical_ids": true, "label": "Akshara 248k", "id": "akshara-248k", "near_duplicates": [] }, { "members": [ "akshara-200k" ], "identical_ids": true, "label": "Akshara 200k", "id": "akshara-200k", "near_duplicates": [] }, { "members": [ "akshara-151k" ], "identical_ids": true, "label": "Akshara 151k", "id": "akshara-151k", "near_duplicates": [] }, { "members": [ "akshara-120k" ], "identical_ids": true, "label": "Akshara 120k", "id": "akshara-120k", "near_duplicates": [] }, { "members": [ "akshara-92k" ], "identical_ids": true, "label": "Akshara 92k", "id": "akshara-92k", "near_duplicates": [] }, { "members": [ "akshara-59k" ], "identical_ids": true, "label": "Akshara 59k", "id": "akshara-59k", "near_duplicates": [] }, { "members": [ "akshara-49k" ], "identical_ids": true, "label": "Akshara 49k", "id": "akshara-49k", "near_duplicates": [] }, { "members": [ "akshara-32k" ], "identical_ids": true, "label": "Akshara 32k", "id": "akshara-32k", "near_duplicates": [] }, { "members": [ "phi-3-5-mini-instruct" ], "identical_ids": true, "label": "Phi-3.5-mini-instruct", "id": "phi-3-5-mini-instruct", "near_duplicates": [] }, { "members": [ "smollm2-135m" ], "identical_ids": true, "label": "SmolLM2-135M", "id": "smollm2-135m", "near_duplicates": [] }, { "members": [ "gpt2" ], "identical_ids": true, "label": "gpt2", "id": "gpt2", "near_duplicates": [] }, { "members": [ "pythia-160m" ], "identical_ids": true, "label": "pythia-160m", "id": "pythia-160m", "near_duplicates": [] }, { "members": [ "p50k_base" ], "identical_ids": true, "label": "OpenAI p50k_base (Codex)", "id": "p50k_base", "near_duplicates": [] }, { "members": [ "sarvam-1" ], "identical_ids": true, "label": "sarvam-1", "id": "sarvam-1", "near_duplicates": [] }, { "members": [ "cl100k_base", "olmo-2-0425-1b", "olmo-3-1025-7b", "phi-4" ], "identical_ids": true, "label": "cl100k_base (GPT-4, OLMo-2-0425-1B, Olmo-3-1025-7B, phi-4)", "id": "cl100k_base", "near_duplicates": [] }, { "members": [ "ernie-4-5-21b-a3b-pt" ], "identical_ids": true, "label": "ERNIE-4.5-21B-A3B-PT", "id": "ernie-4-5-21b-a3b-pt", "near_duplicates": [] }, { "members": [ "hy3" ], "identical_ids": true, "label": "Hy3", "id": "hy3", "near_duplicates": [] }, { "members": [ "lfm3-1-7b" ], "identical_ids": true, "label": "LFM3-1.7B", "id": "lfm3-1-7b", "near_duplicates": [] }, { "members": [ "llama-3-2-1b" ], "identical_ids": true, "label": "Llama-3.2-1B", "id": "llama-3-2-1b", "near_duplicates": [] }, { "members": [ "deepseek-v3" ], "identical_ids": true, "label": "DeepSeek-V3", "id": "deepseek-v3", "near_duplicates": [] }, { "members": [ "mistral-nemo-base-2407" ], "identical_ids": true, "label": "Mistral-Nemo-Base-2407", "id": "mistral-nemo-base-2407", "near_duplicates": [] }, { "members": [ "qwen2-5-0-5b", "qwen3-0-6b" ], "identical_ids": true, "label": "Qwen2.5-0.5B (same tokenizer: Qwen3-0.6B)", "id": "qwen2-5-0-5b", "near_duplicates": [] }, { "members": [ "glm-5-3-flash" ], "identical_ids": true, "label": "GLM-5.3-Flash", "id": "glm-5-3-flash", "near_duplicates": [] }, { "members": [ "seed-oss-36b-instruct" ], "identical_ids": true, "label": "Seed-OSS-36B-Instruct", "id": "seed-oss-36b-instruct", "near_duplicates": [] }, { "members": [ "ling-3-0-tiny" ], "identical_ids": true, "label": "Ling-3.0-tiny", "id": "ling-3-0-tiny", "near_duplicates": [] }, { "members": [ "kimi-k3" ], "identical_ids": true, "label": "Kimi-K3", "id": "kimi-k3", "near_duplicates": [] }, { "members": [ "o200k_base" ], "identical_ids": true, "label": "OpenAI o200k_base (GPT-4o)", "id": "o200k_base", "near_duplicates": [] }, { "members": [ "minimax-m2-5" ], "identical_ids": true, "label": "MiniMax-M2.5", "id": "minimax-m2-5", "near_duplicates": [] }, { "members": [ "mistral-large-4-1t-a52b" ], "identical_ids": true, "label": "Mistral-Large-4-1T-A52B", "id": "mistral-large-4-1t-a52b", "near_duplicates": [] }, { "members": [ "qwen3-8-27b" ], "identical_ids": true, "label": "Qwen3.8-27B", "id": "qwen3-8-27b", "near_duplicates": [] }, { "members": [ "north-micro-vision-instruct" ], "identical_ids": true, "label": "North-Micro-Vision-Instruct", "id": "north-micro-vision-instruct", "near_duplicates": [] }, { "members": [ "falcon-h1-34b-instruct" ], "identical_ids": true, "label": "Falcon-H1-34B-Instruct", "id": "falcon-h1-34b-instruct", "near_duplicates": [] }, { "members": [ "gemma-4-31b-it" ], "identical_ids": true, "label": "gemma-4-31B-it", "id": "gemma-4-31b-it", "near_duplicates": [ { "id": "gemma-3-1b-it", "label": "gemma-3-1b-it", "sets_with_different_counts": 19 } ] }, { "members": [ "sarvam-105b" ], "identical_ids": true, "label": "sarvam-105b", "id": "sarvam-105b", "near_duplicates": [] }, { "members": [ "gemma-3-1b-it" ], "identical_ids": true, "label": "gemma-3-1b-it", "id": "gemma-3-1b-it", "near_duplicates": [ { "id": "gemma-4-31b-it", "label": "gemma-4-31B-it", "sets_with_different_counts": 19 } ] } ], "rankings": { "multilingual": { "description": "multilingual and mixed-language chat, parity with English, no byte fallback", "weights": { "tokens_chat_multilingual": 1, "tokens_chat_mixed_language": 1, "bpt_languages": 1, "parity_mean_abs_log": 1, "byte_fallback_share": 1 }, "ranked": [ { "group": "Akshara 448k", "id": "akshara-448k", "score": { "value": 0.9928571428571429, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.9642857142857143, "tokens_chat_mixed_language": 1.0, "bpt_languages": 1.0, "parity_mean_abs_log": 1.0, "byte_fallback_share": 1.0 }, "values": { "tokens_chat_multilingual": 51424, "tokens_chat_mixed_language": 22958, "bpt_languages": 7.213171840136726, "parity_mean_abs_log": 0.2542721080357902, "byte_fallback_share": 0.0014840604132270476 }, "rank": 1 }, { "group": "Akshara 248k", "id": "akshara-248k", "score": { "value": 0.9285714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.7857142857142857, "tokens_chat_mixed_language": 0.9642857142857143, "bpt_languages": 0.9642857142857143, "parity_mean_abs_log": 0.9642857142857143, "byte_fallback_share": 0.9642857142857143 }, "values": { "tokens_chat_multilingual": 54663, "tokens_chat_mixed_language": 24303, "bpt_languages": 6.59150070810442, "parity_mean_abs_log": 0.32049966978282435, "byte_fallback_share": 0.0025449197164802108 }, "rank": 2 }, { "group": "Akshara 200k", "id": "akshara-200k", "score": { "value": 0.8785714285714287, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.75, "tokens_chat_mixed_language": 0.9285714285714286, "bpt_languages": 0.9285714285714286, "parity_mean_abs_log": 0.8571428571428572, "byte_fallback_share": 0.9285714285714286 }, "values": { "tokens_chat_multilingual": 56012, "tokens_chat_mixed_language": 24928, "bpt_languages": 6.350492189585523, "parity_mean_abs_log": 0.3473345700671691, "byte_fallback_share": 0.003131201977013887 }, "rank": 3 }, { "group": "gemma-3-1b-it", "id": "gemma-3-1b-it", "score": { "value": 0.875, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.8571428571428572, "tokens_chat_mixed_language": 0.875, "bpt_languages": 0.8571428571428572, "parity_mean_abs_log": 0.9285714285714286, "byte_fallback_share": 0.8571428571428572 }, "values": { "tokens_chat_multilingual": 53716, "tokens_chat_mixed_language": 25167, "bpt_languages": 5.966223786730707, "parity_mean_abs_log": 0.32984617060773985, "byte_fallback_share": 0.0036632293192396174 }, "rank": 4 }, { "group": "gemma-4-31B-it", "id": "gemma-4-31b-it", "score": { "value": 0.8607142857142858, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.8214285714285714, "tokens_chat_mixed_language": 0.875, "bpt_languages": 0.8214285714285714, "parity_mean_abs_log": 0.8928571428571429, "byte_fallback_share": 0.8928571428571429 }, "values": { "tokens_chat_multilingual": 53821, "tokens_chat_mixed_language": 25167, "bpt_languages": 5.9660930591060835, "parity_mean_abs_log": 0.32985771705191436, "byte_fallback_share": 0.00366319321237354 }, "rank": 5 }, { "group": "Akshara 151k", "id": "akshara-151k", "score": { "value": 0.7857142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.5714285714285714, "tokens_chat_mixed_language": 0.8214285714285714, "bpt_languages": 0.8928571428571429, "parity_mean_abs_log": 0.8214285714285714, "byte_fallback_share": 0.8214285714285714 }, "values": { "tokens_chat_multilingual": 57827, "tokens_chat_mixed_language": 25793, "bpt_languages": 6.019484072595167, "parity_mean_abs_log": 0.3858641683087437, "byte_fallback_share": 0.004016747638873015 }, "rank": 6 }, { "group": "OpenAI o200k_base (GPT-4o)", "id": "o200k_base", "score": { "value": 0.7642857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.9285714285714286, "tokens_chat_mixed_language": 0.7857142857142857, "bpt_languages": 0.6785714285714286, "parity_mean_abs_log": 0.7142857142857143, "byte_fallback_share": 0.7142857142857143 }, "values": { "tokens_chat_multilingual": 52950, "tokens_chat_mixed_language": 26250, "bpt_languages": 5.326356758143829, "parity_mean_abs_log": 0.4440150070128051, "byte_fallback_share": 0.008041190901993326 }, "rank": 7 }, { "group": "Mistral-Large-4-1T-A52B", "id": "mistral-large-4-1t-a52b", "score": { "value": 0.75, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.8928571428571429, "tokens_chat_mixed_language": 0.75, "bpt_languages": 0.7142857142857143, "parity_mean_abs_log": 0.7857142857142857, "byte_fallback_share": 0.6071428571428572 }, "values": { "tokens_chat_multilingual": 53059, "tokens_chat_mixed_language": 26596, "bpt_languages": 5.417237772859487, "parity_mean_abs_log": 0.4016180425047495, "byte_fallback_share": 0.011690707266487825 }, "rank": 8 }, { "group": "Akshara 120k", "id": "akshara-120k", "score": { "value": 0.7071428571428571, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.5, "tokens_chat_mixed_language": 0.7142857142857143, "bpt_languages": 0.7857142857142857, "parity_mean_abs_log": 0.75, "byte_fallback_share": 0.7857142857142857 }, "values": { "tokens_chat_multilingual": 59769, "tokens_chat_mixed_language": 26805, "bpt_languages": 5.738686309573592, "parity_mean_abs_log": 0.4174629065830746, "byte_fallback_share": 0.004938094263585416 }, "rank": 9 }, { "group": "Akshara 92k", "id": "akshara-92k", "score": { "value": 0.65, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.3928571428571429, "tokens_chat_mixed_language": 0.6785714285714286, "bpt_languages": 0.75, "parity_mean_abs_log": 0.6785714285714286, "byte_fallback_share": 0.75 }, "values": { "tokens_chat_multilingual": 62101, "tokens_chat_mixed_language": 27919, "bpt_languages": 5.418783595302582, "parity_mean_abs_log": 0.4507251644743021, "byte_fallback_share": 0.0063978967049031525 }, "rank": 10 }, { "group": "North-Micro-Vision-Instruct", "id": "north-micro-vision-instruct", "score": { "value": 0.65, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 1.0, "tokens_chat_mixed_language": 0.6428571428571428, "bpt_languages": 0.6071428571428572, "parity_mean_abs_log": 0.6071428571428572, "byte_fallback_share": 0.3928571428571429 }, "values": { "tokens_chat_multilingual": 51377, "tokens_chat_mixed_language": 28350, "bpt_languages": 5.111281463162758, "parity_mean_abs_log": 0.5123200691333425, "byte_fallback_share": 0.05088274291002607 }, "rank": 11 }, { "group": "Mistral-Nemo-Base-2407", "id": "mistral-nemo-base-2407", "score": { "value": 0.5714285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.6071428571428572, "tokens_chat_mixed_language": 0.6071428571428572, "bpt_languages": 0.5357142857142857, "parity_mean_abs_log": 0.5714285714285714, "byte_fallback_share": 0.5357142857142857 }, "values": { "tokens_chat_multilingual": 57645, "tokens_chat_mixed_language": 28794, "bpt_languages": 4.8051629951951735, "parity_mean_abs_log": 0.5139063938443821, "byte_fallback_share": 0.017356979048135063 }, "rank": 12 }, { "group": "LFM3-1.7B", "id": "lfm3-1-7b", "score": { "value": 0.5428571428571429, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.6428571428571428, "tokens_chat_mixed_language": 0.5357142857142857, "bpt_languages": 0.4285714285714286, "parity_mean_abs_log": 0.4285714285714286, "byte_fallback_share": 0.6785714285714286 }, "values": { "tokens_chat_multilingual": 57513, "tokens_chat_mixed_language": 30569, "bpt_languages": 4.417867671524511, "parity_mean_abs_log": 0.5953536933417769, "byte_fallback_share": 0.01021076196166285 }, "rank": 13 }, { "group": "Akshara 59k", "id": "akshara-59k", "score": { "value": 0.5357142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.25, "tokens_chat_mixed_language": 0.5714285714285714, "bpt_languages": 0.5714285714285714, "parity_mean_abs_log": 0.6428571428571428, "byte_fallback_share": 0.6428571428571428 }, "values": { "tokens_chat_multilingual": 66162, "tokens_chat_mixed_language": 29961, "bpt_languages": 4.906114056306499, "parity_mean_abs_log": 0.5022278156012671, "byte_fallback_share": 0.010684610988603281 }, "rank": 14 }, { "group": "DeepSeek-V3", "id": "deepseek-v3", "score": { "value": 0.47857142857142854, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.7142857142857143, "tokens_chat_mixed_language": 0.4642857142857143, "bpt_languages": 0.3928571428571429, "parity_mean_abs_log": 0.3571428571428571, "byte_fallback_share": 0.4642857142857143 }, "values": { "tokens_chat_multilingual": 57084, "tokens_chat_mixed_language": 31914, "bpt_languages": 4.26267324348883, "parity_mean_abs_log": 0.6862448561887082, "byte_fallback_share": 0.03423042314128435 }, "rank": 15 }, { "group": "Akshara 49k", "id": "akshara-49k", "score": { "value": 0.46428571428571425, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.2142857142857143, "tokens_chat_mixed_language": 0.5, "bpt_languages": 0.5, "parity_mean_abs_log": 0.5357142857142857, "byte_fallback_share": 0.5714285714285714 }, "values": { "tokens_chat_multilingual": 68002, "tokens_chat_mixed_language": 31005, "bpt_languages": 4.7026228903209955, "parity_mean_abs_log": 0.5195910932603355, "byte_fallback_share": 0.01301886031912907 }, "rank": 16 }, { "group": "Falcon-H1-34B-Instruct", "id": "falcon-h1-34b-instruct", "score": { "value": 0.4428571428571429, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.5357142857142857, "tokens_chat_mixed_language": 0.4285714285714286, "bpt_languages": 0.4642857142857143, "parity_mean_abs_log": 0.5, "byte_fallback_share": 0.2857142857142857 }, "values": { "tokens_chat_multilingual": 58120, "tokens_chat_mixed_language": 32199, "bpt_languages": 4.620032299561019, "parity_mean_abs_log": 0.555894460840154, "byte_fallback_share": 0.10179611775253003 }, "rank": 17 }, { "group": "Akshara 32k", "id": "akshara-32k", "score": { "value": 0.37142857142857144, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.1428571428571429, "tokens_chat_mixed_language": 0.3928571428571429, "bpt_languages": 0.3571428571428571, "parity_mean_abs_log": 0.4642857142857143, "byte_fallback_share": 0.5 }, "values": { "tokens_chat_multilingual": 72421, "tokens_chat_mixed_language": 33562, "bpt_languages": 4.239961763507105, "parity_mean_abs_log": 0.5635204207046608, "byte_fallback_share": 0.02148318525961295 }, "rank": 18 }, { "group": "sarvam-105b", "id": "sarvam-105b", "score": { "value": 0.36428571428571427, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.4285714285714286, "tokens_chat_mixed_language": 0.2857142857142857, "bpt_languages": 0.6428571428571428, "parity_mean_abs_log": 0.3928571428571429, "byte_fallback_share": 0.0714285714285714 }, "values": { "tokens_chat_multilingual": 61691, "tokens_chat_mixed_language": 36378, "bpt_languages": 5.222810137920554, "parity_mean_abs_log": 0.6278657201181677, "byte_fallback_share": 0.44851763768129654 }, "rank": 19 }, { "group": "ERNIE-4.5-21B-A3B-PT", "id": "ernie-4-5-21b-a3b-pt", "score": { "value": 0.34285714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.2857142857142857, "tokens_chat_mixed_language": 0.3571428571428571, "bpt_languages": 0.3214285714285714, "parity_mean_abs_log": 0.3214285714285714, "byte_fallback_share": 0.4285714285714286 }, "values": { "tokens_chat_multilingual": 66085, "tokens_chat_mixed_language": 33845, "bpt_languages": 3.8813744781985733, "parity_mean_abs_log": 0.6915169267054722, "byte_fallback_share": 0.035451642326111935 }, "rank": 20 }, { "group": "Kimi-K3", "id": "kimi-k3", "score": { "value": 0.3071428571428571, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.3214285714285714, "tokens_chat_mixed_language": 0.3214285714285714, "bpt_languages": 0.25, "parity_mean_abs_log": 0.2857142857142857, "byte_fallback_share": 0.3571428571428571 }, "values": { "tokens_chat_multilingual": 65407, "tokens_chat_mixed_language": 35210, "bpt_languages": 3.592659022298393, "parity_mean_abs_log": 0.8384546893437685, "byte_fallback_share": 0.058563459613022874 }, "rank": 21 }, { "group": "GLM-5.3-Flash", "id": "glm-5-3-flash", "score": { "value": 0.29285714285714287, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.6785714285714286, "tokens_chat_mixed_language": 0.1785714285714286, "bpt_languages": 0.1785714285714286, "parity_mean_abs_log": 0.1785714285714286, "byte_fallback_share": 0.25 }, "values": { "tokens_chat_multilingual": 57233, "tokens_chat_mixed_language": 42375, "bpt_languages": 3.308501833006265, "parity_mean_abs_log": 0.9329646447008982, "byte_fallback_share": 0.17656260475094415 }, "rank": 22 }, { "group": "Hy3", "id": "hy3", "score": { "value": 0.2785714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.3571428571428571, "tokens_chat_mixed_language": 0.25, "bpt_languages": 0.2142857142857143, "parity_mean_abs_log": 0.25, "byte_fallback_share": 0.3214285714285714 }, "values": { "tokens_chat_multilingual": 64276, "tokens_chat_mixed_language": 36821, "bpt_languages": 3.4941530619506898, "parity_mean_abs_log": 0.8561972944417939, "byte_fallback_share": 0.08917222864291212 }, "rank": 23 }, { "group": "Llama-3.2-1B", "id": "llama-3-2-1b", "score": { "value": 0.2785714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.4642857142857143, "tokens_chat_mixed_language": 0.2142857142857143, "bpt_languages": 0.2857142857142857, "parity_mean_abs_log": 0.2142857142857143, "byte_fallback_share": 0.2142857142857143 }, "values": { "tokens_chat_multilingual": 60394, "tokens_chat_mixed_language": 40321, "bpt_languages": 3.6567762973932996, "parity_mean_abs_log": 0.8644443457955898, "byte_fallback_share": 0.18432713640698287 }, "rank": 24 }, { "group": "cl100k_base (GPT-4, OLMo-2-0425-1B, Olmo-3-1025-7B, phi-4)", "id": "cl100k_base", "score": { "value": 0.1571428571428572, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.1785714285714286, "tokens_chat_mixed_language": 0.1428571428571429, "bpt_languages": 0.1428571428571429, "parity_mean_abs_log": 0.1428571428571429, "byte_fallback_share": 0.1785714285714286 }, "values": { "tokens_chat_multilingual": 69668, "tokens_chat_mixed_language": 47553, "bpt_languages": 2.6439619328800035, "parity_mean_abs_log": 1.1498212691865821, "byte_fallback_share": 0.22386126267534295 }, "rank": 25 }, { "group": "Phi-3.5-mini-instruct", "id": "phi-3-5-mini-instruct", "score": { "value": 0.11428571428571425, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.1071428571428571, "tokens_chat_mixed_language": 0.1071428571428571, "bpt_languages": 0.1071428571428571, "parity_mean_abs_log": 0.1071428571428571, "byte_fallback_share": 0.1428571428571429 }, "values": { "tokens_chat_multilingual": 76201, "tokens_chat_mixed_language": 56772, "bpt_languages": 2.3875360774114562, "parity_mean_abs_log": 1.1598479000579562, "byte_fallback_share": 0.2887037126387521 }, "rank": 26 }, { "group": "SmolLM2-135M", "id": "smollm2-135m", "score": { "value": 0.07857142857142854, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.0714285714285714, "tokens_chat_mixed_language": 0.0714285714285714, "bpt_languages": 0.0714285714285714, "parity_mean_abs_log": 0.0714285714285714, "byte_fallback_share": 0.1071428571428571 }, "values": { "tokens_chat_multilingual": 98488, "tokens_chat_mixed_language": 64809, "bpt_languages": 1.9328267604043268, "parity_mean_abs_log": 1.4791129059909334, "byte_fallback_share": 0.4237258034382337 }, "rank": 27 }, { "group": "OpenAI p50k_base (Codex)", "id": "p50k_base", "score": { "value": 0.02857142857142856, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.0357142857142857, "tokens_chat_mixed_language": 0.0357142857142857, "bpt_languages": 0.0357142857142857, "parity_mean_abs_log": 0.0357142857142857, "byte_fallback_share": 0.0 }, "values": { "tokens_chat_multilingual": 110194, "tokens_chat_mixed_language": 71477, "bpt_languages": 1.7883580832246342, "parity_mean_abs_log": 1.6052962627128946, "byte_fallback_share": 0.5307670920608782 }, "rank": 28 }, { "group": "gpt2", "id": "gpt2", "score": { "value": 0.00714285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_chat_multilingual": 0.0, "tokens_chat_mixed_language": 0.0, "bpt_languages": 0.0, "parity_mean_abs_log": 0.0, "byte_fallback_share": 0.0357142857142857 }, "values": { "tokens_chat_multilingual": 111625, "tokens_chat_mixed_language": 71484, "bpt_languages": 1.7870872744910007, "parity_mean_abs_log": 1.6053052794507416, "byte_fallback_share": 0.5303119715109536 }, "rank": 29 } ], "not_ranked": [ { "group": "pythia-160m", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "sarvam-1", "reason": "fails lossless round-trip on 215393 inputs" }, { "group": "Qwen2.5-0.5B (same tokenizer: Qwen3-0.6B)", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Seed-OSS-36B-Instruct", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Ling-3.0-tiny", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "MiniMax-M2.5", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Qwen3.8-27B", "reason": "fails lossless round-trip on 733 inputs" } ] }, "code_heavy": { "description": "code, tool calls, JSON and logs, plus encode speed", "weights": { "tokens_code": 1, "tokens_agent_tool_calls": 1, "tokens_json": 1, "tokens_logs": 1, "encode_mbps_1t_medium": 1 }, "ranked": [ { "group": "Akshara 448k", "id": "akshara-448k", "score": { "value": 0.9142857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 1.0, "tokens_agent_tool_calls": 1.0, "tokens_json": 1.0, "tokens_logs": 1.0, "encode_mbps_1t_medium": 0.5714285714285714 }, "values": { "tokens_code": 213885, "tokens_agent_tool_calls": 118768, "tokens_json": 311319, "tokens_logs": 398575, "encode_mbps_1t_medium": 5.620347250340921 }, "rank": 1 }, { "group": "Akshara 248k", "id": "akshara-248k", "score": { "value": 0.9, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.9642857142857143, "tokens_agent_tool_calls": 0.9642857142857143, "tokens_json": 0.9642857142857143, "tokens_logs": 0.9642857142857143, "encode_mbps_1t_medium": 0.6428571428571428 }, "values": { "tokens_code": 218773, "tokens_agent_tool_calls": 120183, "tokens_json": 314663, "tokens_logs": 401363, "encode_mbps_1t_medium": 5.836628228431435 }, "rank": 2 }, { "group": "Akshara 200k", "id": "akshara-200k", "score": { "value": 0.8571428571428571, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.8928571428571429, "tokens_agent_tool_calls": 0.9285714285714286, "tokens_json": 0.9285714285714286, "tokens_logs": 0.9285714285714286, "encode_mbps_1t_medium": 0.6071428571428572 }, "values": { "tokens_code": 221040, "tokens_agent_tool_calls": 120901, "tokens_json": 316416, "tokens_logs": 403511, "encode_mbps_1t_medium": 5.734110759561312 }, "rank": 3 }, { "group": "Akshara 151k", "id": "akshara-151k", "score": { "value": 0.8142857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.7857142857142857, "tokens_agent_tool_calls": 0.8928571428571429, "tokens_json": 0.8571428571428572, "tokens_logs": 0.75, "encode_mbps_1t_medium": 0.7857142857142857 }, "values": { "tokens_code": 224164, "tokens_agent_tool_calls": 122559, "tokens_json": 320496, "tokens_logs": 410909, "encode_mbps_1t_medium": 6.116414017986229 }, "rank": 4 }, { "group": "Akshara 120k", "id": "akshara-120k", "score": { "value": 0.7285714285714285, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.6428571428571428, "tokens_agent_tool_calls": 0.8571428571428572, "tokens_json": 0.7857142857142857, "tokens_logs": 0.6785714285714286, "encode_mbps_1t_medium": 0.6785714285714286 }, "values": { "tokens_code": 226687, "tokens_agent_tool_calls": 123467, "tokens_json": 322331, "tokens_logs": 415437, "encode_mbps_1t_medium": 5.967619867743156 }, "rank": 5 }, { "group": "OpenAI o200k_base (GPT-4o)", "id": "o200k_base", "score": { "value": 0.7071428571428571, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.7142857142857143, "tokens_agent_tool_calls": 0.7857142857142857, "tokens_json": 0.8214285714285714, "tokens_logs": 0.7857142857142857, "encode_mbps_1t_medium": 0.4285714285714286 }, "values": { "tokens_code": 224729, "tokens_agent_tool_calls": 126331, "tokens_json": 320664, "tokens_logs": 410777, "encode_mbps_1t_medium": 5.101910820251484 }, "rank": 6 }, { "group": "North-Micro-Vision-Instruct", "id": "north-micro-vision-instruct", "score": { "value": 0.7, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.9285714285714286, "tokens_agent_tool_calls": 0.75, "tokens_json": 0.8928571428571429, "tokens_logs": 0.8928571428571429, "encode_mbps_1t_medium": 0.0357142857142857 }, "values": { "tokens_code": 220175, "tokens_agent_tool_calls": 126781, "tokens_json": 318389, "tokens_logs": 407196, "encode_mbps_1t_medium": 4.009868145514843 }, "rank": 7 }, { "group": "Akshara 92k", "id": "akshara-92k", "score": { "value": 0.6642857142857144, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.6071428571428572, "tokens_agent_tool_calls": 0.8214285714285714, "tokens_json": 0.5714285714285714, "tokens_logs": 0.6071428571428572, "encode_mbps_1t_medium": 0.7142857142857143 }, "values": { "tokens_code": 230801, "tokens_agent_tool_calls": 125092, "tokens_json": 326205, "tokens_logs": 419943, "encode_mbps_1t_medium": 6.100163595476987 }, "rank": 8 }, { "group": "Llama-3.2-1B", "id": "llama-3-2-1b", "score": { "value": 0.6214285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.8571428571428572, "tokens_agent_tool_calls": 0.3928571428571429, "tokens_json": 0.7142857142857143, "tokens_logs": 0.8571428571428572, "encode_mbps_1t_medium": 0.2857142857142857 }, "values": { "tokens_code": 222746, "tokens_agent_tool_calls": 138670, "tokens_json": 322854, "tokens_logs": 410572, "encode_mbps_1t_medium": 4.979777107529496 }, "rank": 9 }, { "group": "Mistral-Large-4-1T-A52B", "id": "mistral-large-4-1t-a52b", "score": { "value": 0.6142857142857142, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.5714285714285714, "tokens_agent_tool_calls": 0.7142857142857143, "tokens_json": 0.75, "tokens_logs": 0.6428571428571428, "encode_mbps_1t_medium": 0.3928571428571429 }, "values": { "tokens_code": 233232, "tokens_agent_tool_calls": 126785, "tokens_json": 322537, "tokens_logs": 416106, "encode_mbps_1t_medium": 5.069619965517145 }, "rank": 10 }, { "group": "cl100k_base (GPT-4, OLMo-2-0425-1B, Olmo-3-1025-7B, phi-4)", "id": "cl100k_base", "score": { "value": 0.5857142857142856, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.75, "tokens_agent_tool_calls": 0.25, "tokens_json": 0.6428571428571428, "tokens_logs": 0.8214285714285714, "encode_mbps_1t_medium": 0.4642857142857143 }, "values": { "tokens_code": 224357, "tokens_agent_tool_calls": 148252, "tokens_json": 325029, "tokens_logs": 410578, "encode_mbps_1t_medium": 5.153221493593562 }, "rank": 11 }, { "group": "Akshara 59k", "id": "akshara-59k", "score": { "value": 0.5785714285714285, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.5, "tokens_agent_tool_calls": 0.6785714285714286, "tokens_json": 0.4642857142857143, "tokens_logs": 0.4285714285714286, "encode_mbps_1t_medium": 0.8214285714285714 }, "values": { "tokens_code": 238588, "tokens_agent_tool_calls": 128071, "tokens_json": 334371, "tokens_logs": 436105, "encode_mbps_1t_medium": 6.198963235580143 }, "rank": 12 }, { "group": "Akshara 49k", "id": "akshara-49k", "score": { "value": 0.5428571428571429, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.4285714285714286, "tokens_agent_tool_calls": 0.6071428571428572, "tokens_json": 0.4285714285714286, "tokens_logs": 0.3928571428571429, "encode_mbps_1t_medium": 0.8571428571428572 }, "values": { "tokens_code": 243110, "tokens_agent_tool_calls": 130641, "tokens_json": 338035, "tokens_logs": 440506, "encode_mbps_1t_medium": 6.273736117150162 }, "rank": 13 }, { "group": "Kimi-K3", "id": "kimi-k3", "score": { "value": 0.5285714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.6785714285714286, "tokens_agent_tool_calls": 0.5, "tokens_json": 0.5, "tokens_logs": 0.7142857142857143, "encode_mbps_1t_medium": 0.25 }, "values": { "tokens_code": 225688, "tokens_agent_tool_calls": 133875, "tokens_json": 326493, "tokens_logs": 410951, "encode_mbps_1t_medium": 4.938964989355018 }, "rank": 14 }, { "group": "LFM3-1.7B", "id": "lfm3-1-7b", "score": { "value": 0.5071428571428571, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.3928571428571429, "tokens_agent_tool_calls": 0.6428571428571428, "tokens_json": 0.6785714285714286, "tokens_logs": 0.5, "encode_mbps_1t_medium": 0.3214285714285714 }, "values": { "tokens_code": 243454, "tokens_agent_tool_calls": 129918, "tokens_json": 324359, "tokens_logs": 431342, "encode_mbps_1t_medium": 4.983365371128134 }, "rank": 15 }, { "group": "Akshara 32k", "id": "akshara-32k", "score": { "value": 0.46428571428571425, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.3214285714285714, "tokens_agent_tool_calls": 0.4642857142857143, "tokens_json": 0.3571428571428571, "tokens_logs": 0.2857142857142857, "encode_mbps_1t_medium": 0.8928571428571429 }, "values": { "tokens_code": 253779, "tokens_agent_tool_calls": 135769, "tokens_json": 349737, "tokens_logs": 452118, "encode_mbps_1t_medium": 6.303510437376889 }, "rank": 16 }, { "group": "Hy3", "id": "hy3", "score": { "value": 0.4428571428571429, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.5357142857142857, "tokens_agent_tool_calls": 0.4285714285714286, "tokens_json": 0.6071428571428572, "tokens_logs": 0.5714285714285714, "encode_mbps_1t_medium": 0.0714285714285714 }, "values": { "tokens_code": 235462, "tokens_agent_tool_calls": 136638, "tokens_json": 325834, "tokens_logs": 421768, "encode_mbps_1t_medium": 4.135631116558927 }, "rank": 17 }, { "group": "GLM-5.3-Flash", "id": "glm-5-3-flash", "score": { "value": 0.4357142857142858, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.8214285714285714, "tokens_agent_tool_calls": 0.2857142857142857, "tokens_json": 0.3928571428571429, "tokens_logs": 0.4642857142857143, "encode_mbps_1t_medium": 0.2142857142857143 }, "values": { "tokens_code": 223243, "tokens_agent_tool_calls": 141802, "tokens_json": 340744, "tokens_logs": 431637, "encode_mbps_1t_medium": 4.882575809711876 }, "rank": 18 }, { "group": "DeepSeek-V3", "id": "deepseek-v3", "score": { "value": 0.43571428571428567, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.4642857142857143, "tokens_agent_tool_calls": 0.5357142857142857, "tokens_json": 0.5357142857142857, "tokens_logs": 0.5357142857142857, "encode_mbps_1t_medium": 0.1071428571428571 }, "values": { "tokens_code": 241458, "tokens_agent_tool_calls": 132588, "tokens_json": 326337, "tokens_logs": 427183, "encode_mbps_1t_medium": 4.593484509872189 }, "rank": 19 }, { "group": "Mistral-Nemo-Base-2407", "id": "mistral-nemo-base-2407", "score": { "value": 0.39285714285714285, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.3571428571428571, "tokens_agent_tool_calls": 0.5714285714285714, "tokens_json": 0.2857142857142857, "tokens_logs": 0.25, "encode_mbps_1t_medium": 0.5 }, "values": { "tokens_code": 243869, "tokens_agent_tool_calls": 131769, "tokens_json": 395902, "tokens_logs": 565520, "encode_mbps_1t_medium": 5.466480170437732 }, "rank": 20 }, { "group": "gemma-4-31B-it", "id": "gemma-4-31b-it", "score": { "value": 0.3678571428571429, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.25, "tokens_agent_tool_calls": 0.3392857142857143, "tokens_json": 0.1428571428571429, "tokens_logs": 0.1785714285714286, "encode_mbps_1t_medium": 0.9285714285714286 }, "values": { "tokens_code": 267604, "tokens_agent_tool_calls": 140393, "tokens_json": 422477, "tokens_logs": 574778, "encode_mbps_1t_medium": 6.426287669332646 }, "rank": 21 }, { "group": "gemma-3-1b-it", "id": "gemma-3-1b-it", "score": { "value": 0.3392857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.2857142857142857, "tokens_agent_tool_calls": 0.3392857142857143, "tokens_json": 0.1428571428571429, "tokens_logs": 0.1785714285714286, "encode_mbps_1t_medium": 0.75 }, "values": { "tokens_code": 266710, "tokens_agent_tool_calls": 140393, "tokens_json": 422477, "tokens_logs": 574778, "encode_mbps_1t_medium": 6.103693269272489 }, "rank": 22 }, { "group": "ERNIE-4.5-21B-A3B-PT", "id": "ernie-4-5-21b-a3b-pt", "score": { "value": 0.3071428571428571, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.1785714285714286, "tokens_agent_tool_calls": 0.2142857142857143, "tokens_json": 0.0714285714285714, "tokens_logs": 0.1071428571428571, "encode_mbps_1t_medium": 0.9642857142857143 }, "values": { "tokens_code": 279541, "tokens_agent_tool_calls": 148859, "tokens_json": 426135, "tokens_logs": 575770, "encode_mbps_1t_medium": 6.455317587551054 }, "rank": 23 }, { "group": "sarvam-105b", "id": "sarvam-105b", "score": { "value": 0.25, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.2142857142857143, "tokens_agent_tool_calls": 0.1785714285714286, "tokens_json": 0.1428571428571429, "tokens_logs": 0.1785714285714286, "encode_mbps_1t_medium": 0.5357142857142857 }, "values": { "tokens_code": 270349, "tokens_agent_tool_calls": 156109, "tokens_json": 422477, "tokens_logs": 574778, "encode_mbps_1t_medium": 5.616660252990121 }, "rank": 24 }, { "group": "Phi-3.5-mini-instruct", "id": "phi-3-5-mini-instruct", "score": { "value": 0.24285714285714283, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.0357142857142857, "tokens_agent_tool_calls": 0.1071428571428571, "tokens_json": 0.0357142857142857, "tokens_logs": 0.0357142857142857, "encode_mbps_1t_medium": 1.0 }, "values": { "tokens_code": 300961, "tokens_agent_tool_calls": 174342, "tokens_json": 435188, "tokens_logs": 596201, "encode_mbps_1t_medium": 6.788077586874321 }, "rank": 25 }, { "group": "OpenAI p50k_base (Codex)", "id": "p50k_base", "score": { "value": 0.22857142857142856, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.0714285714285714, "tokens_agent_tool_calls": 0.0357142857142857, "tokens_json": 0.3214285714285714, "tokens_logs": 0.3571428571428571, "encode_mbps_1t_medium": 0.3571428571428571 }, "values": { "tokens_code": 292387, "tokens_agent_tool_calls": 187933, "tokens_json": 362593, "tokens_logs": 447347, "encode_mbps_1t_medium": 5.033180687959843 }, "rank": 26 }, { "group": "SmolLM2-135M", "id": "smollm2-135m", "score": { "value": 0.14285714285714285, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.1428571428571429, "tokens_agent_tool_calls": 0.0714285714285714, "tokens_json": 0.25, "tokens_logs": 0.0714285714285714, "encode_mbps_1t_medium": 0.1785714285714286 }, "values": { "tokens_code": 285563, "tokens_agent_tool_calls": 177092, "tokens_json": 409936, "tokens_logs": 586540, "encode_mbps_1t_medium": 4.853248808026626 }, "rank": 27 }, { "group": "gpt2", "id": "gpt2", "score": { "value": 0.13571428571428573, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.0, "tokens_agent_tool_calls": 0.0, "tokens_json": 0.2142857142857143, "tokens_logs": 0.3214285714285714, "encode_mbps_1t_medium": 0.1428571428571429 }, "values": { "tokens_code": 365096, "tokens_agent_tool_calls": 219131, "tokens_json": 413899, "tokens_logs": 449011, "encode_mbps_1t_medium": 4.814825216097814 }, "rank": 28 }, { "group": "Falcon-H1-34B-Instruct", "id": "falcon-h1-34b-instruct", "score": { "value": 0.05, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_code": 0.1071428571428571, "tokens_agent_tool_calls": 0.1428571428571429, "tokens_json": 0.0, "tokens_logs": 0.0, "encode_mbps_1t_medium": 0.0 }, "values": { "tokens_code": 285645, "tokens_agent_tool_calls": 164832, "tokens_json": 458328, "tokens_logs": 609637, "encode_mbps_1t_medium": 2.765068889391423 }, "rank": 29 } ], "not_ranked": [ { "group": "pythia-160m", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "sarvam-1", "reason": "fails lossless round-trip on 215393 inputs" }, { "group": "Qwen2.5-0.5B (same tokenizer: Qwen3-0.6B)", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Seed-OSS-36B-Instruct", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Ling-3.0-tiny", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "MiniMax-M2.5", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Qwen3.8-27B", "reason": "fails lossless round-trip on 733 inputs" } ] }, "low_memory": { "description": "tokenizer memory and load time, plus embedding size a model would need", "weights": { "rss_load_delta_mb": 1, "rss_peak_large_mb": 1, "load_seconds": 1, "embedding_params_h2048": 1 }, "ranked": [ { "group": "Phi-3.5-mini-instruct", "id": "phi-3-5-mini-instruct", "score": { "value": 0.9732142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.9642857142857143, "rss_peak_large_mb": 1.0, "load_seconds": 0.9642857142857143, "embedding_params_h2048": 0.9642857142857143 }, "values": { "rss_load_delta_mb": 30.879744, "rss_peak_large_mb": 138.862592, "load_seconds": 0.04443212499973015, "embedding_params_h2048": 65667072 }, "rank": 1 }, { "group": "Akshara 32k", "id": "akshara-32k", "score": { "value": 0.9642857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 1.0, "rss_peak_large_mb": 0.8571428571428572, "load_seconds": 1.0, "embedding_params_h2048": 1.0 }, "values": { "rss_load_delta_mb": 24.612864, "rss_peak_large_mb": 176.607232, "load_seconds": 0.03534070899991093, "embedding_params_h2048": 65536000 }, "rank": 2 }, { "group": "Akshara 49k", "id": "akshara-49k", "score": { "value": 0.8839285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.9285714285714286, "rss_peak_large_mb": 0.8214285714285714, "load_seconds": 0.8571428571428572, "embedding_params_h2048": 0.9285714285714286 }, "values": { "rss_load_delta_mb": 33.431552, "rss_peak_large_mb": 178.335744, "load_seconds": 0.05253670900037832, "embedding_params_h2048": 100352000 }, "rank": 3 }, { "group": "Akshara 59k", "id": "akshara-59k", "score": { "value": 0.8392857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.8214285714285714, "rss_peak_large_mb": 0.9285714285714286, "load_seconds": 0.8214285714285714, "embedding_params_h2048": 0.7857142857142857 }, "values": { "rss_load_delta_mb": 38.756352, "rss_peak_large_mb": 172.630016, "load_seconds": 0.06612683299999844, "embedding_params_h2048": 120832000 }, "rank": 4 }, { "group": "SmolLM2-135M", "id": "smollm2-135m", "score": { "value": 0.8125, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.8928571428571429, "rss_peak_large_mb": 0.5714285714285714, "load_seconds": 0.8928571428571429, "embedding_params_h2048": 0.8928571428571429 }, "values": { "rss_load_delta_mb": 34.832384, "rss_peak_large_mb": 207.511552, "load_seconds": 0.050100375000056374, "embedding_params_h2048": 100663296 }, "rank": 5 }, { "group": "Akshara 92k", "id": "akshara-92k", "score": { "value": 0.8035714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.7857142857142857, "rss_peak_large_mb": 0.8928571428571429, "load_seconds": 0.7857142857142857, "embedding_params_h2048": 0.75 }, "values": { "rss_load_delta_mb": 45.416448, "rss_peak_large_mb": 174.1824, "load_seconds": 0.10785962499994639, "embedding_params_h2048": 188416000 }, "rank": 6 }, { "group": "gpt2", "id": "gpt2", "score": { "value": 0.7589285714285715, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.8571428571428572, "rss_peak_large_mb": 0.3928571428571429, "load_seconds": 0.9285714285714286, "embedding_params_h2048": 0.8571428571428572 }, "values": { "rss_load_delta_mb": 35.856384, "rss_peak_large_mb": 240.410624, "load_seconds": 0.04961520899996685, "embedding_params_h2048": 102926336 }, "rank": 7 }, { "group": "Akshara 120k", "id": "akshara-120k", "score": { "value": 0.7053571428571428, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.6785714285714286, "rss_peak_large_mb": 0.7857142857142857, "load_seconds": 0.7142857142857143, "embedding_params_h2048": 0.6428571428571428 }, "values": { "rss_load_delta_mb": 81.068032, "rss_peak_large_mb": 187.088896, "load_seconds": 0.1461671250000336, "embedding_params_h2048": 245760000 }, "rank": 8 }, { "group": "OpenAI p50k_base (Codex)", "id": "p50k_base", "score": { "value": 0.7053571428571428, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.7142857142857143, "rss_peak_large_mb": 0.5357142857142857, "load_seconds": 0.75, "embedding_params_h2048": 0.8214285714285714 }, "values": { "rss_load_delta_mb": 80.216064, "rss_peak_large_mb": 216.608768, "load_seconds": 0.13893637499995748, "embedding_params_h2048": 102975488 }, "rank": 9 }, { "group": "ERNIE-4.5-21B-A3B-PT", "id": "ernie-4-5-21b-a3b-pt", "score": { "value": 0.6696428571428572, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.5, "rss_peak_large_mb": 0.9642857142857143, "load_seconds": 0.5357142857142857, "embedding_params_h2048": 0.6785714285714286 }, "values": { "rss_load_delta_mb": 108.675072, "rss_peak_large_mb": 165.261312, "load_seconds": 0.22060670800010485, "embedding_params_h2048": 211812352 }, "rank": 10 }, { "group": "Hy3", "id": "hy3", "score": { "value": 0.6160714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.5714285714285714, "rss_peak_large_mb": 0.6428571428571428, "load_seconds": 0.6428571428571428, "embedding_params_h2048": 0.6071428571428572 }, "values": { "rss_load_delta_mb": 93.503488, "rss_peak_large_mb": 199.356416, "load_seconds": 0.1811083749998943, "embedding_params_h2048": 247463936 }, "rank": 11 }, { "group": "Mistral-Nemo-Base-2407", "id": "mistral-nemo-base-2407", "score": { "value": 0.6160714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.75, "rss_peak_large_mb": 0.6785714285714286, "load_seconds": 0.5714285714285714, "embedding_params_h2048": 0.4642857142857143 }, "values": { "rss_load_delta_mb": 79.0528, "rss_peak_large_mb": 194.605056, "load_seconds": 0.20511329200007822, "embedding_params_h2048": 268435456 }, "rank": 12 }, { "group": "DeepSeek-V3", "id": "deepseek-v3", "score": { "value": 0.6071428571428572, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.6428571428571428, "rss_peak_large_mb": 0.6071428571428572, "load_seconds": 0.6785714285714286, "embedding_params_h2048": 0.5 }, "values": { "rss_load_delta_mb": 83.18976, "rss_peak_large_mb": 199.45472, "load_seconds": 0.14764133300013782, "embedding_params_h2048": 264765440 }, "rank": 13 }, { "group": "Akshara 151k", "id": "akshara-151k", "score": { "value": 0.5803571428571429, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.5357142857142857, "rss_peak_large_mb": 0.75, "load_seconds": 0.6071428571428572, "embedding_params_h2048": 0.4285714285714286 }, "values": { "rss_load_delta_mb": 93.712384, "rss_peak_large_mb": 188.841984, "load_seconds": 0.199583500000017, "embedding_params_h2048": 309248000 }, "rank": 14 }, { "group": "Akshara 200k", "id": "akshara-200k", "score": { "value": 0.5357142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.6071428571428572, "rss_peak_large_mb": 0.7142857142857143, "load_seconds": 0.5, "embedding_params_h2048": 0.3214285714285714 }, "values": { "rss_load_delta_mb": 84.148224, "rss_peak_large_mb": 191.758336, "load_seconds": 0.2555477079999946, "embedding_params_h2048": 409600000 }, "rank": 15 }, { "group": "cl100k_base (GPT-4, OLMo-2-0425-1B, Olmo-3-1025-7B, phi-4)", "id": "cl100k_base", "score": { "value": 0.5178571428571429, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.4285714285714286, "rss_peak_large_mb": 0.4642857142857143, "load_seconds": 0.4642857142857143, "embedding_params_h2048": 0.7142857142857143 }, "values": { "rss_load_delta_mb": 160.456704, "rss_peak_large_mb": 231.231488, "load_seconds": 0.2929883750002773, "embedding_params_h2048": 205367296 }, "rank": 16 }, { "group": "Llama-3.2-1B", "id": "llama-3-2-1b", "score": { "value": 0.41964285714285715, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.2857142857142857, "rss_peak_large_mb": 0.4285714285714286, "load_seconds": 0.4285714285714286, "embedding_params_h2048": 0.5357142857142857 }, "values": { "rss_load_delta_mb": 179.073024, "rss_peak_large_mb": 239.906816, "load_seconds": 0.3169502499999908, "embedding_params_h2048": 262668288 }, "rank": 17 }, { "group": "LFM3-1.7B", "id": "lfm3-1-7b", "score": { "value": 0.4107142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.3928571428571429, "rss_peak_large_mb": 0.2857142857142857, "load_seconds": 0.3928571428571429, "embedding_params_h2048": 0.5714285714285714 }, "values": { "rss_load_delta_mb": 168.439808, "rss_peak_large_mb": 257.810432, "load_seconds": 0.3344739170001958, "embedding_params_h2048": 262144000 }, "rank": 18 }, { "group": "Akshara 248k", "id": "akshara-248k", "score": { "value": 0.3839285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.4642857142857143, "rss_peak_large_mb": 0.5, "load_seconds": 0.3571428571428571, "embedding_params_h2048": 0.2142857142857143 }, "values": { "rss_load_delta_mb": 119.328768, "rss_peak_large_mb": 220.925952, "load_seconds": 0.33785904199999095, "embedding_params_h2048": 507904000 }, "rank": 19 }, { "group": "GLM-5.3-Flash", "id": "glm-5-3-flash", "score": { "value": 0.3125, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.3214285714285714, "rss_peak_large_mb": 0.2142857142857143, "load_seconds": 0.3214285714285714, "embedding_params_h2048": 0.3928571428571429 }, "values": { "rss_load_delta_mb": 175.747072, "rss_peak_large_mb": 268.546048, "load_seconds": 0.38684520800006794, "embedding_params_h2048": 317194240 }, "rank": 20 }, { "group": "Kimi-K3", "id": "kimi-k3", "score": { "value": 0.2767857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.2142857142857143, "rss_peak_large_mb": 0.25, "load_seconds": 0.2857142857142857, "embedding_params_h2048": 0.3571428571428571 }, "values": { "rss_load_delta_mb": 188.178432, "rss_peak_large_mb": 260.07552, "load_seconds": 0.4387297920002311, "embedding_params_h2048": 335544320 }, "rank": 21 }, { "group": "North-Micro-Vision-Instruct", "id": "north-micro-vision-instruct", "score": { "value": 0.23660714285714285, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.25, "rss_peak_large_mb": 0.3571428571428571, "load_seconds": 0.25, "embedding_params_h2048": 0.0892857142857143 }, "values": { "rss_load_delta_mb": 185.212928, "rss_peak_large_mb": 254.160896, "load_seconds": 0.440281541999866, "embedding_params_h2048": 536870912 }, "rank": 22 }, { "group": "Falcon-H1-34B-Instruct", "id": "falcon-h1-34b-instruct", "score": { "value": 0.23214285714285715, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.3571428571428571, "rss_peak_large_mb": 0.1785714285714286, "load_seconds": 0.2142857142857143, "embedding_params_h2048": 0.1785714285714286 }, "values": { "rss_load_delta_mb": 172.048384, "rss_peak_large_mb": 288.256, "load_seconds": 0.4538142919998336, "embedding_params_h2048": 534773760 }, "rank": 23 }, { "group": "Mistral-Large-4-1T-A52B", "id": "mistral-large-4-1t-a52b", "score": { "value": 0.1785714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.1428571428571429, "rss_peak_large_mb": 0.1428571428571429, "load_seconds": 0.1785714285714286, "embedding_params_h2048": 0.25 }, "values": { "rss_load_delta_mb": 236.31872, "rss_peak_large_mb": 290.353152, "load_seconds": 0.5272523330000354, "embedding_params_h2048": 419430400 }, "rank": 24 }, { "group": "OpenAI o200k_base (GPT-4o)", "id": "o200k_base", "score": { "value": 0.1607142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.1071428571428571, "rss_peak_large_mb": 0.1071428571428571, "load_seconds": 0.1428571428571429, "embedding_params_h2048": 0.2857142857142857 }, "values": { "rss_load_delta_mb": 249.864192, "rss_peak_large_mb": 321.458176, "load_seconds": 0.5957294170000296, "embedding_params_h2048": 409638912 }, "rank": 25 }, { "group": "Akshara 448k", "id": "akshara-448k", "score": { "value": 0.15178571428571427, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.1785714285714286, "rss_peak_large_mb": 0.3214285714285714, "load_seconds": 0.1071428571428571, "embedding_params_h2048": 0.0 }, "values": { "rss_load_delta_mb": 204.791808, "rss_peak_large_mb": 257.273856, "load_seconds": 0.6785901250000279, "embedding_params_h2048": 917504000 }, "rank": 26 }, { "group": "sarvam-105b", "id": "sarvam-105b", "score": { "value": 0.058035714285714274, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.0357142857142857, "rss_peak_large_mb": 0.0714285714285714, "load_seconds": 0.0357142857142857, "embedding_params_h2048": 0.0892857142857143 }, "values": { "rss_load_delta_mb": 312.578048, "rss_peak_large_mb": 372.129792, "load_seconds": 0.693015374999959, "embedding_params_h2048": 536870912 }, "rank": 27 }, { "group": "gemma-3-1b-it", "id": "gemma-3-1b-it", "score": { "value": 0.04910714285714285, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.0, "rss_peak_large_mb": 0.0357142857142857, "load_seconds": 0.0714285714285714, "embedding_params_h2048": 0.0892857142857143 }, "values": { "rss_load_delta_mb": 312.958976, "rss_peak_large_mb": 372.293632, "load_seconds": 0.6804287500003738, "embedding_params_h2048": 536870912 }, "rank": 28 }, { "group": "gemma-4-31B-it", "id": "gemma-4-31b-it", "score": { "value": 0.040178571428571425, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "rss_load_delta_mb": 0.0714285714285714, "rss_peak_large_mb": 0.0, "load_seconds": 0.0, "embedding_params_h2048": 0.0892857142857143 }, "values": { "rss_load_delta_mb": 309.850112, "rss_peak_large_mb": 373.338112, "load_seconds": 0.7190039170000091, "embedding_params_h2048": 536870912 }, "rank": 29 } ], "not_ranked": [ { "group": "pythia-160m", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "sarvam-1", "reason": "fails lossless round-trip on 215393 inputs" }, { "group": "Qwen2.5-0.5B (same tokenizer: Qwen3-0.6B)", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Seed-OSS-36B-Instruct", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Ling-3.0-tiny", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "MiniMax-M2.5", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Qwen3.8-27B", "reason": "fails lossless round-trip on 733 inputs" } ] }, "high_throughput": { "description": "encode speed, small-request latency, and tokens per chat request", "weights": { "encode_mbps_1t_medium": 1, "batch_mbps_4t": 1, "latency_small_p95_us": 1, "tokens_chat_en": 1 }, "ranked": [ { "group": "gemma-4-31B-it", "id": "gemma-4-31b-it", "score": { "value": 0.8125, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.9285714285714286, "batch_mbps_4t": 1.0, "latency_small_p95_us": 0.9642857142857143, "tokens_chat_en": 0.3571428571428571 }, "values": { "encode_mbps_1t_medium": 6.426287669332646, "batch_mbps_4t": 20.89084307099784, "latency_small_p95_us": 53.33300032361876, "tokens_chat_en": 92377 }, "rank": 1 }, { "group": "Akshara 248k", "id": "akshara-248k", "score": { "value": 0.7678571428571428, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.6428571428571428, "batch_mbps_4t": 0.7857142857142857, "latency_small_p95_us": 0.7142857142857143, "tokens_chat_en": 0.9285714285714286 }, "values": { "encode_mbps_1t_medium": 5.836628228431435, "batch_mbps_4t": 18.40640894684117, "latency_small_p95_us": 61.37500008662755, "tokens_chat_en": 86523 }, "rank": 2 }, { "group": "Akshara 151k", "id": "akshara-151k", "score": { "value": 0.7678571428571428, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.7857142857142857, "batch_mbps_4t": 0.8214285714285714, "latency_small_p95_us": 0.7857142857142857, "tokens_chat_en": 0.6785714285714286 }, "values": { "encode_mbps_1t_medium": 6.116414017986229, "batch_mbps_4t": 18.45893944644569, "latency_small_p95_us": 59.750000218627974, "tokens_chat_en": 88295 }, "rank": 3 }, { "group": "Akshara 200k", "id": "akshara-200k", "score": { "value": 0.75, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.6071428571428572, "batch_mbps_4t": 0.8571428571428572, "latency_small_p95_us": 0.6785714285714286, "tokens_chat_en": 0.8571428571428572 }, "values": { "encode_mbps_1t_medium": 5.734110759561312, "batch_mbps_4t": 18.650088442681785, "latency_small_p95_us": 61.70799997562426, "tokens_chat_en": 87229 }, "rank": 4 }, { "group": "Akshara 448k", "id": "akshara-448k", "score": { "value": 0.7321428571428572, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.5714285714285714, "batch_mbps_4t": 0.75, "latency_small_p95_us": 0.6071428571428572, "tokens_chat_en": 1.0 }, "values": { "encode_mbps_1t_medium": 5.620347250340921, "batch_mbps_4t": 18.300612613557618, "latency_small_p95_us": 63.45899964799173, "tokens_chat_en": 85087 }, "rank": 5 }, { "group": "Phi-3.5-mini-instruct", "id": "phi-3-5-mini-instruct", "score": { "value": 0.7321428571428572, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 1.0, "batch_mbps_4t": 0.9285714285714286, "latency_small_p95_us": 1.0, "tokens_chat_en": 0.0 }, "values": { "encode_mbps_1t_medium": 6.788077586874321, "batch_mbps_4t": 20.144123357075937, "latency_small_p95_us": 49.70899999534595, "tokens_chat_en": 105304 }, "rank": 6 }, { "group": "ERNIE-4.5-21B-A3B-PT", "id": "ernie-4-5-21b-a3b-pt", "score": { "value": 0.7142857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.9642857142857143, "batch_mbps_4t": 0.9642857142857143, "latency_small_p95_us": 0.8214285714285714, "tokens_chat_en": 0.1071428571428571 }, "values": { "encode_mbps_1t_medium": 6.455317587551054, "batch_mbps_4t": 20.487115783013383, "latency_small_p95_us": 58.832999911828665, "tokens_chat_en": 97857 }, "rank": 7 }, { "group": "Akshara 59k", "id": "akshara-59k", "score": { "value": 0.6517857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.8214285714285714, "batch_mbps_4t": 0.6071428571428572, "latency_small_p95_us": 0.8928571428571429, "tokens_chat_en": 0.2857142857142857 }, "values": { "encode_mbps_1t_medium": 6.198963235580143, "batch_mbps_4t": 17.667729432326173, "latency_small_p95_us": 58.165999689663295, "tokens_chat_en": 94496 }, "rank": 8 }, { "group": "Akshara 120k", "id": "akshara-120k", "score": { "value": 0.6428571428571428, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.6785714285714286, "batch_mbps_4t": 0.7142857142857143, "latency_small_p95_us": 0.6428571428571428, "tokens_chat_en": 0.5357142857142857 }, "values": { "encode_mbps_1t_medium": 5.967619867743156, "batch_mbps_4t": 18.09350513047292, "latency_small_p95_us": 62.000000013995304, "tokens_chat_en": 89838 }, "rank": 9 }, { "group": "gemma-3-1b-it", "id": "gemma-3-1b-it", "score": { "value": 0.6339285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.75, "batch_mbps_4t": 0.8928571428571429, "latency_small_p95_us": 0.5, "tokens_chat_en": 0.3928571428571429 }, "values": { "encode_mbps_1t_medium": 6.103693269272489, "batch_mbps_4t": 19.8543184165146, "latency_small_p95_us": 64.33400000105394, "tokens_chat_en": 92367 }, "rank": 10 }, { "group": "Akshara 49k", "id": "akshara-49k", "score": { "value": 0.6071428571428572, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.8571428571428572, "batch_mbps_4t": 0.5714285714285714, "latency_small_p95_us": 0.8571428571428572, "tokens_chat_en": 0.1428571428571429 }, "values": { "encode_mbps_1t_medium": 6.273736117150162, "batch_mbps_4t": 17.597606427027582, "latency_small_p95_us": 58.45900022904971, "tokens_chat_en": 96073 }, "rank": 11 }, { "group": "Akshara 32k", "id": "akshara-32k", "score": { "value": 0.5892857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.8928571428571429, "batch_mbps_4t": 0.5, "latency_small_p95_us": 0.9285714285714286, "tokens_chat_en": 0.0357142857142857 }, "values": { "encode_mbps_1t_medium": 6.303510437376889, "batch_mbps_4t": 17.277981622498476, "latency_small_p95_us": 57.66699996456737, "tokens_chat_en": 100294 }, "rank": 12 }, { "group": "OpenAI o200k_base (GPT-4o)", "id": "o200k_base", "score": { "value": 0.5803571428571428, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.4285714285714286, "batch_mbps_4t": 0.6428571428571428, "latency_small_p95_us": 0.3571428571428571, "tokens_chat_en": 0.8928571428571429 }, "values": { "encode_mbps_1t_medium": 5.101910820251484, "batch_mbps_4t": 17.73679441869205, "latency_small_p95_us": 65.54100036737509, "tokens_chat_en": 87171 }, "rank": 13 }, { "group": "sarvam-105b", "id": "sarvam-105b", "score": { "value": 0.5714285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.5357142857142857, "batch_mbps_4t": 0.6785714285714286, "latency_small_p95_us": 0.75, "tokens_chat_en": 0.3214285714285714 }, "values": { "encode_mbps_1t_medium": 5.616660252990121, "batch_mbps_4t": 17.953576871051627, "latency_small_p95_us": 60.2499999331485, "tokens_chat_en": 92483 }, "rank": 14 }, { "group": "Akshara 92k", "id": "akshara-92k", "score": { "value": 0.5625, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.7142857142857143, "batch_mbps_4t": 0.5357142857142857, "latency_small_p95_us": 0.5714285714285714, "tokens_chat_en": 0.4285714285714286 }, "values": { "encode_mbps_1t_medium": 6.100163595476987, "batch_mbps_4t": 17.31074049393309, "latency_small_p95_us": 63.54199967972818, "tokens_chat_en": 91317 }, "rank": 15 }, { "group": "Kimi-K3", "id": "kimi-k3", "score": { "value": 0.48214285714285715, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.25, "batch_mbps_4t": 0.4285714285714286, "latency_small_p95_us": 0.4285714285714286, "tokens_chat_en": 0.8214285714285714 }, "values": { "encode_mbps_1t_medium": 4.938964989355018, "batch_mbps_4t": 16.933337146737262, "latency_small_p95_us": 64.8340001134784, "tokens_chat_en": 87699 }, "rank": 16 }, { "group": "cl100k_base (GPT-4, OLMo-2-0425-1B, Olmo-3-1025-7B, phi-4)", "id": "cl100k_base", "score": { "value": 0.45535714285714285, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.4642857142857143, "batch_mbps_4t": 0.2857142857142857, "latency_small_p95_us": 0.3214285714285714, "tokens_chat_en": 0.75 }, "values": { "encode_mbps_1t_medium": 5.153221493593562, "batch_mbps_4t": 16.422928599858064, "latency_small_p95_us": 65.74999997610576, "tokens_chat_en": 88199 }, "rank": 17 }, { "group": "LFM3-1.7B", "id": "lfm3-1-7b", "score": { "value": 0.41964285714285715, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.3214285714285714, "batch_mbps_4t": 0.3928571428571429, "latency_small_p95_us": 0.4642857142857143, "tokens_chat_en": 0.5 }, "values": { "encode_mbps_1t_medium": 4.983365371128134, "batch_mbps_4t": 16.76986979818239, "latency_small_p95_us": 64.41699997594696, "tokens_chat_en": 89962 }, "rank": 18 }, { "group": "GLM-5.3-Flash", "id": "glm-5-3-flash", "score": { "value": 0.41964285714285715, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.2142857142857143, "batch_mbps_4t": 0.3571428571428571, "latency_small_p95_us": 0.3928571428571429, "tokens_chat_en": 0.7142857142857143 }, "values": { "encode_mbps_1t_medium": 4.882575809711876, "batch_mbps_4t": 16.669548809795145, "latency_small_p95_us": 65.16599978567683, "tokens_chat_en": 88272 }, "rank": 19 }, { "group": "Llama-3.2-1B", "id": "llama-3-2-1b", "score": { "value": 0.4107142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.2857142857142857, "batch_mbps_4t": 0.3214285714285714, "latency_small_p95_us": 0.25, "tokens_chat_en": 0.7857142857142857 }, "values": { "encode_mbps_1t_medium": 4.979777107529496, "batch_mbps_4t": 16.618196262248443, "latency_small_p95_us": 67.41700008205953, "tokens_chat_en": 88175 }, "rank": 20 }, { "group": "Mistral-Nemo-Base-2407", "id": "mistral-nemo-base-2407", "score": { "value": 0.4107142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.5, "batch_mbps_4t": 0.4642857142857143, "latency_small_p95_us": 0.2142857142857143, "tokens_chat_en": 0.4642857142857143 }, "values": { "encode_mbps_1t_medium": 5.466480170437732, "batch_mbps_4t": 17.23201412522955, "latency_small_p95_us": 68.7090000610624, "tokens_chat_en": 90659 }, "rank": 21 }, { "group": "Mistral-Large-4-1T-A52B", "id": "mistral-large-4-1t-a52b", "score": { "value": 0.375, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.3928571428571429, "batch_mbps_4t": 0.25, "latency_small_p95_us": 0.2857142857142857, "tokens_chat_en": 0.5714285714285714 }, "values": { "encode_mbps_1t_medium": 5.069619965517145, "batch_mbps_4t": 16.036505735404884, "latency_small_p95_us": 66.95899992337218, "tokens_chat_en": 89142 }, "rank": 22 }, { "group": "North-Micro-Vision-Instruct", "id": "north-micro-vision-instruct", "score": { "value": 0.33035714285714285, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.0357142857142857, "batch_mbps_4t": 0.2142857142857143, "latency_small_p95_us": 0.1071428571428571, "tokens_chat_en": 0.9642857142857143 }, "values": { "encode_mbps_1t_medium": 4.009868145514843, "batch_mbps_4t": 14.617944898121555, "latency_small_p95_us": 88.54199995766976, "tokens_chat_en": 85590 }, "rank": 23 }, { "group": "OpenAI p50k_base (Codex)", "id": "p50k_base", "score": { "value": 0.30357142857142855, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.3571428571428571, "batch_mbps_4t": 0.1071428571428571, "latency_small_p95_us": 0.5357142857142857, "tokens_chat_en": 0.2142857142857143 }, "values": { "encode_mbps_1t_medium": 5.033180687959843, "batch_mbps_4t": 14.126312341626818, "latency_small_p95_us": 64.29200038837735, "tokens_chat_en": 95593 }, "rank": 24 }, { "group": "DeepSeek-V3", "id": "deepseek-v3", "score": { "value": 0.24999999999999997, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.1071428571428571, "batch_mbps_4t": 0.1785714285714286, "latency_small_p95_us": 0.0714285714285714, "tokens_chat_en": 0.6428571428571428 }, "values": { "encode_mbps_1t_medium": 4.593484509872189, "batch_mbps_4t": 14.352636211190546, "latency_small_p95_us": 89.08300003440672, "tokens_chat_en": 88424 }, "rank": 25 }, { "group": "Hy3", "id": "hy3", "score": { "value": 0.2142857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.0714285714285714, "batch_mbps_4t": 0.1428571428571429, "latency_small_p95_us": 0.0357142857142857, "tokens_chat_en": 0.6071428571428572 }, "values": { "encode_mbps_1t_medium": 4.135631116558927, "batch_mbps_4t": 14.15198592956625, "latency_small_p95_us": 89.50000005825132, "tokens_chat_en": 88615 }, "rank": 26 }, { "group": "SmolLM2-135M", "id": "smollm2-135m", "score": { "value": 0.13392857142857145, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.1785714285714286, "batch_mbps_4t": 0.0357142857142857, "latency_small_p95_us": 0.1428571428571429, "tokens_chat_en": 0.1785714285714286 }, "values": { "encode_mbps_1t_medium": 4.853248808026626, "batch_mbps_4t": 12.374920625155946, "latency_small_p95_us": 80.41699999239427, "tokens_chat_en": 95975 }, "rank": 27 }, { "group": "gpt2", "id": "gpt2", "score": { "value": 0.11607142857142858, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.1428571428571429, "batch_mbps_4t": 0.0714285714285714, "latency_small_p95_us": 0.1785714285714286, "tokens_chat_en": 0.0714285714285714 }, "values": { "encode_mbps_1t_medium": 4.814825216097814, "batch_mbps_4t": 12.407483704422463, "latency_small_p95_us": 73.50000009864743, "tokens_chat_en": 98389 }, "rank": 28 }, { "group": "Falcon-H1-34B-Instruct", "id": "falcon-h1-34b-instruct", "score": { "value": 0.0625, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "encode_mbps_1t_medium": 0.0, "batch_mbps_4t": 0.0, "latency_small_p95_us": 0.0, "tokens_chat_en": 0.25 }, "values": { "encode_mbps_1t_medium": 2.765068889391423, "batch_mbps_4t": 10.159878525556868, "latency_small_p95_us": 124.20799998835719, "tokens_chat_en": 94544 }, "rank": 29 } ], "not_ranked": [ { "group": "pythia-160m", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "sarvam-1", "reason": "fails lossless round-trip on 215393 inputs" }, { "group": "Qwen2.5-0.5B (same tokenizer: Qwen3-0.6B)", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Seed-OSS-36B-Instruct", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Ling-3.0-tiny", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "MiniMax-M2.5", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Qwen3.8-27B", "reason": "fails lossless round-trip on 733 inputs" } ] }, "long_context": { "description": "tokens for long prompts and how encoding scales to 16 MiB", "weights": { "tokens_long_context": 1, "encode_mbps_1t_very_large": 1, "rss_peak_very_large_mb": 1, "throughput_ratio_very_large": 1 }, "ranked": [ { "group": "Akshara 248k", "id": "akshara-248k", "score": { "value": 0.8214285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.9642857142857143, "encode_mbps_1t_very_large": 0.9285714285714286, "rss_peak_very_large_mb": 0.75, "throughput_ratio_very_large": 0.6428571428571428 }, "values": { "tokens_long_context": 491868, "encode_mbps_1t_very_large": 3.6319447567449643, "rss_peak_very_large_mb": 2374.10304, "throughput_ratio_very_large": 0.6222676200366855 }, "rank": 1 }, { "group": "Akshara 200k", "id": "akshara-200k", "score": { "value": 0.8125, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.9285714285714286, "encode_mbps_1t_very_large": 0.8214285714285714, "rss_peak_very_large_mb": 0.8214285714285714, "throughput_ratio_very_large": 0.6785714285714286 }, "values": { "tokens_long_context": 501669, "encode_mbps_1t_very_large": 3.5925404821980336, "rss_peak_very_large_mb": 2359.476224, "throughput_ratio_very_large": 0.6265209433228459 }, "rank": 2 }, { "group": "Akshara 151k", "id": "akshara-151k", "score": { "value": 0.7678571428571429, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.8928571428571429, "encode_mbps_1t_very_large": 0.9642857142857143, "rss_peak_very_large_mb": 0.7857142857142857, "throughput_ratio_very_large": 0.4285714285714286 }, "values": { "tokens_long_context": 515221, "encode_mbps_1t_very_large": 3.6332574759853884, "rss_peak_very_large_mb": 2368.565248, "throughput_ratio_very_large": 0.5940175837183768 }, "rank": 3 }, { "group": "Akshara 120k", "id": "akshara-120k", "score": { "value": 0.7321428571428572, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.8571428571428572, "encode_mbps_1t_very_large": 0.8928571428571429, "rss_peak_very_large_mb": 0.7142857142857143, "throughput_ratio_very_large": 0.4642857142857143 }, "values": { "tokens_long_context": 527532, "encode_mbps_1t_very_large": 3.613083868795755, "rss_peak_very_large_mb": 2378.186752, "throughput_ratio_very_large": 0.6054480595062026 }, "rank": 4 }, { "group": "Akshara 448k", "id": "akshara-448k", "score": { "value": 0.7232142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 1.0, "encode_mbps_1t_very_large": 0.6428571428571428, "rss_peak_very_large_mb": 0.6428571428571428, "throughput_ratio_very_large": 0.6071428571428572 }, "values": { "tokens_long_context": 470151, "encode_mbps_1t_very_large": 3.446423169868346, "rss_peak_very_large_mb": 2394.68544, "throughput_ratio_very_large": 0.6132046680317291 }, "rank": 5 }, { "group": "OpenAI o200k_base (GPT-4o)", "id": "o200k_base", "score": { "value": 0.7232142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.8214285714285714, "encode_mbps_1t_very_large": 0.7142857142857143, "rss_peak_very_large_mb": 0.4642857142857143, "throughput_ratio_very_large": 0.8928571428571429 }, "values": { "tokens_long_context": 539608, "encode_mbps_1t_very_large": 3.490209463670934, "rss_peak_very_large_mb": 2471.8336, "throughput_ratio_very_large": 0.6840984851826348 }, "rank": 6 }, { "group": "gemma-3-1b-it", "id": "gemma-3-1b-it", "score": { "value": 0.7053571428571429, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.6428571428571428, "encode_mbps_1t_very_large": 0.8571428571428572, "rss_peak_very_large_mb": 0.9285714285714286, "throughput_ratio_very_large": 0.3928571428571429 }, "values": { "tokens_long_context": 582365, "encode_mbps_1t_very_large": 3.6097452376447525, "rss_peak_very_large_mb": 1918.242816, "throughput_ratio_very_large": 0.591403446797877 }, "rank": 7 }, { "group": "Akshara 92k", "id": "akshara-92k", "score": { "value": 0.6517857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.7857142857142857, "encode_mbps_1t_very_large": 0.7857142857142857, "rss_peak_very_large_mb": 0.6785714285714286, "throughput_ratio_very_large": 0.3571428571428571 }, "values": { "tokens_long_context": 544398, "encode_mbps_1t_very_large": 3.5534005680066656, "rss_peak_very_large_mb": 2381.4144, "throughput_ratio_very_large": 0.5825090610096689 }, "rank": 8 }, { "group": "Mistral-Large-4-1T-A52B", "id": "mistral-large-4-1t-a52b", "score": { "value": 0.6517857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.7142857142857143, "encode_mbps_1t_very_large": 0.5714285714285714, "rss_peak_very_large_mb": 0.5, "throughput_ratio_very_large": 0.8214285714285714 }, "values": { "tokens_long_context": 547437, "encode_mbps_1t_very_large": 3.3938589958972627, "rss_peak_very_large_mb": 2466.238464, "throughput_ratio_very_large": 0.6694503767504908 }, "rank": 9 }, { "group": "North-Micro-Vision-Instruct", "id": "north-micro-vision-instruct", "score": { "value": 0.6428571428571428, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.75, "encode_mbps_1t_very_large": 0.2857142857142857, "rss_peak_very_large_mb": 0.5714285714285714, "throughput_ratio_very_large": 0.9642857142857143 }, "values": { "tokens_long_context": 545035, "encode_mbps_1t_very_large": 2.9191065167095607, "rss_peak_very_large_mb": 2461.401088, "throughput_ratio_very_large": 0.7279806743707192 }, "rank": 10 }, { "group": "ERNIE-4.5-21B-A3B-PT", "id": "ernie-4-5-21b-a3b-pt", "score": { "value": 0.6339285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.25, "encode_mbps_1t_very_large": 1.0, "rss_peak_very_large_mb": 0.9642857142857143, "throughput_ratio_very_large": 0.3214285714285714 }, "values": { "tokens_long_context": 687756, "encode_mbps_1t_very_large": 3.6967421786623325, "rss_peak_very_large_mb": 1883.11552, "throughput_ratio_very_large": 0.5726661978322217 }, "rank": 11 }, { "group": "Akshara 59k", "id": "akshara-59k", "score": { "value": 0.5446428571428572, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.6785714285714286, "encode_mbps_1t_very_large": 0.6785714285714286, "rss_peak_very_large_mb": 0.6071428571428572, "throughput_ratio_very_large": 0.2142857142857143 }, "values": { "tokens_long_context": 577069, "encode_mbps_1t_very_large": 3.4645631960607575, "rss_peak_very_large_mb": 2407.292928, "throughput_ratio_very_large": 0.558893973781814 }, "rank": 12 }, { "group": "sarvam-105b", "id": "sarvam-105b", "score": { "value": 0.5446428571428572, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.1785714285714286, "encode_mbps_1t_very_large": 0.6071428571428572, "rss_peak_very_large_mb": 0.8928571428571429, "throughput_ratio_very_large": 0.5 }, "values": { "tokens_long_context": 727923, "encode_mbps_1t_very_large": 3.4006473233799785, "rss_peak_very_large_mb": 2000.072704, "throughput_ratio_very_large": 0.6054571881162989 }, "rank": 13 }, { "group": "Akshara 49k", "id": "akshara-49k", "score": { "value": 0.5267857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.5357142857142857, "encode_mbps_1t_very_large": 0.75, "rss_peak_very_large_mb": 0.5357142857142857, "throughput_ratio_very_large": 0.2857142857142857 }, "values": { "tokens_long_context": 592867, "encode_mbps_1t_very_large": 3.5377049935876648, "rss_peak_very_large_mb": 2466.205696, "throughput_ratio_very_large": 0.56389126471495 }, "rank": 14 }, { "group": "Kimi-K3", "id": "kimi-k3", "score": { "value": 0.5089285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.3928571428571429, "encode_mbps_1t_very_large": 0.5, "rss_peak_very_large_mb": 0.2857142857142857, "throughput_ratio_very_large": 0.8571428571428572 }, "values": { "tokens_long_context": 633240, "encode_mbps_1t_very_large": 3.3159529617509573, "rss_peak_very_large_mb": 2572.242944, "throughput_ratio_very_large": 0.6713862051862792 }, "rank": 15 }, { "group": "LFM3-1.7B", "id": "lfm3-1-7b", "score": { "value": 0.48214285714285715, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.5, "encode_mbps_1t_very_large": 0.4642857142857143, "rss_peak_very_large_mb": 0.25, "throughput_ratio_very_large": 0.7142857142857143 }, "values": { "tokens_long_context": 597351, "encode_mbps_1t_very_large": 3.1458846782303604, "rss_peak_very_large_mb": 2572.292096, "throughput_ratio_very_large": 0.6312771478600606 }, "rank": 16 }, { "group": "Hy3", "id": "hy3", "score": { "value": 0.4642857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.3571428571428571, "encode_mbps_1t_very_large": 0.2142857142857143, "rss_peak_very_large_mb": 0.3571428571428571, "throughput_ratio_very_large": 0.9285714285714286 }, "values": { "tokens_long_context": 656516, "encode_mbps_1t_very_large": 2.8731140325847044, "rss_peak_very_large_mb": 2540.376064, "throughput_ratio_very_large": 0.6947220270881639 }, "rank": 17 }, { "group": "DeepSeek-V3", "id": "deepseek-v3", "score": { "value": 0.4642857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.4642857142857143, "encode_mbps_1t_very_large": 0.25, "rss_peak_very_large_mb": 0.3928571428571429, "throughput_ratio_very_large": 0.75 }, "values": { "tokens_long_context": 613548, "encode_mbps_1t_very_large": 2.9006390878341572, "rss_peak_very_large_mb": 2510.675968, "throughput_ratio_very_large": 0.6314681330915962 }, "rank": 18 }, { "group": "gemma-4-31B-it", "id": "gemma-4-31b-it", "score": { "value": 0.4107142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.5714285714285714, "encode_mbps_1t_very_large": 0.0357142857142857, "rss_peak_very_large_mb": 1.0, "throughput_ratio_very_large": 0.0357142857142857 }, "values": { "tokens_long_context": 583505, "encode_mbps_1t_very_large": 2.3881618744868045, "rss_peak_very_large_mb": 1770.811392, "throughput_ratio_very_large": 0.3716238670552402 }, "rank": 19 }, { "group": "GLM-5.3-Flash", "id": "glm-5-3-flash", "score": { "value": 0.4017857142857143, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.2142857142857143, "encode_mbps_1t_very_large": 0.3928571428571429, "rss_peak_very_large_mb": 0.2142857142857143, "throughput_ratio_very_large": 0.7857142857142857 }, "values": { "tokens_long_context": 687842, "encode_mbps_1t_very_large": 3.1044676110377996, "rss_peak_very_large_mb": 2628.591616, "throughput_ratio_very_large": 0.6358257878685136 }, "rank": 20 }, { "group": "Akshara 32k", "id": "akshara-32k", "score": { "value": 0.375, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.4285714285714286, "encode_mbps_1t_very_large": 0.5357142857142857, "rss_peak_very_large_mb": 0.4285714285714286, "throughput_ratio_very_large": 0.1071428571428571 }, "values": { "tokens_long_context": 633139, "encode_mbps_1t_very_large": 3.356025161611074, "rss_peak_very_large_mb": 2486.562816, "throughput_ratio_very_large": 0.5324057435855748 }, "rank": 21 }, { "group": "Mistral-Nemo-Base-2407", "id": "mistral-nemo-base-2407", "score": { "value": 0.3660714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.6071428571428572, "encode_mbps_1t_very_large": 0.3571428571428571, "rss_peak_very_large_mb": 0.3214285714285714, "throughput_ratio_very_large": 0.1785714285714286 }, "values": { "tokens_long_context": 583215, "encode_mbps_1t_very_large": 3.0546923436253532, "rss_peak_very_large_mb": 2571.370496, "throughput_ratio_very_large": 0.5588042485080023 }, "rank": 22 }, { "group": "Falcon-H1-34B-Instruct", "id": "falcon-h1-34b-instruct", "score": { "value": 0.35714285714285715, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.2857142857142857, "encode_mbps_1t_very_large": 0.0, "rss_peak_very_large_mb": 0.1428571428571429, "throughput_ratio_very_large": 1.0 }, "values": { "tokens_long_context": 673656, "encode_mbps_1t_very_large": 2.2328301560909654, "rss_peak_very_large_mb": 2683.24864, "throughput_ratio_very_large": 0.807513391314601 }, "rank": 23 }, { "group": "Llama-3.2-1B", "id": "llama-3-2-1b", "score": { "value": 0.3482142857142857, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.3214285714285714, "encode_mbps_1t_very_large": 0.3214285714285714, "rss_peak_very_large_mb": 0.1785714285714286, "throughput_ratio_very_large": 0.5714285714285714 }, "values": { "tokens_long_context": 664655, "encode_mbps_1t_very_large": 3.0332342045673246, "rss_peak_very_large_mb": 2636.156928, "throughput_ratio_very_large": 0.6091104358829695 }, "rank": 24 }, { "group": "cl100k_base (GPT-4, OLMo-2-0425-1B, Olmo-3-1025-7B, phi-4)", "id": "cl100k_base", "score": { "value": 0.3035714285714286, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.1428571428571429, "encode_mbps_1t_very_large": 0.4285714285714286, "rss_peak_very_large_mb": 0.1071428571428571, "throughput_ratio_very_large": 0.5357142857142857 }, "values": { "tokens_long_context": 762523, "encode_mbps_1t_very_large": 3.1239132248630783, "rss_peak_very_large_mb": 2687.71328, "throughput_ratio_very_large": 0.606205890576739 }, "rank": 25 }, { "group": "Phi-3.5-mini-instruct", "id": "phi-3-5-mini-instruct", "score": { "value": 0.2589285714285714, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.1071428571428571, "encode_mbps_1t_very_large": 0.0714285714285714, "rss_peak_very_large_mb": 0.8571428571428572, "throughput_ratio_very_large": 0.0 }, "values": { "tokens_long_context": 953147, "encode_mbps_1t_very_large": 2.440479566927515, "rss_peak_very_large_mb": 2057.474048, "throughput_ratio_very_large": 0.3595244066812844 }, "rank": 26 }, { "group": "OpenAI p50k_base (Codex)", "id": "p50k_base", "score": { "value": 0.125, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.0357142857142857, "encode_mbps_1t_very_large": 0.1785714285714286, "rss_peak_very_large_mb": 0.0357142857142857, "throughput_ratio_very_large": 0.25 }, "values": { "tokens_long_context": 1122175, "encode_mbps_1t_very_large": 2.8197814033629527, "rss_peak_very_large_mb": 3049.263104, "throughput_ratio_very_large": 0.56023846116002 }, "rank": 27 }, { "group": "SmolLM2-135M", "id": "smollm2-135m", "score": { "value": 0.10714285714285715, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.0714285714285714, "encode_mbps_1t_very_large": 0.1428571428571429, "rss_peak_very_large_mb": 0.0714285714285714, "throughput_ratio_very_large": 0.1428571428571429 }, "values": { "tokens_long_context": 1043079, "encode_mbps_1t_very_large": 2.6634715393753834, "rss_peak_very_large_mb": 3006.599168, "throughput_ratio_very_large": 0.548801770675832 }, "rank": 28 }, { "group": "gpt2", "id": "gpt2", "score": { "value": 0.04464285714285712, "provenance": "measured", "note": "weighted mean of metric percentiles" }, "percentiles": { "tokens_long_context": 0.0, "encode_mbps_1t_very_large": 0.1071428571428571, "rss_peak_very_large_mb": 0.0, "throughput_ratio_very_large": 0.0714285714285714 }, "values": { "tokens_long_context": 1225393, "encode_mbps_1t_very_large": 2.48597633234039, "rss_peak_very_large_mb": 3192.500224, "throughput_ratio_very_large": 0.516317045949833 }, "rank": 29 } ], "not_ranked": [ { "group": "pythia-160m", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "sarvam-1", "reason": "fails lossless round-trip on 215393 inputs" }, { "group": "Qwen2.5-0.5B (same tokenizer: Qwen3-0.6B)", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Seed-OSS-36B-Instruct", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Ling-3.0-tiny", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "MiniMax-M2.5", "reason": "fails lossless round-trip on 733 inputs" }, { "group": "Qwen3.8-27B", "reason": "fails lossless round-trip on 733 inputs" } ] } }, "composite": { "optional": true, "default_weights": { "compression": 1, "runtime": 1, "memory": 1, "correctness": 1, "scalability": 1, "deployment_cost": 1 }, "category_metrics": { "compression": [ "bpt_overall", "bpt_english", "bpt_code", "bpt_languages", "parity_mean_abs_log" ], "runtime": [ "encode_mbps_1t_medium", "batch_mbps_4t", "latency_small_p95_us", "decode_mbps" ], "memory": [ "rss_load_delta_mb", "rss_peak_large_mb", "load_seconds" ], "correctness": [ "byte_fallback_share", "unicode_probe_failures" ], "scalability": [ "throughput_ratio_very_large", "rss_peak_chunked_mb", "encode_mbps_1t_very_large" ], "deployment_cost": [ "embedding_params_h2048", "workload_tokens_total" ] }, "rows": [ { "group": "Akshara 200k", "id": "akshara-200k", "categories": { "compression": 0.9071428571428571, "runtime": 0.7321428571428572, "memory": 0.6071428571428572, "correctness": 0.7232142857142857, "scalability": 0.6904761904761905, "deployment_cost": 0.625 }, "score": { "value": 0.7141865079365081, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Akshara 120k", "id": "akshara-120k", "categories": { "compression": 0.7000000000000001, "runtime": 0.7321428571428572, "memory": 0.7261904761904762, "correctness": 0.6517857142857142, "scalability": 0.6547619047619048, "deployment_cost": 0.75 }, "score": { "value": 0.7024801587301588, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Akshara 151k", "id": "akshara-151k", "categories": { "compression": 0.85, "runtime": 0.8035714285714286, "memory": 0.6309523809523809, "correctness": 0.6696428571428571, "scalability": 0.5714285714285714, "deployment_cost": 0.6607142857142858 }, "score": { "value": 0.6977182539682539, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Akshara 248k", "id": "akshara-248k", "categories": { "compression": 0.9571428571428572, "runtime": 0.7053571428571428, "memory": 0.44047619047619047, "correctness": 0.7410714285714286, "scalability": 0.5357142857142857, "deployment_cost": 0.5892857142857143 }, "score": { "value": 0.6615079365079365, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Akshara 92k", "id": "akshara-92k", "categories": { "compression": 0.6, "runtime": 0.6875, "memory": 0.8214285714285714, "correctness": 0.6339285714285714, "scalability": 0.46428571428571425, "deployment_cost": 0.7321428571428572 }, "score": { "value": 0.656547619047619, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Akshara 49k", "id": "akshara-49k", "categories": { "compression": 0.4, "runtime": 0.8214285714285714, "memory": 0.8690476190476191, "correctness": 0.5446428571428571, "scalability": 0.4880952380952381, "deployment_cost": 0.7678571428571429 }, "score": { "value": 0.6485119047619048, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Akshara 59k", "id": "akshara-59k", "categories": { "compression": 0.47857142857142854, "runtime": 0.8214285714285714, "memory": 0.8571428571428571, "correctness": 0.5803571428571428, "scalability": 0.32142857142857145, "deployment_cost": 0.7321428571428572 }, "score": { "value": 0.6318452380952381, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Akshara 448k", "id": "akshara-448k", "categories": { "compression": 1.0, "runtime": 0.6696428571428572, "memory": 0.20238095238095236, "correctness": 0.7589285714285714, "scalability": 0.5476190476190477, "deployment_cost": 0.5 }, "score": { "value": 0.6130952380952381, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Akshara 32k", "id": "akshara-32k", "categories": { "compression": 0.2571428571428571, "runtime": 0.7946428571428572, "memory": 0.9523809523809524, "correctness": 0.5089285714285714, "scalability": 0.2619047619047619, "deployment_cost": 0.7321428571428572 }, "score": { "value": 0.5845238095238096, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "OpenAI o200k_base (GPT-4o)", "id": "o200k_base", "categories": { "compression": 0.7428571428571429, "runtime": 0.5089285714285714, "memory": 0.11904761904761903, "correctness": 0.6160714285714286, "scalability": 0.8571428571428572, "deployment_cost": 0.5535714285714286 }, "score": { "value": 0.5662698412698413, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "ERNIE-4.5-21B-A3B-PT", "id": "ernie-4-5-21b-a3b-pt", "categories": { "compression": 0.2285714285714286, "runtime": 0.7142857142857143, "memory": 0.6666666666666666, "correctness": 0.4732142857142857, "scalability": 0.5357142857142857, "deployment_cost": 0.44642857142857145 }, "score": { "value": 0.510813492063492, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "LFM3-1.7B", "id": "lfm3-1-7b", "categories": { "compression": 0.42142857142857143, "runtime": 0.4107142857142857, "memory": 0.35714285714285715, "correctness": 0.5982142857142857, "scalability": 0.6607142857142857, "deployment_cost": 0.6071428571428571 }, "score": { "value": 0.5092261904761904, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Mistral-Large-4-1T-A52B", "id": "mistral-large-4-1t-a52b", "categories": { "compression": 0.6071428571428571, "runtime": 0.35714285714285715, "memory": 0.1547619047619048, "correctness": 0.5625, "scalability": 0.7619047619047619, "deployment_cost": 0.5 }, "score": { "value": 0.4905753968253968, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Mistral-Nemo-Base-2407", "id": "mistral-nemo-base-2407", "categories": { "compression": 0.44999999999999996, "runtime": 0.4732142857142857, "memory": 0.6666666666666666, "correctness": 0.5267857142857142, "scalability": 0.3333333333333333, "deployment_cost": 0.4285714285714286 }, "score": { "value": 0.4797619047619048, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "North-Micro-Vision-Instruct", "id": "north-micro-vision-instruct", "categories": { "compression": 0.7571428571428572, "runtime": 0.2232142857142857, "memory": 0.2857142857142857, "correctness": 0.45535714285714285, "scalability": 0.7023809523809524, "deployment_cost": 0.4375 }, "score": { "value": 0.47688492063492066, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "gemma-3-1b-it", "id": "gemma-3-1b-it", "categories": { "compression": 0.6821428571428572, "runtime": 0.5535714285714286, "memory": 0.0357142857142857, "correctness": 0.6875, "scalability": 0.6547619047619048, "deployment_cost": 0.2232142857142857 }, "score": { "value": 0.47281746031746036, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "DeepSeek-V3", "id": "deepseek-v3", "categories": { "compression": 0.4928571428571429, "runtime": 0.24999999999999997, "memory": 0.6428571428571429, "correctness": 0.49107142857142855, "scalability": 0.40476190476190477, "deployment_cost": 0.5357142857142857 }, "score": { "value": 0.4695436507936508, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Phi-3.5-mini-instruct", "id": "phi-3-5-mini-instruct", "categories": { "compression": 0.05714285714285712, "runtime": 0.7321428571428572, "memory": 0.9761904761904762, "correctness": 0.33035714285714285, "scalability": 0.14285714285714282, "deployment_cost": 0.5357142857142857 }, "score": { "value": 0.4624007936507937, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Kimi-K3", "id": "kimi-k3", "categories": { "compression": 0.44285714285714284, "runtime": 0.375, "memory": 0.25, "correctness": 0.43749999999999994, "scalability": 0.7738095238095238, "deployment_cost": 0.4464285714285714 }, "score": { "value": 0.454265873015873, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "gemma-4-31B-it", "id": "gemma-4-31b-it", "categories": { "compression": 0.6535714285714286, "runtime": 0.7589285714285714, "memory": 0.023809523809523798, "correctness": 0.7053571428571428, "scalability": 0.25, "deployment_cost": 0.20535714285714285 }, "score": { "value": 0.43283730158730155, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "cl100k_base (GPT-4, OLMo-2-0425-1B, Olmo-3-1025-7B, phi-4)", "id": "cl100k_base", "categories": { "compression": 0.3642857142857143, "runtime": 0.375, "memory": 0.4523809523809524, "correctness": 0.3482142857142857, "scalability": 0.5, "deployment_cost": 0.48214285714285715 }, "score": { "value": 0.4203373015873016, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Hy3", "id": "hy3", "categories": { "compression": 0.40714285714285714, "runtime": 0.15178571428571427, "memory": 0.619047619047619, "correctness": 0.1607142857142857, "scalability": 0.5952380952380952, "deployment_cost": 0.5535714285714286 }, "score": { "value": 0.4145833333333333, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Llama-3.2-1B", "id": "llama-3-2-1b", "categories": { "compression": 0.4714285714285714, "runtime": 0.3571428571428571, "memory": 0.380952380952381, "correctness": 0.36607142857142855, "scalability": 0.35714285714285715, "deployment_cost": 0.48214285714285715 }, "score": { "value": 0.40248015873015874, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "GLM-5.3-Flash", "id": "glm-5-3-flash", "categories": { "compression": 0.4, "runtime": 0.3214285714285714, "memory": 0.2857142857142857, "correctness": 0.3839285714285714, "scalability": 0.6428571428571429, "deployment_cost": 0.3392857142857143 }, "score": { "value": 0.3955357142857143, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "SmolLM2-135M", "id": "smollm2-135m", "categories": { "compression": 0.09999999999999998, "runtime": 0.14285714285714288, "memory": 0.7857142857142857, "correctness": 0.31249999999999994, "scalability": 0.2619047619047619, "deployment_cost": 0.48214285714285715 }, "score": { "value": 0.34751984126984126, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "sarvam-105b", "id": "sarvam-105b", "categories": { "compression": 0.4571428571428572, "runtime": 0.5, "memory": 0.047619047619047596, "correctness": 0.2946428571428571, "scalability": 0.636904761904762, "deployment_cost": 0.1160714285714286 }, "score": { "value": 0.3420634920634921, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "OpenAI p50k_base (Codex)", "id": "p50k_base", "categories": { "compression": 0.16785714285714284, "runtime": 0.3214285714285714, "memory": 0.6666666666666666, "correctness": 0.2589285714285714, "scalability": 0.17857142857142858, "deployment_cost": 0.42857142857142855 }, "score": { "value": 0.3370039682539682, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "Falcon-H1-34B-Instruct", "id": "falcon-h1-34b-instruct", "categories": { "compression": 0.33571428571428574, "runtime": 0.0625, "memory": 0.25, "correctness": 0.40178571428571425, "scalability": 0.6547619047619048, "deployment_cost": 0.1785714285714286 }, "score": { "value": 0.3138888888888889, "provenance": "measured", "note": "default (equal) weights" } }, { "group": "gpt2", "id": "gpt2", "categories": { "compression": 0.11071428571428572, "runtime": 0.14285714285714288, "memory": 0.7261904761904763, "correctness": 0.27678571428571425, "scalability": 0.0595238095238095, "deployment_cost": 0.4285714285714286 }, "score": { "value": 0.29077380952380955, "provenance": "measured", "note": "default (equal) weights" } } ], "note": "category score = mean percentile of its metrics; composite = weighted mean of category scores; weights are configurable and every raw metric stays visible" }, "limitations": [ "Runtime numbers come from one run per tokenizer. Repeating gpt2 on the same machine changed encode throughput by up to 8% and p95 latency by 26%, so close speed or latency differences are within noise.", "No GPU, Apple Silicon or second CPU type was available: every runtime number comes from one machine (see environment). evaluation/run_runtime.sh repeats the runtime phase anywhere.", "Tokenizers run on the CPU; GPU numbers would only describe model inference, which this evaluation does not measure.", "Model quality (bits per byte, downstream accuracy) was not measured; phase 5 is a plan and a script.", "Undertrained tokens need model weights; only self-reachability (a vocabulary-only check) was measured.", "Costs use a unit rate of 1 USD per million tokens, so they compare tokenizers, not providers. No price is assumed for Akshara.", "Constructed scenarios (mixed-language chat, tool calls, RAG prompts, JSON, logs, long context) are synthetic text built from corpus pieces and seeded templates.", "OASST2 has no Indic conversations, so Indic chat is covered only by the constructed mixed-language scenario and FLORES-200.", "FLORES-200 comes from the SIB-200 copy (1,004 sentences per language); the official FLORES repositories need a terms acceptance.", "The 41 test sets are not stored in the repository; their sources and hashes are in the config, and runs refuse files that differ.", "Words per token uses whitespace words and is unavailable for Chinese, Japanese and Thai." ], "findings": [ "Multilingual: Akshara 448k ranks first (0.99); 29 ranked, 7 not ranked.", "Code heavy: Akshara 448k ranks first (0.91); 29 ranked, 7 not ranked.", "Low memory: Phi-3.5-mini-instruct ranks first (0.97); 29 ranked, 7 not ranked.", "High throughput: gemma-4-31B-it ranks first (0.81); 29 ranked, 7 not ranked.", "Long context: Akshara 248k ranks first (0.82); 29 ranked, 7 not ranked.", "Identical outputs on every input: cl100k_base (GPT-4, OLMo-2-0425-1B, Olmo-3-1025-7B, phi-4); Qwen2.5-0.5B (same tokenizer: Qwen3-0.6B).", "Near duplicates (same provider, headline scores within 0.01, but not identical): gemma-4-31B-it vs gemma-3-1b-it (token counts differ on 19 of 41 sets).", "7 tokenizers normalize text to NFC, so the 733 inputs not already in NFC decode to canonically equivalent but different text: pythia-160m, Qwen2.5-0.5B, Qwen3-0.6B, Seed-OSS-36B-Instruct, Ling-3.0-tiny, MiniMax-M2.5, Qwen3.8-27B. The frozen rule requires byte-exact round-trip, so they are not ranked; their other measurements are reported.", "sarvam-1 is not lossless: 215,393 inputs do not decode back exactly (0 of them only by NFC normalization; e.g. ' ret ; just return until the proper paging code is installed into this location'); it is not ranked.", "Metadata checks: Kimi-K3: the earlier BPT benchmark listed a vocabulary of 163,840; the tokenizer's own files give 163,584; gemma-3-1b-it: the tokenizer has 262,145 IDs but the model config has 262,144 embedding rows, so 1 IDs have no embedding.", "Byte fallback (under 1.2 bytes per token, or at least 50% of tokens are single bytes of a multi-byte character): Phi-3.5-mini-instruct on gu, te; SmolLM2-135M on gu, ko, te, zh; gpt2 on bn, gu, ko, ta, te, ur, zh; pythia-160m on ko; OpenAI p50k_base (Codex) on bn, gu, ko, ta, te, ur, zh; sarvam-1 on ar, ja, ko, ru, th, ur, vi, zh; sarvam-105b on ja, ko, th, vi.", "Normalize NFD input to NFC, so decoded text differs from the input: pythia-160m, Qwen2.5-0.5B, Qwen3-0.6B, Seed-OSS-36B-Instruct, Ling-3.0-tiny, MiniMax-M2.5, Qwen3.8-27B.", "Same-backend conversions: every tiktoken and Akshara tokenizer gives identical IDs on the tokenizers library.", "Single-thread encode on the tokenizers library ranges from 2.77 MB/s (Falcon-H1-34B-Instruct) to 6.79 MB/s (Phi-3.5-mini-instruct).", "Memory added by loading ranges from 24.6 MB (Akshara 32k) to 313.0 MB (gemma-3-1b-it).", "CPU cost of encoding (4-thread batch) ranges from 0.17 CPU-seconds per MB (gemma-4-31B-it) to 0.36 (Falcon-H1-34B-Instruct).", "Peak RAM encoding one 16 MiB input ranges from 1,771 MB (gemma-4-31B-it) to 3,193 MB (gpt2); working memory is 84–189 MB per MB of input on the tokenizers library.", "Akshara 448k on the tokenizers library: 5.62 MB/s single-thread, 204.8 MB to load, 0.68 s to load; its native runtime encodes at 36.18 MB/s." ], "tokenizers": [ { "id": "akshara-448k", "label": "Akshara 448k", "provider": "Akshara", "family": "akshara", "metrics": { "bpt_overall": { "value": 5.409025106334158, "provenance": "measured" }, "bpt_english": { "value": 4.936344898660254, "provenance": "measured" }, "bpt_code": { "value": 4.077558580205492, "provenance": "measured" }, "bpt_languages": { "value": 7.213171840136726, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.2542721080357902, "provenance": "measured" }, "byte_fallback_share": { "value": 0.0014840604132270476, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 5.620347250340921, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.4628430832448185, "q3": 5.638177126563517, "n": 5 }, "batch_mbps_4t": { "value": 18.300612613557618, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.190271144082107, "q3": 18.779172937414017, "n": 5 }, "latency_small_p95_us": { "value": 63.45899964799173, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 61.58399992273189, "q3": 65.66699994436931, "n": 5 }, "decode_mbps": { "value": 43.08721156622042, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 38.015456932945334, "q3": 44.60498553642126, "n": 5 }, "rss_load_delta_mb": { "value": 204.791808, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 204.791808, "q3": 205.451264, "n": 5 }, "rss_peak_large_mb": { "value": 257.273856, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 256.765952, "q3": 258.854912, "n": 5 }, "load_seconds": { "value": 0.6785901250000279, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.6715609589996347, "q3": 0.6974505419998422, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6132046680317291, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2394.68544, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2393.84576, "q3": 2396.40576, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.57184, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.769024, "q3": 18.415616, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.446423169868346, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.396524588100035, "q3": 3.5321811815725868, "n": 5 }, "embedding_params_h2048": { "value": 917504000, "provenance": "estimated", "note": "448000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.19461342897082637, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19197595901550035, "q3": 0.19861731913171873, "n": 5 }, "cpu_cores_used_batch": { "value": 3.602732836677368, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.586244613345892, "q3": 3.605149734177987, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 130.41162109375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 130.26196289062503, "q3": 130.486083984375, "n": 5 }, "workload_tokens_total": { "value": 1886759, "provenance": "measured" }, "tokens_chat_en": { "value": 85087, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 51424, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 22958, "provenance": "measured" }, "tokens_code": { "value": 213885, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 118768, "provenance": "measured" }, "tokens_rag": { "value": 206733, "provenance": "measured" }, "tokens_json": { "value": 311319, "provenance": "measured" }, "tokens_logs": { "value": 398575, "provenance": "measured" }, "tokens_tech_docs": { "value": 7859, "provenance": "measured" }, "tokens_long_context": { "value": 470151, "provenance": "measured" } }, "parity": { "de": { "value": 1.2129238943584935, "provenance": "measured" }, "fr": { "value": 1.2960228160650102, "provenance": "measured" }, "es": { "value": 1.2541412720737615, "provenance": "measured" }, "ru": { "value": 1.3212611345522738, "provenance": "measured" }, "zh": { "value": 0.9644085013283326, "provenance": "measured" }, "ja": { "value": 1.0327004219409284, "provenance": "measured" }, "ko": { "value": 1.2520706360368807, "provenance": "measured" }, "ar": { "value": 1.1923347398030941, "provenance": "measured" }, "ur": { "value": 1.4810517268323176, "provenance": "measured" }, "hi": { "value": 1.5077355836849509, "provenance": "measured" }, "mr": { "value": 1.3952180028129395, "provenance": "measured" }, "gu": { "value": 1.4637834036568214, "provenance": "measured" }, "bn": { "value": 1.3837709017033912, "provenance": "measured" }, "ta": { "value": 1.4569854664791373, "provenance": "measured" }, "te": { "value": 1.502891076730739, "provenance": "measured" }, "th": { "value": 1.2362869198312236, "provenance": "measured" }, "vi": { "value": 1.365994686669792, "provenance": "measured" }, "tr": { "value": 1.1913970932958275, "provenance": "measured" }, "id": { "value": 1.082786372870761, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 25.49402390438247, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 30.922310756972113, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2129238943584935, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 33.04083665338646, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2960228160650102, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 31.973107569721115, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2541412720737615, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 33.68426294820717, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3212611345522738, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 24.586653386454184, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 0.9644085013283326, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 26.327689243027887, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0327004219409284, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 31.9203187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2520706360368807, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 30.397410358565736, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1923347398030941, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 37.75796812749004, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4810517268323176, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 38.43824701195219, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5077355836849509, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 35.569721115537845, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3952180028129395, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 37.31772908366534, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4637834036568214, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 35.27788844621514, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3837709017033912, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 37.14442231075697, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4569854664791373, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 38.31474103585657, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.502891076730739, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 31.51792828685259, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2362869198312236, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 34.82470119521912, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.365994686669792, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 30.37350597609562, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1913970932958275, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 27.60458167330677, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.082786372870761, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (exported from Akshara)" } }, "vocabulary": { "size": { "value": 448000, "provenance": "measured" }, "used_share": { "value": 0.5442388392857143, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 243819, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 204181, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 36, "provenance": "measured", "note": "of 446411 entries whose text is valid UTF-8; 1589 are byte pieces" }, "unreachable_examples": [ "️◾", "️◾️◾", "️◾️◾️◾️◾", "️⬜", "️⬜️⬜", "️◽", "️◽️◽", "️⬜️⬜️⬜️⬜", "️◾️◾️◾️◾️◾️◾️◾️◾", " \n \t", "️\n", "️◽️◽️◽️◽" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (exported from Akshara)", "load_seconds": { "value": 0.6785901250000279, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.6715609589996347, "q3": 0.6974505419998422, "n": 5 }, "rss_load_delta_mb": { "value": 204.791808, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 204.791808, "q3": 205.451264, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 204.705792, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 204.705792, "q3": 205.365248, "n": 5 }, "latency_small_p50_us": { "value": 46.3749997834384, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 45.91649985741242, "q3": 46.50000028050272, "n": 5 }, "latency_small_p95_us": { "value": 63.45899964799173, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 61.58399992273189, "q3": 65.66699994436931, "n": 5 }, "encode_mbps_1t_medium": { "value": 5.620347250340921, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.4628430832448185, "q3": 5.638177126563517, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1163362.370163222, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1130760.3950570768, "q3": 1167052.9974746888, "n": 5 }, "encode_mbps_1t_large": { "value": 4.823923846802096, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.748523975841772, "q3": 4.825493345102941, "n": 5 }, "latency_large_p50_ms": { "value": 215.85337499982415, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 215.79125000016575, "q3": 221.6181249998499, "n": 5 }, "decode_mbps": { "value": 43.08721156622042, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 38.015456932945334, "q3": 44.60498553642126, "n": 5 }, "decode_tokens_per_s": { "value": 8525159.594298717, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 7521671.1785922935, "q3": 8825463.671858909, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.606393794805339, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.553036519257405, "q3": 5.63421448183548, "n": 5 }, "batch_mbps_4t": { "value": 18.300612613557618, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.190271144082107, "q3": 18.779172937414017, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3776073.164559497, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3753305.759411766, "q3": 3874817.334205398, "n": 5 }, "batch_cpu_utilization": { "value": 3.602732836677368, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.586244613345892, "q3": 3.605149734177987, "n": 5 }, "cpu_cores_used_batch": { "value": 3.602732836677368, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.586244613345892, "q3": 3.605149734177987, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.19461342897082637, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19197595901550035, "q3": 0.19861731913171873, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 130.41162109375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 130.26196289062503, "q3": 130.486083984375, "n": 5 }, "rss_peak_large_mb": { "value": 257.273856, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 256.765952, "q3": 258.854912, "n": 5 }, "rss_peak_very_large_mb": { "value": 2394.68544, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2393.84576, "q3": 2396.40576, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.57184, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.769024, "q3": 18.415616, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.446423169868346, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.396524588100035, "q3": 3.5321811815725868, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6132046680317291, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "akshara-native (Rust)", "load_seconds": { "value": 0.8073415829999249, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.7976622080000197, "q3": 0.8185076249999952, "n": 5 }, "rss_load_delta_mb": { "value": 93.540352, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 93.536256, "q3": 93.540352, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 93.323264, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 93.323264, "q3": 93.32736, "n": 5 }, "latency_small_p50_us": { "value": 10.041999757959275, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9.999999974752427, "q3": 10.708999980124645, "n": 5 }, "latency_small_p95_us": { "value": 19.458000224403804, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 19.250000150350388, "q3": 20.166000012977747, "n": 5 }, "encode_mbps_1t_medium": { "value": 36.18286900914715, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 34.1841535617166, "q3": 36.61716876583317, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 7489535.143443936, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 7075818.65287236, "q3": 7579431.367252689, "n": 5 }, "encode_mbps_1t_large": { "value": 33.956435283557305, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 33.942915278666575, "q3": 35.03544665343124, "n": 5 }, "latency_large_p50_ms": { "value": 30.04879100035396, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 29.61841700016521, "q3": 30.643167000107496, "n": 5 }, "decode_mbps": { "value": 112.98578979403972, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 112.44912130252726, "q3": 113.2155370183665, "n": 5 }, "decode_tokens_per_s": { "value": 22355168.851010635, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 22248984.570964023, "q3": 22400626.231113378, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 33.47440188594872, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 32.82133203812969, "q3": 33.809270367391214, "n": 5 }, "batch_mbps_4t": { "value": 96.03981959090198, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 94.16075364712611, "q3": 97.95374011367092, "n": 5 }, "batch_tokens_per_s_4t": { "value": 19816461.51111228, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 19428742.76996625, "q3": 20211372.00278475, "n": 5 }, "batch_cpu_utilization": { "value": 3.1593753279251393, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.1459098183159018, "q3": 3.192017136422293, "n": 5 }, "cpu_cores_used_batch": { "value": 3.1593753279251393, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.1459098183159018, "q3": 3.192017136422293, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.03312668939821483, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.032212086727031373, "q3": 0.03355299533566941, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 8.2236328125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 8.223388671875, "q3": 8.2236328125, "n": 5 }, "rss_peak_large_mb": { "value": 88.43264, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 87.703552, "q3": 88.604672, "n": 5 }, "rss_peak_very_large_mb": { "value": 222.539776, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 222.53568, "q3": 222.605312, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.707008, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.686528, "q3": 17.887232, "n": 5 }, "encode_mbps_1t_very_large": { "value": 33.95617030318472, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 33.39287446831044, "q3": 35.55375593841288, "n": 5 }, "throughput_ratio_very_large": { "value": 0.9384598632739845, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.7412542962092106, "provenance": "measured" }, "chars_per_token": { "value": 2.738784854842974, "provenance": "measured" }, "tokens_per_word": { "value": 2.405328245927369, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.5701657380233365, "provenance": "measured" }, "chars_per_token": { "value": 3.5559099995746672, "provenance": "measured" }, "tokens_per_word": { "value": 2.3748884661357934, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.9995082037319496, "provenance": "measured" }, "chars_per_token": { "value": 3.9988667303388405, "provenance": "measured" }, "tokens_per_word": { "value": 2.6353237288453952, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 5.044786903520814, "provenance": "measured" }, "chars_per_token": { "value": 5.01683508634304, "provenance": "measured" }, "tokens_per_word": { "value": 2.433278914981791, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.290062858751184, "provenance": "measured" }, "chars_per_token": { "value": 4.281585029894155, "provenance": "measured" }, "tokens_per_word": { "value": 2.493449874526817, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.728079953872766, "provenance": "measured" }, "chars_per_token": { "value": 3.7251047472611956, "provenance": "measured" }, "tokens_per_word": { "value": 2.254489046034387, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.843028904070197, "provenance": "measured" }, "chars_per_token": { "value": 3.7833787995367887, "provenance": "measured" }, "tokens_per_word": { "value": 3.1351608187134503, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.7219867537330815, "provenance": "measured" }, "chars_per_token": { "value": 4.719434378813425, "provenance": "measured" }, "tokens_per_word": { "value": 2.09049306745599, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.132617495419489, "provenance": "measured" }, "chars_per_token": { "value": 4.117295155311262, "provenance": "measured" }, "tokens_per_word": { "value": 2.4386417007834744, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.765509268330916, "provenance": "measured" }, "chars_per_token": { "value": 4.749676686130814, "provenance": "measured" }, "tokens_per_word": { "value": 2.725010679196924, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.855985018028099, "provenance": "measured" }, "chars_per_token": { "value": 3.824785527788139, "provenance": "measured" }, "tokens_per_word": { "value": 2.512063715155774, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 4.0434815041981285, "provenance": "measured" }, "chars_per_token": { "value": 4.037737569312634, "provenance": "measured" }, "tokens_per_word": { "value": 2.832468599600892, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 4.252620132993183, "provenance": "measured" }, "chars_per_token": { "value": 4.252218644138681, "provenance": "measured" }, "tokens_per_word": { "value": 2.5432905037440436, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.360578741506437, "provenance": "measured" }, "chars_per_token": { "value": 4.348659941529576, "provenance": "measured" }, "tokens_per_word": { "value": 2.726119208350771, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.2167408139726477, "provenance": "measured" }, "chars_per_token": { "value": 3.2016394793211402, "provenance": "measured" }, "tokens_per_word": { "value": 2.7906655937464078, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.212066636650158, "provenance": "measured" }, "chars_per_token": { "value": 4.122187014178553, "provenance": "measured" }, "tokens_per_word": { "value": 2.479107872456931, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.382981410158117, "provenance": "measured" }, "chars_per_token": { "value": 4.38150216024606, "provenance": "measured" }, "tokens_per_word": { "value": 2.3049272349272347, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.630454334237705, "provenance": "measured" }, "chars_per_token": { "value": 3.6238514232608336, "provenance": "measured" }, "tokens_per_word": { "value": 2.407194919535253, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.6156496970859746, "provenance": "measured" }, "chars_per_token": { "value": 3.601415320587085, "provenance": "measured" }, "tokens_per_word": { "value": 2.492587551500883, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.820904741362407, "provenance": "measured" }, "chars_per_token": { "value": 4.802835405310012, "provenance": "measured" }, "tokens_per_word": { "value": 2.1924250892051074, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.400266778459545, "provenance": "measured" }, "chars_per_token": { "value": 4.306040626548268, "provenance": "measured" }, "tokens_per_word": { "value": 2.8442743788752547, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.936344898660254, "provenance": "measured" }, "chars_per_token": { "value": 4.916270795504736, "provenance": "measured" }, "tokens_per_word": { "value": 1.2439753243938696, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 7.214051771013796, "provenance": "measured" }, "chars_per_token": { "value": 4.000644924695558, "provenance": "measured" }, "tokens_per_word": { "value": 1.5295152992147305, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 11.690768291913049, "provenance": "measured" }, "chars_per_token": { "value": 4.391574251383599, "provenance": "measured" }, "tokens_per_word": { "value": 1.6311742697284866, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 4.196140092184389, "provenance": "measured" }, "chars_per_token": { "value": 4.090228354473551, "provenance": "measured" }, "tokens_per_word": { "value": 2.1069077383153867, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.8296995692877225, "provenance": "measured" }, "chars_per_token": { "value": 4.722261773395508, "provenance": "measured" }, "tokens_per_word": { "value": 1.3217747247576928, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 4.598908067170958, "provenance": "measured" }, "chars_per_token": { "value": 4.452396585686146, "provenance": "measured" }, "tokens_per_word": { "value": 1.4416177176125828, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 8.616133947095436, "provenance": "measured" }, "chars_per_token": { "value": 3.347640041493776, "provenance": "measured" }, "tokens_per_word": { "value": 1.902670696354777, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 9.197875434946702, "provenance": "measured" }, "chars_per_token": { "value": 3.651343041773294, "provenance": "measured" }, "tokens_per_word": { "value": 1.5140205151075927, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.985512639029323, "provenance": "measured" }, "chars_per_token": { "value": 4.97393731041456, "provenance": "measured" }, "tokens_per_word": { "value": 1.5004490727255073, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 5.928718130311615, "provenance": "measured" }, "chars_per_token": { "value": 2.080240793201133, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 4.546506996397894, "provenance": "measured" }, "chars_per_token": { "value": 1.9621432529786644, "provenance": "measured" }, "tokens_per_word": { "value": 2.2834997528423133, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 9.930422892473544, "provenance": "measured" }, "chars_per_token": { "value": 3.7732947564503982, "provenance": "measured" }, "tokens_per_word": { "value": 1.8598702638950317, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 7.375746138815279, "provenance": "measured" }, "chars_per_token": { "value": 4.0580716828608905, "provenance": "measured" }, "tokens_per_word": { "value": 1.8235466393322595, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 10.973319450771132, "provenance": "measured" }, "chars_per_token": { "value": 4.180776989653153, "provenance": "measured" }, "tokens_per_word": { "value": 2.2892541463899096, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 9.22593829255387, "provenance": "measured" }, "chars_per_token": { "value": 3.5146970277519296, "provenance": "measured" }, "tokens_per_word": { "value": 2.2714161382568694, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 12.362165319832538, "provenance": "measured" }, "chars_per_token": { "value": 4.41995910816863, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 4.92409215301904, "provenance": "measured" }, "chars_per_token": { "value": 4.501251868097743, "provenance": "measured" }, "tokens_per_word": { "value": 1.7338762598643804, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 6.369553682703817, "provenance": "measured" }, "chars_per_token": { "value": 3.597537049147962, "provenance": "measured" }, "tokens_per_word": { "value": 1.3299145018625793, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.986860777668331, "provenance": "measured" }, "chars_per_token": { "value": 3.8179625022956074, "provenance": "measured" }, "tokens_per_word": { "value": 1.2032946612425293, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 5.097851315409366, "provenance": "measured" }, "chars_per_token": { "value": 1.9147100720212777, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 85087, "provenance": "measured" }, "tokens_per_request_mean": { "value": 425.435, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.425435, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 51424, "provenance": "measured" }, "tokens_per_request_mean": { "value": 315.4846625766871, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3154846625766871, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 22958, "provenance": "measured" }, "tokens_per_request_mean": { "value": 120.83157894736843, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.12083157894736843, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 213885, "provenance": "measured" }, "tokens_per_request_mean": { "value": 509.25, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.50925, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 118768, "provenance": "measured" }, "tokens_per_request_mean": { "value": 791.7866666666666, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.7917866666666666, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 206733, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1033.665, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.033665, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 311319, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1556.595, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.556595, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 398575, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1992.875, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.992875, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 7859, "provenance": "measured" }, "tokens_per_request_mean": { "value": 314.36, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.31436, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 470151, "provenance": "measured" }, "tokens_per_request_mean": { "value": 78358.5, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 78.3585, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 448000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 917504000, "provenance": "estimated", "note": "448000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1835008000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1835008000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 3670016000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 448000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 1835008000, "provenance": "estimated", "note": "448000 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 3670016000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 3670016000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 7340032000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "akshara-248k", "label": "Akshara 248k", "provider": "Akshara", "family": "akshara", "metrics": { "bpt_overall": { "value": 5.140763130612541, "provenance": "measured" }, "bpt_english": { "value": 4.839411791591593, "provenance": "measured" }, "bpt_code": { "value": 3.9913768921416097, "provenance": "measured" }, "bpt_languages": { "value": 6.59150070810442, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.32049966978282435, "provenance": "measured" }, "byte_fallback_share": { "value": 0.0025449197164802108, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 5.836628228431435, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.729098732557851, "q3": 5.852981716836457, "n": 5 }, "batch_mbps_4t": { "value": 18.40640894684117, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.07314929157135, "q3": 18.44548721846948, "n": 5 }, "latency_small_p95_us": { "value": 61.37500008662755, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 60.58300004951889, "q3": 65.91700002900325, "n": 5 }, "decode_mbps": { "value": 41.01579788879382, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.03215785134015, "q3": 43.44499781195673, "n": 5 }, "rss_load_delta_mb": { "value": 119.328768, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 119.328768, "q3": 119.328768, "n": 5 }, "rss_peak_large_mb": { "value": 220.925952, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 220.91776, "q3": 220.971008, "n": 5 }, "load_seconds": { "value": 0.33785904199999095, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.33691641699988395, "q3": 0.34502274999999827, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6222676200366855, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2374.10304, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2373.025792, "q3": 2375.737344, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.403328, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.616896, "q3": 18.522112, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.6319447567449643, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.6174148114076146, "q3": 3.689625088228187, "n": 5 }, "embedding_params_h2048": { "value": 507904000, "provenance": "estimated", "note": "248000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.19559169142126062, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19464037112459176, "q3": 0.19919049557111526, "n": 5 }, "cpu_cores_used_batch": { "value": 3.599999563918848, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5902364777768137, "q3": 3.600140658904089, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 131.10302734375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 131.0390625, "q3": 131.19409179687503, "n": 5 }, "workload_tokens_total": { "value": 1943521, "provenance": "measured" }, "tokens_chat_en": { "value": 86523, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 54663, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 24303, "provenance": "measured" }, "tokens_code": { "value": 218773, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 120183, "provenance": "measured" }, "tokens_rag": { "value": 223147, "provenance": "measured" }, "tokens_json": { "value": 314663, "provenance": "measured" }, "tokens_logs": { "value": 401363, "provenance": "measured" }, "tokens_tech_docs": { "value": 8035, "provenance": "measured" }, "tokens_long_context": { "value": 491868, "provenance": "measured" } }, "parity": { "de": { "value": 1.2823836183756423, "provenance": "measured" }, "fr": { "value": 1.3447350256921542, "provenance": "measured" }, "es": { "value": 1.2998696219035202, "provenance": "measured" }, "ru": { "value": 1.4355395352404325, "provenance": "measured" }, "zh": { "value": 1.0240816013497969, "provenance": "measured" }, "ja": { "value": 1.1046859421734796, "provenance": "measured" }, "ko": { "value": 1.3632563846920775, "provenance": "measured" }, "ar": { "value": 1.2786256614771072, "provenance": "measured" }, "ur": { "value": 1.573471892016259, "provenance": "measured" }, "hi": { "value": 1.6162282383618376, "provenance": "measured" }, "mr": { "value": 1.523621443362221, "provenance": "measured" }, "gu": { "value": 1.5921849835110053, "provenance": "measured" }, "bn": { "value": 1.484699746913107, "provenance": "measured" }, "ta": { "value": 1.6077153155916866, "provenance": "measured" }, "te": { "value": 1.651238591916558, "provenance": "measured" }, "th": { "value": 1.3725362374415215, "provenance": "measured" }, "vi": { "value": 1.3933967328782881, "provenance": "measured" }, "tr": { "value": 1.2854896847917785, "provenance": "measured" }, "id": { "value": 1.1630876600966331, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 25.97410358565737, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 33.308764940239044, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2823836183756423, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 34.92828685258964, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3447350256921542, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 33.76294820717131, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2998696219035202, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 37.286852589641434, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4355395352404325, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 26.599601593625497, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0240816013497969, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 28.693227091633467, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1046859421734796, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 35.4093625498008, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3632563846920775, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 33.211155378486055, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2786256614771072, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 40.8695219123506, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.573471892016259, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 41.9800796812749, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6162282383618376, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 39.57470119521912, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.523621443362221, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 41.35557768924303, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5921849835110053, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 38.56374501992032, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.484699746913107, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 41.7589641434263, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6077153155916866, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 42.8894422310757, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.651238591916558, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 35.6503984063745, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3725362374415215, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 36.19223107569721, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3933967328782881, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 33.3894422310757, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2854896847917785, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 30.2101593625498, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1630876600966331, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (exported from Akshara)" } }, "vocabulary": { "size": { "value": 248000, "provenance": "measured" }, "used_share": { "value": 0.7113104838709677, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 176405, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 71595, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 24, "provenance": "measured", "note": "of 246660 entries whose text is valid UTF-8; 1340 are byte pieces" }, "unreachable_examples": [ "️◾", "️◾️◾", "️◾️◾️◾️◾", "️⬜", "️⬜️⬜", "️◽", "️◽️◽", "️⬜️⬜️⬜️⬜", "️◾️◾️◾️◾️◾️◾️◾️◾", " \n \t", "️\n", "️◽️◽️◽️◽" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (exported from Akshara)", "load_seconds": { "value": 0.33785904199999095, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.33691641699988395, "q3": 0.34502274999999827, "n": 5 }, "rss_load_delta_mb": { "value": 119.328768, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 119.328768, "q3": 119.328768, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 119.250944, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 119.250944, "q3": 119.250944, "n": 5 }, "latency_small_p50_us": { "value": 44.75000002912566, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 44.74999991543882, "q3": 46.00000011123484, "n": 5 }, "latency_small_p95_us": { "value": 61.37500008662755, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 60.58300004951889, "q3": 65.91700002900325, "n": 5 }, "encode_mbps_1t_medium": { "value": 5.836628228431435, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.729098732557851, "q3": 5.852981716836457, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1257138.28674862, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1233977.7493755228, "q3": 1260660.6280030631, "n": 5 }, "encode_mbps_1t_large": { "value": 4.92427116435717, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.88432846298033, "q3": 4.926202875879603, "n": 5 }, "latency_large_p50_ms": { "value": 210.24804199998925, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 207.20229099993048, "q3": 211.22887499996068, "n": 5 }, "decode_mbps": { "value": 41.01579788879382, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.03215785134015, "q3": 43.44499781195673, "n": 5 }, "decode_tokens_per_s": { "value": 8474591.851217687, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 8271354.360421153, "q3": 8976507.672278423, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.728144128412491, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.484624536931851, "q3": 5.794024870745297, "n": 5 }, "batch_mbps_4t": { "value": 18.40640894684117, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.07314929157135, "q3": 18.44548721846948, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3951050.638346849, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3879514.3719576877, "q3": 3959439.033416656, "n": 5 }, "batch_cpu_utilization": { "value": 3.599999563918848, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5902364777768137, "q3": 3.600140658904089, "n": 5 }, "cpu_cores_used_batch": { "value": 3.599999563918848, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5902364777768137, "q3": 3.600140658904089, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.19559169142126062, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19464037112459176, "q3": 0.19919049557111526, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 131.10302734375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 131.0390625, "q3": 131.19409179687503, "n": 5 }, "rss_peak_large_mb": { "value": 220.925952, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 220.91776, "q3": 220.971008, "n": 5 }, "rss_peak_very_large_mb": { "value": 2374.10304, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2373.025792, "q3": 2375.737344, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.403328, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.616896, "q3": 18.522112, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.6319447567449643, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.6174148114076146, "q3": 3.689625088228187, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6222676200366855, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "akshara-native (Rust)", "load_seconds": { "value": 0.41601233400001547, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.41419583400011106, "q3": 0.4196818330001406, "n": 5 }, "rss_load_delta_mb": { "value": 72.159232, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 72.155136, "q3": 72.159232, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 71.938048, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 71.933952, "q3": 71.938048, "n": 5 }, "latency_small_p50_us": { "value": 9.542000043438748, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9.459000011702301, "q3": 9.56250005401671, "n": 5 }, "latency_small_p95_us": { "value": 15.874999917286914, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 15.624999832652975, "q3": 16.2080000336573, "n": 5 }, "encode_mbps_1t_medium": { "value": 41.148355701987875, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 38.179145933126726, "q3": 41.25354307372325, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 8862852.209386276, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 8223320.764905955, "q3": 8885508.281884924, "n": 5 }, "encode_mbps_1t_large": { "value": 39.193522357746424, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 39.05755762604685, "q3": 40.45741723304305, "n": 5 }, "latency_large_p50_ms": { "value": 26.05112499986717, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 26.046459000099276, "q3": 26.71374999999898, "n": 5 }, "decode_mbps": { "value": 110.3553940136642, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 109.80790261679124, "q3": 110.50125141473536, "n": 5 }, "decode_tokens_per_s": { "value": 22801383.14953108, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 22688261.709272105, "q3": 22831519.87747939, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 38.382632429702454, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 38.28435747503214, "q3": 39.18722359495706, "n": 5 }, "batch_mbps_4t": { "value": 106.473221629358, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 104.02521484246931, "q3": 109.93066708625297, "n": 5 }, "batch_tokens_per_s_4t": { "value": 22855142.005182706, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 22329662.059260182, "q3": 23597304.266108602, "n": 5 }, "batch_cpu_utilization": { "value": 3.102361224422478, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0547019656329772, "q3": 3.1542357859680386, "n": 5 }, "cpu_cores_used_batch": { "value": 3.102361224422478, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.0547019656329772, "q3": 3.1542357859680386, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.028916322200501585, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.028385108761659932, "q3": 0.02962468626100365, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 9.595458984375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 9.590576171875, "q3": 9.661865234375, "n": 5 }, "rss_peak_large_mb": { "value": 63.01696, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 63.01696, "q3": 63.021056, "n": 5 }, "rss_peak_very_large_mb": { "value": 218.144768, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 218.058752, "q3": 219.295744, "n": 5 }, "rss_peak_chunked_mb": { "value": 19.304448, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 19.214336, "q3": 19.39456, "n": 5 }, "encode_mbps_1t_very_large": { "value": 37.248202771272716, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.989390086012065, "q3": 38.45133801622376, "n": 5 }, "throughput_ratio_very_large": { "value": 0.9052172835541338, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.683500459804294, "provenance": "measured" }, "chars_per_token": { "value": 2.6810830456114845, "provenance": "measured" }, "tokens_per_word": { "value": 2.4570953076798183, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.5042130238448106, "provenance": "measured" }, "chars_per_token": { "value": 3.4902206358152252, "provenance": "measured" }, "tokens_per_word": { "value": 2.4195861883196685, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.9275920235737134, "provenance": "measured" }, "chars_per_token": { "value": 3.926962084677786, "provenance": "measured" }, "tokens_per_word": { "value": 2.683577828283777, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.9353623685990975, "provenance": "measured" }, "chars_per_token": { "value": 4.908016843550879, "provenance": "measured" }, "tokens_per_word": { "value": 2.4872284314956676, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.21708474462591, "provenance": "measured" }, "chars_per_token": { "value": 4.208751131828699, "provenance": "measured" }, "tokens_per_word": { "value": 2.5365998894134663, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.6480376742471545, "provenance": "measured" }, "chars_per_token": { "value": 3.645126345643163, "provenance": "measured" }, "tokens_per_word": { "value": 2.3039552135330004, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.787279509194936, "provenance": "measured" }, "chars_per_token": { "value": 3.7284947266028907, "provenance": "measured" }, "tokens_per_word": { "value": 3.181310916179337, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.626146475017111, "provenance": "measured" }, "chars_per_token": { "value": 4.623645904631531, "provenance": "measured" }, "tokens_per_word": { "value": 2.1338019940800748, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.062205245478686, "provenance": "measured" }, "chars_per_token": { "value": 4.047143970044983, "provenance": "measured" }, "tokens_per_word": { "value": 2.4809119058999562, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.631908734239897, "provenance": "measured" }, "chars_per_token": { "value": 4.616520016759989, "provenance": "measured" }, "tokens_per_word": { "value": 2.8036095685604443, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.7524974099533415, "provenance": "measured" }, "chars_per_token": { "value": 3.722135257151932, "provenance": "measured" }, "tokens_per_word": { "value": 2.58134223471539, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.9548770996822102, "provenance": "measured" }, "chars_per_token": { "value": 3.9492590310655684, "provenance": "measured" }, "tokens_per_word": { "value": 2.895926751966193, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 4.129248661950669, "provenance": "measured" }, "chars_per_token": { "value": 4.128858820567381, "provenance": "measured" }, "tokens_per_word": { "value": 2.6192775697753574, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.2687469915731375, "provenance": "measured" }, "chars_per_token": { "value": 4.257079195952318, "provenance": "measured" }, "tokens_per_word": { "value": 2.784765058742896, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.1424639232508915, "provenance": "measured" }, "chars_per_token": { "value": 3.127711289426877, "provenance": "measured" }, "tokens_per_word": { "value": 2.8566271985285665, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.1492531068245535, "provenance": "measured" }, "chars_per_token": { "value": 4.060713837399055, "provenance": "measured" }, "tokens_per_word": { "value": 2.5166378838296617, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.32231838680697, "provenance": "measured" }, "chars_per_token": { "value": 4.320859610577907, "provenance": "measured" }, "tokens_per_word": { "value": 2.337276507276507, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.5268116522384907, "provenance": "measured" }, "chars_per_token": { "value": 3.5203972420220317, "provenance": "measured" }, "tokens_per_word": { "value": 2.4779353395395827, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.5430715943684787, "provenance": "measured" }, "chars_per_token": { "value": 3.529122949100822, "provenance": "measured" }, "tokens_per_word": { "value": 2.5436469982342556, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.711248443039188, "provenance": "measured" }, "chars_per_token": { "value": 4.6935901121011785, "provenance": "measured" }, "tokens_per_word": { "value": 2.2434547095997592, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.295047206660268, "provenance": "measured" }, "chars_per_token": { "value": 4.203074198900371, "provenance": "measured" }, "tokens_per_word": { "value": 2.9139530850279667, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.839411791591593, "provenance": "measured" }, "chars_per_token": { "value": 4.819731875882541, "provenance": "measured" }, "tokens_per_word": { "value": 1.2688920701685842, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 6.5817527343209195, "provenance": "measured" }, "chars_per_token": { "value": 3.6499953851123723, "provenance": "measured" }, "tokens_per_word": { "value": 1.676453522107462, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 10.559511138143389, "provenance": "measured" }, "chars_per_token": { "value": 3.966623583973238, "provenance": "measured" }, "tokens_per_word": { "value": 1.8059245529125048, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.940703792150047, "provenance": "measured" }, "chars_per_token": { "value": 3.8412393373746507, "provenance": "measured" }, "tokens_per_word": { "value": 2.243477433876132, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.583460716668497, "provenance": "measured" }, "chars_per_token": { "value": 4.481500561612715, "provenance": "measured" }, "tokens_per_word": { "value": 1.3927848875505786, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 4.349285471248724, "provenance": "measured" }, "chars_per_token": { "value": 4.210726437563797, "provenance": "measured" }, "tokens_per_word": { "value": 1.5243578273103304, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 7.753515169194865, "provenance": "measured" }, "chars_per_token": { "value": 3.012485414235706, "provenance": "measured" }, "tokens_per_word": { "value": 2.1143526799481895, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 8.488819791348082, "provenance": "measured" }, "chars_per_token": { "value": 3.369864410235498, "provenance": "measured" }, "tokens_per_word": { "value": 1.6404838889508306, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.63754223068299, "provenance": "measured" }, "chars_per_token": { "value": 4.626774817345242, "provenance": "measured" }, "tokens_per_word": { "value": 1.613032818720264, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 5.467242111221169, "provenance": "measured" }, "chars_per_token": { "value": 1.9183202533987875, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 4.171800637210891, "provenance": "measured" }, "chars_per_token": { "value": 1.800430633794424, "provenance": "measured" }, "tokens_per_word": { "value": 2.488601087493821, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 8.964062108690207, "provenance": "measured" }, "chars_per_token": { "value": 3.406103538334943, "provenance": "measured" }, "tokens_per_word": { "value": 2.0603715170278636, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 6.7619754601227, "provenance": "measured" }, "chars_per_token": { "value": 3.7203803680981595, "provenance": "measured" }, "tokens_per_word": { "value": 1.9890662371259824, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 9.815498942548361, "provenance": "measured" }, "chars_per_token": { "value": 3.7396534663070686, "provenance": "measured" }, "tokens_per_word": { "value": 2.559290892839408, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 8.199753510848765, "provenance": "measured" }, "chars_per_token": { "value": 3.1237635001459476, "provenance": "measured" }, "tokens_per_word": { "value": 2.5556798872725768, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 10.92155378554594, "provenance": "measured" }, "chars_per_token": { "value": 3.9048839649744536, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 4.47205231707532, "provenance": "measured" }, "chars_per_token": { "value": 4.088029472422485, "provenance": "measured" }, "tokens_per_word": { "value": 1.9091383284817687, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.972129023301016, "provenance": "measured" }, "chars_per_token": { "value": 3.3730707823310273, "provenance": "measured" }, "tokens_per_word": { "value": 1.4184157408471043, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.865620342240937, "provenance": "measured" }, "chars_per_token": { "value": 3.725140292721068, "provenance": "measured" }, "tokens_per_word": { "value": 1.2332780874893274, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.732234171421174, "provenance": "measured" }, "chars_per_token": { "value": 1.7773873482333578, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 86523, "provenance": "measured" }, "tokens_per_request_mean": { "value": 432.615, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.432615, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 54663, "provenance": "measured" }, "tokens_per_request_mean": { "value": 335.3558282208589, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33535582822085885, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 24303, "provenance": "measured" }, "tokens_per_request_mean": { "value": 127.91052631578947, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.12791052631578947, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 218773, "provenance": "measured" }, "tokens_per_request_mean": { "value": 520.8880952380953, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5208880952380953, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 120183, "provenance": "measured" }, "tokens_per_request_mean": { "value": 801.22, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.80122, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 223147, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1115.735, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.115735, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 314663, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1573.315, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.573315, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 401363, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2006.815, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.006815, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8035, "provenance": "measured" }, "tokens_per_request_mean": { "value": 321.4, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3214, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 491868, "provenance": "measured" }, "tokens_per_request_mean": { "value": 81978.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 81.978, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 248000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 507904000, "provenance": "estimated", "note": "248000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1015808000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1015808000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2031616000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 248000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 1015808000, "provenance": "estimated", "note": "248000 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 2031616000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 2031616000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 4063232000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "akshara-200k", "label": "Akshara 200k", "provider": "Akshara", "family": "akshara", "metrics": { "bpt_overall": { "value": 5.031530165012163, "provenance": "measured" }, "bpt_english": { "value": 4.791828140370768, "provenance": "measured" }, "bpt_code": { "value": 3.952270165080198, "provenance": "measured" }, "bpt_languages": { "value": 6.350492189585523, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.3473345700671691, "provenance": "measured" }, "byte_fallback_share": { "value": 0.003131201977013887, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 5.734110759561312, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.429966938050952, "q3": 5.844051229124779, "n": 5 }, "batch_mbps_4t": { "value": 18.650088442681785, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.585399546469404, "q3": 18.674387758522915, "n": 5 }, "latency_small_p95_us": { "value": 61.70799997562426, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 59.04099998588208, "q3": 61.70899996504886, "n": 5 }, "decode_mbps": { "value": 45.14105379322546, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 44.88950725625062, "q3": 45.64243052736785, "n": 5 }, "rss_load_delta_mb": { "value": 84.148224, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 84.148224, "q3": 84.148224, "n": 5 }, "rss_peak_large_mb": { "value": 191.758336, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 191.75424, "q3": 191.758336, "n": 5 }, "load_seconds": { "value": 0.2555477079999946, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.2500330009997924, "q3": 0.2636542090003786, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6265209433228459, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2359.476224, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2359.369728, "q3": 2361.46688, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.129472, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.068032, "q3": 17.199104, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.5925404821980336, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5421169844501246, "q3": 3.593587164699474, "n": 5 }, "embedding_params_h2048": { "value": 409600000, "provenance": "estimated", "note": "200000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.1927506055427878, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19189894737243662, "q3": 0.19449302377037084, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5789323405534805, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.572936159223736, "q3": 3.6132156532798265, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 134.06909179687503, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 134.06494140625, "q3": 134.18994140625, "n": 5 }, "workload_tokens_total": { "value": 1970192, "provenance": "measured" }, "tokens_chat_en": { "value": 87229, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 56012, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 24928, "provenance": "measured" }, "tokens_code": { "value": 221040, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 120901, "provenance": "measured" }, "tokens_rag": { "value": 230398, "provenance": "measured" }, "tokens_json": { "value": 316416, "provenance": "measured" }, "tokens_logs": { "value": 403511, "provenance": "measured" }, "tokens_tech_docs": { "value": 8088, "provenance": "measured" }, "tokens_long_context": { "value": 501669, "provenance": "measured" } }, "parity": { "de": { "value": 1.3120987654320988, "provenance": "measured" }, "fr": { "value": 1.3652421652421654, "provenance": "measured" }, "es": { "value": 1.3228110161443494, "provenance": "measured" }, "ru": { "value": 1.4845584045584046, "provenance": "measured" }, "zh": { "value": 1.0424311490978158, "provenance": "measured" }, "ja": { "value": 1.1342830009496676, "provenance": "measured" }, "ko": { "value": 1.4117758784425452, "provenance": "measured" }, "ar": { "value": 1.3158594491927824, "provenance": "measured" }, "ur": { "value": 1.6084330484330485, "provenance": "measured" }, "hi": { "value": 1.6679202279202279, "provenance": "measured" }, "mr": { "value": 1.5828679962013295, "provenance": "measured" }, "gu": { "value": 1.6444824311490978, "provenance": "measured" }, "bn": { "value": 1.5250902184235517, "provenance": "measured" }, "ta": { "value": 1.6701994301994303, "provenance": "measured" }, "te": { "value": 1.7077682811016144, "provenance": "measured" }, "th": { "value": 1.4143589743589744, "provenance": "measured" }, "vi": { "value": 1.4033048433048434, "provenance": "measured" }, "tr": { "value": 1.3233048433048433, "provenance": "measured" }, "id": { "value": 1.191604938271605, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.220119521912352, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 34.40338645418327, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3120987654320988, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 35.79681274900398, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3652421652421654, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 34.68426294820717, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3228110161443494, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 38.92529880478088, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4845584045584046, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 27.33266932270916, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0424311490978158, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 29.741035856573706, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1342830009496676, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 37.016932270916335, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4117758784425452, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 34.50199203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3158594491927824, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 42.17330677290837, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6084330484330485, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 43.733067729083665, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6679202279202279, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 41.50298804780876, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5828679962013295, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 43.11852589641434, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6444824311490978, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 39.98804780876494, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5250902184235517, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 43.79282868525896, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6701994301994303, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 44.77788844621514, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7077682811016144, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 37.08466135458168, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4143589743589744, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 36.79482071713147, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4033048433048434, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 34.69721115537848, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3233048433048433, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 31.24402390438247, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.191604938271605, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (exported from Akshara)" } }, "vocabulary": { "size": { "value": 200000, "provenance": "measured" }, "used_share": { "value": 0.766055, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 153211, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 46789, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 20, "provenance": "measured", "note": "of 198748 entries whose text is valid UTF-8; 1252 are byte pieces" }, "unreachable_examples": [ "️◾", "️◾️◾", "️◾️◾️◾️◾", "️⬜", "️⬜️⬜", "️◽", "️◽️◽", "️⬜️⬜️⬜️⬜", "️◾️◾️◾️◾️◾️◾️◾️◾", " \n \t", "️\n", "️◽️◽️◽️◽" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (exported from Akshara)", "load_seconds": { "value": 0.2555477079999946, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.2500330009997924, "q3": 0.2636542090003786, "n": 5 }, "rss_load_delta_mb": { "value": 84.148224, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 84.148224, "q3": 84.148224, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 84.074496, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 84.074496, "q3": 84.074496, "n": 5 }, "latency_small_p50_us": { "value": 45.083500026521506, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 44.29199998412514, "q3": 46.041000018703926, "n": 5 }, "latency_small_p95_us": { "value": 61.70799997562426, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 59.04099998588208, "q3": 61.70899996504886, "n": 5 }, "encode_mbps_1t_medium": { "value": 5.734110759561312, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.429966938050952, "q3": 5.844051229124779, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1257624.863474719, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1190918.9960713293, "q3": 1281737.3848093573, "n": 5 }, "encode_mbps_1t_large": { "value": 4.852403690848972, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.750938472496363, "q3": 4.899374160069841, "n": 5 }, "latency_large_p50_ms": { "value": 212.5544999998965, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 209.5327920001182, "q3": 220.840083999974, "n": 5 }, "decode_mbps": { "value": 45.14105379322546, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 44.88950725625062, "q3": 45.64243052736785, "n": 5 }, "decode_tokens_per_s": { "value": 9516362.703259533, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9463333.190621953, "q3": 9622059.900204252, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.746981436458913, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.740828055934775, "q3": 5.821680816875852, "n": 5 }, "batch_mbps_4t": { "value": 18.650088442681785, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.585399546469404, "q3": 18.674387758522915, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4075020.475018993, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4060886.0339206574, "q3": 4080329.8444563486, "n": 5 }, "batch_cpu_utilization": { "value": 3.5789323405534805, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.572936159223736, "q3": 3.6132156532798265, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5789323405534805, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.572936159223736, "q3": 3.6132156532798265, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.1927506055427878, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19189894737243662, "q3": 0.19449302377037084, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 134.06909179687503, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 134.06494140625, "q3": 134.18994140625, "n": 5 }, "rss_peak_large_mb": { "value": 191.758336, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 191.75424, "q3": 191.758336, "n": 5 }, "rss_peak_very_large_mb": { "value": 2359.476224, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2359.369728, "q3": 2361.46688, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.129472, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.068032, "q3": 17.199104, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.5925404821980336, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5421169844501246, "q3": 3.593587164699474, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6265209433228459, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "akshara-native (Rust)", "load_seconds": { "value": 0.3114650829999732, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.3081390420002208, "q3": 0.3124225839999326, "n": 5 }, "rss_load_delta_mb": { "value": 51.34336, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 51.34336, "q3": 51.34336, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 51.11808, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 51.11808, "q3": 51.11808, "n": 5 }, "latency_small_p50_us": { "value": 9.207999937643763, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9.124999905907316, "q3": 10.000000202126103, "n": 5 }, "latency_small_p95_us": { "value": 16.041999970184406, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 15.95799994902336, "q3": 16.95800006018544, "n": 5 }, "encode_mbps_1t_medium": { "value": 37.12108302421878, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.41212202025844, "q3": 40.28306496977799, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 8141523.407535022, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 7986031.645481451, "q3": 8835020.146495407, "n": 5 }, "encode_mbps_1t_large": { "value": 36.57678351385389, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.19861712323029, "q3": 37.078793551043844, "n": 5 }, "latency_large_p50_ms": { "value": 28.81829199986896, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 28.734792000022935, "q3": 28.980584000237286, "n": 5 }, "decode_mbps": { "value": 108.66943691211007, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 108.54476329906092, "q3": 110.9778271687773, "n": 5 }, "decode_tokens_per_s": { "value": 22909030.461531166, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 22882747.527890842, "q3": 23395671.27066559, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 36.708620791445895, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.34383579124641, "q3": 38.08049101315523, "n": 5 }, "batch_mbps_4t": { "value": 107.02999409867677, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 105.7416243454292, "q3": 107.85708361876581, "n": 5 }, "batch_tokens_per_s_4t": { "value": 23385916.84075434, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 23104409.69723518, "q3": 23566634.843212876, "n": 5 }, "batch_cpu_utilization": { "value": 3.0971002054103325, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0693066236596565, "q3": 3.109076699784603, "n": 5 }, "cpu_cores_used_batch": { "value": 3.0971002054103325, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.0693066236596565, "q3": 3.109076699784603, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.029026475076958005, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.02871485211261057, "q3": 0.029048648707932997, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 9.591064453125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 9.586181640625, "q3": 9.624755859375, "n": 5 }, "rss_peak_large_mb": { "value": 57.91744, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 57.901056, "q3": 58.642432, "n": 5 }, "rss_peak_very_large_mb": { "value": 211.120128, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 211.030016, "q3": 211.68128, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.096704, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.842752, "q3": 17.567744, "n": 5 }, "encode_mbps_1t_very_large": { "value": 36.45649341088134, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.937514418620964, "q3": 38.282477449847576, "n": 5 }, "throughput_ratio_very_large": { "value": 0.9820967073373414, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.6446460842147537, "provenance": "measured" }, "chars_per_token": { "value": 2.6422636717372074, "provenance": "measured" }, "tokens_per_word": { "value": 2.4931942414894195, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.4704313671444322, "provenance": "measured" }, "chars_per_token": { "value": 3.456573869900772, "provenance": "measured" }, "tokens_per_word": { "value": 2.4431387733800234, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.8865155387481733, "provenance": "measured" }, "chars_per_token": { "value": 3.885892188030281, "provenance": "measured" }, "tokens_per_word": { "value": 2.711940494750089, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.876954547722397, "provenance": "measured" }, "chars_per_token": { "value": 4.84993264481365, "provenance": "measured" }, "tokens_per_word": { "value": 2.5170161999246514, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.192550052511377, "provenance": "measured" }, "chars_per_token": { "value": 4.184264924066881, "provenance": "measured" }, "tokens_per_word": { "value": 2.551444004933861, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.614537756874203, "provenance": "measured" }, "chars_per_token": { "value": 3.611653162990735, "provenance": "measured" }, "tokens_per_word": { "value": 2.3253085135884635, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.7628683185194203, "provenance": "measured" }, "chars_per_token": { "value": 3.7044624375989286, "provenance": "measured" }, "tokens_per_word": { "value": 3.201949317738791, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.584848320399414, "provenance": "measured" }, "chars_per_token": { "value": 4.582370072900273, "provenance": "measured" }, "tokens_per_word": { "value": 2.1530222776133354, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.029330386446537, "provenance": "measured" }, "chars_per_token": { "value": 4.014390999808896, "provenance": "measured" }, "tokens_per_word": { "value": 2.5011533905525885, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.573290208540965, "provenance": "measured" }, "chars_per_token": { "value": 4.558096240997386, "provenance": "measured" }, "tokens_per_word": { "value": 2.839545066211021, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.7101872967213727, "provenance": "measured" }, "chars_per_token": { "value": 3.680167482896557, "provenance": "measured" }, "tokens_per_word": { "value": 2.6107792613414538, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.9198264432927563, "provenance": "measured" }, "chars_per_token": { "value": 3.9142581656020248, "provenance": "measured" }, "tokens_per_word": { "value": 2.9218218100716045, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 4.074155394576575, "provenance": "measured" }, "chars_per_token": { "value": 4.0737707545355475, "provenance": "measured" }, "tokens_per_word": { "value": 2.654697072838666, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.23057180477907, "provenance": "measured" }, "chars_per_token": { "value": 4.219008353659853, "provenance": "measured" }, "tokens_per_word": { "value": 2.8098937957554693, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.1142661157815814, "provenance": "measured" }, "chars_per_token": { "value": 3.0996458595881125, "provenance": "measured" }, "tokens_per_word": { "value": 2.8824922404874123, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.113554906580825, "provenance": "measured" }, "chars_per_token": { "value": 4.025777386917997, "provenance": "measured" }, "tokens_per_word": { "value": 2.5384777389326483, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.293446664133806, "provenance": "measured" }, "chars_per_token": { "value": 4.291997632069553, "provenance": "measured" }, "tokens_per_word": { "value": 2.352993762993763, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.485923804314973, "provenance": "measured" }, "chars_per_token": { "value": 3.479583759121461, "provenance": "measured" }, "tokens_per_word": { "value": 2.5070000721656927, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.5022479539687645, "provenance": "measured" }, "chars_per_token": { "value": 3.4884600264465173, "provenance": "measured" }, "tokens_per_word": { "value": 2.573296792230724, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.672656606355481, "provenance": "measured" }, "chars_per_token": { "value": 4.655142922305003, "provenance": "measured" }, "tokens_per_word": { "value": 2.2619835776621815, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.24486389505728, "provenance": "measured" }, "chars_per_token": { "value": 4.15396549949264, "provenance": "measured" }, "tokens_per_word": { "value": 2.948402202662273, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.791828140370768, "provenance": "measured" }, "chars_per_token": { "value": 4.772341727981021, "provenance": "measured" }, "tokens_per_word": { "value": 1.2814923796820008, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 6.333248220966507, "provenance": "measured" }, "chars_per_token": { "value": 3.512184020338155, "provenance": "measured" }, "tokens_per_word": { "value": 1.7422343429654559, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 10.075024482245839, "provenance": "measured" }, "chars_per_token": { "value": 3.784628776613108, "provenance": "measured" }, "tokens_per_word": { "value": 1.8927676518037964, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.842023328434974, "provenance": "measured" }, "chars_per_token": { "value": 3.7450495958853782, "provenance": "measured" }, "tokens_per_word": { "value": 2.3011000390984115, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.467297181614645, "provenance": "measured" }, "chars_per_token": { "value": 4.367921111550112, "provenance": "measured" }, "tokens_per_word": { "value": 1.4290015996988803, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 4.2451077255178715, "provenance": "measured" }, "chars_per_token": { "value": 4.109867574411557, "provenance": "measured" }, "tokens_per_word": { "value": 1.5617665746035452, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 7.43657144455947, "provenance": "measured" }, "chars_per_token": { "value": 2.889342771606838, "provenance": "measured" }, "tokens_per_word": { "value": 2.2044655523345464, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 8.16848696903306, "provenance": "measured" }, "chars_per_token": { "value": 3.24269971550963, "provenance": "measured" }, "tokens_per_word": { "value": 1.7048165904783141, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.489416255117055, "provenance": "measured" }, "chars_per_token": { "value": 4.478992759640459, "provenance": "measured" }, "tokens_per_word": { "value": 1.6662540052432275, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 5.314524235274201, "provenance": "measured" }, "chars_per_token": { "value": 1.864735321814392, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 4.03990920981765, "provenance": "measured" }, "chars_per_token": { "value": 1.7435100407786412, "provenance": "measured" }, "tokens_per_word": { "value": 2.5698467622343055, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 8.57696191698759, "provenance": "measured" }, "chars_per_token": { "value": 3.259015832263586, "provenance": "measured" }, "tokens_per_word": { "value": 2.153361344537815, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 6.5088875503431005, "provenance": "measured" }, "chars_per_token": { "value": 3.5811335908065525, "provenance": "measured" }, "tokens_per_word": { "value": 2.0664079660272368, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 9.375458690092294, "provenance": "measured" }, "chars_per_token": { "value": 3.5720004447903926, "provenance": "measured" }, "tokens_per_word": { "value": 2.6794120568080246, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 7.836435490120109, "provenance": "measured" }, "chars_per_token": { "value": 2.9853545137543587, "provenance": "measured" }, "tokens_per_word": { "value": 2.6741680135936012, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 10.42649739690256, "provenance": "measured" }, "chars_per_token": { "value": 3.72788188342722, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 4.314547124247475, "provenance": "measured" }, "chars_per_token": { "value": 3.944049522125098, "provenance": "measured" }, "tokens_per_word": { "value": 1.9788325957833455, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.793174268270785, "provenance": "measured" }, "chars_per_token": { "value": 3.271996767821871, "provenance": "measured" }, "tokens_per_word": { "value": 1.462231484976287, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.8026047109896295, "provenance": "measured" }, "chars_per_token": { "value": 3.6768952488142133, "provenance": "measured" }, "tokens_per_word": { "value": 1.2494600974335794, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.6131754015901345, "provenance": "measured" }, "chars_per_token": { "value": 1.7326698757819965, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 87229, "provenance": "measured" }, "tokens_per_request_mean": { "value": 436.145, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.436145, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 56012, "provenance": "measured" }, "tokens_per_request_mean": { "value": 343.6319018404908, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3436319018404908, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 24928, "provenance": "measured" }, "tokens_per_request_mean": { "value": 131.2, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.1312, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 221040, "provenance": "measured" }, "tokens_per_request_mean": { "value": 526.2857142857143, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5262857142857144, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 120901, "provenance": "measured" }, "tokens_per_request_mean": { "value": 806.0066666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8060066666666666, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 230398, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1151.99, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.15199, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 316416, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1582.08, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.58208, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 403511, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2017.555, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.017555, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8088, "provenance": "measured" }, "tokens_per_request_mean": { "value": 323.52, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.32352, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 501669, "provenance": "measured" }, "tokens_per_request_mean": { "value": 83611.5, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 83.6115, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 200000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 409600000, "provenance": "estimated", "note": "200000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 819200000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 819200000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1638400000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 200000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 819200000, "provenance": "estimated", "note": "200000 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1638400000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1638400000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 3276800000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "akshara-151k", "label": "Akshara 151k", "provider": "Akshara", "family": "akshara", "metrics": { "bpt_overall": { "value": 4.8772340657185085, "provenance": "measured" }, "bpt_english": { "value": 4.718526666869941, "provenance": "measured" }, "bpt_code": { "value": 3.893691457690418, "provenance": "measured" }, "bpt_languages": { "value": 6.019484072595167, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.3858641683087437, "provenance": "measured" }, "byte_fallback_share": { "value": 0.004016747638873015, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 6.116414017986229, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.0974718698952675, "q3": 6.128655757761346, "n": 5 }, "batch_mbps_4t": { "value": 18.45893944644569, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.424939107275005, "q3": 18.56070840705968, "n": 5 }, "latency_small_p95_us": { "value": 59.750000218627974, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 59.29200005994062, "q3": 71.33400004022405, "n": 5 }, "decode_mbps": { "value": 46.21559142784512, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 46.12766144661114, "q3": 46.52376839601841, "n": 5 }, "rss_load_delta_mb": { "value": 93.712384, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 93.36832, "q3": 93.71648, "n": 5 }, "rss_peak_large_mb": { "value": 188.841984, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 188.833792, "q3": 188.850176, "n": 5 }, "load_seconds": { "value": 0.199583500000017, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.1833465839999917, "q3": 0.20041354199975103, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5940175837183768, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2368.565248, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2368.323584, "q3": 2368.7168, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.649664, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.010688, "q3": 17.743872, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.6332574759853884, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.6305750428938315, "q3": 3.6573276145629863, "n": 5 }, "embedding_params_h2048": { "value": 309248000, "provenance": "estimated", "note": "151000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.19385141902893324, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19280472827645798, "q3": 0.1945450006687853, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5890226977657993, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5785923410417104, "q3": 3.591094386953844, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 136.063720703125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 136.041259765625, "q3": 136.068115234375, "n": 5 }, "workload_tokens_total": { "value": 2014225, "provenance": "measured" }, "tokens_chat_en": { "value": 88295, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 57827, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 25793, "provenance": "measured" }, "tokens_code": { "value": 224164, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 122559, "provenance": "measured" }, "tokens_rag": { "value": 240716, "provenance": "measured" }, "tokens_json": { "value": 320496, "provenance": "measured" }, "tokens_logs": { "value": 410909, "provenance": "measured" }, "tokens_tech_docs": { "value": 8245, "provenance": "measured" }, "tokens_long_context": { "value": 515221, "provenance": "measured" } }, "parity": { "de": { "value": 1.3506542196228395, "provenance": "measured" }, "fr": { "value": 1.3981179469875904, "provenance": "measured" }, "es": { "value": 1.355640535372849, "provenance": "measured" }, "ru": { "value": 1.5464702133243355, "provenance": "measured" }, "zh": { "value": 1.0758069958384884, "provenance": "measured" }, "ja": { "value": 1.1759832039890525, "provenance": "measured" }, "ko": { "value": 1.4799985003561653, "provenance": "measured" }, "ar": { "value": 1.3658006223521915, "provenance": "measured" }, "ur": { "value": 1.655306864619653, "provenance": "measured" }, "hi": { "value": 1.7308889138829529, "provenance": "measured" }, "mr": { "value": 1.6650170584486184, "provenance": "measured" }, "gu": { "value": 1.7316387358002474, "provenance": "measured" }, "bn": { "value": 1.5851235331608744, "provenance": "measured" }, "ta": { "value": 1.7440857796273386, "provenance": "measured" }, "te": { "value": 1.8236418850523002, "provenance": "measured" }, "th": { "value": 1.4969069845911596, "provenance": "measured" }, "vi": { "value": 1.4215873729989128, "provenance": "measured" }, "tr": { "value": 1.3717617065946837, "provenance": "measured" }, "id": { "value": 1.2410677464102275, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.566733067729082, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 35.88247011952191, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3506542196228395, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 37.14342629482072, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3981179469875904, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 36.014940239043824, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.355640535372849, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 41.08466135458168, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5464702133243355, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 28.580677290836654, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0758069958384884, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 31.24203187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1759832039890525, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 39.31872509960159, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4799985003561653, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 36.28486055776892, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3658006223521915, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 43.97609561752988, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.655306864619653, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 45.98406374501992, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7308889138829529, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 44.23406374501992, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6650170584486184, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 46.00398406374502, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7316387358002474, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 42.11155378486056, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5851235331608744, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 46.33466135458168, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7440857796273386, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 48.44820717131474, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8236418850523002, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 39.76792828685259, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4969069845911596, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 37.766932270916335, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4215873729989128, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 36.44322709163347, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3717617065946837, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 32.971115537848604, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2410677464102275, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (exported from Akshara)" } }, "vocabulary": { "size": { "value": 151000, "provenance": "measured" }, "used_share": { "value": 0.8287019867549669, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 125134, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 25866, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 17, "provenance": "measured", "note": "of 149839 entries whose text is valid UTF-8; 1161 are byte pieces" }, "unreachable_examples": [ "️◾", "️◾️◾", "️◾️◾️◾️◾", "️⬜", "️⬜️⬜", "️◽", "️◽️◽", "️⬜️⬜️⬜️⬜", "️◾️◾️◾️◾️◾️◾️◾️◾", " \n \t", "️\n", "️◽️◽️◽️◽" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (exported from Akshara)", "load_seconds": { "value": 0.199583500000017, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.1833465839999917, "q3": 0.20041354199975103, "n": 5 }, "rss_load_delta_mb": { "value": 93.712384, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 93.36832, "q3": 93.71648, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 93.626368, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 93.282304, "q3": 93.630464, "n": 5 }, "latency_small_p50_us": { "value": 44.45899958227528, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 44.16699994180817, "q3": 48.37499977838888, "n": 5 }, "latency_small_p95_us": { "value": 59.750000218627974, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 59.29200005994062, "q3": 71.33400004022405, "n": 5 }, "encode_mbps_1t_medium": { "value": 6.116414017986229, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.0974718698952675, "q3": 6.128655757761346, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1375434.5977129862, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1371174.963593599, "q3": 1378187.4709444877, "n": 5 }, "encode_mbps_1t_large": { "value": 4.990565721193887, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.913126611370299, "q3": 4.991362323914697, "n": 5 }, "latency_large_p50_ms": { "value": 206.4570420002383, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 206.30549999987124, "q3": 206.74149999990732, "n": 5 }, "decode_mbps": { "value": 46.21559142784512, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 46.12766144661114, "q3": 46.52376839601841, "n": 5 }, "decode_tokens_per_s": { "value": 10015315.960634524, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9996260.777799834, "q3": 10082100.558920251, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.8044436208649035, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.746288877959993, "q3": 5.850315307212808, "n": 5 }, "batch_mbps_4t": { "value": 18.45893944644569, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.424939107275005, "q3": 18.56070840705968, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4134633.8565118304, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4127018.09105164, "q3": 4157429.1742665367, "n": 5 }, "batch_cpu_utilization": { "value": 3.5890226977657993, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5785923410417104, "q3": 3.591094386953844, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5890226977657993, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5785923410417104, "q3": 3.591094386953844, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.19385141902893324, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19280472827645798, "q3": 0.1945450006687853, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 136.063720703125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 136.041259765625, "q3": 136.068115234375, "n": 5 }, "rss_peak_large_mb": { "value": 188.841984, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 188.833792, "q3": 188.850176, "n": 5 }, "rss_peak_very_large_mb": { "value": 2368.565248, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2368.323584, "q3": 2368.7168, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.649664, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.010688, "q3": 17.743872, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.6332574759853884, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.6305750428938315, "q3": 3.6573276145629863, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5940175837183768, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "akshara-native (Rust)", "load_seconds": { "value": 0.23353245799989963, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.23245729200016285, "q3": 0.23717933299997185, "n": 5 }, "rss_load_delta_mb": { "value": 45.367296, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 45.3632, "q3": 45.367296, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 45.13792, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 45.133824, "q3": 45.13792, "n": 5 }, "latency_small_p50_us": { "value": 8.584000170230865, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 8.541000170225743, "q3": 9.583000064594671, "n": 5 }, "latency_small_p95_us": { "value": 14.416999874811154, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.20800026608049, "q3": 15.250000160449417, "n": 5 }, "encode_mbps_1t_medium": { "value": 41.239480939779185, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 41.192756585857424, "q3": 43.16628297529017, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 9273768.70000909, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9263261.515120888, "q3": 9707060.196430417, "n": 5 }, "encode_mbps_1t_large": { "value": 36.267520144232755, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.707981379798255, "q3": 37.40348824548492, "n": 5 }, "latency_large_p50_ms": { "value": 28.68925000029776, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 28.423207999821898, "q3": 29.07358300035412, "n": 5 }, "decode_mbps": { "value": 110.06677227775391, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 104.97068436158962, "q3": 111.11994214786446, "n": 5 }, "decode_tokens_per_s": { "value": 23852415.755621836, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 22748049.69501992, "q3": 24080646.720183775, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 36.003180511804125, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.733897904822875, "q3": 36.78132116849989, "n": 5 }, "batch_mbps_4t": { "value": 111.58481435588936, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 103.20651842996239, "q3": 115.39791728780762, "n": 5 }, "batch_tokens_per_s_4t": { "value": 24993979.34789173, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 23117317.57673656, "q3": 25848079.580812175, "n": 5 }, "batch_cpu_utilization": { "value": 3.04510686041378, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0164652470623974, "q3": 3.0527870962058725, "n": 5 }, "cpu_cores_used_batch": { "value": 3.04510686041378, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.0164652470623974, "q3": 3.0527870962058725, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.027358445804903996, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.026594051601615668, "q3": 0.028802831358091977, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 8.943359375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 8.941650390625, "q3": 8.96630859375, "n": 5 }, "rss_peak_large_mb": { "value": 46.661632, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 46.641152, "q3": 47.104, "n": 5 }, "rss_peak_very_large_mb": { "value": 196.38272, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 196.325376, "q3": 197.156864, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.3296, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.473536, "q3": 18.55488, "n": 5 }, "encode_mbps_1t_very_large": { "value": 40.88607117563489, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.78357568805734, "q3": 40.934241209296154, "n": 5 }, "throughput_ratio_very_large": { "value": 0.991430305229584, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.603154847118651, "provenance": "measured" }, "chars_per_token": { "value": 2.600809811756159, "provenance": "measured" }, "tokens_per_word": { "value": 2.532932835417005, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.4317672310994514, "provenance": "measured" }, "chars_per_token": { "value": 3.418064120472897, "provenance": "measured" }, "tokens_per_word": { "value": 2.470664489301167, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.8388952740606954, "provenance": "measured" }, "chars_per_token": { "value": 3.838279561065047, "provenance": "measured" }, "tokens_per_word": { "value": 2.7455812468303313, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.813030547732063, "provenance": "measured" }, "chars_per_token": { "value": 4.786362830638331, "provenance": "measured" }, "tokens_per_word": { "value": 2.550445811879945, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.142995988765248, "provenance": "measured" }, "chars_per_token": { "value": 4.134808786827994, "provenance": "measured" }, "tokens_per_word": { "value": 2.5819616349793715, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.568338018219547, "provenance": "measured" }, "chars_per_token": { "value": 3.565490294191232, "provenance": "measured" }, "tokens_per_word": { "value": 2.355414586799778, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.722024840045164, "provenance": "measured" }, "chars_per_token": { "value": 3.664252916823485, "provenance": "measured" }, "tokens_per_word": { "value": 3.237085769980507, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.5149768866958215, "provenance": "measured" }, "chars_per_token": { "value": 4.512536406794154, "provenance": "measured" }, "tokens_per_word": { "value": 2.1863413304253, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.982998365543354, "provenance": "measured" }, "chars_per_token": { "value": 3.9682307622810113, "provenance": "measured" }, "tokens_per_word": { "value": 2.530247927014277, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.492113040265977, "provenance": "measured" }, "chars_per_token": { "value": 4.477188769855929, "provenance": "measured" }, "tokens_per_word": { "value": 2.890858607432721, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.6471673428588227, "provenance": "measured" }, "chars_per_token": { "value": 3.617657435173752, "provenance": "measured" }, "tokens_per_word": { "value": 2.65589130944015, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.8752680997092606, "provenance": "measured" }, "chars_per_token": { "value": 3.8697631190124397, "provenance": "measured" }, "tokens_per_word": { "value": 2.9554173025002934, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.994829889211912, "provenance": "measured" }, "chars_per_token": { "value": 3.994452738272963, "provenance": "measured" }, "tokens_per_word": { "value": 2.707411504424779, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.17450563587878, "provenance": "measured" }, "chars_per_token": { "value": 4.163095430806063, "provenance": "measured" }, "tokens_per_word": { "value": 2.8476323913847796, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.0438117808475473, "provenance": "measured" }, "chars_per_token": { "value": 3.0295222800835697, "provenance": "measured" }, "tokens_per_word": { "value": 2.9492125531670306, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.055547375997906, "provenance": "measured" }, "chars_per_token": { "value": 3.9690076560659597, "provenance": "measured" }, "tokens_per_word": { "value": 2.5747862347837076, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.239691486205873, "provenance": "measured" }, "chars_per_token": { "value": 4.238260596436736, "provenance": "measured" }, "tokens_per_word": { "value": 2.382827442827443, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.4288980597300185, "provenance": "measured" }, "chars_per_token": { "value": 3.422661730457496, "provenance": "measured" }, "tokens_per_word": { "value": 2.5486938009670204, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.4236703093993683, "provenance": "measured" }, "chars_per_token": { "value": 3.410191732580564, "provenance": "measured" }, "tokens_per_word": { "value": 2.632357268981754, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.599446247673211, "provenance": "measured" }, "chars_per_token": { "value": 4.582206964932137, "provenance": "measured" }, "tokens_per_word": { "value": 2.2979880486651476, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.1743893444401055, "provenance": "measured" }, "chars_per_token": { "value": 4.085000072309717, "provenance": "measured" }, "tokens_per_word": { "value": 2.998178901270433, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.718526666869941, "provenance": "measured" }, "chars_per_token": { "value": 4.699338341702767, "provenance": "measured" }, "tokens_per_word": { "value": 1.301400136137065, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 6.029997780291099, "provenance": "measured" }, "chars_per_token": { "value": 3.344012599490524, "provenance": "measured" }, "tokens_per_word": { "value": 1.8298518432555801, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 9.385201452825408, "provenance": "measured" }, "chars_per_token": { "value": 3.525500464566264, "provenance": "measured" }, "tokens_per_word": { "value": 2.031888236707514, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.705598887397742, "provenance": "measured" }, "chars_per_token": { "value": 3.612068545511722, "provenance": "measured" }, "tokens_per_word": { "value": 2.3858167869559246, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.317948921950832, "provenance": "measured" }, "chars_per_token": { "value": 4.221895138833638, "provenance": "measured" }, "tokens_per_word": { "value": 1.478427590100687, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 4.120014181882645, "provenance": "measured" }, "chars_per_token": { "value": 3.988759246426327, "provenance": "measured" }, "tokens_per_word": { "value": 1.609185565539866, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 6.953952618290753, "provenance": "measured" }, "chars_per_token": { "value": 2.701830121790092, "provenance": "measured" }, "tokens_per_word": { "value": 2.3574600629124776, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 7.7859491639004474, "provenance": "measured" }, "chars_per_token": { "value": 3.09084108653981, "provenance": "measured" }, "tokens_per_word": { "value": 1.788577321886498, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.293065858670289, "provenance": "measured" }, "chars_per_token": { "value": 4.283098248180266, "provenance": "measured" }, "tokens_per_word": { "value": 1.7424628604718906, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 5.091907180438551, "provenance": "measured" }, "chars_per_token": { "value": 1.7866244943888567, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.870805319650861, "provenance": "measured" }, "chars_per_token": { "value": 1.6705296060391601, "provenance": "measured" }, "tokens_per_word": { "value": 2.6821156697973305, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 8.043095838081634, "provenance": "measured" }, "chars_per_token": { "value": 3.05616101953357, "provenance": "measured" }, "tokens_per_word": { "value": 2.2962922010909628, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 6.19172864750691, "provenance": "measured" }, "chars_per_token": { "value": 3.4066355077185806, "provenance": "measured" }, "tokens_per_word": { "value": 2.17225557670718, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 8.83526905007248, "provenance": "measured" }, "chars_per_token": { "value": 3.366191033410762, "provenance": "measured" }, "tokens_per_word": { "value": 2.843231701261297, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 7.325867114832952, "provenance": "measured" }, "chars_per_token": { "value": 2.7908492944278636, "provenance": "measured" }, "tokens_per_word": { "value": 2.860541257408098, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 9.656090772202534, "provenance": "measured" }, "chars_per_token": { "value": 3.4524312896405918, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 4.0738647311966085, "provenance": "measured" }, "chars_per_token": { "value": 3.724034941229366, "provenance": "measured" }, "tokens_per_word": { "value": 2.0957412798034696, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.5479783323660286, "provenance": "measured" }, "chars_per_token": { "value": 3.13350959781604, "provenance": "measured" }, "tokens_per_word": { "value": 1.5268556049690665, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.702831636870321, "provenance": "measured" }, "chars_per_token": { "value": 3.600508545292807, "provenance": "measured" }, "tokens_per_word": { "value": 1.2759680578574657, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.4390298908800725, "provenance": "measured" }, "chars_per_token": { "value": 1.6672622868344813, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 88295, "provenance": "measured" }, "tokens_per_request_mean": { "value": 441.475, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.441475, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 57827, "provenance": "measured" }, "tokens_per_request_mean": { "value": 354.76687116564415, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.35476687116564415, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 25793, "provenance": "measured" }, "tokens_per_request_mean": { "value": 135.75263157894736, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.13575263157894735, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 224164, "provenance": "measured" }, "tokens_per_request_mean": { "value": 533.7238095238096, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5337238095238096, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 122559, "provenance": "measured" }, "tokens_per_request_mean": { "value": 817.06, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.81706, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 240716, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1203.58, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.20358, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 320496, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1602.48, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.60248, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 410909, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2054.545, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.054545, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8245, "provenance": "measured" }, "tokens_per_request_mean": { "value": 329.8, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3298, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 515221, "provenance": "measured" }, "tokens_per_request_mean": { "value": 85870.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 85.87016666666668, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 151000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 309248000, "provenance": "estimated", "note": "151000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 618496000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 618496000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1236992000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 151000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 618496000, "provenance": "estimated", "note": "151000 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1236992000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1236992000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2473984000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "akshara-120k", "label": "Akshara 120k", "provider": "Akshara", "family": "akshara", "metrics": { "bpt_overall": { "value": 4.746921724007486, "provenance": "measured" }, "bpt_english": { "value": 4.655252367244555, "provenance": "measured" }, "bpt_code": { "value": 3.846826495204314, "provenance": "measured" }, "bpt_languages": { "value": 5.738686309573592, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.4174629065830746, "provenance": "measured" }, "byte_fallback_share": { "value": 0.004938094263585416, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 5.967619867743156, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.956208157401425, "q3": 6.016314594826091, "n": 5 }, "batch_mbps_4t": { "value": 18.09350513047292, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.71619448610454, "q3": 18.279818245514086, "n": 5 }, "latency_small_p95_us": { "value": 62.000000013995304, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 59.959000054732314, "q3": 66.79100010842376, "n": 5 }, "decode_mbps": { "value": 46.44357459654735, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 45.12073969902209, "q3": 47.21170720911533, "n": 5 }, "rss_load_delta_mb": { "value": 81.068032, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 81.068032, "q3": 81.068032, "n": 5 }, "rss_peak_large_mb": { "value": 187.088896, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 186.998784, "q3": 187.138048, "n": 5 }, "load_seconds": { "value": 0.1461671250000336, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.14010229199993773, "q3": 0.15664995900033318, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6054480595062026, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2378.186752, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2377.9328, "q3": 2378.264576, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.088512, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.826368, "q3": 17.661952, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.613083868795755, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.4992375717937434, "q3": 3.6358303119393023, "n": 5 }, "embedding_params_h2048": { "value": 245760000, "provenance": "estimated", "note": "120000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.19794448056600464, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19622614337851274, "q3": 0.203311929818789, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5869782359774, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.581509474669802, "q3": 3.5968366058587065, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 137.31005859375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 137.30590820312503, "q3": 137.3251953125, "n": 5 }, "workload_tokens_total": { "value": 2050796, "provenance": "measured" }, "tokens_chat_en": { "value": 89838, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 59769, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 26805, "provenance": "measured" }, "tokens_code": { "value": 226687, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 123467, "provenance": "measured" }, "tokens_rag": { "value": 250575, "provenance": "measured" }, "tokens_json": { "value": 322331, "provenance": "measured" }, "tokens_logs": { "value": 415437, "provenance": "measured" }, "tokens_tech_docs": { "value": 8355, "provenance": "measured" }, "tokens_long_context": { "value": 527532, "provenance": "measured" } }, "parity": { "de": { "value": 1.3859051708183736, "provenance": "measured" }, "fr": { "value": 1.4287300588518341, "provenance": "measured" }, "es": { "value": 1.3785394381315468, "provenance": "measured" }, "ru": { "value": 1.599252322611689, "provenance": "measured" }, "zh": { "value": 1.1020098456527372, "provenance": "measured" }, "ja": { "value": 1.220527815819669, "provenance": "measured" }, "ko": { "value": 1.5180071806640263, "provenance": "measured" }, "ar": { "value": 1.410260206536625, "provenance": "measured" }, "ur": { "value": 1.6974497538586815, "provenance": "measured" }, "hi": { "value": 1.7922049080208757, "provenance": "measured" }, "mr": { "value": 1.7308361402080172, "provenance": "measured" }, "gu": { "value": 1.7945737868749305, "provenance": "measured" }, "bn": { "value": 1.6544397971647482, "provenance": "measured" }, "ta": { "value": 1.8156716141688567, "provenance": "measured" }, "te": { "value": 1.8886626938594218, "provenance": "measured" }, "th": { "value": 1.5791908798164118, "provenance": "measured" }, "vi": { "value": 1.4477551171484622, "provenance": "measured" }, "tr": { "value": 1.4087796572528408, "provenance": "measured" }, "id": { "value": 1.282895954399082, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.909362549800797, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 37.29382470119522, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3859051708183736, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 38.44621513944223, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4287300588518341, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 37.09561752988048, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3785394381315468, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 43.03486055776892, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.599252322611689, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 29.654382470119522, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1020098456527372, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 32.84362549800797, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.220527815819669, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 40.84860557768924, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5180071806640263, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 37.949203187250994, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.410260206536625, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 45.677290836653384, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6974497538586815, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 48.22709163346614, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7922049080208757, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 46.57569721115538, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7308361402080172, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 48.29083665338646, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7945737868749305, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 44.5199203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6544397971647482, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 48.85856573705179, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8156716141688567, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 50.822709163346616, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8886626938594218, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 42.495019920318725, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5791908798164118, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 38.95816733067729, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4477551171484622, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 37.9093625498008, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4087796572528408, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 34.52191235059761, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.282895954399082, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (exported from Akshara)" } }, "vocabulary": { "size": { "value": 120000, "provenance": "measured" }, "used_share": { "value": 0.8698, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 104376, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 15624, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 16, "provenance": "measured", "note": "of 118893 entries whose text is valid UTF-8; 1107 are byte pieces" }, "unreachable_examples": [ "️◾", "️◾️◾", "️◾️◾️◾️◾", "️⬜", "️⬜️⬜", "️◽", "️◽️◽", "️⬜️⬜️⬜️⬜", "️◾️◾️◾️◾️◾️◾️◾️◾", " \n \t", "️\n", "️◽️◽️◽️◽" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (exported from Akshara)", "load_seconds": { "value": 0.1461671250000336, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.14010229199993773, "q3": 0.15664995900033318, "n": 5 }, "rss_load_delta_mb": { "value": 81.068032, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 81.068032, "q3": 81.068032, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 81.002496, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 81.002496, "q3": 81.002496, "n": 5 }, "latency_small_p50_us": { "value": 46.771000000944696, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 45.10399992341263, "q3": 47.04150001089147, "n": 5 }, "latency_small_p95_us": { "value": 62.000000013995304, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 59.959000054732314, "q3": 66.79100010842376, "n": 5 }, "encode_mbps_1t_medium": { "value": 5.967619867743156, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.956208157401425, "q3": 6.016314594826091, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1372957.156139522, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1370331.6890815147, "q3": 1384160.244723658, "n": 5 }, "encode_mbps_1t_large": { "value": 4.873229762428417, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.836300885772165, "q3": 4.94713991323193, "n": 5 }, "latency_large_p50_ms": { "value": 211.81587499995658, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 211.2760420000086, "q3": 221.4414160000615, "n": 5 }, "decode_mbps": { "value": 46.44357459654735, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 45.12073969902209, "q3": 47.21170720911533, "n": 5 }, "decode_tokens_per_s": { "value": 10317098.422712717, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10023240.381997181, "q3": 10487733.431632733, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.620777487688762, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.510117753378362, "q3": 5.663579343266078, "n": 5 }, "batch_mbps_4t": { "value": 18.09350513047292, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.71619448610454, "q3": 18.279818245514086, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4145271.3076912956, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4058828.4113642266, "q3": 4187956.1498187617, "n": 5 }, "batch_cpu_utilization": { "value": 3.5869782359774, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.581509474669802, "q3": 3.5968366058587065, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5869782359774, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.581509474669802, "q3": 3.5968366058587065, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.19794448056600464, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19622614337851274, "q3": 0.203311929818789, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 137.31005859375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 137.30590820312503, "q3": 137.3251953125, "n": 5 }, "rss_peak_large_mb": { "value": 187.088896, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 186.998784, "q3": 187.138048, "n": 5 }, "rss_peak_very_large_mb": { "value": 2378.186752, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2377.9328, "q3": 2378.264576, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.088512, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.826368, "q3": 17.661952, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.613083868795755, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.4992375717937434, "q3": 3.6358303119393023, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6054480595062026, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "akshara-native (Rust)", "load_seconds": { "value": 0.2049808749998192, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.19523945800006004, "q3": 0.2066935839998223, "n": 5 }, "rss_load_delta_mb": { "value": 41.598976, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 41.598976, "q3": 41.603072, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 41.365504, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 41.365504, "q3": 41.3696, "n": 5 }, "latency_small_p50_us": { "value": 8.541999704902992, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 8.374999993066012, "q3": 9.66599998264428, "n": 5 }, "latency_small_p95_us": { "value": 14.457999895967077, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.292000005549198, "q3": 14.87499997665509, "n": 5 }, "encode_mbps_1t_medium": { "value": 40.06823361453194, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.20469114167294, "q3": 40.9232847662627, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 9218410.23626498, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 8329533.531025575, "q3": 9415129.971042262, "n": 5 }, "encode_mbps_1t_large": { "value": 39.245744235603574, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 39.030673815998426, "q3": 39.906656538715396, "n": 5 }, "latency_large_p50_ms": { "value": 26.31687499979307, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 26.304291000087687, "q3": 26.458166999987043, "n": 5 }, "decode_mbps": { "value": 107.71359840311378, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 100.94939648822275, "q3": 107.86364610800392, "n": 5 }, "decode_tokens_per_s": { "value": 23927783.462905336, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 22425165.77007554, "q3": 23961115.456682768, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 36.83746482677672, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.04855248101739, "q3": 37.58045161453272, "n": 5 }, "batch_mbps_4t": { "value": 109.33602708549441, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 108.50220877359422, "q3": 109.78547342923949, "n": 5 }, "batch_tokens_per_s_4t": { "value": 25049181.609987587, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 24858151.563613307, "q3": 25152150.991521925, "n": 5 }, "batch_cpu_utilization": { "value": 2.9744001655419723, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.953907515248812, "q3": 3.0248478812764845, "n": 5 }, "cpu_cores_used_batch": { "value": 2.9744001655419723, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 2.953907515248812, "q3": 3.0248478812764845, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.027261644792260502, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.02709283908548316, "q3": 0.027518668170658754, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 10.009033203125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 10.002197265625, "q3": 10.02783203125, "n": 5 }, "rss_peak_large_mb": { "value": 44.07296, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 44.068864, "q3": 44.089344, "n": 5 }, "rss_peak_very_large_mb": { "value": 208.60928, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 208.494592, "q3": 208.928768, "n": 5 }, "rss_peak_chunked_mb": { "value": 16.994304, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.994304, "q3": 18.2272, "n": 5 }, "encode_mbps_1t_very_large": { "value": 40.885564715103776, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.03593255645141, "q3": 41.06080588236472, "n": 5 }, "throughput_ratio_very_large": { "value": 1.0203984809621207, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.572958960089968, "provenance": "measured" }, "chars_per_token": { "value": 2.570641126498804, "provenance": "measured" }, "tokens_per_word": { "value": 2.5626589814363805, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3893640933838522, "provenance": "measured" }, "chars_per_token": { "value": 3.3758302992778835, "provenance": "measured" }, "tokens_per_word": { "value": 2.501574100574084, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.8081015785116117, "provenance": "measured" }, "chars_per_token": { "value": 3.8074908044572933, "provenance": "measured" }, "tokens_per_word": { "value": 2.7677830162099215, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.755753194056574, "provenance": "measured" }, "chars_per_token": { "value": 4.729402835485409, "provenance": "measured" }, "tokens_per_word": { "value": 2.581162878312194, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.110661632506293, "provenance": "measured" }, "chars_per_token": { "value": 4.102538328266483, "provenance": "measured" }, "tokens_per_word": { "value": 2.6022712772744674, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.53339356756678, "provenance": "measured" }, "chars_per_token": { "value": 3.530573731073578, "provenance": "measured" }, "tokens_per_word": { "value": 2.3787090959511925, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.6805237145324083, "provenance": "measured" }, "chars_per_token": { "value": 3.623395956768988, "provenance": "measured" }, "tokens_per_word": { "value": 3.2735867446393763, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.462589465714713, "provenance": "measured" }, "chars_per_token": { "value": 4.460177302779269, "provenance": "measured" }, "tokens_per_word": { "value": 2.2120073220127745, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.945440640458214, "provenance": "measured" }, "chars_per_token": { "value": 3.9308122884665453, "provenance": "measured" }, "tokens_per_word": { "value": 2.5543340468422038, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.431541545189504, "provenance": "measured" }, "chars_per_token": { "value": 4.416818513119534, "provenance": "measured" }, "tokens_per_word": { "value": 2.930371636052969, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.582265504869298, "provenance": "measured" }, "chars_per_token": { "value": 3.553280729997618, "provenance": "measured" }, "tokens_per_word": { "value": 2.7040095260404464, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.8369552393761355, "provenance": "measured" }, "chars_per_token": { "value": 3.8315046837026028, "provenance": "measured" }, "tokens_per_word": { "value": 2.9849278084282194, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.937775918955342, "provenance": "measured" }, "chars_per_token": { "value": 3.9374041544681444, "provenance": "measured" }, "tokens_per_word": { "value": 2.746638869979578, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.125065509903835, "provenance": "measured" }, "chars_per_token": { "value": 4.113790439868748, "provenance": "measured" }, "tokens_per_word": { "value": 2.8817621049183906, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.006915671929149, "provenance": "measured" }, "chars_per_token": { "value": 2.9927993839045053, "provenance": "measured" }, "tokens_per_word": { "value": 2.985400620761007, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.011675418510757, "provenance": "measured" }, "chars_per_token": { "value": 3.926071864911443, "provenance": "measured" }, "tokens_per_word": { "value": 2.602944273619477, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.191889303922499, "provenance": "measured" }, "chars_per_token": { "value": 4.1904745473210205, "provenance": "measured" }, "tokens_per_word": { "value": 2.41, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.373966524806887, "provenance": "measured" }, "chars_per_token": { "value": 3.3678301025987505, "provenance": "measured" }, "tokens_per_word": { "value": 2.590189074114166, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.3727895370848393, "provenance": "measured" }, "chars_per_token": { "value": 3.3595112717260367, "provenance": "measured" }, "tokens_per_word": { "value": 2.6720681283107712, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.537939754143748, "provenance": "measured" }, "chars_per_token": { "value": 4.520931005205065, "provenance": "measured" }, "tokens_per_word": { "value": 2.3291346029835345, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.115789623778188, "provenance": "measured" }, "chars_per_token": { "value": 4.027655190606218, "provenance": "measured" }, "tokens_per_word": { "value": 3.0408663226813513, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.655252367244555, "provenance": "measured" }, "chars_per_token": { "value": 4.63632135286985, "provenance": "measured" }, "tokens_per_word": { "value": 1.3190887973847032, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 5.752024118008853, "provenance": "measured" }, "chars_per_token": { "value": 3.189858740257514, "provenance": "measured" }, "tokens_per_word": { "value": 1.9182816912305134, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 8.810952690593627, "provenance": "measured" }, "chars_per_token": { "value": 3.3097870045834457, "provenance": "measured" }, "tokens_per_word": { "value": 2.1643153811828513, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.5979521098171436, "provenance": "measured" }, "chars_per_token": { "value": 3.50713880240139, "provenance": "measured" }, "tokens_per_word": { "value": 2.4571978062620863, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.192853831891224, "provenance": "measured" }, "chars_per_token": { "value": 4.0995828182941905, "provenance": "measured" }, "tokens_per_word": { "value": 1.522536934224146, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.988408465096687, "provenance": "measured" }, "chars_per_token": { "value": 3.861346209349673, "provenance": "measured" }, "tokens_per_word": { "value": 1.6622839434135968, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 6.599719414992675, "provenance": "measured" }, "chars_per_token": { "value": 2.564199339507859, "provenance": "measured" }, "tokens_per_word": { "value": 2.4839943255412322, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 7.393609779198485, "provenance": "measured" }, "chars_per_token": { "value": 2.9350914580003553, "provenance": "measured" }, "tokens_per_word": { "value": 1.8834875682907795, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.107633659011237, "provenance": "measured" }, "chars_per_token": { "value": 4.0980965837143275, "provenance": "measured" }, "tokens_per_word": { "value": 1.8211234100398097, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.898534187659089, "provenance": "measured" }, "chars_per_token": { "value": 1.7187746862108313, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.733146107358692, "provenance": "measured" }, "chars_per_token": { "value": 1.6111198009242802, "provenance": "measured" }, "tokens_per_word": { "value": 2.7810182896688085, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 7.59095040445966, "provenance": "measured" }, "chars_per_token": { "value": 2.8843578634835034, "provenance": "measured" }, "tokens_per_word": { "value": 2.433067964027716, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.9348151498508495, "provenance": "measured" }, "chars_per_token": { "value": 3.265283925090729, "provenance": "measured" }, "tokens_per_word": { "value": 2.2662908185678723, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 8.33096190899659, "provenance": "measured" }, "chars_per_token": { "value": 3.174052665382145, "provenance": "measured" }, "tokens_per_word": { "value": 3.0153441255338165, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 6.962519277373871, "provenance": "measured" }, "chars_per_token": { "value": 2.652428949107733, "provenance": "measured" }, "tokens_per_word": { "value": 3.0098222056446597, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 9.031096632857732, "provenance": "measured" }, "chars_per_token": { "value": 3.2289713643540976, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.8994489828854237, "provenance": "measured" }, "chars_per_token": { "value": 3.5645965740107437, "provenance": "measured" }, "tokens_per_word": { "value": 2.189480237586445, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.3395984401022005, "provenance": "measured" }, "chars_per_token": { "value": 3.01581620514518, "provenance": "measured" }, "tokens_per_word": { "value": 1.5864417349065427, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.581609019096557, "provenance": "measured" }, "chars_per_token": { "value": 3.5076999769921007, "provenance": "measured" }, "tokens_per_word": { "value": 1.3097282908944805, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.289205702647658, "provenance": "measured" }, "chars_per_token": { "value": 1.6109895820237528, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 89838, "provenance": "measured" }, "tokens_per_request_mean": { "value": 449.19, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.44919, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 59769, "provenance": "measured" }, "tokens_per_request_mean": { "value": 366.680981595092, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.36668098159509205, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 26805, "provenance": "measured" }, "tokens_per_request_mean": { "value": 141.07894736842104, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.14107894736842103, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 226687, "provenance": "measured" }, "tokens_per_request_mean": { "value": 539.7309523809524, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5397309523809525, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 123467, "provenance": "measured" }, "tokens_per_request_mean": { "value": 823.1133333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8231133333333334, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 250575, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1252.875, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.252875, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 322331, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1611.655, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.611655, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 415437, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2077.185, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.077185, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8355, "provenance": "measured" }, "tokens_per_request_mean": { "value": 334.2, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3342, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 527532, "provenance": "measured" }, "tokens_per_request_mean": { "value": 87922.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 87.922, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 120000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 245760000, "provenance": "estimated", "note": "120000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 491520000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 491520000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 983040000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 120000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 491520000, "provenance": "estimated", "note": "120000 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 983040000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 983040000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1966080000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "akshara-92k", "label": "Akshara 92k", "provider": "Akshara", "family": "akshara", "metrics": { "bpt_overall": { "value": 4.588550438102257, "provenance": "measured" }, "bpt_english": { "value": 4.564852758982179, "provenance": "measured" }, "bpt_code": { "value": 3.782014960022011, "provenance": "measured" }, "bpt_languages": { "value": 5.418783595302582, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.4507251644743021, "provenance": "measured" }, "byte_fallback_share": { "value": 0.0063978967049031525, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 6.100163595476987, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.008446439600618, "q3": 6.168005805384802, "n": 5 }, "batch_mbps_4t": { "value": 17.31074049393309, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.257512822256132, "q3": 17.90969294083765, "n": 5 }, "latency_small_p95_us": { "value": 63.54199967972818, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 58.37499975314131, "q3": 67.50000000010914, "n": 5 }, "decode_mbps": { "value": 46.85866863218416, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 46.77488202423642, "q3": 47.64206157670607, "n": 5 }, "rss_load_delta_mb": { "value": 45.416448, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 45.416448, "q3": 45.416448, "n": 5 }, "rss_peak_large_mb": { "value": 174.1824, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 174.174208, "q3": 174.211072, "n": 5 }, "load_seconds": { "value": 0.10785962499994639, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.10639008299995112, "q3": 0.11712995800007775, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5825090610096689, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2381.4144, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2381.316096, "q3": 2381.471744, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.747968, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.6128, "q3": 18.276352, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.5534005680066656, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5477630581524116, "q3": 3.576103728874091, "n": 5 }, "embedding_params_h2048": { "value": 188416000, "provenance": "estimated", "note": "92000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.20255731108722086, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.20001926483207288, "q3": 0.20291375816579718, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5482269309547316, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.495635393329438, "q3": 3.5822836153945126, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 139.429443359375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 139.414794921875, "q3": 139.46240234375, "n": 5 }, "workload_tokens_total": { "value": 2099319, "provenance": "measured" }, "tokens_chat_en": { "value": 91317, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 62101, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 27919, "provenance": "measured" }, "tokens_code": { "value": 230801, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 125092, "provenance": "measured" }, "tokens_rag": { "value": 263064, "provenance": "measured" }, "tokens_json": { "value": 326205, "provenance": "measured" }, "tokens_logs": { "value": 419943, "provenance": "measured" }, "tokens_tech_docs": { "value": 8479, "provenance": "measured" }, "tokens_long_context": { "value": 544398, "provenance": "measured" } }, "parity": { "de": { "value": 1.4196729632718175, "provenance": "measured" }, "fr": { "value": 1.458540299481527, "provenance": "measured" }, "es": { "value": 1.403248613175737, "provenance": "measured" }, "ru": { "value": 1.6559588122258075, "provenance": "measured" }, "zh": { "value": 1.1288205648816214, "provenance": "measured" }, "ja": { "value": 1.254704325441427, "provenance": "measured" }, "ko": { "value": 1.568833617345274, "provenance": "measured" }, "ar": { "value": 1.4557847793771075, "provenance": "measured" }, "ur": { "value": 1.7443892534715928, "provenance": "measured" }, "hi": { "value": 1.8560965882310285, "provenance": "measured" }, "mr": { "value": 1.8183169573256952, "provenance": "measured" }, "gu": { "value": 1.8737536710054024, "provenance": "measured" }, "bn": { "value": 1.7316993582538704, "provenance": "measured" }, "ta": { "value": 1.893368623327653, "provenance": "measured" }, "te": { "value": 1.98491715311265, "provenance": "measured" }, "th": { "value": 1.6568289764693085, "provenance": "measured" }, "vi": { "value": 1.4770312896559226, "provenance": "measured" }, "tr": { "value": 1.4537181392987926, "provenance": "measured" }, "id": { "value": 1.3177912330952468, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 27.471115537848604, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 39.0, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4196729632718175, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 40.06772908366534, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.458540299481527, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 38.548804780876495, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.403248613175737, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 45.4910358565737, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6559588122258075, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 31.00996015936255, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1288205648816214, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 34.46812749003984, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.254704325441427, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 43.09760956175299, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.568833617345274, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 39.99203187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4557847793771075, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 47.9203187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7443892534715928, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 50.9890438247012, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8560965882310285, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 49.951195219123505, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8183169573256952, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 51.47410358565737, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8737536710054024, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 47.57171314741036, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7316993582538704, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 52.01294820717131, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.893368623327653, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 54.52788844621514, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.98491715311265, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 45.514940239043824, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6568289764693085, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 40.57569721115538, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4770312896559226, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 39.93525896414343, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4537181392987926, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 36.201195219123505, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3177912330952468, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (exported from Akshara)" } }, "vocabulary": { "size": { "value": 92000, "provenance": "measured" }, "used_share": { "value": 0.9075869565217392, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 83498, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 8502, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 12, "provenance": "measured", "note": "of 90954 entries whose text is valid UTF-8; 1046 are byte pieces" }, "unreachable_examples": [ "️◾", "️◾️◾", "️◾️◾️◾️◾", "️⬜", "️⬜️⬜", "️◽", "️◽️◽", "️⬜️⬜️⬜️⬜", "️◾️◾️◾️◾️◾️◾️◾️◾", " \n \t", "️\n", "️◽️◽️◽️◽" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (exported from Akshara)", "load_seconds": { "value": 0.10785962499994639, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.10639008299995112, "q3": 0.11712995800007775, "n": 5 }, "rss_load_delta_mb": { "value": 45.416448, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 45.416448, "q3": 45.416448, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 42.299392, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 42.299392, "q3": 42.811392, "n": 5 }, "latency_small_p50_us": { "value": 46.041000132390764, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 43.790999825432664, "q3": 47.14599998578706, "n": 5 }, "latency_small_p95_us": { "value": 63.54199967972818, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 58.37499975314131, "q3": 67.50000000010914, "n": 5 }, "encode_mbps_1t_medium": { "value": 6.100163595476987, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.008446439600618, "q3": 6.168005805384802, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1444635.3253728573, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1422914.9499685038, "q3": 1460701.6572753096, "n": 5 }, "encode_mbps_1t_large": { "value": 4.917470404771271, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.740993795094809, "q3": 4.917889201571319, "n": 5 }, "latency_large_p50_ms": { "value": 213.69991700021274, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 211.13120900008653, "q3": 221.49045799994838, "n": 5 }, "decode_mbps": { "value": 46.85866863218416, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 46.77488202423642, "q3": 47.64206157670607, "n": 5 }, "decode_tokens_per_s": { "value": 10747373.183679001, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10728156.121613555, "q3": 10927049.998449948, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.734698298317507, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.49027721235168, "q3": 5.777182240169107, "n": 5 }, "batch_mbps_4t": { "value": 17.31074049393309, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.257512822256132, "q3": 17.90969294083765, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4081342.156199448, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4068792.7022712557, "q3": 4222556.789505697, "n": 5 }, "batch_cpu_utilization": { "value": 3.5482269309547316, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.495635393329438, "q3": 3.5822836153945126, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5482269309547316, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.495635393329438, "q3": 3.5822836153945126, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.20255731108722086, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.20001926483207288, "q3": 0.20291375816579718, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 139.429443359375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 139.414794921875, "q3": 139.46240234375, "n": 5 }, "rss_peak_large_mb": { "value": 174.1824, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 174.174208, "q3": 174.211072, "n": 5 }, "rss_peak_very_large_mb": { "value": 2381.4144, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2381.316096, "q3": 2381.471744, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.747968, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.6128, "q3": 18.276352, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.5534005680066656, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5477630581524116, "q3": 3.576103728874091, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5825090610096689, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "akshara-native (Rust)", "load_seconds": { "value": 0.15286491599999863, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.1514836250000826, "q3": 0.15466254200009644, "n": 5 }, "rss_load_delta_mb": { "value": 30.691328, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 30.552064, "q3": 30.818304, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 30.45376, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 30.314496, "q3": 30.580736, "n": 5 }, "latency_small_p50_us": { "value": 8.041499995670165, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 7.749999895168003, "q3": 8.416499895247398, "n": 5 }, "latency_small_p95_us": { "value": 13.541000043915119, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.95900005970907, "q3": 13.9579997266992, "n": 5 }, "encode_mbps_1t_medium": { "value": 39.1210378725301, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 38.182779392110085, "q3": 41.60118715472255, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 9264609.44716472, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9042411.907060971, "q3": 9851956.197653497, "n": 5 }, "encode_mbps_1t_large": { "value": 37.40353227453945, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.69141119578563, "q3": 38.02523453289593, "n": 5 }, "latency_large_p50_ms": { "value": 27.947042000050715, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 27.850834000219038, "q3": 29.79058300024917, "n": 5 }, "decode_mbps": { "value": 99.18681068335604, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 98.29075211288611, "q3": 102.35190719021891, "n": 5 }, "decode_tokens_per_s": { "value": 22749209.49377513, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 22543692.0061387, "q3": 23475147.176482793, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 35.2394585319425, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 32.53316061621905, "q3": 37.01807470164324, "n": 5 }, "batch_mbps_4t": { "value": 102.9317037703785, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 98.31125080890793, "q3": 109.62298731222458, "n": 5 }, "batch_tokens_per_s_4t": { "value": 24268141.61732203, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 23178780.393346924, "q3": 25845741.23577805, "n": 5 }, "batch_cpu_utilization": { "value": 2.980744039552697, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.957263856035104, "q3": 3.0143524199970737, "n": 5 }, "cpu_cores_used_batch": { "value": 2.980744039552697, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 2.957263856035104, "q3": 3.0143524199970737, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.028730349811679108, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.027497448244241825, "q3": 0.03031946003155331, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 10.306640625, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 10.306396484375002, "q3": 10.3076171875, "n": 5 }, "rss_peak_large_mb": { "value": 32.530432, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 32.247808, "q3": 33.243136, "n": 5 }, "rss_peak_very_large_mb": { "value": 197.81632, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 197.660672, "q3": 198.36928, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.424384, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.088512, "q3": 17.465344, "n": 5 }, "encode_mbps_1t_very_large": { "value": 37.9858662744024, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.767618526361396, "q3": 39.24025684823028, "n": 5 }, "throughput_ratio_very_large": { "value": 0.9709830909438935, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.530039353318035, "provenance": "measured" }, "chars_per_token": { "value": 2.5277601835795944, "provenance": "measured" }, "tokens_per_word": { "value": 2.6061319478270284, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3447275093973725, "provenance": "measured" }, "chars_per_token": { "value": 3.331371950004649, "provenance": "measured" }, "tokens_per_word": { "value": 2.534958500984865, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.748248244905048, "provenance": "measured" }, "chars_per_token": { "value": 3.7476470706112606, "provenance": "measured" }, "tokens_per_word": { "value": 2.8119799395180225, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.681994098938575, "provenance": "measured" }, "chars_per_token": { "value": 4.65605241981837, "provenance": "measured" }, "tokens_per_word": { "value": 2.6218259449956047, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.037542141595762, "provenance": "measured" }, "chars_per_token": { "value": 4.029563332798202, "provenance": "measured" }, "tokens_per_word": { "value": 2.6493981540555485, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.4765246940575105, "provenance": "measured" }, "chars_per_token": { "value": 3.4737502419580317, "provenance": "measured" }, "tokens_per_word": { "value": 2.4176199389905713, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.6252602715622158, "provenance": "measured" }, "chars_per_token": { "value": 3.568990292970468, "provenance": "measured" }, "tokens_per_word": { "value": 3.3234892787524366, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.3854139631456, "provenance": "measured" }, "chars_per_token": { "value": 4.383043515875076, "provenance": "measured" }, "tokens_per_word": { "value": 2.250934725035052, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.8778607641498875, "provenance": "measured" }, "chars_per_token": { "value": 3.8634829753546467, "provenance": "measured" }, "tokens_per_word": { "value": 2.5988486876285877, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.361759747480182, "provenance": "measured" }, "chars_per_token": { "value": 4.347268553391442, "provenance": "measured" }, "tokens_per_word": { "value": 2.9772533105510464, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.5164156893313963, "provenance": "measured" }, "chars_per_token": { "value": 3.4879637175353437, "provenance": "measured" }, "tokens_per_word": { "value": 2.754645896775201, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.7839208842350205, "provenance": "measured" }, "chars_per_token": { "value": 3.7785456660849333, "provenance": "measured" }, "tokens_per_word": { "value": 3.0267637046601714, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.862919401896426, "provenance": "measured" }, "chars_per_token": { "value": 3.862554704595186, "provenance": "measured" }, "tokens_per_word": { "value": 2.799863852961198, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.054735600794439, "provenance": "measured" }, "chars_per_token": { "value": 4.043652763985436, "provenance": "measured" }, "tokens_per_word": { "value": 2.931746638280838, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.95695374269183, "provenance": "measured" }, "chars_per_token": { "value": 2.94307200630093, "provenance": "measured" }, "tokens_per_word": { "value": 3.0358432003678586, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.952291749701076, "provenance": "measured" }, "chars_per_token": { "value": 3.8679553607014747, "provenance": "measured" }, "tokens_per_word": { "value": 2.6420538309254034, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.118697078343123, "provenance": "measured" }, "chars_per_token": { "value": 4.117307024012341, "provenance": "measured" }, "tokens_per_word": { "value": 2.452827442827443, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.3083837038554793, "provenance": "measured" }, "chars_per_token": { "value": 3.3023665608031965, "provenance": "measured" }, "tokens_per_word": { "value": 2.6415349642779824, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.2917173318643225, "provenance": "measured" }, "chars_per_token": { "value": 3.278758237993188, "provenance": "measured" }, "tokens_per_word": { "value": 2.7378788993525602, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.455899810604344, "provenance": "measured" }, "chars_per_token": { "value": 4.439198557330699, "provenance": "measured" }, "tokens_per_word": { "value": 2.37201754008856, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.051008378594585, "provenance": "measured" }, "chars_per_token": { "value": 3.9642611539163264, "provenance": "measured" }, "tokens_per_word": { "value": 3.089493994710142, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.564852758982179, "provenance": "measured" }, "chars_per_token": { "value": 4.546289362976679, "provenance": "measured" }, "tokens_per_word": { "value": 1.345211241380784, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 5.450428501820058, "provenance": "measured" }, "chars_per_token": { "value": 3.0226050235508803, "provenance": "measured" }, "tokens_per_word": { "value": 2.024428455378902, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 8.22681771064712, "provenance": "measured" }, "chars_per_token": { "value": 3.0903598400710797, "provenance": "measured" }, "tokens_per_word": { "value": 2.317989908351354, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.449474521833422, "provenance": "measured" }, "chars_per_token": { "value": 3.362408829847325, "provenance": "measured" }, "tokens_per_word": { "value": 2.5629642936396393, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.0338555989951095, "provenance": "measured" }, "chars_per_token": { "value": 3.944121538255712, "provenance": "measured" }, "tokens_per_word": { "value": 1.582549167215583, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.8259652798563306, "provenance": "measured" }, "chars_per_token": { "value": 3.7040781203232567, "provenance": "measured" }, "tokens_per_word": { "value": 1.7328613477522334, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 6.1872897631379855, "provenance": "measured" }, "chars_per_token": { "value": 2.4039573997555723, "provenance": "measured" }, "tokens_per_word": { "value": 2.6495713316474436, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 6.957552884816243, "provenance": "measured" }, "chars_per_token": { "value": 2.7619869929115546, "provenance": "measured" }, "tokens_per_word": { "value": 2.0015330583119635, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.8996140461879154, "provenance": "measured" }, "chars_per_token": { "value": 3.8905599493831065, "provenance": "measured" }, "tokens_per_word": { "value": 1.9182687639576659, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.694434319360718, "provenance": "measured" }, "chars_per_token": { "value": 1.6471610822935652, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.5720301508905123, "provenance": "measured" }, "chars_per_token": { "value": 1.5415867314312148, "provenance": "measured" }, "tokens_per_word": { "value": 2.906455758774098, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 7.076611308817714, "provenance": "measured" }, "chars_per_token": { "value": 2.688922781449472, "provenance": "measured" }, "tokens_per_word": { "value": 2.609907120743034, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.618835452330217, "provenance": "measured" }, "chars_per_token": { "value": 3.091434630560455, "provenance": "measured" }, "tokens_per_word": { "value": 2.3937374920681407, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 7.807528474858783, "provenance": "measured" }, "chars_per_token": { "value": 2.9746272803037317, "provenance": "measured" }, "tokens_per_word": { "value": 3.217499255139537, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 6.466640236338291, "provenance": "measured" }, "chars_per_token": { "value": 2.4635197524075045, "provenance": "measured" }, "tokens_per_word": { "value": 3.240623316341332, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 8.36364235182526, "provenance": "measured" }, "chars_per_token": { "value": 2.990330272570382, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.6950502836419776, "provenance": "measured" }, "chars_per_token": { "value": 3.377749943562893, "provenance": "measured" }, "tokens_per_word": { "value": 2.310595480473154, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.072820543058463, "provenance": "measured" }, "chars_per_token": { "value": 2.8651394990025256, "provenance": "measured" }, "tokens_per_word": { "value": 1.6698721630536455, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.4410776413214785, "provenance": "measured" }, "chars_per_token": { "value": 3.4001085372743765, "provenance": "measured" }, "tokens_per_word": { "value": 1.35117271859776, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.1172192410114485, "provenance": "measured" }, "chars_per_token": { "value": 1.5463929137469126, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 91317, "provenance": "measured" }, "tokens_per_request_mean": { "value": 456.585, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.456585, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 62101, "provenance": "measured" }, "tokens_per_request_mean": { "value": 380.9877300613497, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.38098773006134967, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 27919, "provenance": "measured" }, "tokens_per_request_mean": { "value": 146.94210526315788, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.1469421052631579, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 230801, "provenance": "measured" }, "tokens_per_request_mean": { "value": 549.5261904761904, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5495261904761904, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 125092, "provenance": "measured" }, "tokens_per_request_mean": { "value": 833.9466666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8339466666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 263064, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1315.32, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.31532, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 326205, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1631.025, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.631025, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 419943, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2099.715, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.099715, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8479, "provenance": "measured" }, "tokens_per_request_mean": { "value": 339.16, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33916, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 544398, "provenance": "measured" }, "tokens_per_request_mean": { "value": 90733.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 90.733, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 92000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 188416000, "provenance": "estimated", "note": "92000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 376832000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 376832000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 753664000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 92000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 376832000, "provenance": "estimated", "note": "92000 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 753664000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 753664000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1507328000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "akshara-59k", "label": "Akshara 59k", "provider": "Akshara", "family": "akshara", "metrics": { "bpt_overall": { "value": 4.3190455822943425, "provenance": "measured" }, "bpt_english": { "value": 4.393184939957679, "provenance": "measured" }, "bpt_code": { "value": 3.6578377506188486, "provenance": "measured" }, "bpt_languages": { "value": 4.906114056306499, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.5022278156012671, "provenance": "measured" }, "byte_fallback_share": { "value": 0.010684610988603281, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 6.198963235580143, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.142785722277156, "q3": 6.2555780795380285, "n": 5 }, "batch_mbps_4t": { "value": 17.667729432326173, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.166497951509918, "q3": 17.810078165382766, "n": 5 }, "latency_small_p95_us": { "value": 58.165999689663295, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.45899989051395, "q3": 58.5830000545684, "n": 5 }, "decode_mbps": { "value": 48.26985164104428, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 47.21170933430155, "q3": 49.15305904263578, "n": 5 }, "rss_load_delta_mb": { "value": 38.756352, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 38.756352, "q3": 38.756352, "n": 5 }, "rss_peak_large_mb": { "value": 172.630016, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 172.593152, "q3": 172.6464, "n": 5 }, "load_seconds": { "value": 0.06612683299999844, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.06587079200016888, "q3": 0.06826133300000947, "n": 5 }, "throughput_ratio_very_large": { "value": 0.558893973781814, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2407.292928, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2406.936576, "q3": 2407.387136, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.341888, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.686528, "q3": 18.604032, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.4645631960607575, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.4580087763252116, "q3": 3.5054022623286794, "n": 5 }, "embedding_params_h2048": { "value": 120832000, "provenance": "estimated", "note": "59000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.202374438238212, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.20022431131205665, "q3": 0.20833628385680988, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5754968188117333, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5660106350776615, "q3": 3.5764043900531157, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 141.488037109375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 141.466796875, "q3": 141.49389648437503, "n": 5 }, "workload_tokens_total": { "value": 2200740, "provenance": "measured" }, "tokens_chat_en": { "value": 94496, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 66162, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 29961, "provenance": "measured" }, "tokens_code": { "value": 238588, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 128071, "provenance": "measured" }, "tokens_rag": { "value": 287148, "provenance": "measured" }, "tokens_json": { "value": 334371, "provenance": "measured" }, "tokens_logs": { "value": 436105, "provenance": "measured" }, "tokens_tech_docs": { "value": 8769, "provenance": "measured" }, "tokens_long_context": { "value": 577069, "provenance": "measured" } }, "parity": { "de": { "value": 1.475993463825053, "provenance": "measured" }, "fr": { "value": 1.4984528734832945, "provenance": "measured" }, "es": { "value": 1.450092132253242, "provenance": "measured" }, "ru": { "value": 1.7522511559990266, "provenance": "measured" }, "zh": { "value": 1.1768591593366478, "provenance": "measured" }, "ja": { "value": 1.3128672252546674, "provenance": "measured" }, "ko": { "value": 1.663526057782568, "provenance": "measured" }, "ar": { "value": 1.5272050898724054, "provenance": "measured" }, "ur": { "value": 1.8325974342036644, "provenance": "measured" }, "hi": { "value": 1.9370719326913048, "provenance": "measured" }, "mr": { "value": 1.9379063380036854, "provenance": "measured" }, "gu": { "value": 1.9846678023850086, "provenance": "measured" }, "bn": { "value": 1.8246705837360497, "provenance": "measured" }, "ta": { "value": 2.0259360984598267, "provenance": "measured" }, "te": { "value": 2.1382331467510345, "provenance": "measured" }, "th": { "value": 1.7950491951465424, "provenance": "measured" }, "vi": { "value": 1.5236588672947884, "provenance": "measured" }, "tr": { "value": 1.550186002850885, "provenance": "measured" }, "id": { "value": 1.3808364913256614, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 28.64840637450199, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 42.28486055776892, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.475993463825053, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 42.92828685258964, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4984528734832945, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 41.54282868525896, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.450092132253242, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 50.199203187250994, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7522511559990266, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 33.71513944223108, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1768591593366478, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 37.61155378486056, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3128672252546674, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 47.657370517928285, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.663526057782568, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 43.75199203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5272050898724054, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 52.50099601593625, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8325974342036644, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 55.49402390438247, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9370719326913048, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 55.51792828685259, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9379063380036854, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 56.85756972111554, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9846678023850086, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 52.27390438247012, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8246705837360497, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 58.0398406374502, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0259360984598267, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 61.256972111553786, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.1382331467510345, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 51.42529880478088, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7950491951465424, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 43.6503984063745, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5236588672947884, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 44.41035856573705, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.550186002850885, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 39.558764940239044, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3808364913256614, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (exported from Akshara)" } }, "vocabulary": { "size": { "value": 59000, "provenance": "measured" }, "used_share": { "value": 0.9475593220338983, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 55906, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 3094, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 11, "provenance": "measured", "note": "of 58036 entries whose text is valid UTF-8; 964 are byte pieces" }, "unreachable_examples": [ "️◾", "️◾️◾", "️◾️◾️◾️◾", "️⬜", "️⬜️⬜", "️◽", "️◽️◽", "️⬜️⬜️⬜️⬜", "️◾️◾️◾️◾️◾️◾️◾️◾", " \n \t", "️\n" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (exported from Akshara)", "load_seconds": { "value": 0.06612683299999844, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.06587079200016888, "q3": 0.06826133300000947, "n": 5 }, "rss_load_delta_mb": { "value": 38.756352, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 38.756352, "q3": 38.756352, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 33.693696, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 33.693696, "q3": 33.693696, "n": 5 }, "latency_small_p50_us": { "value": 43.83299983601319, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 43.667000227287645, "q3": 44.0409999100666, "n": 5 }, "latency_small_p95_us": { "value": 58.165999689663295, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.45899989051395, "q3": 58.5830000545684, "n": 5 }, "encode_mbps_1t_medium": { "value": 6.198963235580143, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.142785722277156, "q3": 6.2555780795380285, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1551504.471206105, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1537444.1098588149, "q3": 1565674.289641732, "n": 5 }, "encode_mbps_1t_large": { "value": 4.912054854698267, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.763107249873185, "q3": 4.921053674655524, "n": 5 }, "latency_large_p50_ms": { "value": 210.94562500002212, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 210.84304199985127, "q3": 216.82166600021446, "n": 5 }, "decode_mbps": { "value": 48.26985164104428, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 47.21170933430155, "q3": 49.15305904263578, "n": 5 }, "decode_tokens_per_s": { "value": 11772225.098242236, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 11514161.545166042, "q3": 11987624.897215886, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.551808061409028, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.52257937899791, "q3": 5.559749175873513, "n": 5 }, "batch_mbps_4t": { "value": 17.667729432326173, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.166497951509918, "q3": 17.810078165382766, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4400239.477698756, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4275405.182618503, "q3": 4435692.166579648, "n": 5 }, "batch_cpu_utilization": { "value": 3.5754968188117333, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5660106350776615, "q3": 3.5764043900531157, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5754968188117333, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5660106350776615, "q3": 3.5764043900531157, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.202374438238212, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.20022431131205665, "q3": 0.20833628385680988, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 141.488037109375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 141.466796875, "q3": 141.49389648437503, "n": 5 }, "rss_peak_large_mb": { "value": 172.630016, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 172.593152, "q3": 172.6464, "n": 5 }, "rss_peak_very_large_mb": { "value": 2407.292928, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2406.936576, "q3": 2407.387136, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.341888, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.686528, "q3": 18.604032, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.4645631960607575, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.4580087763252116, "q3": 3.5054022623286794, "n": 5 }, "throughput_ratio_very_large": { "value": 0.558893973781814, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "akshara-native (Rust)", "load_seconds": { "value": 0.11027433399999609, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.10760424999989482, "q3": 0.11044145899995783, "n": 5 }, "rss_load_delta_mb": { "value": 25.432064, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 25.432064, "q3": 25.43616, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 25.182208, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 25.182208, "q3": 25.186304, "n": 5 }, "latency_small_p50_us": { "value": 7.9580000829082564, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 7.541499996932544, "q3": 7.999999979801942, "n": 5 }, "latency_small_p95_us": { "value": 11.792000009336334, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 11.416999996072263, "q3": 12.166999994178695, "n": 5 }, "encode_mbps_1t_medium": { "value": 43.147832703931286, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 41.94876540731698, "q3": 43.53332102375039, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 10799234.13301826, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10499126.163140938, "q3": 10895715.88796145, "n": 5 }, "encode_mbps_1t_large": { "value": 39.1324188881637, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 38.76428978357709, "q3": 40.45504967985997, "n": 5 }, "latency_large_p50_ms": { "value": 26.48416600004566, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 26.039500000024418, "q3": 26.929582999997592, "n": 5 }, "decode_mbps": { "value": 97.06023132026147, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 96.48454810324738, "q3": 98.91740955818551, "n": 5 }, "decode_tokens_per_s": { "value": 23671398.447348937, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 23530998.753014524, "q3": 24124333.758337304, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 38.154162330240474, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.12897891737149, "q3": 39.09027023757751, "n": 5 }, "batch_mbps_4t": { "value": 111.90822234931201, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 111.18645773404667, "q3": 113.30136477754456, "n": 5 }, "batch_tokens_per_s_4t": { "value": 27871322.10433107, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 27691562.890456893, "q3": 28218291.438123886, "n": 5 }, "batch_cpu_utilization": { "value": 2.9378113687036254, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.9257790427707264, "q3": 2.9592666675475647, "n": 5 }, "cpu_cores_used_batch": { "value": 2.9378113687036254, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 2.9257790427707264, "q3": 2.9592666675475647, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.025885925967253132, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.025822981466420893, "q3": 0.026422384781164147, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 10.853759765625002, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 10.755126953125002, "q3": 10.855712890625, "n": 5 }, "rss_peak_large_mb": { "value": 29.229056, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 28.823552, "q3": 29.241344, "n": 5 }, "rss_peak_very_large_mb": { "value": 204.423168, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 203.907072, "q3": 204.455936, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.260544, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.199104, "q3": 17.809408, "n": 5 }, "encode_mbps_1t_very_large": { "value": 40.27647498355967, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.17939874158913, "q3": 40.50928143169784, "n": 5 }, "throughput_ratio_very_large": { "value": 0.9334530255534712, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.4456170990098514, "provenance": "measured" }, "chars_per_token": { "value": 2.44341398051821, "provenance": "measured" }, "tokens_per_word": { "value": 2.696095145315798, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.2484439198384902, "provenance": "measured" }, "chars_per_token": { "value": 3.235472822616537, "provenance": "measured" }, "tokens_per_word": { "value": 2.610094446034445, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.634720144057312, "provenance": "measured" }, "chars_per_token": { "value": 3.6341371783162653, "provenance": "measured" }, "tokens_per_word": { "value": 2.899810289449464, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.549714679351721, "provenance": "measured" }, "chars_per_token": { "value": 4.524505925174311, "provenance": "measured" }, "tokens_per_word": { "value": 2.6980534974255934, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.925830243902439, "provenance": "measured" }, "chars_per_token": { "value": 3.918072195121951, "provenance": "measured" }, "tokens_per_word": { "value": 2.7247883969205904, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.3736138359004326, "provenance": "measured" }, "chars_per_token": { "value": 3.37092151215372, "provenance": "measured" }, "tokens_per_word": { "value": 2.4913685524126454, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.5086035010040373, "provenance": "measured" }, "chars_per_token": { "value": 3.454144226607347, "provenance": "measured" }, "tokens_per_word": { "value": 3.4339912280701754, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.259962687194312, "provenance": "measured" }, "chars_per_token": { "value": 4.257660050086559, "provenance": "measured" }, "tokens_per_word": { "value": 2.3172223087708366, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.7572015185558225, "provenance": "measured" }, "chars_per_token": { "value": 3.7432710932052373, "provenance": "measured" }, "tokens_per_word": { "value": 2.6823084436501174, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.217368685730141, "provenance": "measured" }, "chars_per_token": { "value": 4.203357206027711, "provenance": "measured" }, "tokens_per_word": { "value": 3.0791862451943612, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.4023490531108154, "provenance": "measured" }, "chars_per_token": { "value": 3.374820015632928, "provenance": "measured" }, "tokens_per_word": { "value": 2.8469977356133365, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.676138804114389, "provenance": "measured" }, "chars_per_token": { "value": 3.6709166949248333, "provenance": "measured" }, "tokens_per_word": { "value": 3.1155065148491605, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.715693081246209, "provenance": "measured" }, "chars_per_token": { "value": 3.7153422835468572, "provenance": "measured" }, "tokens_per_word": { "value": 2.910802416609939, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.9138293241100564, "provenance": "measured" }, "chars_per_token": { "value": 3.9031316273998424, "provenance": "measured" }, "tokens_per_word": { "value": 3.03729582522093, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.8622911642023006, "provenance": "measured" }, "chars_per_token": { "value": 2.848853831435881, "provenance": "measured" }, "tokens_per_word": { "value": 3.136245545464996, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.834698407542325, "provenance": "measured" }, "chars_per_token": { "value": 3.75287129631778, "provenance": "measured" }, "tokens_per_word": { "value": 2.7230740069921233, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.004318052591244, "provenance": "measured" }, "chars_per_token": { "value": 4.002966601016885, "provenance": "measured" }, "tokens_per_word": { "value": 2.522889812889813, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.1787917366652665, "provenance": "measured" }, "chars_per_token": { "value": 3.1730102897942043, "provenance": "measured" }, "tokens_per_word": { "value": 2.7492242188063796, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.118858328612439, "provenance": "measured" }, "chars_per_token": { "value": 3.106579759775434, "provenance": "measured" }, "tokens_per_word": { "value": 2.889622572101236, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.307490823719044, "provenance": "measured" }, "chars_per_token": { "value": 4.2913458253396755, "provenance": "measured" }, "tokens_per_word": { "value": 2.4537423154636517, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.879057672537175, "provenance": "measured" }, "chars_per_token": { "value": 3.795992554914227, "provenance": "measured" }, "tokens_per_word": { "value": 3.226444955122924, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.393184939957679, "provenance": "measured" }, "chars_per_token": { "value": 4.3753196470179745, "provenance": "measured" }, "tokens_per_word": { "value": 1.3977766314317894, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 4.948260458499943, "provenance": "measured" }, "chars_per_token": { "value": 2.744121295179939, "provenance": "measured" }, "tokens_per_word": { "value": 2.229875053189432, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 7.181821707430581, "provenance": "measured" }, "chars_per_token": { "value": 2.6978127100677387, "provenance": "measured" }, "tokens_per_word": { "value": 2.6552706552706553, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.209362185302681, "provenance": "measured" }, "chars_per_token": { "value": 3.1283569951704533, "provenance": "measured" }, "tokens_per_word": { "value": 2.7547155854722982, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.763421399348173, "provenance": "measured" }, "chars_per_token": { "value": 3.679703210595659, "provenance": "measured" }, "tokens_per_word": { "value": 1.6962689376117437, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.5936575320321342, "provenance": "measured" }, "chars_per_token": { "value": 3.4791712058701565, "provenance": "measured" }, "tokens_per_word": { "value": 1.8448801234423828, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 5.590352196363407, "provenance": "measured" }, "chars_per_token": { "value": 2.1720283103198055, "provenance": "measured" }, "tokens_per_word": { "value": 2.9324924443347933, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 6.3142322712738395, "provenance": "measured" }, "chars_per_token": { "value": 2.5066036424301403, "provenance": "measured" }, "tokens_per_word": { "value": 2.20545768759059, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.565282783115734, "provenance": "measured" }, "chars_per_token": { "value": 3.557004934315183, "provenance": "measured" }, "tokens_per_word": { "value": 2.098152733275075, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.348640295057272, "provenance": "measured" }, "chars_per_token": { "value": 1.5258304979091453, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.2803361134539095, "provenance": "measured" }, "chars_per_token": { "value": 1.4156998719270297, "provenance": "measured" }, "tokens_per_word": { "value": 3.1649036085022244, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 6.357476339093147, "provenance": "measured" }, "chars_per_token": { "value": 2.415670751820558, "provenance": "measured" }, "tokens_per_word": { "value": 2.9051304732419285, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.12285153889772, "provenance": "measured" }, "chars_per_token": { "value": 2.818548574509421, "provenance": "measured" }, "tokens_per_word": { "value": 2.625494215844194, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 6.977380246334635, "provenance": "measured" }, "chars_per_token": { "value": 2.65834517192392, "provenance": "measured" }, "tokens_per_word": { "value": 3.6003078756579603, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 5.766750681431977, "provenance": "measured" }, "chars_per_token": { "value": 2.196891073526225, "provenance": "measured" }, "tokens_per_word": { "value": 3.6339259811844666, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 7.341277550359629, "provenance": "measured" }, "chars_per_token": { "value": 2.6247947454844005, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.3249762458635037, "provenance": "measured" }, "chars_per_token": { "value": 3.039454801612005, "provenance": "measured" }, "tokens_per_word": { "value": 2.567767663340681, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 4.585058002167309, "provenance": "measured" }, "chars_per_token": { "value": 2.5896502105132257, "provenance": "measured" }, "tokens_per_word": { "value": 1.8475146462740206, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.160313105004388, "provenance": "measured" }, "chars_per_token": { "value": 3.1851539757928604, "provenance": "measured" }, "tokens_per_word": { "value": 1.4423584953041033, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.784716418793495, "provenance": "measured" }, "chars_per_token": { "value": 1.421507650665227, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 94496, "provenance": "measured" }, "tokens_per_request_mean": { "value": 472.48, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.47248, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 66162, "provenance": "measured" }, "tokens_per_request_mean": { "value": 405.90184049079755, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.4059018404907975, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 29961, "provenance": "measured" }, "tokens_per_request_mean": { "value": 157.68947368421053, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.15768947368421055, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 238588, "provenance": "measured" }, "tokens_per_request_mean": { "value": 568.0666666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5680666666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 128071, "provenance": "measured" }, "tokens_per_request_mean": { "value": 853.8066666666666, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8538066666666666, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 287148, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1435.74, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.43574, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 334371, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1671.855, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.671855, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 436105, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2180.525, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.180525, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8769, "provenance": "measured" }, "tokens_per_request_mean": { "value": 350.76, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.35076, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 577069, "provenance": "measured" }, "tokens_per_request_mean": { "value": 96178.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 96.17816666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 59000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 120832000, "provenance": "estimated", "note": "59000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 241664000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 241664000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 483328000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 59000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 241664000, "provenance": "estimated", "note": "59000 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 483328000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 483328000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 966656000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "akshara-49k", "label": "Akshara 49k", "provider": "Akshara", "family": "akshara", "metrics": { "bpt_overall": { "value": 4.202067673684221, "provenance": "measured" }, "bpt_english": { "value": 4.3059243547292745, "provenance": "measured" }, "bpt_code": { "value": 3.597655776002392, "provenance": "measured" }, "bpt_languages": { "value": 4.7026228903209955, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.5195910932603355, "provenance": "measured" }, "byte_fallback_share": { "value": 0.01301886031912907, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 6.273736117150162, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.253023908066624, "q3": 6.294651485427605, "n": 5 }, "batch_mbps_4t": { "value": 17.597606427027582, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.457215779760034, "q3": 17.756775096730202, "n": 5 }, "latency_small_p95_us": { "value": 58.45900022904971, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.16700025004684, "q3": 58.91699993298971, "n": 5 }, "decode_mbps": { "value": 49.309272169518415, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 49.11152353280939, "q3": 49.31400755671072, "n": 5 }, "rss_load_delta_mb": { "value": 33.431552, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 33.427456, "q3": 33.431552, "n": 5 }, "rss_peak_large_mb": { "value": 178.335744, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 178.065408, "q3": 178.495488, "n": 5 }, "load_seconds": { "value": 0.05253670900037832, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.05247358399992663, "q3": 0.054333542000222224, "n": 5 }, "throughput_ratio_very_large": { "value": 0.56389126471495, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2466.205696, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2465.374208, "q3": 2466.316288, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.555456, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.145856, "q3": 17.8176, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.5377049935876648, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.420759465150558, "q3": 3.5703086268281754, "n": 5 }, "embedding_params_h2048": { "value": 100352000, "provenance": "estimated", "note": "49000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.20279120713008603, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19897257407959762, "q3": 0.20407107864700794, "n": 5 }, "cpu_cores_used_batch": { "value": 3.561898862847338, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5331112483489044, "q3": 3.562512854349198, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 145.688720703125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 145.63916015625, "q3": 145.6953125, "n": 5 }, "workload_tokens_total": { "value": 2246930, "provenance": "measured" }, "tokens_chat_en": { "value": 96073, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 68002, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 31005, "provenance": "measured" }, "tokens_code": { "value": 243110, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 130641, "provenance": "measured" }, "tokens_rag": { "value": 297775, "provenance": "measured" }, "tokens_json": { "value": 338035, "provenance": "measured" }, "tokens_logs": { "value": 440506, "provenance": "measured" }, "tokens_tech_docs": { "value": 8916, "provenance": "measured" }, "tokens_long_context": { "value": 592867, "provenance": "measured" } }, "parity": { "de": { "value": 1.4940008837225112, "provenance": "measured" }, "fr": { "value": 1.506678902824513, "provenance": "measured" }, "es": { "value": 1.4605553856089188, "provenance": "measured" }, "ru": { "value": 1.7901159036062677, "provenance": "measured" }, "zh": { "value": 1.1900003398932735, "provenance": "measured" }, "ja": { "value": 1.3307161551272901, "provenance": "measured" }, "ko": { "value": 1.6828115971584923, "provenance": "measured" }, "ar": { "value": 1.5512729003093029, "provenance": "measured" }, "ur": { "value": 1.865300295707148, "provenance": "measured" }, "hi": { "value": 1.962611739913667, "provenance": "measured" }, "mr": { "value": 1.9902450630502022, "provenance": "measured" }, "gu": { "value": 2.0254240168587065, "provenance": "measured" }, "bn": { "value": 1.867577580639679, "provenance": "measured" }, "ta": { "value": 2.0854831582882976, "provenance": "measured" }, "te": { "value": 2.183848271642704, "provenance": "measured" }, "th": { "value": 1.848917439923864, "provenance": "measured" }, "vi": { "value": 1.5550797049726386, "provenance": "measured" }, "tr": { "value": 1.5758131946568776, "provenance": "measured" }, "id": { "value": 1.401006084089596, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 29.30378486055777, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 43.77988047808765, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4940008837225112, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 44.15139442231076, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.506678902824513, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 42.79980079681275, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4605553856089188, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 52.45717131474104, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7901159036062677, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 34.87151394422311, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1900003398932735, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 38.995019920318725, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3307161551272901, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 49.31274900398407, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6828115971584923, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 45.45816733067729, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5512729003093029, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 54.66035856573705, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.865300295707148, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 57.51195219123506, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.962611739913667, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 58.32171314741036, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9902450630502022, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 59.352589641434264, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0254240168587065, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 54.72709163346614, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.867577580639679, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 61.11254980079681, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0854831582882976, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 63.995019920318725, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.183848271642704, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 54.180278884462155, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.848917439923864, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 45.569721115537845, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5550797049726386, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 46.177290836653384, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5758131946568776, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 41.05478087649402, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.401006084089596, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (exported from Akshara)" } }, "vocabulary": { "size": { "value": 49000, "provenance": "measured" }, "used_share": { "value": 0.9587346938775511, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 46978, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 2022, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 8, "provenance": "measured", "note": "of 48067 entries whose text is valid UTF-8; 933 are byte pieces" }, "unreachable_examples": [ "️◾", "️◾️◾", "️◾️◾️◾️◾", "️⬜", "️⬜️⬜", "️◽", "️◽️◽", "️⬜️⬜️⬜️⬜" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (exported from Akshara)", "load_seconds": { "value": 0.05253670900037832, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.05247358399992663, "q3": 0.054333542000222224, "n": 5 }, "rss_load_delta_mb": { "value": 33.431552, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 33.427456, "q3": 33.431552, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 33.36192, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 33.36192, "q3": 33.366016, "n": 5 }, "latency_small_p50_us": { "value": 44.729500132234534, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 43.77050004222838, "q3": 45.16699982559658, "n": 5 }, "latency_small_p95_us": { "value": 58.45900022904971, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.16700025004684, "q3": 58.91699993298971, "n": 5 }, "encode_mbps_1t_medium": { "value": 6.273736117150162, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.253023908066624, "q3": 6.294651485427605, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1611142.7727815737, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1605823.7212068574, "q3": 1616513.9971702038, "n": 5 }, "encode_mbps_1t_large": { "value": 4.77098737359956, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.754955536824789, "q3": 4.777184871939625, "n": 5 }, "latency_large_p50_ms": { "value": 219.9035419998836, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 217.36158400017302, "q3": 221.30637500004013, "n": 5 }, "decode_mbps": { "value": 49.309272169518415, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 49.11152353280939, "q3": 49.31400755671072, "n": 5 }, "decode_tokens_per_s": { "value": 12375871.43347554, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12326239.557027586, "q3": 12377059.943070209, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.667138748632376, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.579460334934711, "q3": 5.669191683960934, "n": 5 }, "batch_mbps_4t": { "value": 17.597606427027582, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.457215779760034, "q3": 17.756775096730202, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4496210.719330612, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4460340.73122992, "q3": 4536878.515935005, "n": 5 }, "batch_cpu_utilization": { "value": 3.561898862847338, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5331112483489044, "q3": 3.562512854349198, "n": 5 }, "cpu_cores_used_batch": { "value": 3.561898862847338, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5331112483489044, "q3": 3.562512854349198, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.20279120713008603, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.19897257407959762, "q3": 0.20407107864700794, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 145.688720703125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 145.63916015625, "q3": 145.6953125, "n": 5 }, "rss_peak_large_mb": { "value": 178.335744, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 178.065408, "q3": 178.495488, "n": 5 }, "rss_peak_very_large_mb": { "value": 2466.205696, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2465.374208, "q3": 2466.316288, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.555456, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.145856, "q3": 17.8176, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.5377049935876648, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.420759465150558, "q3": 3.5703086268281754, "n": 5 }, "throughput_ratio_very_large": { "value": 0.56389126471495, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "akshara-native (Rust)", "load_seconds": { "value": 0.09296616699975857, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.09286824999992405, "q3": 0.09580345900008069, "n": 5 }, "rss_load_delta_mb": { "value": 20.840448, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 20.836352, "q3": 20.844544, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 20.5824, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 20.5824, "q3": 20.586496, "n": 5 }, "latency_small_p50_us": { "value": 7.916500180726871, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 7.70799988458748, "q3": 8.374999993066012, "n": 5 }, "latency_small_p95_us": { "value": 12.834000244765775, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.792000234185252, "q3": 13.958999943497474, "n": 5 }, "encode_mbps_1t_medium": { "value": 39.730417183112266, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.78823476444552, "q3": 40.083570953849474, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 10203070.914822804, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9704303.81515082, "q3": 10293763.467837334, "n": 5 }, "encode_mbps_1t_large": { "value": 37.34634919662313, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.22172203571927, "q3": 37.38522913812649, "n": 5 }, "latency_large_p50_ms": { "value": 28.235334000044077, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 28.167957999812643, "q3": 28.24212500036083, "n": 5 }, "decode_mbps": { "value": 95.15622305737604, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 94.86281507110628, "q3": 96.38846820944634, "n": 5 }, "decode_tokens_per_s": { "value": 23882753.300528146, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 23809112.393739946, "q3": 24192027.944131028, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 37.21347425266274, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.02928212571014, "q3": 39.02782724167151, "n": 5 }, "batch_mbps_4t": { "value": 104.66337423945855, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 103.02778553968231, "q3": 104.76936728546434, "n": 5 }, "batch_tokens_per_s_4t": { "value": 26741624.61401588, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 26323729.630692743, "q3": 26768705.971449465, "n": 5 }, "batch_cpu_utilization": { "value": 2.9631479528634874, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.9451766088628752, "q3": 2.996321705979906, "n": 5 }, "cpu_cores_used_batch": { "value": 2.9631479528634874, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 2.9451766088628752, "q3": 2.996321705979906, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.02859921543494531, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.027978592193785127, "q3": 0.02908727374253457, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 12.04150390625, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 12.04052734375, "q3": 12.084716796875, "n": 5 }, "rss_peak_large_mb": { "value": 22.052864, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 21.87264, "q3": 23.216128, "n": 5 }, "rss_peak_very_large_mb": { "value": 215.801856, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 215.7568, "q3": 216.645632, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.854464, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.05984, "q3": 18.468864, "n": 5 }, "encode_mbps_1t_very_large": { "value": 37.222763151174306, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.41999350930412, "q3": 37.49484321245102, "n": 5 }, "throughput_ratio_very_large": { "value": 0.9368832695518773, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.399733100904601, "provenance": "measured" }, "chars_per_token": { "value": 2.3975713167186186, "provenance": "measured" }, "tokens_per_word": { "value": 2.7476457217080696, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.198192706051196, "provenance": "measured" }, "chars_per_token": { "value": 3.18542226286411, "provenance": "measured" }, "tokens_per_word": { "value": 2.65110523746191, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.5843388501018825, "provenance": "measured" }, "chars_per_token": { "value": 3.5837639649192288, "provenance": "measured" }, "tokens_per_word": { "value": 2.9405698829805216, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.474571305629561, "provenance": "measured" }, "chars_per_token": { "value": 4.449778900821226, "provenance": "measured" }, "tokens_per_word": { "value": 2.7433630541253295, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.883217144930557, "provenance": "measured" }, "chars_per_token": { "value": 3.875543306235573, "provenance": "measured" }, "tokens_per_word": { "value": 2.754689294372847, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.3176889132145178, "provenance": "measured" }, "chars_per_token": { "value": 3.3150412205384328, "provenance": "measured" }, "tokens_per_word": { "value": 2.533364531336661, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.4267112038198464, "provenance": "measured" }, "chars_per_token": { "value": 3.373523032037644, "provenance": "measured" }, "tokens_per_word": { "value": 3.5160575048732943, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.195337134911899, "provenance": "measured" }, "chars_per_token": { "value": 4.193069429845978, "provenance": "measured" }, "tokens_per_word": { "value": 2.352917121046892, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.704590421915464, "provenance": "measured" }, "chars_per_token": { "value": 3.6908550606174035, "provenance": "measured" }, "tokens_per_word": { "value": 2.720401504603171, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.155966438250824, "provenance": "measured" }, "chars_per_token": { "value": 4.142158956919975, "provenance": "measured" }, "tokens_per_word": { "value": 3.124679624092268, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.3504540697478142, "provenance": "measured" }, "chars_per_token": { "value": 3.323344924209176, "provenance": "measured" }, "tokens_per_word": { "value": 2.891094713828375, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.6275365659600096, "provenance": "measured" }, "chars_per_token": { "value": 3.6223834982860286, "provenance": "measured" }, "tokens_per_word": { "value": 3.157248503345463, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.6499396967608546, "provenance": "measured" }, "chars_per_token": { "value": 3.649595106822881, "provenance": "measured" }, "tokens_per_word": { "value": 2.9632402995234854, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.8454914148816566, "provenance": "measured" }, "chars_per_token": { "value": 3.834980506650499, "provenance": "measured" }, "tokens_per_word": { "value": 3.0912713576068565, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.8203409419242993, "provenance": "measured" }, "chars_per_token": { "value": 2.807100548974285, "provenance": "measured" }, "tokens_per_word": { "value": 3.18289458558455, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.771460084433119, "provenance": "measured" }, "chars_per_token": { "value": 3.69098239075039, "provenance": "measured" }, "tokens_per_word": { "value": 2.7687334147677016, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.9431005801923154, "provenance": "measured" }, "chars_per_token": { "value": 3.941769789426705, "provenance": "measured" }, "tokens_per_word": { "value": 2.5620582120582123, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.1129312117628913, "provenance": "measured" }, "chars_per_token": { "value": 3.1072695491234383, "provenance": "measured" }, "tokens_per_word": { "value": 2.8073897669048136, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.0180597474591933, "provenance": "measured" }, "chars_per_token": { "value": 3.006178010471204, "provenance": "measured" }, "tokens_per_word": { "value": 2.9861315479693937, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.231500047330964, "provenance": "measured" }, "chars_per_token": { "value": 4.2156398716040036, "provenance": "measured" }, "tokens_per_word": { "value": 2.4978074889299684, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.839609715866766, "provenance": "measured" }, "chars_per_token": { "value": 3.7573893263807596, "provenance": "measured" }, "tokens_per_word": { "value": 3.2595932879503966, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.3059243547292745, "provenance": "measured" }, "chars_per_token": { "value": 4.288413915031239, "provenance": "measured" }, "tokens_per_word": { "value": 1.4261029086324941, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 4.756021309056349, "provenance": "measured" }, "chars_per_token": { "value": 2.6375126095257153, "provenance": "measured" }, "tokens_per_word": { "value": 2.3200069629801554, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 6.782778028739912, "provenance": "measured" }, "chars_per_token": { "value": 2.5479140976961676, "provenance": "measured" }, "tokens_per_word": { "value": 2.81148525726839, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.1211817110112965, "provenance": "measured" }, "chars_per_token": { "value": 3.042402220464686, "provenance": "measured" }, "tokens_per_word": { "value": 2.8325425591495565, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.6591356526429344, "provenance": "measured" }, "chars_per_token": { "value": 3.577737324703344, "provenance": "measured" }, "tokens_per_word": { "value": 1.7446127787710548, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.5037483725073666, "provenance": "measured" }, "chars_per_token": { "value": 3.392126361954362, "provenance": "measured" }, "tokens_per_word": { "value": 1.8922213145576432, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 5.329934325963804, "provenance": "measured" }, "chars_per_token": { "value": 2.0708477465283, "provenance": "measured" }, "tokens_per_word": { "value": 3.0757725282180965, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 6.089872985689558, "provenance": "measured" }, "chars_per_token": { "value": 2.417538213998391, "provenance": "measured" }, "tokens_per_word": { "value": 2.2867097781246515, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.4264572619874913, "provenance": "measured" }, "chars_per_token": { "value": 3.4185017373175817, "provenance": "measured" }, "tokens_per_word": { "value": 2.183160986503544, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.183652901097803, "provenance": "measured" }, "chars_per_token": { "value": 1.4679404954272102, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.1666787286653397, "provenance": "measured" }, "chars_per_token": { "value": 1.3666485736686569, "provenance": "measured" }, "tokens_per_word": { "value": 3.278497281265447, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 6.084964542451914, "provenance": "measured" }, "chars_per_token": { "value": 2.3121235671264815, "provenance": "measured" }, "tokens_per_word": { "value": 3.035235146690255, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 4.902117931703152, "provenance": "measured" }, "chars_per_token": { "value": 2.697102854448902, "provenance": "measured" }, "tokens_per_word": { "value": 2.74371552691951, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 6.637029165190696, "provenance": "measured" }, "chars_per_token": { "value": 2.52867320029913, "provenance": "measured" }, "tokens_per_word": { "value": 3.784933955705631, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 5.511788879320681, "provenance": "measured" }, "chars_per_token": { "value": 2.0997612792814397, "provenance": "measured" }, "tokens_per_word": { "value": 3.802022462596875, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 6.961631247669803, "provenance": "measured" }, "chars_per_token": { "value": 2.4890562975634363, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.198184740474615, "provenance": "measured" }, "chars_per_token": { "value": 2.923551101446535, "provenance": "measured" }, "tokens_per_word": { "value": 2.6695663879120324, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 4.3576204224282025, "provenance": "measured" }, "chars_per_token": { "value": 2.4611929966739265, "provenance": "measured" }, "tokens_per_word": { "value": 1.9439421041403417, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.003833625993593, "provenance": "measured" }, "chars_per_token": { "value": 3.0653525997614035, "provenance": "measured" }, "tokens_per_word": { "value": 1.498729345587866, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.6732030835044, "provenance": "measured" }, "chars_per_token": { "value": 1.3796241799572213, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 96073, "provenance": "measured" }, "tokens_per_request_mean": { "value": 480.365, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.480365, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 68002, "provenance": "measured" }, "tokens_per_request_mean": { "value": 417.19018404907973, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.4171901840490797, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 31005, "provenance": "measured" }, "tokens_per_request_mean": { "value": 163.18421052631578, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.1631842105263158, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 243110, "provenance": "measured" }, "tokens_per_request_mean": { "value": 578.8333333333334, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5788333333333334, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 130641, "provenance": "measured" }, "tokens_per_request_mean": { "value": 870.94, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.87094, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 297775, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1488.875, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.488875, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 338035, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1690.175, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.690175, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 440506, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2202.53, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.20253, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8916, "provenance": "measured" }, "tokens_per_request_mean": { "value": 356.64, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.35664, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 592867, "provenance": "measured" }, "tokens_per_request_mean": { "value": 98811.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 98.81116666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 49000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 100352000, "provenance": "estimated", "note": "49000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 200704000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 200704000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 401408000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 49000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 200704000, "provenance": "estimated", "note": "49000 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 401408000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 401408000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 802816000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "akshara-32k", "label": "Akshara 32k", "provider": "Akshara", "family": "akshara", "metrics": { "bpt_overall": { "value": 3.928255846680546, "provenance": "measured" }, "bpt_english": { "value": 4.092740614758017, "provenance": "measured" }, "bpt_code": { "value": 3.452065161776516, "provenance": "measured" }, "bpt_languages": { "value": 4.239961763507105, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.5635204207046608, "provenance": "measured" }, "byte_fallback_share": { "value": 0.02148318525961295, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 6.303510437376889, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.114958198355796, "q3": 6.381434276626128, "n": 5 }, "batch_mbps_4t": { "value": 17.277981622498476, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.969863289683705, "q3": 17.50034971107797, "n": 5 }, "latency_small_p95_us": { "value": 57.66699996456737, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.45799990108935, "q3": 58.54200003341248, "n": 5 }, "decode_mbps": { "value": 46.35135997715321, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 45.87138545053736, "q3": 47.085390975300854, "n": 5 }, "rss_load_delta_mb": { "value": 24.612864, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 24.612864, "q3": 24.612864, "n": 5 }, "rss_peak_large_mb": { "value": 176.607232, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 176.533504, "q3": 176.771072, "n": 5 }, "load_seconds": { "value": 0.03534070899991093, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.03528599999981452, "q3": 0.0369946250000055, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5324057435855748, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2486.562816, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2485.219328, "q3": 2487.762944, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.046976, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.772544, "q3": 18.419712, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.356025161611074, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.250074850960608, "q3": 3.463857998948457, "n": 5 }, "embedding_params_h2048": { "value": 65536000, "provenance": "estimated", "note": "32000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.20501333875037522, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.20308780924764394, "q3": 0.20746221362934414, "n": 5 }, "cpu_cores_used_batch": { "value": 3.531046994243594, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.521034285318306, "q3": 3.5422166992960378, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 147.15576171875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 147.07568359375, "q3": 147.227294921875, "n": 5 }, "workload_tokens_total": { "value": 2367401, "provenance": "measured" }, "tokens_chat_en": { "value": 100294, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 72421, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 33562, "provenance": "measured" }, "tokens_code": { "value": 253779, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 135769, "provenance": "measured" }, "tokens_rag": { "value": 327277, "provenance": "measured" }, "tokens_json": { "value": 349737, "provenance": "measured" }, "tokens_logs": { "value": 452118, "provenance": "measured" }, "tokens_tech_docs": { "value": 9305, "provenance": "measured" }, "tokens_long_context": { "value": 633139, "provenance": "measured" } }, "parity": { "de": { "value": 1.52453207048742, "provenance": "measured" }, "fr": { "value": 1.532295995618698, "provenance": "measured" }, "es": { "value": 1.494861634612287, "provenance": "measured" }, "ru": { "value": 1.8897909216842241, "provenance": "measured" }, "zh": { "value": 1.227312264424471, "provenance": "measured" }, "ja": { "value": 1.3930930060242905, "provenance": "measured" }, "ko": { "value": 1.7583840726780708, "provenance": "measured" }, "ar": { "value": 1.604845204729229, "provenance": "measured" }, "ur": { "value": 1.9250346316162494, "provenance": "measured" }, "hi": { "value": 2.062594632904868, "provenance": "measured" }, "mr": { "value": 2.1060210689088623, "provenance": "measured" }, "gu": { "value": 2.1504461840791214, "provenance": "measured" }, "bn": { "value": 1.9545762056634772, "provenance": "measured" }, "ta": { "value": 2.2252826906349665, "provenance": "measured" }, "te": { "value": 2.314487291002223, "provenance": "measured" }, "th": { "value": 1.9620179762249927, "provenance": "measured" }, "vi": { "value": 1.6598047743307238, "provenance": "measured" }, "tr": { "value": 1.641538610225186, "provenance": "measured" }, "id": { "value": 1.4495022711897168, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 30.91733067729084, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 47.13446215139442, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.52453207048742, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 47.374501992031874, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.532295995618698, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 46.21713147410359, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.494861634612287, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 58.427290836653384, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8897909216842241, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 37.94521912350598, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.227312264424471, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 43.070717131474105, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3930930060242905, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 54.364541832669325, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7583840726780708, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 49.61752988047809, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.604845204729229, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 59.516932270916335, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9250346316162494, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 63.7699203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.062594632904868, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 65.11254980079681, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.1060210689088623, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 66.48605577689243, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.1504461840791214, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 60.430278884462155, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9545762056634772, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 68.79980079681275, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.2252826906349665, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 71.55776892430279, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.314487291002223, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 60.66035856573705, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9620179762249927, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 51.31673306772908, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6598047743307238, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 50.75199203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.641538610225186, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 44.81474103585657, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4495022711897168, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (exported from Akshara)" } }, "vocabulary": { "size": { "value": 32000, "provenance": "measured" }, "used_share": { "value": 0.97453125, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 31185, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 815, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 5, "provenance": "measured", "note": "of 31147 entries whose text is valid UTF-8; 853 are byte pieces" }, "unreachable_examples": [ "️◾", "️◾️◾", "️◾️◾️◾️◾", "️⬜", "️⬜️⬜" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (exported from Akshara)", "load_seconds": { "value": 0.03534070899991093, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.03528599999981452, "q3": 0.0369946250000055, "n": 5 }, "rss_load_delta_mb": { "value": 24.612864, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 24.612864, "q3": 24.612864, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 20.586496, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 20.586496, "q3": 20.586496, "n": 5 }, "latency_small_p50_us": { "value": 43.91699985717423, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 43.45850015852193, "q3": 44.16699994180817, "n": 5 }, "latency_small_p95_us": { "value": 57.66699996456737, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.45799990108935, "q3": 58.54200003341248, "n": 5 }, "encode_mbps_1t_medium": { "value": 6.303510437376889, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.114958198355796, "q3": 6.381434276626128, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1723203.8987007975, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1671658.976769151, "q3": 1744506.1024379109, "n": 5 }, "encode_mbps_1t_large": { "value": 4.754877551870971, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.738138473676185, "q3": 4.783772733380019, "n": 5 }, "latency_large_p50_ms": { "value": 218.17012500014243, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 216.40087499986294, "q3": 218.8113339998381, "n": 5 }, "decode_mbps": { "value": 46.35135997715321, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 45.87138545053736, "q3": 47.085390975300854, "n": 5 }, "decode_tokens_per_s": { "value": 12438394.83638852, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12309593.595682763, "q3": 12635372.171716094, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.4308370914924105, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.414361992561935, "q3": 5.718681389103371, "n": 5 }, "batch_mbps_4t": { "value": 17.277981622498476, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.969863289683705, "q3": 17.50034971107797, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4698031.115023103, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4614251.102617722, "q3": 4758494.901937561, "n": 5 }, "batch_cpu_utilization": { "value": 3.531046994243594, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.521034285318306, "q3": 3.5422166992960378, "n": 5 }, "cpu_cores_used_batch": { "value": 3.531046994243594, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.521034285318306, "q3": 3.5422166992960378, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.20501333875037522, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.20308780924764394, "q3": 0.20746221362934414, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 147.15576171875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 147.07568359375, "q3": 147.227294921875, "n": 5 }, "rss_peak_large_mb": { "value": 176.607232, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 176.533504, "q3": 176.771072, "n": 5 }, "rss_peak_very_large_mb": { "value": 2486.562816, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2485.219328, "q3": 2487.762944, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.046976, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.772544, "q3": 18.419712, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.356025161611074, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.250074850960608, "q3": 3.463857998948457, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5324057435855748, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "akshara-native (Rust)", "load_seconds": { "value": 0.07983766699999251, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.0760976669998854, "q3": 0.08004070800006957, "n": 5 }, "rss_load_delta_mb": { "value": 18.190336, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 18.18624, "q3": 18.190336, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 17.924096, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 17.92, "q3": 17.924096, "n": 5 }, "latency_small_p50_us": { "value": 7.708000111961155, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 7.666999977118394, "q3": 7.709000044542336, "n": 5 }, "latency_small_p95_us": { "value": 11.12500001454464, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10.916999940491223, "q3": 12.334000075497897, "n": 5 }, "encode_mbps_1t_medium": { "value": 42.53190270744731, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 41.40957775792447, "q3": 42.55086535378701, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 11627035.648272077, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 11320223.317612762, "q3": 11632219.506762818, "n": 5 }, "encode_mbps_1t_large": { "value": 39.4530397549227, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.98512721114527, "q3": 39.57433097421705, "n": 5 }, "latency_large_p50_ms": { "value": 26.734708000049068, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 26.5385409998089, "q3": 27.2648750000144, "n": 5 }, "decode_mbps": { "value": 92.08316318702454, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 91.72270818743597, "q3": 92.39182152658293, "n": 5 }, "decode_tokens_per_s": { "value": 24710531.515544824, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 24613803.358963385, "q3": 24793360.03015289, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 40.447332733056164, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.18940420210774, "q3": 40.972305257620725, "n": 5 }, "batch_mbps_4t": { "value": 110.03894128296493, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 105.7101972536139, "q3": 113.27802850463881, "n": 5 }, "batch_tokens_per_s_4t": { "value": 29920530.14678538, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 28743507.588063568, "q3": 30801265.691259068, "n": 5 }, "batch_cpu_utilization": { "value": 2.8963933518245875, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.889344662570933, "q3": 2.913886901442137, "n": 5 }, "cpu_cores_used_batch": { "value": 2.8963933518245875, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 2.889344662570933, "q3": 2.913886901442137, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.02632153052414888, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.025935518604272456, "q3": 0.027332695781836273, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 12.583251953125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 12.562255859375, "q3": 12.589599609375002, "n": 5 }, "rss_peak_large_mb": { "value": 23.158784, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 23.150592, "q3": 23.187456, "n": 5 }, "rss_peak_very_large_mb": { "value": 224.940032, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 224.587776, "q3": 225.050624, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.149376, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.584128, "q3": 18.407424, "n": 5 }, "encode_mbps_1t_very_large": { "value": 40.64601302257395, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.55632211316989, "q3": 40.72431399820588, "n": 5 }, "throughput_ratio_very_large": { "value": 0.9556594094121461, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.291452915093408, "provenance": "measured" }, "chars_per_token": { "value": 2.289388674418933, "provenance": "measured" }, "tokens_per_word": { "value": 2.877482816474536, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.0780593940801495, "provenance": "measured" }, "chars_per_token": { "value": 3.065768645450712, "provenance": "measured" }, "tokens_per_word": { "value": 2.754574992844997, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.4211829117358357, "provenance": "measured" }, "chars_per_token": { "value": 3.4206341948188927, "provenance": "measured" }, "tokens_per_word": { "value": 3.080805424594752, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.291783384118231, "provenance": "measured" }, "chars_per_token": { "value": 4.268003758375118, "provenance": "measured" }, "tokens_per_word": { "value": 2.8602034409142285, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.7420638758490368, "provenance": "measured" }, "chars_per_token": { "value": 3.734668977882262, "provenance": "measured" }, "tokens_per_word": { "value": 2.8585981030156096, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.2019148233740506, "provenance": "measured" }, "chars_per_token": { "value": 3.199359524595576, "provenance": "measured" }, "tokens_per_word": { "value": 2.624965335551858, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.248610472373694, "provenance": "measured" }, "chars_per_token": { "value": 3.1981867157217003, "provenance": "measured" }, "tokens_per_word": { "value": 3.7088206627680314, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.046022700062258, "provenance": "measured" }, "chars_per_token": { "value": 4.043835703909455, "provenance": "measured" }, "tokens_per_word": { "value": 2.4397491821155946, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.5687677079883726, "provenance": "measured" }, "chars_per_token": { "value": 3.555535931118225, "provenance": "measured" }, "tokens_per_word": { "value": 2.8239364907832667, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 3.972152808401522, "provenance": "measured" }, "chars_per_token": { "value": 3.9589560161366717, "provenance": "measured" }, "tokens_per_word": { "value": 3.2692759504485265, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.223582834201671, "provenance": "measured" }, "chars_per_token": { "value": 3.197500227370172, "provenance": "measured" }, "tokens_per_word": { "value": 3.00488014367143, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.501719125722284, "provenance": "measured" }, "chars_per_token": { "value": 3.4967447869935038, "provenance": "measured" }, "tokens_per_word": { "value": 3.27068904801033, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.495846970488737, "provenance": "measured" }, "chars_per_token": { "value": 3.4955169284083722, "provenance": "measured" }, "tokens_per_word": { "value": 3.093856364874064, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.6943221316604964, "provenance": "measured" }, "chars_per_token": { "value": 3.684224415474745, "provenance": "measured" }, "tokens_per_word": { "value": 3.2177641913981234, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.7158199321128484, "provenance": "measured" }, "chars_per_token": { "value": 2.703070224250181, "provenance": "measured" }, "tokens_per_word": { "value": 3.305391424301644, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.6218105583029576, "provenance": "measured" }, "chars_per_token": { "value": 3.544526176614535, "provenance": "measured" }, "tokens_per_word": { "value": 2.883134661555958, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.786904409358001, "provenance": "measured" }, "chars_per_token": { "value": 3.7856263345750403, "provenance": "measured" }, "tokens_per_word": { "value": 2.667733887733888, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 2.977258618676206, "provenance": "measured" }, "chars_per_token": { "value": 2.971843711393432, "provenance": "measured" }, "tokens_per_word": { "value": 2.935321498159775, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.8264985318973426, "provenance": "measured" }, "chars_per_token": { "value": 2.815370948296251, "provenance": "measured" }, "tokens_per_word": { "value": 3.1885116244849914, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.074215545741534, "provenance": "measured" }, "chars_per_token": { "value": 4.058944891415125, "provenance": "measured" }, "tokens_per_word": { "value": 2.5942349855982116, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.713378746068196, "provenance": "measured" }, "chars_per_token": { "value": 3.6338614332670796, "provenance": "measured" }, "tokens_per_word": { "value": 3.370398473745827, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.092740614758017, "provenance": "measured" }, "chars_per_token": { "value": 4.076097106458642, "provenance": "measured" }, "tokens_per_word": { "value": 1.5003861286708933, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 4.329756067942744, "provenance": "measured" }, "chars_per_token": { "value": 2.401121753517813, "provenance": "measured" }, "tokens_per_word": { "value": 2.5484120536923136, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 6.0157117952160775, "provenance": "measured" }, "chars_per_token": { "value": 2.259769791339563, "provenance": "measured" }, "tokens_per_word": { "value": 3.1699790615453267, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 2.895866504215875, "provenance": "measured" }, "chars_per_token": { "value": 2.8227740318717376, "provenance": "measured" }, "tokens_per_word": { "value": 3.052930795811186, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.414232600859341, "provenance": "measured" }, "chars_per_token": { "value": 3.338282198554344, "provenance": "measured" }, "tokens_per_word": { "value": 1.86975392867225, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.269821577028842, "provenance": "measured" }, "chars_per_token": { "value": 3.165651979279913, "provenance": "measured" }, "tokens_per_word": { "value": 2.0275930032027594, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 4.72903174151306, "provenance": "measured" }, "chars_per_token": { "value": 1.8373781225535548, "provenance": "measured" }, "tokens_per_word": { "value": 3.466600875840375, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 5.451362292273615, "provenance": "measured" }, "chars_per_token": { "value": 2.1640642901567975, "provenance": "measured" }, "tokens_per_word": { "value": 2.554549002118408, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.1178842252765877, "provenance": "measured" }, "chars_per_token": { "value": 3.1106451433398927, "provenance": "measured" }, "tokens_per_word": { "value": 2.399225652976017, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.8511393608538307, "provenance": "measured" }, "chars_per_token": { "value": 1.3512697049622768, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 2.897992637278332, "provenance": "measured" }, "chars_per_token": { "value": 1.2506912900216909, "provenance": "measured" }, "tokens_per_word": { "value": 3.5824616905585764, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 5.434668025941913, "provenance": "measured" }, "chars_per_token": { "value": 2.0650283061839794, "provenance": "measured" }, "tokens_per_word": { "value": 3.398422526905499, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 4.420655356355032, "provenance": "measured" }, "chars_per_token": { "value": 2.432206312918622, "provenance": "measured" }, "tokens_per_word": { "value": 3.042539171181725, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 5.904306722689076, "provenance": "measured" }, "chars_per_token": { "value": 2.2495098039215686, "provenance": "measured" }, "tokens_per_word": { "value": 4.254642963551495, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 4.914289601819366, "provenance": "measured" }, "chars_per_token": { "value": 1.8721390182033764, "provenance": "measured" }, "tokens_per_word": { "value": 4.264287786480998, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 6.0851632815420444, "provenance": "measured" }, "chars_per_token": { "value": 2.1756846130989467, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.9201657458563535, "provenance": "measured" }, "chars_per_token": { "value": 2.669406077348066, "provenance": "measured" }, "tokens_per_word": { "value": 2.9237266746310846, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 3.9456606484949477, "provenance": "measured" }, "chars_per_token": { "value": 2.228517267209882, "provenance": "measured" }, "tokens_per_word": { "value": 2.146905820765709, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 3.5941160545077158, "provenance": "measured" }, "chars_per_token": { "value": 2.7516710284872, "provenance": "measured" }, "tokens_per_word": { "value": 1.6695796293506102, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.3674492669702314, "provenance": "measured" }, "chars_per_token": { "value": 1.26478561840339, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 100294, "provenance": "measured" }, "tokens_per_request_mean": { "value": 501.47, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.50147, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 72421, "provenance": "measured" }, "tokens_per_request_mean": { "value": 444.3006134969325, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.4443006134969325, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 33562, "provenance": "measured" }, "tokens_per_request_mean": { "value": 176.6421052631579, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.1766421052631579, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 253779, "provenance": "measured" }, "tokens_per_request_mean": { "value": 604.2357142857143, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6042357142857143, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 135769, "provenance": "measured" }, "tokens_per_request_mean": { "value": 905.1266666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9051266666666666, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 327277, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1636.385, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.636385, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 349737, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1748.685, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.748685, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 452118, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2260.59, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.26059, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9305, "provenance": "measured" }, "tokens_per_request_mean": { "value": 372.2, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3722, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 633139, "provenance": "measured" }, "tokens_per_request_mean": { "value": 105523.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 105.52316666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 32000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 65536000, "provenance": "estimated", "note": "32000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 131072000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 131072000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 262144000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 32000, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 131072000, "provenance": "estimated", "note": "32000 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 262144000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 262144000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 524288000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "phi-3-5-mini-instruct", "label": "Phi-3.5-mini-instruct", "provider": "Microsoft", "family": "published", "metrics": { "bpt_overall": { "value": 3.169534137397006, "provenance": "measured" }, "bpt_english": { "value": 4.044230548871319, "provenance": "measured" }, "bpt_code": { "value": 3.076835785908242, "provenance": "measured" }, "bpt_languages": { "value": 2.3875360774114562, "provenance": "measured" }, "parity_mean_abs_log": { "value": 1.1598479000579562, "provenance": "measured" }, "byte_fallback_share": { "value": 0.2887037126387521, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 6.788077586874321, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.781886443853547, "q3": 6.87281856579862, "n": 5 }, "batch_mbps_4t": { "value": 20.144123357075937, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 20.137997757743218, "q3": 20.48385563393946, "n": 5 }, "latency_small_p95_us": { "value": 49.70899999534595, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 49.333000106344116, "q3": 50.7919999108708, "n": 5 }, "decode_mbps": { "value": 16.656788731075494, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.40120282645318, "q3": 16.842082066379636, "n": 5 }, "rss_load_delta_mb": { "value": 30.879744, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 30.879744, "q3": 30.879744, "n": 5 }, "rss_peak_large_mb": { "value": 138.862592, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 138.768384, "q3": 138.932224, "n": 5 }, "load_seconds": { "value": 0.04443212499973015, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.043551125000021784, "q3": 0.046170334000009916, "n": 5 }, "throughput_ratio_very_large": { "value": 0.3595244066812844, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2057.474048, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2055.565312, "q3": 2057.8304, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.625088, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.965632, "q3": 17.768448, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.440479566927515, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.419795930492611, "q3": 2.4461253510318297, "n": 5 }, "embedding_params_h2048": { "value": 65667072, "provenance": "estimated", "note": "32064 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.169624223714293, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.1672564137227595, "q3": 0.17031208312679666, "n": 5 }, "cpu_cores_used_batch": { "value": 3.429526846427324, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.414632821740462, "q3": 3.4297443481240077, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 120.52392578125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 120.410400390625, "q3": 120.545166015625, "n": 5 }, "workload_tokens_total": { "value": 3325522, "provenance": "measured" }, "tokens_chat_en": { "value": 105304, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 76201, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 56772, "provenance": "measured" }, "tokens_code": { "value": 300961, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 174342, "provenance": "measured" }, "tokens_rag": { "value": 617290, "provenance": "measured" }, "tokens_json": { "value": 435188, "provenance": "measured" }, "tokens_logs": { "value": 596201, "provenance": "measured" }, "tokens_tech_docs": { "value": 10116, "provenance": "measured" }, "tokens_long_context": { "value": 953147, "provenance": "measured" } }, "parity": { "de": { "value": 1.413212053790531, "provenance": "measured" }, "fr": { "value": 1.4621601282596604, "provenance": "measured" }, "es": { "value": 1.4444917056571671, "provenance": "measured" }, "ru": { "value": 1.6590648823741125, "provenance": "measured" }, "zh": { "value": 2.0176357033013774, "provenance": "measured" }, "ja": { "value": 2.2470634427248632, "provenance": "measured" }, "ko": { "value": 3.196315806694369, "provenance": "measured" }, "ar": { "value": 3.4685403919772275, "provenance": "measured" }, "ur": { "value": 4.397735824362791, "provenance": "measured" }, "hi": { "value": 4.6131924222098615, "provenance": "measured" }, "mr": { "value": 4.590779700945588, "provenance": "measured" }, "gu": { "value": 9.99685894709289, "provenance": "measured" }, "bn": { "value": 5.394921964466839, "provenance": "measured" }, "ta": { "value": 5.910381834244021, "provenance": "measured" }, "te": { "value": 10.780878840427969, "provenance": "measured" }, "th": { "value": 4.365180119752642, "provenance": "measured" }, "vi": { "value": 2.9183326244151426, "provenance": "measured" }, "tr": { "value": 2.08762228838792, "provenance": "measured" }, "id": { "value": 1.7560448908811308, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 30.441235059760956, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 43.0199203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.413212053790531, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 44.50996015936255, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4621601282596604, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 43.97211155378486, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4444917056571671, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 50.50398406374502, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6590648823741125, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 61.419322709163346, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0176357033013774, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 68.40338645418326, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.2470634427248632, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 97.29980079681275, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.196315806694369, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 105.58665338645419, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.4685403919772275, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 133.87250996015936, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.397735824362791, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 140.4312749003984, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.6131924222098615, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 139.74900398406373, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.590779700945588, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 304.3167330677291, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 9.99685894709289, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 164.2280876494024, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.394921964466839, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 179.91932270916334, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.910381834244021, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 328.1832669322709, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 10.780878840427969, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 132.88147410358565, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.365180119752642, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 88.83764940239044, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.9183326244151426, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 63.54980079681275, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.08762228838792, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 53.45617529880478, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7560448908811308, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 32,064 embedding rows for 32,011 tokenizer IDs (53 unused rows, usually padding)" ], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [ { "set": "wiki_gu", "bytes_per_token": { "value": 1.081700177644471, "provenance": "measured" }, "split_byte_token_share": { "value": 0.8841872002376733, "provenance": "measured" } }, { "set": "wiki_te", "bytes_per_token": { "value": 1.066041773398184, "provenance": "measured" }, "split_byte_token_share": { "value": 0.902970772957554, "provenance": "measured" } } ], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>", "", "" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 32011, "provenance": "measured" }, "used_share": { "value": 0.9136859204648402, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 29248, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 2763, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": null, "provenance": "unavailable", "note": "not applicable: the tokenizer prepends a word-boundary marker before encoding, so a token's own text does not encode to itself by design" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.04443212499973015, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.043551125000021784, "q3": 0.046170334000009916, "n": 5 }, "rss_load_delta_mb": { "value": 30.879744, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 30.879744, "q3": 30.879744, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 30.728192, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 30.728192, "q3": 30.728192, "n": 5 }, "latency_small_p50_us": { "value": 40.45799983032339, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.208999962487724, "q3": 40.541999851484434, "n": 5 }, "latency_small_p95_us": { "value": 49.70899999534595, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 49.333000106344116, "q3": 50.7919999108708, "n": 5 }, "encode_mbps_1t_medium": { "value": 6.788077586874321, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.781886443853547, "q3": 6.87281856579862, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 2696022.943857687, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2693564.0056061577, "q3": 2729679.5455303956, "n": 5 }, "encode_mbps_1t_large": { "value": 4.46978792683956, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.400469165490519, "q3": 4.471886383551747, "n": 5 }, "latency_large_p50_ms": { "value": 231.38024999991558, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 230.12920900009703, "q3": 234.9109159999898, "n": 5 }, "decode_mbps": { "value": 16.656788731075494, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.40120282645318, "q3": 16.842082066379636, "n": 5 }, "decode_tokens_per_s": { "value": 6867084.711713509, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6761714.457788939, "q3": 6943475.488506853, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 6.535101133782265, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.464198450063571, "q3": 6.647994973786918, "n": 5 }, "batch_mbps_4t": { "value": 20.144123357075937, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 20.137997757743218, "q3": 20.48385563393946, "n": 5 }, "batch_tokens_per_s_4t": { "value": 7953400.790466946, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 7950982.251535699, "q3": 8087535.540907916, "n": 5 }, "batch_cpu_utilization": { "value": 3.429526846427324, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.414632821740462, "q3": 3.4297443481240077, "n": 5 }, "cpu_cores_used_batch": { "value": 3.429526846427324, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.414632821740462, "q3": 3.4297443481240077, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.169624223714293, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.1672564137227595, "q3": 0.17031208312679666, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 120.52392578125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 120.410400390625, "q3": 120.545166015625, "n": 5 }, "rss_peak_large_mb": { "value": 138.862592, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 138.768384, "q3": 138.932224, "n": 5 }, "rss_peak_very_large_mb": { "value": 2057.474048, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2055.565312, "q3": 2057.8304, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.625088, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.965632, "q3": 17.768448, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.440479566927515, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.419795930492611, "q3": 2.4461253510318297, "n": 5 }, "throughput_ratio_very_large": { "value": 0.3595244066812844, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 1.886647309330236, "provenance": "measured" }, "chars_per_token": { "value": 1.8849477351916377, "provenance": "measured" }, "tokens_per_word": { "value": 3.494885533365806, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 2.549865324638503, "provenance": "measured" }, "chars_per_token": { "value": 2.5396836647899876, "provenance": "measured" }, "tokens_per_word": { "value": 3.3251738244751596, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.0396409670272524, "provenance": "measured" }, "chars_per_token": { "value": 3.039153444885622, "provenance": "measured" }, "tokens_per_word": { "value": 3.4675144161235183, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 3.945938963345713, "provenance": "measured" }, "chars_per_token": { "value": 3.924075569191022, "provenance": "measured" }, "tokens_per_word": { "value": 3.1108878563355518, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.452894819940415, "provenance": "measured" }, "chars_per_token": { "value": 3.4460713648283154, "provenance": "measured" }, "tokens_per_word": { "value": 3.097996682403981, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 2.6009836892099916, "provenance": "measured" }, "chars_per_token": { "value": 2.598907965522605, "provenance": "measured" }, "tokens_per_word": { "value": 3.231437188019967, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 2.8104045060047858, "provenance": "measured" }, "chars_per_token": { "value": 2.7667824238531797, "provenance": "measured" }, "tokens_per_word": { "value": 4.287110136452242, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 3.6715315250063374, "provenance": "measured" }, "chars_per_token": { "value": 3.6695469525223627, "provenance": "measured" }, "tokens_per_word": { "value": 2.688600249260009, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.2083068699057904, "provenance": "measured" }, "chars_per_token": { "value": 3.196411559225151, "provenance": "measured" }, "tokens_per_word": { "value": 3.141212410897982, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 3.5860599537002904, "provenance": "measured" }, "chars_per_token": { "value": 3.5741458883203823, "provenance": "measured" }, "tokens_per_word": { "value": 3.621262281076463, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 2.7110755378782083, "provenance": "measured" }, "chars_per_token": { "value": 2.6891397226744758, "provenance": "measured" }, "tokens_per_word": { "value": 3.5729288670258454, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.1849709473134427, "provenance": "measured" }, "chars_per_token": { "value": 3.1804465626428153, "provenance": "measured" }, "tokens_per_word": { "value": 3.595961967367062, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.2211226558540296, "provenance": "measured" }, "chars_per_token": { "value": 3.220818550430816, "provenance": "measured" }, "tokens_per_word": { "value": 3.3577263444520082, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.2571848798849956, "provenance": "measured" }, "chars_per_token": { "value": 3.2482819939672436, "provenance": "measured" }, "tokens_per_word": { "value": 3.649610907921951, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.463083522583901, "provenance": "measured" }, "chars_per_token": { "value": 2.451520312894272, "provenance": "measured" }, "tokens_per_word": { "value": 3.6445568456144386, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.283108197589723, "provenance": "measured" }, "chars_per_token": { "value": 3.213051251489869, "provenance": "measured" }, "tokens_per_word": { "value": 3.1805736910829365, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.3722284294607103, "provenance": "measured" }, "chars_per_token": { "value": 3.3710903072236063, "provenance": "measured" }, "tokens_per_word": { "value": 2.9957796257796256, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 2.695737102788135, "provenance": "measured" }, "chars_per_token": { "value": 2.690834214480494, "provenance": "measured" }, "tokens_per_word": { "value": 3.241863318178538, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.5216379927024204, "provenance": "measured" }, "chars_per_token": { "value": 2.51171060825172, "provenance": "measured" }, "tokens_per_word": { "value": 3.5739957327839904, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 3.7885568114401065, "provenance": "measured" }, "chars_per_token": { "value": 3.7743568407183967, "provenance": "measured" }, "tokens_per_word": { "value": 2.7898413653755214, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.362571498468098, "provenance": "measured" }, "chars_per_token": { "value": 3.290566279516781, "provenance": "measured" }, "tokens_per_word": { "value": 3.7220222867796906, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.044230548871319, "provenance": "measured" }, "chars_per_token": { "value": 4.027784311242183, "provenance": "measured" }, "tokens_per_word": { "value": 1.5183830823751339, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 2.015623840666504, "provenance": "measured" }, "chars_per_token": { "value": 1.1177900497828845, "provenance": "measured" }, "tokens_per_word": { "value": 5.474236973424626, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 2.121531623043847, "provenance": "measured" }, "chars_per_token": { "value": 0.7969419474082164, "provenance": "measured" }, "tokens_per_word": { "value": 8.98863831393952, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 2.8091536669061803, "provenance": "measured" }, "chars_per_token": { "value": 2.738249850583898, "provenance": "measured" }, "tokens_per_word": { "value": 3.1471685352889582, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.4708726377386245, "provenance": "measured" }, "chars_per_token": { "value": 3.393662264573274, "provenance": "measured" }, "tokens_per_word": { "value": 1.8392420250305825, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.377427537189209, "provenance": "measured" }, "chars_per_token": { "value": 3.2698298412027373, "provenance": "measured" }, "tokens_per_word": { "value": 1.962993218448931, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.081700177644471, "provenance": "measured" }, "chars_per_token": { "value": 0.42027466724728396, "provenance": "measured" }, "tokens_per_word": { "value": 15.155461666563868, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 2.420664760889578, "provenance": "measured" }, "chars_per_token": { "value": 0.9609477203352875, "provenance": "measured" }, "tokens_per_word": { "value": 5.7528710001114955, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 2.5876924497961618, "provenance": "measured" }, "chars_per_token": { "value": 2.5816843634409414, "provenance": "measured" }, "tokens_per_word": { "value": 2.8908025050975823, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 2.4487775150525315, "provenance": "measured" }, "chars_per_token": { "value": 0.8592155620018038, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 1.628059197980844, "provenance": "measured" }, "chars_per_token": { "value": 0.7026240965424435, "provenance": "measured" }, "tokens_per_word": { "value": 6.376885813148789, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 2.4741358454421123, "provenance": "measured" }, "chars_per_token": { "value": 0.940105362424027, "provenance": "measured" }, "tokens_per_word": { "value": 7.464949137549756, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 4.833074622677085, "provenance": "measured" }, "chars_per_token": { "value": 2.6591158234453243, "provenance": "measured" }, "tokens_per_word": { "value": 2.782911114365207, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.3278268352053892, "provenance": "measured" }, "chars_per_token": { "value": 0.8868897795406753, "provenance": "measured" }, "tokens_per_word": { "value": 10.79148872777833, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.066041773398184, "provenance": "measured" }, "chars_per_token": { "value": 0.406117376207252, "provenance": "measured" }, "tokens_per_word": { "value": 19.657714783041154, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 2.6902971001902696, "provenance": "measured" }, "chars_per_token": { "value": 0.9618867620698277, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.2604310521485784, "provenance": "measured" }, "chars_per_token": { "value": 2.0663239395208355, "provenance": "measured" }, "tokens_per_word": { "value": 3.7770523800710065, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 1.733244913034719, "provenance": "measured" }, "chars_per_token": { "value": 0.978940299509771, "provenance": "measured" }, "tokens_per_word": { "value": 4.887342665375716, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 2.015417677362329, "provenance": "measured" }, "chars_per_token": { "value": 1.5430126209039414, "provenance": "measured" }, "tokens_per_word": { "value": 2.977379338054342, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 2.001212244451049, "provenance": "measured" }, "chars_per_token": { "value": 0.7516384852419015, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 105304, "provenance": "measured" }, "tokens_per_request_mean": { "value": 526.52, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.52652, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 76201, "provenance": "measured" }, "tokens_per_request_mean": { "value": 467.4907975460123, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.46749079754601225, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 56772, "provenance": "measured" }, "tokens_per_request_mean": { "value": 298.8, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.2988, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 300961, "provenance": "measured" }, "tokens_per_request_mean": { "value": 716.5738095238095, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.7165738095238094, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 174342, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1162.28, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.16228, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 617290, "provenance": "measured" }, "tokens_per_request_mean": { "value": 3086.45, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.985, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 3.08645, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 435188, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2175.94, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.17594, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 596201, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2981.005, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.981005, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 10116, "provenance": "measured" }, "tokens_per_request_mean": { "value": 404.64, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.40464, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 953147, "provenance": "measured" }, "tokens_per_request_mean": { "value": 158857.83333333334, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 0.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 158.85783333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 32064, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 65667072, "provenance": "estimated", "note": "32064 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 131334144, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 131334144, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 262668288, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 32064, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 131334144, "provenance": "estimated", "note": "32064 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 262668288, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 262668288, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 525336576, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 197001216, "provenance": "estimated", "note": "32064 rows x 3072 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 394002432, "provenance": "estimated", "note": "32064 rows x 3072 hidden x 2 matrices from the published config" } } } }, { "id": "smollm2-135m", "label": "SmolLM2-135M", "provider": "Hugging Face", "family": "published", "metrics": { "bpt_overall": { "value": 3.2038999542952644, "provenance": "measured" }, "bpt_english": { "value": 4.542947535289986, "provenance": "measured" }, "bpt_code": { "value": 3.1359255671914794, "provenance": "measured" }, "bpt_languages": { "value": 1.9328267604043268, "provenance": "measured" }, "parity_mean_abs_log": { "value": 1.4791129059909334, "provenance": "measured" }, "byte_fallback_share": { "value": 0.4237258034382337, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.853248808026626, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.824940646454353, "q3": 4.899484800901388, "n": 5 }, "batch_mbps_4t": { "value": 12.374920625155946, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.127153555075383, "q3": 12.633565211752764, "n": 5 }, "latency_small_p95_us": { "value": 80.41699999239427, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 79.62500012581586, "q3": 81.95899999918765, "n": 5 }, "decode_mbps": { "value": 32.44802029987467, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 32.22535120544784, "q3": 33.72718610168552, "n": 5 }, "rss_load_delta_mb": { "value": 34.832384, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 34.832384, "q3": 34.832384, "n": 5 }, "rss_peak_large_mb": { "value": 207.511552, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 207.245312, "q3": 208.134144, "n": 5 }, "load_seconds": { "value": 0.050100375000056374, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.04943820800008325, "q3": 0.05065804199966806, "n": 5 }, "throughput_ratio_very_large": { "value": 0.548801770675832, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 3006.599168, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 3003.424768, "q3": 3007.803392, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.543168, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.969728, "q3": 17.702912, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.6634715393753834, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.637972201463825, "q3": 2.681014800559647, "n": 5 }, "embedding_params_h2048": { "value": 100663296, "provenance": "estimated", "note": "49152 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.2793891808416967, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.27692719252505443, "q3": 0.28669884942697854, "n": 5 }, "cpu_cores_used_batch": { "value": 3.4985777456728875, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.457418936443337, "q3": 3.5161412931353415, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 177.715576171875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 177.5263671875, "q3": 177.787353515625, "n": 5 }, "workload_tokens_total": { "value": 3513597, "provenance": "measured" }, "tokens_chat_en": { "value": 95975, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 98488, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 64809, "provenance": "measured" }, "tokens_code": { "value": 285563, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 177092, "provenance": "measured" }, "tokens_rag": { "value": 742587, "provenance": "measured" }, "tokens_json": { "value": 409936, "provenance": "measured" }, "tokens_logs": { "value": 586540, "provenance": "measured" }, "tokens_tech_docs": { "value": 9528, "provenance": "measured" }, "tokens_long_context": { "value": 1043079, "provenance": "measured" } }, "parity": { "de": { "value": 2.0652062609713284, "provenance": "measured" }, "fr": { "value": 1.9164716208308952, "provenance": "measured" }, "es": { "value": 1.925650965476887, "provenance": "measured" }, "ru": { "value": 3.938085137507314, "provenance": "measured" }, "zh": { "value": 2.676162960795787, "provenance": "measured" }, "ja": { "value": 3.6496489174956115, "provenance": "measured" }, "ko": { "value": 4.796884142773552, "provenance": "measured" }, "ar": { "value": 4.0426053247513165, "provenance": "measured" }, "ur": { "value": 5.546408718548859, "provenance": "measured" }, "hi": { "value": 5.18124634289058, "provenance": "measured" }, "mr": { "value": 5.470487126974839, "provenance": "measured" }, "gu": { "value": 11.881655939145698, "provenance": "measured" }, "bn": { "value": 8.536022527794032, "provenance": "measured" }, "ta": { "value": 9.530354008191924, "provenance": "measured" }, "te": { "value": 12.809574312463429, "provenance": "measured" }, "th": { "value": 8.266200994733762, "provenance": "measured" }, "vi": { "value": 3.831224400234055, "provenance": "measured" }, "tr": { "value": 2.3890433001755413, "provenance": "measured" }, "id": { "value": 1.9151916325336453, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 27.235059760956176, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 56.24601593625498, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0652062609713284, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 52.19521912350598, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9164716208308952, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 52.44521912350598, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.925650965476887, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 107.25398406374502, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.938085137507314, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 72.88545816733068, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.676162960795787, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 99.39840637450199, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.6496489174956115, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 130.64342629482073, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.796884142773552, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 110.10059760956176, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.0426053247513165, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 151.05677290836653, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.546408718548859, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 141.11155378486055, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.18124634289058, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 148.98904382470118, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.470487126974839, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 323.59760956175296, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 11.881655939145698, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 232.47908366533864, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.536022527794032, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 259.5597609561753, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 9.530354008191924, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 348.8695219123506, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 12.809574312463429, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 225.1304780876494, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.266200994733762, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 104.34362549800797, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.831224400234055, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 65.06573705179282, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.3890433001755413, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 52.16035856573705, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9151916325336453, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [ { "set": "wiki_gu", "bytes_per_token": { "value": 1.0117643700038066, "provenance": "measured" }, "split_byte_token_share": { "value": 0.9269623144271032, "provenance": "measured" } }, { "set": "wiki_ko", "bytes_per_token": { "value": 1.179067375663562, "provenance": "measured" }, "split_byte_token_share": { "value": 0.7803621632788711, "provenance": "measured" } }, { "set": "wiki_te", "bytes_per_token": { "value": 1.010956362487654, "provenance": "measured" }, "split_byte_token_share": { "value": 0.9358313040070696, "provenance": "measured" } }, { "set": "wiki_zh", "bytes_per_token": { "value": 1.6462281297146368, "provenance": "measured" }, "split_byte_token_share": { "value": 0.5461531780472034, "provenance": "measured" } } ], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>", "<|im_start|>user", "<|im_end|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 49152, "provenance": "measured" }, "used_share": { "value": 0.8539225260416666, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 41972, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 7180, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 48822 entries whose text is valid UTF-8; 313 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.050100375000056374, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.04943820800008325, "q3": 0.05065804199966806, "n": 5 }, "rss_load_delta_mb": { "value": 34.832384, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 34.832384, "q3": 34.832384, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 34.75456, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 34.709504, "q3": 34.75456, "n": 5 }, "latency_small_p50_us": { "value": 55.08300000656163, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 54.95849995895696, "q3": 55.79100002250925, "n": 5 }, "latency_small_p95_us": { "value": 80.41699999239427, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 79.62500012581586, "q3": 81.95899999918765, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.853248808026626, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.824940646454353, "q3": 4.899484800901388, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 2082829.99518042, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2070681.2077674132, "q3": 2102672.726642535, "n": 5 }, "encode_mbps_1t_large": { "value": 3.659594777290573, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.585758120725134, "q3": 3.666833886344759, "n": 5 }, "latency_large_p50_ms": { "value": 286.3078330001372, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 285.0805000002765, "q3": 296.4327090000438, "n": 5 }, "decode_mbps": { "value": 32.44802029987467, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 32.22535120544784, "q3": 33.72718610168552, "n": 5 }, "decode_tokens_per_s": { "value": 14594575.35858241, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14494422.534200326, "q3": 15169922.683877384, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.336084915036293, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.324993395914808, "q3": 4.339820999802098, "n": 5 }, "batch_mbps_4t": { "value": 12.374920625155946, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.127153555075383, "q3": 12.633565211752764, "n": 5 }, "batch_tokens_per_s_4t": { "value": 5259749.39423336, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5154440.2180178715, "q3": 5369681.873711967, "n": 5 }, "batch_cpu_utilization": { "value": 3.4985777456728875, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.457418936443337, "q3": 3.5161412931353415, "n": 5 }, "cpu_cores_used_batch": { "value": 3.4985777456728875, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.457418936443337, "q3": 3.5161412931353415, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.2793891808416967, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.27692719252505443, "q3": 0.28669884942697854, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 177.715576171875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 177.5263671875, "q3": 177.787353515625, "n": 5 }, "rss_peak_large_mb": { "value": 207.511552, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 207.245312, "q3": 208.134144, "n": 5 }, "rss_peak_very_large_mb": { "value": 3006.599168, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 3003.424768, "q3": 3007.803392, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.543168, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.969728, "q3": 17.702912, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.6634715393753834, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.637972201463825, "q3": 2.681014800559647, "n": 5 }, "throughput_ratio_very_large": { "value": 0.548801770675832, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 1.9607899152225259, "provenance": "measured" }, "chars_per_token": { "value": 1.9590235501925297, "provenance": "measured" }, "tokens_per_word": { "value": 3.3627347513124426, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 2.7310583054965076, "provenance": "measured" }, "chars_per_token": { "value": 2.7201531386924644, "provenance": "measured" }, "tokens_per_word": { "value": 3.10456404989983, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.1401526597947376, "provenance": "measured" }, "chars_per_token": { "value": 3.1396490167769087, "provenance": "measured" }, "tokens_per_word": { "value": 3.356524352448393, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 3.946703704900029, "provenance": "measured" }, "chars_per_token": { "value": 3.9248360735165866, "provenance": "measured" }, "tokens_per_word": { "value": 3.110285068441542, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.4227728027436783, "provenance": "measured" }, "chars_per_token": { "value": 3.416008873404284, "provenance": "measured" }, "tokens_per_word": { "value": 3.125260516354047, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 2.867558793440917, "provenance": "measured" }, "chars_per_token": { "value": 2.8652703286046277, "provenance": "measured" }, "tokens_per_word": { "value": 2.9310350804215197, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 2.7801448346433673, "provenance": "measured" }, "chars_per_token": { "value": 2.7369924321649854, "provenance": "measured" }, "tokens_per_word": { "value": 4.333771929824562, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 3.7585917978452845, "provenance": "measured" }, "chars_per_token": { "value": 3.756560166682732, "provenance": "measured" }, "tokens_per_word": { "value": 2.626324193799657, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.2957191303697764, "provenance": "measured" }, "chars_per_token": { "value": 3.2834997247575486, "provenance": "measured" }, "tokens_per_word": { "value": 3.0578981275587607, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 3.5829957128335077, "provenance": "measured" }, "chars_per_token": { "value": 3.571091827865109, "provenance": "measured" }, "tokens_per_word": { "value": 3.6243592481845366, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 2.9171506842872597, "provenance": "measured" }, "chars_per_token": { "value": 2.8935474768376994, "provenance": "measured" }, "tokens_per_word": { "value": 3.320527836339502, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.2884086496980154, "provenance": "measured" }, "chars_per_token": { "value": 3.283737327437446, "provenance": "measured" }, "tokens_per_word": { "value": 3.482850099776969, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.283064922317935, "provenance": "measured" }, "chars_per_token": { "value": 3.282754968940089, "provenance": "measured" }, "tokens_per_word": { "value": 3.294375425459496, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.4103729508410665, "provenance": "measured" }, "chars_per_token": { "value": 3.401051354911456, "provenance": "measured" }, "tokens_per_word": { "value": 3.4856766805563137, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.475686848896413, "provenance": "measured" }, "chars_per_token": { "value": 2.464064471540076, "provenance": "measured" }, "tokens_per_word": { "value": 3.626002988849293, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.267120895630572, "provenance": "measured" }, "chars_per_token": { "value": 3.197405096238114, "provenance": "measured" }, "tokens_per_word": { "value": 3.1961374836780254, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.3584308413217316, "provenance": "measured" }, "chars_per_token": { "value": 3.3572973757507483, "provenance": "measured" }, "tokens_per_word": { "value": 3.008087318087318, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 2.6996282693625964, "provenance": "measured" }, "chars_per_token": { "value": 2.6947183039720004, "provenance": "measured" }, "tokens_per_word": { "value": 3.237190589593707, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.526031973522088, "provenance": "measured" }, "chars_per_token": { "value": 2.5160872904991933, "provenance": "measured" }, "tokens_per_word": { "value": 3.5677788404944084, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 3.758095703946011, "provenance": "measured" }, "chars_per_token": { "value": 3.744009905152055, "provenance": "measured" }, "tokens_per_word": { "value": 2.8124543226860412, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.383957513907044, "provenance": "measured" }, "chars_per_token": { "value": 3.311494340463197, "provenance": "measured" }, "tokens_per_word": { "value": 3.6984997615227853, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.542947535289986, "provenance": "measured" }, "chars_per_token": { "value": 4.524473218902922, "provenance": "measured" }, "tokens_per_word": { "value": 1.351697592571678, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 1.944455805963434, "provenance": "measured" }, "chars_per_token": { "value": 1.0783229034589008, "provenance": "measured" }, "tokens_per_word": { "value": 5.674596727399327, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 1.4723840368071495, "provenance": "measured" }, "chars_per_token": { "value": 0.5530931468946618, "provenance": "measured" }, "tokens_per_word": { "value": 12.951566951566951, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 2.478614576513215, "provenance": "measured" }, "chars_per_token": { "value": 2.416053658348714, "provenance": "measured" }, "tokens_per_word": { "value": 3.5668635676772373, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 2.9300742340396813, "provenance": "measured" }, "chars_per_token": { "value": 2.8648940477797273, "provenance": "measured" }, "tokens_per_word": { "value": 2.178707537404724, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 2.951152924879513, "provenance": "measured" }, "chars_per_token": { "value": 2.857135436206747, "provenance": "measured" }, "tokens_per_word": { "value": 2.2465346663036008, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.0117643700038066, "provenance": "measured" }, "chars_per_token": { "value": 0.3931023981728207, "provenance": "measured" }, "tokens_per_word": { "value": 16.203046937642632, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 2.3785653413824788, "provenance": "measured" }, "chars_per_token": { "value": 0.9442352280247378, "provenance": "measured" }, "tokens_per_word": { "value": 5.854693945813358, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 2.6579066528093427, "provenance": "measured" }, "chars_per_token": { "value": 2.6517355436052905, "provenance": "measured" }, "tokens_per_word": { "value": 2.8144358675599572, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 1.6479846449136277, "provenance": "measured" }, "chars_per_token": { "value": 0.5782371179684039, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 1.179067375663562, "provenance": "measured" }, "chars_per_token": { "value": 0.5088519819277649, "provenance": "measured" }, "tokens_per_word": { "value": 8.805219970341078, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 2.3236173774338194, "provenance": "measured" }, "chars_per_token": { "value": 0.8829123755558543, "provenance": "measured" }, "tokens_per_word": { "value": 7.948510983340705, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 2.358044764688012, "provenance": "measured" }, "chars_per_token": { "value": 1.297375818853199, "provenance": "measured" }, "tokens_per_word": { "value": 5.703885390735588, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 1.6092973096780967, "provenance": "measured" }, "chars_per_token": { "value": 0.6131338098737391, "provenance": "measured" }, "tokens_per_word": { "value": 15.60974277485351, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.010956362487654, "provenance": "measured" }, "chars_per_token": { "value": 0.38513213613080455, "provenance": "measured" }, "tokens_per_word": { "value": 20.72883252517717, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 1.6121519771658421, "provenance": "measured" }, "chars_per_token": { "value": 0.5764075815904681, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.222276724711818, "provenance": "measured" }, "chars_per_token": { "value": 2.031445989979328, "provenance": "measured" }, "tokens_per_word": { "value": 3.8419006915582776, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 1.572964374845734, "provenance": "measured" }, "chars_per_token": { "value": 0.888413521164529, "provenance": "measured" }, "tokens_per_word": { "value": 5.385348802861972, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 1.7162014639807868, "provenance": "measured" }, "chars_per_token": { "value": 1.313931374464222, "provenance": "measured" }, "tokens_per_word": { "value": 3.4964793330319925, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 1.6462281297146368, "provenance": "measured" }, "chars_per_token": { "value": 0.6183094377981581, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 95975, "provenance": "measured" }, "tokens_per_request_mean": { "value": 479.875, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.479875, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 98488, "provenance": "measured" }, "tokens_per_request_mean": { "value": 604.2208588957055, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6042208588957055, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 64809, "provenance": "measured" }, "tokens_per_request_mean": { "value": 341.1, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3411, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 285563, "provenance": "measured" }, "tokens_per_request_mean": { "value": 679.9119047619048, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6799119047619049, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 177092, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1180.6133333333332, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.1806133333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 742587, "provenance": "measured" }, "tokens_per_request_mean": { "value": 3712.935, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.98, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 3.712935, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 409936, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2049.68, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.04968, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 586540, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2932.7, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.9327, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9528, "provenance": "measured" }, "tokens_per_request_mean": { "value": 381.12, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.38112, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 1043079, "provenance": "measured" }, "tokens_per_request_mean": { "value": 173846.5, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 0.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 173.8465, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 49152, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 100663296, "provenance": "estimated", "note": "49152 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 201326592, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 201326592, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 402653184, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 49152, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 201326592, "provenance": "estimated", "note": "49152 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 402653184, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 402653184, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 805306368, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 28311552, "provenance": "estimated", "note": "49152 rows x 576 hidden x 1 matrix from the published config" }, "bf16_bytes": { "value": 56623104, "provenance": "estimated", "note": "49152 rows x 576 hidden x 1 matrix from the published config" } } } }, { "id": "gpt2", "label": "gpt2", "provider": "OpenAI", "family": "published", "metrics": { "bpt_overall": { "value": 2.9504405681956523, "provenance": "measured" }, "bpt_english": { "value": 4.659133648920304, "provenance": "measured" }, "bpt_code": { "value": 2.4051007811756513, "provenance": "measured" }, "bpt_languages": { "value": 1.7870872744910007, "provenance": "measured" }, "parity_mean_abs_log": { "value": 1.6053052794507416, "provenance": "measured" }, "byte_fallback_share": { "value": 0.5303119715109536, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.814825216097814, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.813821967040831, "q3": 4.819629790355922, "n": 5 }, "batch_mbps_4t": { "value": 12.407483704422463, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.314218491580338, "q3": 12.439353137060534, "n": 5 }, "latency_small_p95_us": { "value": 73.50000009864743, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 72.45799997690483, "q3": 74.1250000828586, "n": 5 }, "decode_mbps": { "value": 29.806463251233527, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 29.531230642237514, "q3": 29.983672418984067, "n": 5 }, "rss_load_delta_mb": { "value": 35.856384, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 35.856384, "q3": 35.897344, "n": 5 }, "rss_peak_large_mb": { "value": 240.410624, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 239.464448, "q3": 242.450432, "n": 5 }, "load_seconds": { "value": 0.04961520899996685, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.04851966699993682, "q3": 0.053656749999987596, "n": 5 }, "throughput_ratio_very_large": { "value": 0.516317045949833, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 3192.500224, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 3187.777536, "q3": 3193.532416, "n": 5 }, "rss_peak_chunked_mb": { "value": 24.22784, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 23.94112, "q3": 24.879104, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.48597633234039, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.472561425238269, "q3": 2.5227557875029145, "n": 5 }, "embedding_params_h2048": { "value": 102926336, "provenance": "estimated", "note": "50257 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.276965340707377, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.275581515393626, "q3": 0.2774910703450097, "n": 5 }, "cpu_cores_used_batch": { "value": 3.4413917119159008, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.417085669690939, "q3": 3.4452696797853495, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 189.016357421875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 188.75244140625, "q3": 189.07861328125, "n": 5 }, "workload_tokens_total": { "value": 3789874, "provenance": "measured" }, "tokens_chat_en": { "value": 98389, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 111625, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 71484, "provenance": "measured" }, "tokens_code": { "value": 365096, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 219131, "provenance": "measured" }, "tokens_rag": { "value": 826240, "provenance": "measured" }, "tokens_json": { "value": 413899, "provenance": "measured" }, "tokens_logs": { "value": 449011, "provenance": "measured" }, "tokens_tech_docs": { "value": 9606, "provenance": "measured" }, "tokens_long_context": { "value": 1225393, "provenance": "measured" } }, "parity": { "de": { "value": 2.1316640348898415, "provenance": "measured" }, "fr": { "value": 1.979848108880367, "provenance": "measured" }, "es": { "value": 1.9711632453567938, "provenance": "measured" }, "ru": { "value": 5.76825325212422, "provenance": "measured" }, "zh": { "value": 3.2036243326565907, "provenance": "measured" }, "ja": { "value": 2.9771035416196705, "provenance": "measured" }, "ko": { "value": 5.050003759681179, "provenance": "measured" }, "ar": { "value": 4.428114895856831, "provenance": "measured" }, "ur": { "value": 6.294721407624634, "provenance": "measured" }, "hi": { "value": 7.472779908263779, "provenance": "measured" }, "mr": { "value": 7.8587487781036165, "provenance": "measured" }, "gu": { "value": 12.186141815174073, "provenance": "measured" }, "bn": { "value": 9.615422212196405, "provenance": "measured" }, "ta": { "value": 15.53128054740958, "provenance": "measured" }, "te": { "value": 13.138055492894203, "provenance": "measured" }, "th": { "value": 8.991014361982105, "provenance": "measured" }, "vi": { "value": 4.525791412888187, "provenance": "measured" }, "tr": { "value": 2.428603654410106, "provenance": "measured" }, "id": { "value": 1.9725167305812468, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.49203187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 56.47211155378486, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.1316640348898415, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 52.45019920318725, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.979848108880367, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 52.22011952191235, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9711632453567938, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 152.81274900398407, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.76825325212422, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 84.87051792828686, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.2036243326565907, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 78.86952191235059, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.9771035416196705, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 133.78486055776892, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.050003759681179, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 117.3097609561753, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.428114895856831, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 166.75996015936255, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.294721407624634, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 197.96912350597609, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.472779908263779, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 208.1942231075697, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.8587487781036165, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 322.83565737051794, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 12.186141815174073, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 254.7320717131474, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 9.615422212196405, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 411.4551792828685, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 15.53128054740958, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 348.05378486055776, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 13.138055492894203, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 238.1902390438247, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.991014361982105, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 119.89741035856574, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.525791412888187, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 64.33864541832669, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.428603654410106, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 52.25597609561753, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9725167305812468, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [ { "set": "wiki_bn", "bytes_per_token": { "value": 1.3305384116126961, "provenance": "measured" }, "split_byte_token_share": { "value": 0.5931437001698029, "provenance": "measured" } }, { "set": "wiki_gu", "bytes_per_token": { "value": 1.0123327715409298, "provenance": "measured" }, "split_byte_token_share": { "value": 0.9276544691061788, "provenance": "measured" } }, { "set": "wiki_ko", "bytes_per_token": { "value": 1.1703891437411118, "provenance": "measured" }, "split_byte_token_share": { "value": 0.8007806056715393, "provenance": "measured" } }, { "set": "wiki_ta", "bytes_per_token": { "value": 1.0205056927757021, "provenance": "measured" }, "split_byte_token_share": { "value": 0.9471834679534249, "provenance": "measured" } }, { "set": "wiki_te", "bytes_per_token": { "value": 1.0140420821267278, "provenance": "measured" }, "split_byte_token_share": { "value": 0.939108863237099, "provenance": "measured" } }, { "set": "wiki_ur", "bytes_per_token": { "value": 1.4178493637195146, "provenance": "measured" }, "split_byte_token_share": { "value": 0.5735918191331862, "provenance": "measured" } }, { "set": "wiki_zh", "bytes_per_token": { "value": 1.437360970677452, "provenance": "measured" }, "split_byte_token_share": { "value": 0.6673238961914392, "provenance": "measured" } } ], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 50257, "provenance": "measured" }, "used_share": { "value": 0.8435242851742046, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 42393, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 7864, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 49912 entries whose text is valid UTF-8; 344 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.04961520899996685, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.04851966699993682, "q3": 0.053656749999987596, "n": 5 }, "rss_load_delta_mb": { "value": 35.856384, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 35.856384, "q3": 35.897344, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 35.20512, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 35.033088, "q3": 35.262464, "n": 5 }, "latency_small_p50_us": { "value": 58.93749994356767, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 58.708999858936295, "q3": 58.97950006783503, "n": 5 }, "latency_small_p95_us": { "value": 73.50000009864743, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 72.45799997690483, "q3": 74.1250000828586, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.814825216097814, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.813821967040831, "q3": 4.819629790355922, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 2426769.9482909082, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2426264.290338051, "q3": 2429191.5515476423, "n": 5 }, "encode_mbps_1t_large": { "value": 3.5422790680463763, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5001950919587532, "q3": 3.5430554599923445, "n": 5 }, "latency_large_p50_ms": { "value": 293.7437079999654, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 293.56479199987007, "q3": 297.578625000142, "n": 5 }, "decode_mbps": { "value": 29.806463251233527, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 29.531230642237514, "q3": 29.983672418984067, "n": 5 }, "decode_tokens_per_s": { "value": 16088072.181684256, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 15939515.0703303, "q3": 16183721.029989295, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.233808084876881, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.144181295105138, "q3": 4.30372077488126, "n": 5 }, "batch_mbps_4t": { "value": 12.407483704422463, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.314218491580338, "q3": 12.439353137060534, "n": 5 }, "batch_tokens_per_s_4t": { "value": 6210429.018021555, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6163746.145167952, "q3": 6226380.910762934, "n": 5 }, "batch_cpu_utilization": { "value": 3.4413917119159008, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.417085669690939, "q3": 3.4452696797853495, "n": 5 }, "cpu_cores_used_batch": { "value": 3.4413917119159008, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.417085669690939, "q3": 3.4452696797853495, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.276965340707377, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.275581515393626, "q3": 0.2774910703450097, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 189.016357421875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 188.75244140625, "q3": 189.07861328125, "n": 5 }, "rss_peak_large_mb": { "value": 240.410624, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 239.464448, "q3": 242.450432, "n": 5 }, "rss_peak_very_large_mb": { "value": 3192.500224, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 3187.777536, "q3": 3193.532416, "n": 5 }, "rss_peak_chunked_mb": { "value": 24.22784, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 23.94112, "q3": 24.879104, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.48597633234039, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.472561425238269, "q3": 2.5227557875029145, "n": 5 }, "throughput_ratio_very_large": { "value": 0.516317045949833, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.0055682678694895, "provenance": "measured" }, "chars_per_token": { "value": 2.00376156454746, "provenance": "measured" }, "tokens_per_word": { "value": 3.287654922335877, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 2.322101574567167, "provenance": "measured" }, "chars_per_token": { "value": 2.312829379625147, "provenance": "measured" }, "tokens_per_word": { "value": 3.6513240963652587, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 2.4610064382576358, "provenance": "measured" }, "chars_per_token": { "value": 2.4606117221900603, "provenance": "measured" }, "tokens_per_word": { "value": 4.282800202858806, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 2.1996174531380093, "provenance": "measured" }, "chars_per_token": { "value": 2.187429960170121, "provenance": "measured" }, "tokens_per_word": { "value": 5.580685671229436, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 2.6525320494228204, "provenance": "measured" }, "chars_per_token": { "value": 2.647290235143359, "provenance": "measured" }, "tokens_per_word": { "value": 4.032771894007061, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 2.701574939136151, "provenance": "measured" }, "chars_per_token": { "value": 2.6994189382670655, "provenance": "measured" }, "tokens_per_word": { "value": 3.1111168885191347, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 2.0877409276108847, "provenance": "measured" }, "chars_per_token": { "value": 2.0553357681183897, "provenance": "measured" }, "tokens_per_word": { "value": 5.771076998050682, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 2.7997415164369035, "provenance": "measured" }, "chars_per_token": { "value": 2.7982281724991163, "provenance": "measured" }, "tokens_per_word": { "value": 3.5257828322168563, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 2.413487284128801, "provenance": "measured" }, "chars_per_token": { "value": 2.4045388941422385, "provenance": "measured" }, "tokens_per_word": { "value": 4.175689436605083, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 2.3453686291528038, "provenance": "measured" }, "chars_per_token": { "value": 2.3375765466030183, "provenance": "measured" }, "tokens_per_word": { "value": 5.536896625373772, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 2.5877355833915665, "provenance": "measured" }, "chars_per_token": { "value": 2.5667977346447084, "provenance": "measured" }, "tokens_per_word": { "value": 3.7432263605840554, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 2.239893111475365, "provenance": "measured" }, "chars_per_token": { "value": 2.236711249466246, "provenance": "measured" }, "tokens_per_word": { "value": 5.113205775325742, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 2.389976073068805, "provenance": "measured" }, "chars_per_token": { "value": 2.389750435996634, "provenance": "measured" }, "tokens_per_word": { "value": 4.525421204901293, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 2.287685767512221, "provenance": "measured" }, "chars_per_token": { "value": 2.281432820211132, "provenance": "measured" }, "tokens_per_word": { "value": 5.196280728569609, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.3695971451806717, "provenance": "measured" }, "chars_per_token": { "value": 2.358472817313411, "provenance": "measured" }, "tokens_per_word": { "value": 3.7883434877572135, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 2.4977833976483863, "provenance": "measured" }, "chars_per_token": { "value": 2.444484186557314, "provenance": "measured" }, "tokens_per_word": { "value": 4.1805736910829365, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 2.287947416496379, "provenance": "measured" }, "chars_per_token": { "value": 2.2871752375391976, "provenance": "measured" }, "tokens_per_word": { "value": 4.415509355509355, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 2.483043028059995, "provenance": "measured" }, "chars_per_token": { "value": 2.4785269783988273, "provenance": "measured" }, "tokens_per_word": { "value": 3.519556902648481, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.400766313561398, "provenance": "measured" }, "chars_per_token": { "value": 2.3913147863239486, "provenance": "measured" }, "tokens_per_word": { "value": 3.7539361389052384, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 2.4826968130225793, "provenance": "measured" }, "chars_per_token": { "value": 2.4733913640586502, "provenance": "measured" }, "tokens_per_word": { "value": 4.257254632217015, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 2.4912526755506454, "provenance": "measured" }, "chars_per_token": { "value": 2.4379056480011045, "provenance": "measured" }, "tokens_per_word": { "value": 5.023804361965052, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.659133648920304, "provenance": "measured" }, "chars_per_token": { "value": 4.640186850954636, "provenance": "measured" }, "tokens_per_word": { "value": 1.3179899331829517, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 1.8464346863540295, "provenance": "measured" }, "chars_per_token": { "value": 1.0239640345284062, "provenance": "measured" }, "tokens_per_word": { "value": 5.975842327182701, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 1.3305384116126961, "provenance": "measured" }, "chars_per_token": { "value": 0.49980960044833705, "provenance": "measured" }, "tokens_per_word": { "value": 14.33230357326743, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 2.7026628031308855, "provenance": "measured" }, "chars_per_token": { "value": 2.6344468456943866, "provenance": "measured" }, "tokens_per_word": { "value": 3.2711739033952214, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 2.9992318489781935, "provenance": "measured" }, "chars_per_token": { "value": 2.9325132354078938, "provenance": "measured" }, "tokens_per_word": { "value": 2.1284699350710454, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.002307651656166, "provenance": "measured" }, "chars_per_token": { "value": 2.9066604816121853, "provenance": "measured" }, "tokens_per_word": { "value": 2.2082571543418785, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.0123327715409298, "provenance": "measured" }, "chars_per_token": { "value": 0.393323240113882, "provenance": "measured" }, "tokens_per_word": { "value": 16.19394929994449, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 1.687225606873168, "provenance": "measured" }, "chars_per_token": { "value": 0.66978940116444, "provenance": "measured" }, "tokens_per_word": { "value": 8.253651466161221, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 2.6620509143681486, "provenance": "measured" }, "chars_per_token": { "value": 2.6558701830463964, "provenance": "measured" }, "tokens_per_word": { "value": 2.8100543742110884, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 1.9647904678826948, "provenance": "measured" }, "chars_per_token": { "value": 0.6893964583145896, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 1.1703891437411118, "provenance": "measured" }, "chars_per_token": { "value": 0.5051067035793735, "provenance": "measured" }, "tokens_per_word": { "value": 8.870509144834404, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 1.6817939260507249, "provenance": "measured" }, "chars_per_token": { "value": 0.6390366524478036, "provenance": "measured" }, "tokens_per_word": { "value": 10.981903287630843, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 1.717446179927949, "provenance": "measured" }, "chars_per_token": { "value": 0.9449240223881528, "provenance": "measured" }, "tokens_per_word": { "value": 7.831405281397959, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 1.0205056927757021, "provenance": "measured" }, "chars_per_token": { "value": 0.3888073009545788, "provenance": "measured" }, "tokens_per_word": { "value": 24.6159499453769, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.0140420821267278, "provenance": "measured" }, "chars_per_token": { "value": 0.3863076663912531, "provenance": "measured" }, "tokens_per_word": { "value": 20.665754900741845, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 1.5084802547165328, "provenance": "measured" }, "chars_per_token": { "value": 0.5393408734496032, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.256089210603541, "provenance": "measured" }, "chars_per_token": { "value": 2.0623549393969034, "provenance": "measured" }, "tokens_per_word": { "value": 3.784321313792465, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 1.4178493637195146, "provenance": "measured" }, "chars_per_token": { "value": 0.8008042431517103, "provenance": "measured" }, "tokens_per_word": { "value": 5.9745146626844114, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 1.5231262285928453, "provenance": "measured" }, "chars_per_token": { "value": 1.1661121267053698, "provenance": "measured" }, "tokens_per_word": { "value": 3.9397016724423684, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 1.437360970677452, "provenance": "measured" }, "chars_per_token": { "value": 0.5398606898101337, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 98389, "provenance": "measured" }, "tokens_per_request_mean": { "value": 491.945, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.491945, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 111625, "provenance": "measured" }, "tokens_per_request_mean": { "value": 684.8159509202454, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6848159509202454, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 71484, "provenance": "measured" }, "tokens_per_request_mean": { "value": 376.2315789473684, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3762315789473684, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 365096, "provenance": "measured" }, "tokens_per_request_mean": { "value": 869.2761904761904, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8692761904761904, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 219131, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1460.8733333333332, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.4608733333333332, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 826240, "provenance": "measured" }, "tokens_per_request_mean": { "value": 4131.2, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.965, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 4.1312, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 413899, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2069.495, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.069495, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 449011, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2245.055, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.245055, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9606, "provenance": "measured" }, "tokens_per_request_mean": { "value": 384.24, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.38424, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 1225393, "provenance": "measured" }, "tokens_per_request_mean": { "value": 204232.16666666666, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 0.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 204.23216666666664, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 50257, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 102926336, "provenance": "estimated", "note": "50257 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 205852672, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 205852672, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 411705344, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 50257, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 205852672, "provenance": "estimated", "note": "50257 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 411705344, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 411705344, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 823410688, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 77194752, "provenance": "estimated", "note": "50257 rows x 768 hidden x 2 matrices from the published config (tie_word_embeddings not stated; counted as untied)" }, "bf16_bytes": { "value": 154389504, "provenance": "estimated", "note": "50257 rows x 768 hidden x 2 matrices from the published config (tie_word_embeddings not stated; counted as untied)" } } } }, { "id": "pythia-160m", "label": "pythia-160m", "provider": "EleutherAI", "family": "published", "metrics": { "bpt_overall": { "value": 3.4933320609124494, "provenance": "measured" }, "bpt_english": { "value": 4.666125467074576, "provenance": "measured" }, "bpt_code": { "value": 3.3694991089406896, "provenance": "measured" }, "bpt_languages": { "value": 2.4443716067220826, "provenance": "measured" }, "parity_mean_abs_log": { "value": 1.2298206190490917, "provenance": "measured" }, "byte_fallback_share": { "value": 0.22827136960640465, "provenance": "measured" }, "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.724995212054687, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.715288721532525, "q3": 4.767939642189444, "n": 5 }, "batch_mbps_4t": { "value": 13.191500272194986, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.986857161371269, "q3": 13.255276234210921, "n": 5 }, "latency_small_p95_us": { "value": 73.3329999320631, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 71.54200000059063, "q3": 73.3329999320631, "n": 5 }, "decode_mbps": { "value": 36.310007055338886, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.04379219659033, "q3": 37.0972005070531, "n": 5 }, "rss_load_delta_mb": { "value": 37.154816, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 37.154816, "q3": 37.154816, "n": 5 }, "rss_peak_large_mb": { "value": 197.931008, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 197.246976, "q3": 198.172672, "n": 5 }, "load_seconds": { "value": 0.04969120799978555, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.04936454200003482, "q3": 0.05005616699963866, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5650348376134589, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2817.212416, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2817.040384, "q3": 2821.394432, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.65376, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.108992, "q3": 17.87904, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.669786902367691, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.651050663925692, "q3": 2.688321338143425, "n": 5 }, "embedding_params_h2048": { "value": 103022592, "provenance": "estimated", "note": "50304 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.2681402355793314, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.26810614064138066, "q3": 0.2726636562982291, "n": 5 }, "cpu_cores_used_batch": { "value": 3.537306295162905, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.53672222724792, "q3": 3.5438245302255855, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 166.404541015625, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 166.400390625, "q3": 166.663330078125, "n": 5 }, "workload_tokens_total": { "value": 2911101, "provenance": "measured" }, "tokens_chat_en": { "value": 95595, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 80268, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 50433, "provenance": "measured" }, "tokens_code": { "value": 278014, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 163904, "provenance": "measured" }, "tokens_rag": { "value": 564582, "provenance": "measured" }, "tokens_json": { "value": 354130, "provenance": "measured" }, "tokens_logs": { "value": 440123, "provenance": "measured" }, "tokens_tech_docs": { "value": 9278, "provenance": "measured" }, "tokens_long_context": { "value": 874774, "provenance": "measured" } }, "parity": { "de": { "value": 1.810446597616661, "provenance": "measured" }, "fr": { "value": 1.7409139844823105, "provenance": "measured" }, "es": { "value": 1.7121431488287486, "provenance": "measured" }, "ru": { "value": 2.690388684708765, "provenance": "measured" }, "zh": { "value": 2.0962616475479825, "provenance": "measured" }, "ja": { "value": 2.2065560381631215, "provenance": "measured" }, "ko": { "value": 3.5779039982180643, "provenance": "measured" }, "ar": { "value": 3.0174852433455843, "provenance": "measured" }, "ur": { "value": 4.485280469243048, "provenance": "measured" }, "hi": { "value": 4.769870438430412, "provenance": "measured" }, "mr": { "value": 5.1300441771541, "provenance": "measured" }, "gu": { "value": 6.946393436537105, "provenance": "measured" }, "bn": { "value": 6.571927089133905, "provenance": "measured" }, "ta": { "value": 7.808182054423284, "provenance": "measured" }, "te": { "value": 7.3585774213906525, "provenance": "measured" }, "th": { "value": 5.106322159112002, "provenance": "measured" }, "vi": { "value": 3.2113078665033226, "provenance": "measured" }, "tr": { "value": 2.192077811189071, "provenance": "measured" }, "id": { "value": 1.8740023016668523, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.8296812749004, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 48.57370517928287, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.810446597616661, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 46.70816733067729, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7409139844823105, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 45.93625498007968, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7121431488287486, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 72.18227091633466, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.690388684708765, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 56.24203187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0962616475479825, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 59.201195219123505, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.2065560381631215, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 95.99402390438247, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.5779039982180643, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 80.95816733067728, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.0174852433455843, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 120.33864541832669, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.485280469243048, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 127.97410358565737, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.769870438430412, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 137.6374501992032, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.1300441771541, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 186.3695219123506, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.946393436537105, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 176.32270916334662, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.571927089133905, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 209.49103585657372, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.808182054423284, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 197.42828685258965, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.3585774213906525, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 137.00099601593627, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.106322159112002, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 86.15836653386454, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.2113078665033226, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 58.81274900398407, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.192077811189071, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 50.278884462151396, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8740023016668523, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 50,304 embedding rows for 50,277 tokenizer IDs (27 unused rows, usually padding)" ], "roundtrip_failures": { "value": 733, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 733, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [ "code_cs" ], "roundtrip_example": " \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\",", "byte_fallback_sets": [ { "set": "wiki_ko", "bytes_per_token": { "value": 1.5454556157634018, "provenance": "measured" }, "split_byte_token_share": { "value": 0.5991952882863526, "provenance": "measured" } } ], "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "unicode_failed_probes": [ "latin_accents (NFD)", "vietnamese (NFD)", "devanagari_nukta_precomposed (NFC)", "tamil (NFD)", "telugu (NFD)", "japanese (NFD)", "korean (NFD)" ], "nfd_normalized_to_nfc": [ "japanese", "korean", "latin_accents", "tamil", "telugu", "vietnamese" ], "special_strings_parsed_by_default": [ "<|endoftext|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 50277, "provenance": "measured" }, "used_share": { "value": 0.8713129263878115, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 43807, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 6470, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 221, "provenance": "measured", "note": "of 49791 entries whose text is valid UTF-8; 484 are byte pieces" }, "unreachable_examples": [ " ", " ", " ", " ", " ", " ", "\n ", " ", "\n ", "\n ", " ", "\n " ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.04969120799978555, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.04936454200003482, "q3": 0.05005616699963866, "n": 5 }, "rss_load_delta_mb": { "value": 37.154816, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 37.154816, "q3": 37.154816, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 36.18816, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 36.12672, "q3": 36.507648, "n": 5 }, "latency_small_p50_us": { "value": 57.66699996456737, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.66699996456737, "q3": 59.0420000321501, "n": 5 }, "latency_small_p95_us": { "value": 73.3329999320631, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 71.54200000059063, "q3": 73.3329999320631, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.724995212054687, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.715288721532525, "q3": 4.767939642189444, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1716173.5992222328, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1712648.0881840973, "q3": 1731771.5192037986, "n": 5 }, "encode_mbps_1t_large": { "value": 3.6251791367277635, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.6072191849046247, "q3": 3.6399818417301164, "n": 5 }, "latency_large_p50_ms": { "value": 289.9741249998442, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 287.78833399996984, "q3": 290.3223749999597, "n": 5 }, "decode_mbps": { "value": 36.310007055338886, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.04379219659033, "q3": 37.0972005070531, "n": 5 }, "decode_tokens_per_s": { "value": 13541586.048341624, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13442302.911007158, "q3": 13835164.836327886, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.240223533529752, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.213187571799668, "q3": 4.247286322542315, "n": 5 }, "batch_mbps_4t": { "value": 13.191500272194986, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.986857161371269, "q3": 13.255276234210921, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4747661.386094408, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4674009.703181897, "q3": 4770614.55032716, "n": 5 }, "batch_cpu_utilization": { "value": 3.537306295162905, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.53672222724792, "q3": 3.5438245302255855, "n": 5 }, "cpu_cores_used_batch": { "value": 3.537306295162905, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.53672222724792, "q3": 3.5438245302255855, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.2681402355793314, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.26810614064138066, "q3": 0.2726636562982291, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 166.404541015625, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 166.400390625, "q3": 166.663330078125, "n": 5 }, "rss_peak_large_mb": { "value": 197.931008, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 197.246976, "q3": 198.172672, "n": 5 }, "rss_peak_very_large_mb": { "value": 2817.212416, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2817.040384, "q3": 2821.394432, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.65376, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.108992, "q3": 17.87904, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.669786902367691, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.651050663925692, "q3": 2.688321338143425, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5650348376134589, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.3977966610410504, "provenance": "measured" }, "chars_per_token": { "value": 2.39563662128451, "provenance": "measured" }, "tokens_per_word": { "value": 2.749864696649889, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.019521437007992, "provenance": "measured" }, "chars_per_token": { "value": 3.0074644315340757, "provenance": "measured" }, "tokens_per_word": { "value": 2.8079765652620416, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.328285793934649, "provenance": "measured" }, "chars_per_token": { "value": 3.3277519766069386, "provenance": "measured" }, "tokens_per_word": { "value": 3.1667950186893066, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.237713730794575, "provenance": "measured" }, "chars_per_token": { "value": 4.214233690562897, "provenance": "measured" }, "tokens_per_word": { "value": 2.8966972246640714, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.607963389353934, "provenance": "measured" }, "chars_per_token": { "value": 3.6008334947243084, "provenance": "measured" }, "tokens_per_word": { "value": 2.9648462421845094, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 2.9939310123417155, "provenance": "measured" }, "chars_per_token": { "value": 2.9915416957356564, "provenance": "measured" }, "tokens_per_word": { "value": 2.807317665002773, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.1038679790092147, "provenance": "measured" }, "chars_per_token": { "value": 3.0556908629823987, "provenance": "measured" }, "tokens_per_word": { "value": 3.8817738791423, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 3.9736765777983334, "provenance": "measured" }, "chars_per_token": { "value": 3.971528686886107, "provenance": "measured" }, "tokens_per_word": { "value": 2.4841680947188034, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.49704338294681, "provenance": "measured" }, "chars_per_token": { "value": 3.4840775354794045, "provenance": "measured" }, "tokens_per_word": { "value": 2.8818554001537855, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 3.8132143809090766, "provenance": "measured" }, "chars_per_token": { "value": 3.8005456341428996, "provenance": "measured" }, "tokens_per_word": { "value": 3.405542503203759, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.0447179663388084, "provenance": "measured" }, "chars_per_token": { "value": 3.0200825883406863, "provenance": "measured" }, "tokens_per_word": { "value": 3.1814047005543844, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.5155983136958167, "provenance": "measured" }, "chars_per_token": { "value": 3.510604258999027, "provenance": "measured" }, "tokens_per_word": { "value": 3.257776734358493, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.456043395803169, "provenance": "measured" }, "chars_per_token": { "value": 3.4557171115688154, "provenance": "measured" }, "tokens_per_word": { "value": 3.1294885976855005, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.468389980781278, "provenance": "measured" }, "chars_per_token": { "value": 3.458909806503219, "provenance": "measured" }, "tokens_per_word": { "value": 3.427370489655549, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.744357678547581, "provenance": "measured" }, "chars_per_token": { "value": 2.7314739971744464, "provenance": "measured" }, "tokens_per_word": { "value": 3.271019657431889, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.4304622473760316, "provenance": "measured" }, "chars_per_token": { "value": 3.357260971542831, "provenance": "measured" }, "tokens_per_word": { "value": 3.0439534981677268, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.5577487681483055, "provenance": "measured" }, "chars_per_token": { "value": 3.556548033064144, "provenance": "measured" }, "tokens_per_word": { "value": 2.8395634095634095, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 2.955532505567589, "provenance": "measured" }, "chars_per_token": { "value": 2.9501571127856248, "provenance": "measured" }, "tokens_per_word": { "value": 2.9568990401962907, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.988114186745702, "provenance": "measured" }, "chars_per_token": { "value": 2.9763503418162305, "provenance": "measured" }, "tokens_per_word": { "value": 3.016057239552678, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 3.9752372790919526, "provenance": "measured" }, "chars_per_token": { "value": 3.960337607321293, "provenance": "measured" }, "tokens_per_word": { "value": 2.6588280813378615, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.6502646205208946, "provenance": "measured" }, "chars_per_token": { "value": 3.572098816953418, "provenance": "measured" }, "tokens_per_word": { "value": 3.4286736330919654, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.666125467074576, "provenance": "measured" }, "chars_per_token": { "value": 4.647150236233622, "provenance": "measured" }, "tokens_per_word": { "value": 1.3160150300203595, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 2.629921768033229, "provenance": "measured" }, "chars_per_token": { "value": 1.4584568433378358, "provenance": "measured" }, "tokens_per_word": { "value": 4.195563034312019, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 1.9430708912026917, "provenance": "measured" }, "chars_per_token": { "value": 0.729904133017624, "provenance": "measured" }, "tokens_per_word": { "value": 9.814196958775272, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 2.9480503463780066, "provenance": "measured" }, "chars_per_token": { "value": 2.8736407394095718, "provenance": "measured" }, "tokens_per_word": { "value": 2.998890450477106, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.373380820964173, "provenance": "measured" }, "chars_per_token": { "value": 3.298339176052311, "provenance": "measured" }, "tokens_per_word": { "value": 1.8923967253222922, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.2840608380648972, "provenance": "measured" }, "chars_per_token": { "value": 3.179437607584205, "provenance": "measured" }, "tokens_per_word": { "value": 2.0188016234229327, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.7327081540587759, "provenance": "measured" }, "chars_per_token": { "value": 0.6732118177788208, "provenance": "measured" }, "tokens_per_word": { "value": 9.46129649047061, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 2.590028720437131, "provenance": "measured" }, "chars_per_token": { "value": 1.0281812809107593, "provenance": "measured" }, "tokens_per_word": { "value": 5.3766863641431595, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 2.765838860517506, "provenance": "measured" }, "chars_per_token": { "value": 2.7594171550633195, "provenance": "measured" }, "tokens_per_word": { "value": 2.7046072434216915, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 2.681048333288852, "provenance": "measured" }, "chars_per_token": { "value": 0.9407136566228081, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 1.5454556157634018, "provenance": "measured" }, "chars_per_token": { "value": 0.6669747372324863, "provenance": "measured" }, "tokens_per_word": { "value": 6.7177261492832425, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 2.506108874496762, "provenance": "measured" }, "chars_per_token": { "value": 0.9522542572078718, "provenance": "measured" }, "tokens_per_word": { "value": 7.369711042311661, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 3.457758453015102, "provenance": "measured" }, "chars_per_token": { "value": 1.9024287713090018, "provenance": "measured" }, "tokens_per_word": { "value": 3.8898081710352908, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 1.9793914873969967, "provenance": "measured" }, "chars_per_token": { "value": 0.7541377448410245, "provenance": "measured" }, "tokens_per_word": { "value": 12.691131194756183, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.7712692357542448, "provenance": "measured" }, "chars_per_token": { "value": 0.6747795748079489, "provenance": "measured" }, "tokens_per_word": { "value": 11.831033196568445, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 2.550208078925381, "provenance": "measured" }, "chars_per_token": { "value": 0.9117994408381002, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.471255186924081, "provenance": "measured" }, "chars_per_token": { "value": 2.2590442422707526, "provenance": "measured" }, "tokens_per_word": { "value": 3.4548299708906125, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 1.9755333073090082, "provenance": "measured" }, "chars_per_token": { "value": 1.115785284009583, "provenance": "measured" }, "tokens_per_word": { "value": 4.287936721532074, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 2.0946710706246185, "provenance": "measured" }, "chars_per_token": { "value": 1.603689366685601, "provenance": "measured" }, "tokens_per_word": { "value": 2.864728039777008, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 2.1433004845647083, "provenance": "measured" }, "chars_per_token": { "value": 0.8050056330123843, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 95595, "provenance": "measured" }, "tokens_per_request_mean": { "value": 477.975, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.477975, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 80268, "provenance": "measured" }, "tokens_per_request_mean": { "value": 492.44171779141107, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.4924417177914111, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 50433, "provenance": "measured" }, "tokens_per_request_mean": { "value": 265.43684210526317, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.26543684210526314, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 278014, "provenance": "measured" }, "tokens_per_request_mean": { "value": 661.9380952380952, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6619380952380952, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 163904, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1092.6933333333334, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.0926933333333335, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 564582, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2822.91, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.82291, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 354130, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1770.65, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.77065, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 440123, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2200.615, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.200615, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9278, "provenance": "measured" }, "tokens_per_request_mean": { "value": 371.12, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.37112, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 874774, "provenance": "measured" }, "tokens_per_request_mean": { "value": 145795.66666666666, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 0.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 145.79566666666665, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 50304, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 103022592, "provenance": "estimated", "note": "50304 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 206045184, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 206045184, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 412090368, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 50304, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 206045184, "provenance": "estimated", "note": "50304 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 412090368, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 412090368, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 824180736, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 77266944, "provenance": "estimated", "note": "50304 rows x 768 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 154533888, "provenance": "estimated", "note": "50304 rows x 768 hidden x 2 matrices from the published config" } } } }, { "id": "p50k_base", "label": "OpenAI p50k_base (Codex)", "provider": "OpenAI", "family": "published", "metrics": { "bpt_overall": { "value": 3.2164760343916257, "provenance": "measured" }, "bpt_english": { "value": 4.659133648920304, "provenance": "measured" }, "bpt_code": { "value": 3.201936371029939, "provenance": "measured" }, "bpt_languages": { "value": 1.7883580832246342, "provenance": "measured" }, "parity_mean_abs_log": { "value": 1.6052962627128946, "provenance": "measured" }, "byte_fallback_share": { "value": 0.5307670920608782, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 5.033180687959843, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.988326943952645, "q3": 5.0736678747005595, "n": 5 }, "batch_mbps_4t": { "value": 14.126312341626818, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.121296533748598, "q3": 14.12667908683695, "n": 5 }, "latency_small_p95_us": { "value": 64.29200038837735, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 63.499999669147655, "q3": 64.45799999710289, "n": 5 }, "decode_mbps": { "value": 33.18989830935828, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 32.07674185188733, "q3": 33.241454751372615, "n": 5 }, "rss_load_delta_mb": { "value": 80.216064, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 80.216064, "q3": 80.216064, "n": 5 }, "rss_peak_large_mb": { "value": 216.608768, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 216.608768, "q3": 216.612864, "n": 5 }, "load_seconds": { "value": 0.13893637499995748, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.13823958300008599, "q3": 0.14304120799988596, "n": 5 }, "throughput_ratio_very_large": { "value": 0.56023846116002, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 3049.263104, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 3049.189376, "q3": 3049.435136, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.120704, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.89952, "q3": 18.14528, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.8197814033629527, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.80170416160013, "q3": 2.8225520514154203, "n": 5 }, "embedding_params_h2048": { "value": 102975488, "provenance": "estimated", "note": "50281 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.24275214339138776, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2424166778130887, "q3": 0.2463328271546391, "n": 5 }, "cpu_cores_used_batch": { "value": 3.4797744563824042, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.4232377921248203, "q3": 3.481527017858286, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 176.546875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 176.54248046875003, "q3": 176.553955078125, "n": 5 }, "workload_tokens_total": { "value": 3522582, "provenance": "measured" }, "tokens_chat_en": { "value": 95593, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 110194, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 71477, "provenance": "measured" }, "tokens_code": { "value": 292387, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 187933, "provenance": "measured" }, "tokens_rag": { "value": 823394, "provenance": "measured" }, "tokens_json": { "value": 362593, "provenance": "measured" }, "tokens_logs": { "value": 447347, "provenance": "measured" }, "tokens_tech_docs": { "value": 9489, "provenance": "measured" }, "tokens_long_context": { "value": 1122175, "provenance": "measured" } }, "parity": { "de": { "value": 2.1316640348898415, "provenance": "measured" }, "fr": { "value": 1.979848108880367, "provenance": "measured" }, "es": { "value": 1.9711632453567938, "provenance": "measured" }, "ru": { "value": 5.76821565531243, "provenance": "measured" }, "zh": { "value": 3.2036243326565907, "provenance": "measured" }, "ja": { "value": 2.9771035416196705, "provenance": "measured" }, "ko": { "value": 5.0499285660575985, "provenance": "measured" }, "ar": { "value": 4.428114895856831, "provenance": "measured" }, "ur": { "value": 6.294646214001053, "provenance": "measured" }, "hi": { "value": 7.472779908263779, "provenance": "measured" }, "mr": { "value": 7.858598390856455, "provenance": "measured" }, "gu": { "value": 12.185878637491541, "provenance": "measured" }, "bn": { "value": 9.615384615384615, "provenance": "measured" }, "ta": { "value": 15.530453417550191, "provenance": "measured" }, "te": { "value": 13.137529137529137, "provenance": "measured" }, "th": { "value": 8.991014361982105, "provenance": "measured" }, "vi": { "value": 4.525791412888187, "provenance": "measured" }, "tr": { "value": 2.428603654410106, "provenance": "measured" }, "id": { "value": 1.9725167305812468, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.49203187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 56.47211155378486, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.1316640348898415, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 52.45019920318725, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.979848108880367, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 52.22011952191235, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9711632453567938, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 152.8117529880478, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.76821565531243, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 84.87051792828686, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.2036243326565907, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 78.86952191235059, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.9771035416196705, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 133.78286852589642, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.0499285660575985, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 117.3097609561753, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.428114895856831, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 166.75796812749005, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.294646214001053, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 197.96912350597609, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.472779908263779, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 208.1902390438247, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.858598390856455, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 322.82868525896413, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 12.185878637491541, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 254.73107569721117, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 9.615384615384615, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 411.4332669322709, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 15.530453417550191, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 348.0398406374502, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 13.137529137529137, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 238.1902390438247, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.991014361982105, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 119.89741035856574, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.525791412888187, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 64.33864541832669, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.428603654410106, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 52.25597609561753, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9725167305812468, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [ { "set": "wiki_bn", "bytes_per_token": { "value": 1.33957277920174, "provenance": "measured" }, "split_byte_token_share": { "value": 0.5971711511428, "provenance": "measured" } }, { "set": "wiki_gu", "bytes_per_token": { "value": 1.0123481947145003, "provenance": "measured" }, "split_byte_token_share": { "value": 0.927668602182073, "provenance": "measured" } }, { "set": "wiki_ko", "bytes_per_token": { "value": 1.1704074058388076, "provenance": "measured" }, "split_byte_token_share": { "value": 0.800793100604299, "provenance": "measured" } }, { "set": "wiki_ta", "bytes_per_token": { "value": 1.0206642334440994, "provenance": "measured" }, "split_byte_token_share": { "value": 0.9473306176471181, "provenance": "measured" } }, { "set": "wiki_te", "bytes_per_token": { "value": 1.014070553316086, "provenance": "measured" }, "split_byte_token_share": { "value": 0.9391352305316567, "provenance": "measured" } }, { "set": "wiki_ur", "bytes_per_token": { "value": 1.418344131757258, "provenance": "measured" }, "split_byte_token_share": { "value": 0.5737919778426233, "provenance": "measured" } }, { "set": "wiki_zh", "bytes_per_token": { "value": 1.4374053798988284, "provenance": "measured" }, "split_byte_token_share": { "value": 0.667344514070483, "provenance": "measured" } } ], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (converted from tiktoken)" } }, "vocabulary": { "size": { "value": 50281, "provenance": "measured" }, "used_share": { "value": 0.8435989737674271, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 42417, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 7864, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 49936 entries whose text is valid UTF-8; 344 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (converted from tiktoken)", "load_seconds": { "value": 0.13893637499995748, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.13823958300008599, "q3": 0.14304120799988596, "n": 5 }, "rss_load_delta_mb": { "value": 80.216064, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 80.216064, "q3": 80.216064, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 80.150528, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 80.150528, "q3": 80.150528, "n": 5 }, "latency_small_p50_us": { "value": 51.58300018592854, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 51.37499999818829, "q3": 51.66650021237729, "n": 5 }, "latency_small_p95_us": { "value": 64.29200038837735, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 63.499999669147655, "q3": 64.45799999710289, "n": 5 }, "encode_mbps_1t_medium": { "value": 5.033180687959843, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.988326943952645, "q3": 5.0736678747005595, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 2292986.1851135455, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2272551.986197311, "q3": 2311431.094133533, "n": 5 }, "encode_mbps_1t_large": { "value": 3.9353353841259424, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.846703179458672, "q3": 3.9778548368528464, "n": 5 }, "latency_large_p50_ms": { "value": 266.34208299998363, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 263.9800829997512, "q3": 267.46425000010277, "n": 5 }, "decode_mbps": { "value": 33.18989830935828, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 32.07674185188733, "q3": 33.241454751372615, "n": 5 }, "decode_tokens_per_s": { "value": 16071101.703741122, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 15532092.801892588, "q3": 16096066.191892684, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.823116205166725, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.693890175674108, "q3": 4.827981347241175, "n": 5 }, "batch_mbps_4t": { "value": 14.126312341626818, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.121296533748598, "q3": 14.12667908683695, "n": 5 }, "batch_tokens_per_s_4t": { "value": 6385018.097128765, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6382750.978626806, "q3": 6385183.864014495, "n": 5 }, "batch_cpu_utilization": { "value": 3.4797744563824042, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.4232377921248203, "q3": 3.481527017858286, "n": 5 }, "cpu_cores_used_batch": { "value": 3.4797744563824042, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.4232377921248203, "q3": 3.481527017858286, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.24275214339138776, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2424166778130887, "q3": 0.2463328271546391, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 176.546875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 176.54248046875003, "q3": 176.553955078125, "n": 5 }, "rss_peak_large_mb": { "value": 216.608768, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 216.608768, "q3": 216.612864, "n": 5 }, "rss_peak_very_large_mb": { "value": 3049.263104, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 3049.189376, "q3": 3049.435136, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.120704, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.89952, "q3": 18.14528, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.8197814033629527, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.80170416160013, "q3": 2.8225520514154203, "n": 5 }, "throughput_ratio_very_large": { "value": 0.56023846116002, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "tiktoken", "load_seconds": { "value": 0.03768454200007909, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.03750716600006854, "q3": 0.04000120800037621, "n": 5 }, "rss_load_delta_mb": { "value": 26.29632, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 26.29632, "q3": 26.29632, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 26.157056, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 26.157056, "q3": 26.157056, "n": 5 }, "latency_small_p50_us": { "value": 14.74999999118154, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.541999917128123, "q3": 14.791000012337463, "n": 5 }, "latency_small_p95_us": { "value": 20.0829999812413, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 19.583999801398022, "q3": 20.166000012977747, "n": 5 }, "encode_mbps_1t_medium": { "value": 19.471356296940503, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 19.126967563461037, "q3": 19.823081075696624, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 8870643.388804251, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 8713748.840975638, "q3": 9030880.048016425, "n": 5 }, "encode_mbps_1t_large": { "value": 18.185000262235548, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.000239491631866, "q3": 18.34225941848224, "n": 5 }, "latency_large_p50_ms": { "value": 56.90858300022228, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 56.85708300006809, "q3": 58.737333999943075, "n": 5 }, "decode_mbps": { "value": 160.8849104287948, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 155.67390986114853, "q3": 161.1063421911218, "n": 5 }, "decode_tokens_per_s": { "value": 77903153.96565716, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 75379900.67574626, "q3": 78010374.91402, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 53.06085523276197, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 49.64950641426258, "q3": 54.41751455310093, "n": 5 }, "batch_mbps_4t": { "value": 57.774697014819, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 56.93341749470764, "q3": 60.09178194137764, "n": 5 }, "batch_tokens_per_s_4t": { "value": 26113855.978443418, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 25733602.10674995, "q3": 27161166.049951963, "n": 5 }, "batch_cpu_utilization": { "value": 3.938516148533554, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.938023079866714, "q3": 3.9403325832777463, "n": 5 }, "cpu_cores_used_batch": { "value": 3.938516148533554, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.938023079866714, "q3": 3.9403325832777463, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.06811548694604964, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.06554167676997433, "q3": 0.06936555519553209, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 14.79443359375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 14.742919921875002, "q3": 14.8232421875, "n": 5 }, "rss_peak_large_mb": { "value": 25.731072, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 25.68192, "q3": 25.731072, "n": 5 }, "rss_peak_very_large_mb": { "value": 265.601024, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 264.740864, "q3": 266.084352, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.157568, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.69472, "q3": 18.501632, "n": 5 }, "encode_mbps_1t_very_large": { "value": 19.071817024517756, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.246350546304548, "q3": 19.27641159142505, "n": 5 }, "throughput_ratio_very_large": { "value": 0.9794806655309612, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.2921427059570565, "provenance": "measured" }, "chars_per_token": { "value": 2.290077843888902, "provenance": "measured" }, "tokens_per_word": { "value": 2.8766168750338257, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 2.78594385285576, "provenance": "measured" }, "chars_per_token": { "value": 2.7748195270363714, "provenance": "measured" }, "tokens_per_word": { "value": 3.0434014040640416, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.1850685102566723, "provenance": "measured" }, "chars_per_token": { "value": 3.1845576632723724, "provenance": "measured" }, "tokens_per_word": { "value": 3.309190630928455, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.021492105024973, "provenance": "measured" }, "chars_per_token": { "value": 3.9992100910863715, "provenance": "measured" }, "tokens_per_word": { "value": 3.052442546778852, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.4779154508871155, "provenance": "measured" }, "chars_per_token": { "value": 3.471042551131885, "provenance": "measured" }, "tokens_per_word": { "value": 3.0757092424822425, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 2.8186230346711616, "provenance": "measured" }, "chars_per_token": { "value": 2.816373623179982, "provenance": "measured" }, "tokens_per_word": { "value": 2.9819224902939547, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 2.9378238034127095, "provenance": "measured" }, "chars_per_token": { "value": 2.8922239650173487, "provenance": "measured" }, "tokens_per_word": { "value": 4.101169590643275, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 3.8114393557749424, "provenance": "measured" }, "chars_per_token": { "value": 3.809379158928397, "provenance": "measured" }, "tokens_per_word": { "value": 2.5899088643090824, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.2923628414113364, "provenance": "measured" }, "chars_per_token": { "value": 3.280155879777043, "provenance": "measured" }, "tokens_per_word": { "value": 3.061015399322513, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 3.6489024920856403, "provenance": "measured" }, "chars_per_token": { "value": 3.6367796432161557, "provenance": "measured" }, "tokens_per_word": { "value": 3.558895771038018, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 2.9297754633319752, "provenance": "measured" }, "chars_per_token": { "value": 2.9060701064526984, "provenance": "measured" }, "tokens_per_word": { "value": 3.30621925509487, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.2510662685930587, "provenance": "measured" }, "chars_per_token": { "value": 3.2464479927493737, "provenance": "measured" }, "tokens_per_word": { "value": 3.5228547951637514, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.322194487643592, "provenance": "measured" }, "chars_per_token": { "value": 3.3218808400527973, "provenance": "measured" }, "tokens_per_word": { "value": 3.255573519400953, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.3187992765992034, "provenance": "measured" }, "chars_per_token": { "value": 3.3097279796255656, "provenance": "measured" }, "tokens_per_word": { "value": 3.5818549047437087, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.629545071892784, "provenance": "measured" }, "chars_per_token": { "value": 2.6172003906118464, "provenance": "measured" }, "tokens_per_word": { "value": 3.4138406713415335, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.281131089066977, "provenance": "measured" }, "chars_per_token": { "value": 3.211116331703185, "provenance": "measured" }, "tokens_per_word": { "value": 3.1824902068152143, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.4336348219332957, "provenance": "measured" }, "chars_per_token": { "value": 3.4324759751271907, "provenance": "measured" }, "tokens_per_word": { "value": 2.942203742203742, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 2.7908197363569323, "provenance": "measured" }, "chars_per_token": { "value": 2.7857439159292037, "provenance": "measured" }, "tokens_per_word": { "value": 3.1314137259147, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.8016742047813183, "provenance": "measured" }, "chars_per_token": { "value": 2.7906443515601325, "provenance": "measured" }, "tokens_per_word": { "value": 3.2167635373749266, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 3.784969825728185, "provenance": "measured" }, "chars_per_token": { "value": 3.7707832994642527, "provenance": "measured" }, "tokens_per_word": { "value": 2.7924852757835, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.425339393364029, "provenance": "measured" }, "chars_per_token": { "value": 3.3519900792708976, "provenance": "measured" }, "tokens_per_word": { "value": 3.653817803408056, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.659133648920304, "provenance": "measured" }, "chars_per_token": { "value": 4.640186850954636, "provenance": "measured" }, "tokens_per_word": { "value": 1.3179899331829517, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 1.8464705444446963, "provenance": "measured" }, "chars_per_token": { "value": 1.02398392009244, "provenance": "measured" }, "tokens_per_word": { "value": 5.975726277513442, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 1.33957277920174, "provenance": "measured" }, "chars_per_token": { "value": 0.5032033120582937, "provenance": "measured" }, "tokens_per_word": { "value": 14.235643428414512, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 2.7026802645053123, "provenance": "measured" }, "chars_per_token": { "value": 2.6344638663388884, "provenance": "measured" }, "tokens_per_word": { "value": 3.271152769118595, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 2.999331299599885, "provenance": "measured" }, "chars_per_token": { "value": 2.9326104737272587, "provenance": "measured" }, "tokens_per_word": { "value": 2.128399360120448, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.0023958003029985, "provenance": "measured" }, "chars_per_token": { "value": 2.9067458220296185, "provenance": "measured" }, "tokens_per_word": { "value": 2.2081923211576613, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.0123481947145003, "provenance": "measured" }, "chars_per_token": { "value": 0.3933292325037088, "provenance": "measured" }, "tokens_per_word": { "value": 16.193702584345896, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 1.691836465166051, "provenance": "measured" }, "chars_per_token": { "value": 0.6716198048770576, "provenance": "measured" }, "tokens_per_word": { "value": 8.231157319656594, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 2.665620026295758, "provenance": "measured" }, "chars_per_token": { "value": 2.659431008234724, "provenance": "measured" }, "tokens_per_word": { "value": 2.8062918729973783, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 1.9654824281650238, "provenance": "measured" }, "chars_per_token": { "value": 0.6896392500909799, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 1.1704074058388076, "provenance": "measured" }, "chars_per_token": { "value": 0.5051145849819112, "provenance": "measured" }, "tokens_per_word": { "value": 8.870370736529907, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 1.681968918873561, "provenance": "measured" }, "chars_per_token": { "value": 0.6391031450340684, "provenance": "measured" }, "tokens_per_word": { "value": 10.980760725342769, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 1.7175907024001347, "provenance": "measured" }, "chars_per_token": { "value": 0.9450035374013969, "provenance": "measured" }, "tokens_per_word": { "value": 7.8307463269390345, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 1.0206642334440994, "provenance": "measured" }, "chars_per_token": { "value": 0.3888677041152936, "provenance": "measured" }, "tokens_per_word": { "value": 24.612126328334494, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.014070553316086, "provenance": "measured" }, "chars_per_token": { "value": 0.3863185127248661, "provenance": "measured" }, "tokens_per_word": { "value": 20.665174686062414, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 1.5085160983723418, "provenance": "measured" }, "chars_per_token": { "value": 0.5393536889628133, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.256219624443624, "provenance": "measured" }, "chars_per_token": { "value": 2.062474154375636, "provenance": "measured" }, "tokens_per_word": { "value": 3.7841025727314026, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 1.418344131757258, "provenance": "measured" }, "chars_per_token": { "value": 0.8010836891592616, "provenance": "measured" }, "tokens_per_word": { "value": 5.97243054301984, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 1.5278787305273442, "provenance": "measured" }, "chars_per_token": { "value": 1.1697506630485657, "provenance": "measured" }, "tokens_per_word": { "value": 3.9274471397719855, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 1.4374053798988284, "provenance": "measured" }, "chars_per_token": { "value": 0.5398773695401218, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 95593, "provenance": "measured" }, "tokens_per_request_mean": { "value": 477.965, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.477965, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 110194, "provenance": "measured" }, "tokens_per_request_mean": { "value": 676.036809815951, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6760368098159509, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 71477, "provenance": "measured" }, "tokens_per_request_mean": { "value": 376.19473684210527, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3761947368421053, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 292387, "provenance": "measured" }, "tokens_per_request_mean": { "value": 696.1595238095238, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6961595238095238, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 187933, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1252.8866666666668, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.2528866666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 823394, "provenance": "measured" }, "tokens_per_request_mean": { "value": 4116.97, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.965, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 4.11697, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 362593, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1812.965, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.812965, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 447347, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2236.735, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.236735, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9489, "provenance": "measured" }, "tokens_per_request_mean": { "value": 379.56, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.37956, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 1122175, "provenance": "measured" }, "tokens_per_request_mean": { "value": 187029.16666666666, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 0.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 187.02916666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 50281, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 102975488, "provenance": "estimated", "note": "50281 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 205950976, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 205950976, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 411901952, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 50281, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 205950976, "provenance": "estimated", "note": "50281 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 411901952, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 411901952, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 823803904, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "sarvam-1", "label": "sarvam-1", "provider": "Sarvam", "family": "published", "metrics": { "bpt_overall": { "value": 3.498430160918236, "provenance": "measured" }, "bpt_english": { "value": 4.019946287696525, "provenance": "measured" }, "bpt_code": { "value": 2.76623855985143, "provenance": "measured" }, "bpt_languages": { "value": 3.709105635206752, "provenance": "measured" }, "parity_mean_abs_log": { "value": 1.023875625012277, "provenance": "measured" }, "byte_fallback_share": { "value": 0.648164667630094, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 6.321151415335212, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.306436262401096, "q3": 6.3230548212430735, "n": 5 }, "batch_mbps_4t": { "value": 16.82872400173529, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.379430910362814, "q3": 17.234958655190237, "n": 5 }, "latency_small_p95_us": { "value": 54.99999997482519, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 53.874999935032974, "q3": 56.12499990093056, "n": 5 }, "decode_mbps": { "value": 15.417834483448631, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.422796936108409, "q3": 15.65840171959222, "n": 5 }, "rss_load_delta_mb": { "value": 85.950464, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 85.950464, "q3": 85.958656, "n": 5 }, "rss_peak_large_mb": { "value": 205.59872, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 205.500416, "q3": 205.668352, "n": 5 }, "load_seconds": { "value": 0.13710370900025737, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.13492304100009278, "q3": 0.14654233300007036, "n": 5 }, "throughput_ratio_very_large": { "value": 0.49375695380053597, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2489.655296, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2487.9104, "q3": 2519.396352, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.637376, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.629184, "q3": 17.682432, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.121112467347861, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0925840438443206, "q3": 3.145814188820669, "n": 5 }, "embedding_params_h2048": { "value": 139460608, "provenance": "estimated", "note": "68096 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.19074854124927193, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.1873500150566107, "q3": 0.20099705043022803, "n": 5 }, "cpu_cores_used_batch": { "value": 3.2861305919883934, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.2289697635499537, "q3": 3.2922173007086304, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 142.76440429687503, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 142.666748046875, "q3": 144.54345703125, "n": 5 }, "workload_tokens_total": { "value": 3771660, "provenance": "measured" }, "tokens_chat_en": { "value": 108523, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 150547, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 56370, "provenance": "measured" }, "tokens_code": { "value": 338417, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 199457, "provenance": "measured" }, "tokens_rag": { "value": 720300, "provenance": "measured" }, "tokens_json": { "value": 474694, "provenance": "measured" }, "tokens_logs": { "value": 609290, "provenance": "measured" }, "tokens_tech_docs": { "value": 11080, "provenance": "measured" }, "tokens_long_context": { "value": 1102982, "provenance": "measured" } }, "parity": { "de": { "value": 2.155707116201261, "provenance": "measured" }, "fr": { "value": 2.121670312701068, "provenance": "measured" }, "es": { "value": 2.069682151589242, "provenance": "measured" }, "ru": { "value": 8.376045553982756, "provenance": "measured" }, "zh": { "value": 3.8516921889074767, "provenance": "measured" }, "ja": { "value": 5.378361858190709, "provenance": "measured" }, "ko": { "value": 5.0014155192381935, "provenance": "measured" }, "ar": { "value": 6.7943636597606485, "provenance": "measured" }, "ur": { "value": 7.413299446660662, "provenance": "measured" }, "hi": { "value": 1.1485651782267405, "provenance": "measured" }, "mr": { "value": 1.0724810191738514, "provenance": "measured" }, "gu": { "value": 1.2122313730536611, "provenance": "measured" }, "bn": { "value": 1.2796293913267276, "provenance": "measured" }, "ta": { "value": 1.157540857032557, "provenance": "measured" }, "te": { "value": 1.1668060738643675, "provenance": "measured" }, "th": { "value": 11.536835671084802, "provenance": "measured" }, "vi": { "value": 4.361922532492601, "provenance": "measured" }, "tr": { "value": 2.6842426972075666, "provenance": "measured" }, "id": { "value": 1.9476257881868486, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 30.9601593625498, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 66.7410358565737, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.155707116201261, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 65.68725099601593, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.121670312701068, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 64.07768924302789, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.069682151589242, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 259.32370517928285, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.376045553982756, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 119.24900398406375, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.8516921889074767, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 166.51494023904382, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.378361858190709, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 154.84462151394422, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.0014155192381935, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 210.35458167330677, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.7943636597606485, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 229.51693227091633, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.413299446660662, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 35.559760956175296, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1485651782267405, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 33.20418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0724810191738514, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 37.53087649402391, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2122313730536611, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 39.61752988047809, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2796293913267276, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 35.83764940239044, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.157540857032557, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 36.124501992031874, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1668060738643675, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 357.1822709163347, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 11.536835671084802, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 135.04581673306774, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.361922532492601, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 83.10458167330677, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.6842426972075666, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 60.298804780876495, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9476257881868486, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 215393, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [ "code_asm", "code_c", "code_cpp", "code_cs", "code_dart", "code_go", "code_html", "code_java", "code_js", "code_kotlin", "code_lua", "code_php", "code_ps", "code_py", "code_r", "code_ruby", "code_rust", "code_shell", "code_sql", "code_swift", "code_ts", "en_fineweb", "wiki_ar", "wiki_bn", "wiki_de", "wiki_es", "wiki_fr", "wiki_gu", "wiki_hi", "wiki_id", "wiki_ja", "wiki_ko", "wiki_mr", "wiki_ru", "wiki_ta", "wiki_te", "wiki_th", "wiki_tr", "wiki_ur", "wiki_vi", "wiki_zh" ], "roundtrip_example": " ret ; just return until the proper paging code is installed into this location", "byte_fallback_sets": [ { "set": "wiki_ar", "bytes_per_token": { "value": 1.0024407344343875, "provenance": "measured" }, "split_byte_token_share": { "value": 0.8928870778135658, "provenance": "measured" } }, { "set": "wiki_ja", "bytes_per_token": { "value": 1.0058575445173383, "provenance": "measured" }, "split_byte_token_share": { "value": 0.9793514046091366, "provenance": "measured" } }, { "set": "wiki_ko", "bytes_per_token": { "value": 1.0102571305431989, "provenance": "measured" }, "split_byte_token_share": { "value": 0.8607793187310481, "provenance": "measured" } }, { "set": "wiki_ru", "bytes_per_token": { "value": 1.0066856519698013, "provenance": "measured" }, "split_byte_token_share": { "value": 0.900065943179265, "provenance": "measured" } }, { "set": "wiki_th", "bytes_per_token": { "value": 1.010280092743623, "provenance": "measured" }, "split_byte_token_share": { "value": 0.9732240184978016, "provenance": "measured" } }, { "set": "wiki_ur", "bytes_per_token": { "value": 1.0026474655810051, "provenance": "measured" }, "split_byte_token_share": { "value": 0.8724224601423377, "provenance": "measured" } }, { "set": "wiki_vi", "bytes_per_token": { "value": 1.3377971658395886, "provenance": "measured" }, "split_byte_token_share": { "value": 0.5186518031494652, "provenance": "measured" } }, { "set": "wiki_zh", "bytes_per_token": { "value": 1.0359201073655957, "provenance": "measured" }, "split_byte_token_share": { "value": 0.9666526994228897, "provenance": "measured" } } ], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "", "", "" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 68096, "provenance": "measured" }, "used_share": { "value": 0.6212259163533834, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 42303, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 25793, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": null, "provenance": "unavailable", "note": "not applicable: the tokenizer prepends a word-boundary marker before encoding, so a token's own text does not encode to itself by design" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.13710370900025737, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.13492304100009278, "q3": 0.14654233300007036, "n": 5 }, "rss_load_delta_mb": { "value": 85.950464, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 85.950464, "q3": 85.958656, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 85.8112, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 85.8112, "q3": 85.819392, "n": 5 }, "latency_small_p50_us": { "value": 44.833000060862105, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 44.66600000796461, "q3": 46.29199975170195, "n": 5 }, "latency_small_p95_us": { "value": 54.99999997482519, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 53.874999935032974, "q3": 56.12499990093056, "n": 5 }, "encode_mbps_1t_medium": { "value": 6.321151415335212, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.306436262401096, "q3": 6.3230548212430735, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 2993720.938215191, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2986751.786781636, "q3": 2994622.3983679195, "n": 5 }, "encode_mbps_1t_large": { "value": 4.603257207598636, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.54755514210928, "q3": 4.644340626886743, "n": 5 }, "latency_large_p50_ms": { "value": 226.38479200008987, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 225.6512090000342, "q3": 226.47124999991775, "n": 5 }, "decode_mbps": { "value": 15.417834483448631, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.422796936108409, "q3": 15.65840171959222, "n": 5 }, "decode_tokens_per_s": { "value": 7034462.155334811, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6580471.42288687, "q3": 7144222.129751597, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.890873349036755, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.8544732759220155, "q3": 5.895183094138746, "n": 5 }, "batch_mbps_4t": { "value": 16.82872400173529, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.379430910362814, "q3": 17.234958655190237, "n": 5 }, "batch_tokens_per_s_4t": { "value": 7812354.393479591, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 7603780.240383858, "q3": 8000939.640903886, "n": 5 }, "batch_cpu_utilization": { "value": 3.2861305919883934, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.2289697635499537, "q3": 3.2922173007086304, "n": 5 }, "cpu_cores_used_batch": { "value": 3.2861305919883934, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.2289697635499537, "q3": 3.2922173007086304, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.19074854124927193, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.1873500150566107, "q3": 0.20099705043022803, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 142.76440429687503, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 142.666748046875, "q3": 144.54345703125, "n": 5 }, "rss_peak_large_mb": { "value": 205.59872, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 205.500416, "q3": 205.668352, "n": 5 }, "rss_peak_very_large_mb": { "value": 2489.655296, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2487.9104, "q3": 2519.396352, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.637376, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.629184, "q3": 17.682432, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.121112467347861, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0925840438443206, "q3": 3.145814188820669, "n": 5 }, "throughput_ratio_very_large": { "value": 0.49375695380053597, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 1.8676812699532046, "provenance": "measured" }, "chars_per_token": { "value": 1.8659987812496408, "provenance": "measured" }, "tokens_per_word": { "value": 3.530375602099908, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 2.5425406778035247, "provenance": "measured" }, "chars_per_token": { "value": 2.532388265406576, "provenance": "measured" }, "tokens_per_word": { "value": 3.3347531103217225, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 2.901952773496892, "provenance": "measured" }, "chars_per_token": { "value": 2.901487334898586, "provenance": "measured" }, "tokens_per_word": { "value": 3.632036664850955, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 3.4329383503431226, "provenance": "measured" }, "chars_per_token": { "value": 3.413917355604099, "provenance": "measured" }, "tokens_per_word": { "value": 3.5757629034283562, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.0568777120075845, "provenance": "measured" }, "chars_per_token": { "value": 3.0508368480546473, "provenance": "measured" }, "tokens_per_word": { "value": 3.4993407341244525, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 2.52251080674778, "provenance": "measured" }, "chars_per_token": { "value": 2.5204977086053444, "provenance": "measured" }, "tokens_per_word": { "value": 3.3319640876317247, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 2.36153974735535, "provenance": "measured" }, "chars_per_token": { "value": 2.3248847816223703, "provenance": "measured" }, "tokens_per_word": { "value": 5.1019736842105265, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 3.3766761479073546, "provenance": "measured" }, "chars_per_token": { "value": 3.374850953564125, "provenance": "measured" }, "tokens_per_word": { "value": 2.9233720205639506, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 2.9089480345756327, "provenance": "measured" }, "chars_per_token": { "value": 2.898162644639997, "provenance": "measured" }, "tokens_per_word": { "value": 3.4644734927991023, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 3.1116840246679804, "provenance": "measured" }, "chars_per_token": { "value": 3.10134599145321, "provenance": "measured" }, "tokens_per_word": { "value": 4.173323366082871, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 2.561024577049722, "provenance": "measured" }, "chars_per_token": { "value": 2.540302852011272, "provenance": "measured" }, "tokens_per_word": { "value": 3.782267509955493, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 2.8165909366465938, "provenance": "measured" }, "chars_per_token": { "value": 2.8125898512150482, "provenance": "measured" }, "tokens_per_word": { "value": 4.0662753844347925, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 2.8724406779661016, "provenance": "measured" }, "chars_per_token": { "value": 2.872169491525424, "provenance": "measured" }, "tokens_per_word": { "value": 3.7653165418652144, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 2.897896992090987, "provenance": "measured" }, "chars_per_token": { "value": 2.8899761502372407, "provenance": "measured" }, "tokens_per_word": { "value": 4.102098003893953, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.2358202630659725, "provenance": "measured" }, "chars_per_token": { "value": 2.2253239651154133, "provenance": "measured" }, "tokens_per_word": { "value": 4.015013219910334, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 2.785587073648964, "provenance": "measured" }, "chars_per_token": { "value": 2.726146533852455, "provenance": "measured" }, "tokens_per_word": { "value": 3.7486415904974515, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.04014714990897, "provenance": "measured" }, "chars_per_token": { "value": 3.039121104625337, "provenance": "measured" }, "tokens_per_word": { "value": 3.323014553014553, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 2.3912455386012805, "provenance": "measured" }, "chars_per_token": { "value": 2.3868964461843007, "provenance": "measured" }, "tokens_per_word": { "value": 3.654669120300209, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.2695489453157567, "provenance": "measured" }, "chars_per_token": { "value": 2.2606140050209826, "provenance": "measured" }, "tokens_per_word": { "value": 3.9709755738669807, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 3.3245191170007775, "provenance": "measured" }, "chars_per_token": { "value": 3.312058415875055, "provenance": "measured" }, "tokens_per_word": { "value": 3.17924852757835, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 2.8128389407264844, "provenance": "measured" }, "chars_per_token": { "value": 2.752605549735669, "provenance": "measured" }, "tokens_per_word": { "value": 4.449442830507739, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.019946287696525, "provenance": "measured" }, "chars_per_token": { "value": 4.003598804261816, "provenance": "measured" }, "tokens_per_word": { "value": 1.5275555460591983, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 1.0024407344343875, "provenance": "measured" }, "chars_per_token": { "value": 0.5559163648695903, "provenance": "measured" }, "tokens_per_word": { "value": 11.007137054659394, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 9.243261679755092, "provenance": "measured" }, "chars_per_token": { "value": 3.4721815519765737, "provenance": "measured" }, "tokens_per_word": { "value": 2.0630899667044247, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 2.1030856051179567, "provenance": "measured" }, "chars_per_token": { "value": 2.0500031421640714, "provenance": "measured" }, "tokens_per_word": { "value": 4.203766128094851, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 2.372704777560157, "provenance": "measured" }, "chars_per_token": { "value": 2.3199234051482933, "provenance": "measured" }, "tokens_per_word": { "value": 2.6905053166462785, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 2.376402457728739, "provenance": "measured" }, "chars_per_token": { "value": 2.300695302986642, "provenance": "measured" }, "tokens_per_word": { "value": 2.7898756499526716, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 6.743470042750764, "provenance": "measured" }, "chars_per_token": { "value": 2.6200509964606935, "provenance": "measured" }, "tokens_per_word": { "value": 2.431042990193055, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 8.52139726074127, "provenance": "measured" }, "chars_per_token": { "value": 3.3827969093793175, "provenance": "measured" }, "tokens_per_word": { "value": 1.6342122867655258, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 2.2973192586960085, "provenance": "measured" }, "chars_per_token": { "value": 2.2919853587988848, "provenance": "measured" }, "tokens_per_word": { "value": 3.2561899213515875, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 1.0058575445173383, "provenance": "measured" }, "chars_per_token": { "value": 0.3529305745800591, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 1.0102571305431989, "provenance": "measured" }, "chars_per_token": { "value": 0.4359982760478397, "provenance": "measured" }, "tokens_per_word": { "value": 10.276539792387544, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 9.138488191848271, "provenance": "measured" }, "chars_per_token": { "value": 3.4723807786997356, "provenance": "measured" }, "tokens_per_word": { "value": 2.021045260209347, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 1.0066856519698013, "provenance": "measured" }, "chars_per_token": { "value": 0.5538697320807173, "provenance": "measured" }, "tokens_per_word": { "value": 13.360692146239078, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 9.747788889531186, "provenance": "measured" }, "chars_per_token": { "value": 3.7138562922712297, "provenance": "measured" }, "tokens_per_word": { "value": 2.5770682292183933, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 7.786356636895596, "provenance": "measured" }, "chars_per_token": { "value": 2.9662765629812133, "provenance": "measured" }, "tokens_per_word": { "value": 2.6913672344481743, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 1.010280092743623, "provenance": "measured" }, "chars_per_token": { "value": 0.36121476959702353, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 1.7308463753087093, "provenance": "measured" }, "chars_per_token": { "value": 1.5822156121655364, "provenance": "measured" }, "tokens_per_word": { "value": 4.93271188437011, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 1.0026474655810051, "provenance": "measured" }, "chars_per_token": { "value": 0.5662973552537527, "provenance": "measured" }, "tokens_per_word": { "value": 8.448594450005743, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 1.3377971658395886, "provenance": "measured" }, "chars_per_token": { "value": 1.0242233827191458, "provenance": "measured" }, "tokens_per_word": { "value": 4.485480387725378, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 1.0359201073655957, "provenance": "measured" }, "chars_per_token": { "value": 0.3890828783857915, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 108523, "provenance": "measured" }, "tokens_per_request_mean": { "value": 542.615, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.542615, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 150547, "provenance": "measured" }, "tokens_per_request_mean": { "value": 923.601226993865, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9236012269938649, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 56370, "provenance": "measured" }, "tokens_per_request_mean": { "value": 296.6842105263158, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.2966842105263158, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 338417, "provenance": "measured" }, "tokens_per_request_mean": { "value": 805.754761904762, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8057547619047619, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 199457, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1329.7133333333334, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.3297133333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 720300, "provenance": "measured" }, "tokens_per_request_mean": { "value": 3601.5, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.97, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 3.6015, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 474694, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2373.47, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.37347, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 609290, "provenance": "measured" }, "tokens_per_request_mean": { "value": 3046.45, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 3.04645, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 11080, "provenance": "measured" }, "tokens_per_request_mean": { "value": 443.2, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.4432, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 1102982, "provenance": "measured" }, "tokens_per_request_mean": { "value": 183830.33333333334, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 0.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 183.83033333333336, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 68096, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 139460608, "provenance": "estimated", "note": "68096 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 278921216, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 278921216, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 557842432, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 68096, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 278921216, "provenance": "estimated", "note": "68096 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 557842432, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 557842432, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1115684864, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 278921216, "provenance": "estimated", "note": "68096 rows x 2048 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 557842432, "provenance": "estimated", "note": "68096 rows x 2048 hidden x 2 matrices from the published config" } } } }, { "id": "cl100k_base", "label": "OpenAI cl100k_base (GPT-4)", "provider": "OpenAI", "family": "published", "metrics": { "bpt_overall": { "value": 3.7358904392533057, "provenance": "measured" }, "bpt_english": { "value": 4.691402879543303, "provenance": "measured" }, "bpt_code": { "value": 3.8723065053366117, "provenance": "measured" }, "bpt_languages": { "value": 2.6439619328800035, "provenance": "measured" }, "parity_mean_abs_log": { "value": 1.1498212691865821, "provenance": "measured" }, "byte_fallback_share": { "value": 0.22386126267534295, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 5.153221493593562, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.101702636960398, "q3": 5.19896650664927, "n": 5 }, "batch_mbps_4t": { "value": 16.422928599858064, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.343199716368368, "q3": 16.71586612937975, "n": 5 }, "latency_small_p95_us": { "value": 65.74999997610576, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 62.791000345896464, "q3": 84.12500000076761, "n": 5 }, "decode_mbps": { "value": 36.939341497354796, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.73547012610242, "q3": 38.24051348435033, "n": 5 }, "rss_load_delta_mb": { "value": 160.456704, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 160.452608, "q3": 160.456704, "n": 5 }, "rss_peak_large_mb": { "value": 231.231488, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 231.231488, "q3": 231.297024, "n": 5 }, "load_seconds": { "value": 0.2929883750002773, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.2891960420001851, "q3": 0.2957059999998819, "n": 5 }, "throughput_ratio_very_large": { "value": 0.606205890576739, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2687.71328, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2687.438848, "q3": 2688.016384, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.260544, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.047552, "q3": 17.330176, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.1239132248630783, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.096698741071744, "q3": 3.2022921105990445, "n": 5 }, "embedding_params_h2048": { "value": 205367296, "provenance": "estimated", "note": "100277 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.21258265340158997, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2100391233452332, "q3": 0.2144655046253479, "n": 5 }, "cpu_cores_used_batch": { "value": 3.519135287307303, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5109858677711996, "q3": 3.5221516695946185, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 148.69970703125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 148.696044921875, "q3": 148.7119140625, "n": 5 }, "workload_tokens_total": { "value": 2604726, "provenance": "measured" }, "tokens_chat_en": { "value": 88199, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 69668, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 47553, "provenance": "measured" }, "tokens_code": { "value": 224357, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 148252, "provenance": "measured" }, "tokens_rag": { "value": 520245, "provenance": "measured" }, "tokens_json": { "value": 325029, "provenance": "measured" }, "tokens_logs": { "value": 410578, "provenance": "measured" }, "tokens_tech_docs": { "value": 8322, "provenance": "measured" }, "tokens_long_context": { "value": 762523, "provenance": "measured" } }, "parity": { "de": { "value": 1.5819281180756375, "provenance": "measured" }, "fr": { "value": 1.5913546400270402, "provenance": "measured" }, "es": { "value": 1.5378750891951778, "provenance": "measured" }, "ru": { "value": 2.508806850189657, "provenance": "measured" }, "zh": { "value": 1.896984264092838, "provenance": "measured" }, "ja": { "value": 2.2897434934465015, "provenance": "measured" }, "ko": { "value": 2.3872009614301275, "provenance": "measured" }, "ar": { "value": 3.0752243962894807, "provenance": "measured" }, "ur": { "value": 4.410147594546888, "provenance": "measured" }, "hi": { "value": 4.804859728846659, "provenance": "measured" }, "mr": { "value": 5.066849438539828, "provenance": "measured" }, "gu": { "value": 7.692605250309835, "provenance": "measured" }, "bn": { "value": 5.84868742254103, "provenance": "measured" }, "ta": { "value": 7.664738798963458, "provenance": "measured" }, "te": { "value": 8.425996169301836, "provenance": "measured" }, "th": { "value": 4.362564314417696, "provenance": "measured" }, "vi": { "value": 2.447966349945544, "provenance": "measured" }, "tr": { "value": 1.9202688999887332, "provenance": "measured" }, "id": { "value": 1.5519209824614113, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.520916334661354, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 41.95418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5819281180756375, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 42.20418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5913546400270402, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 40.78585657370518, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5378750891951778, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 66.53585657370517, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.508806850189657, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 50.309760956175296, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.896984264092838, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 60.72609561752988, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.2897434934465015, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 63.310756972111555, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.3872009614301275, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 81.55776892430279, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.0752243962894807, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 116.96115537848605, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.410147594546888, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 127.4292828685259, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.804859728846659, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 134.37749003984064, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.066849438539828, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 204.01494023904382, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.692605250309835, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 155.1125498007968, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.84868742254103, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 203.27589641434264, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.664738798963458, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 223.46513944223108, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.425996169301836, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 115.699203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.362564314417696, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 64.92231075697211, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.447966349945544, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 50.927290836653384, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9202688999887332, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 41.158366533864545, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5519209824614113, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>", "<|fim_prefix|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (converted from tiktoken)" } }, "vocabulary": { "size": { "value": 100277, "provenance": "measured" }, "used_share": { "value": 0.789592827866809, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 79178, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 21099, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 99483 entries whose text is valid UTF-8; 789 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (converted from tiktoken)", "load_seconds": { "value": 0.2929883750002773, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.2891960420001851, "q3": 0.2957059999998819, "n": 5 }, "rss_load_delta_mb": { "value": 160.456704, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 160.452608, "q3": 160.456704, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 160.391168, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 160.387072, "q3": 160.391168, "n": 5 }, "latency_small_p50_us": { "value": 49.20899982607807, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 47.70800001097086, "q3": 52.04200002140169, "n": 5 }, "latency_small_p95_us": { "value": 65.74999997610576, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 62.791000345896464, "q3": 84.12500000076761, "n": 5 }, "encode_mbps_1t_medium": { "value": 5.153221493593562, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.101702636960398, "q3": 5.19896650664927, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1634102.286883091, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1617765.4999729, "q3": 1648608.1703466217, "n": 5 }, "encode_mbps_1t_large": { "value": 4.418223669768888, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.328596435860958, "q3": 4.468618029275175, "n": 5 }, "latency_large_p50_ms": { "value": 234.56470900009663, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 234.38987600002292, "q3": 236.46970800018607, "n": 5 }, "decode_mbps": { "value": 36.939341497354796, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.73547012610242, "q3": 38.24051348435033, "n": 5 }, "decode_tokens_per_s": { "value": 12112501.15588933, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 11717748.764949154, "q3": 12539158.658639424, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.38932356081766, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.373200810014367, "q3": 5.397790871566799, "n": 5 }, "batch_mbps_4t": { "value": 16.422928599858064, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.343199716368368, "q3": 16.71586612937975, "n": 5 }, "batch_tokens_per_s_4t": { "value": 5168614.311085754, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5143522.0843427535, "q3": 5260807.435968059, "n": 5 }, "batch_cpu_utilization": { "value": 3.519135287307303, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5109858677711996, "q3": 3.5221516695946185, "n": 5 }, "cpu_cores_used_batch": { "value": 3.519135287307303, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5109858677711996, "q3": 3.5221516695946185, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.21258265340158997, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2100391233452332, "q3": 0.2144655046253479, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 148.69970703125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 148.696044921875, "q3": 148.7119140625, "n": 5 }, "rss_peak_large_mb": { "value": 231.231488, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 231.231488, "q3": 231.297024, "n": 5 }, "rss_peak_very_large_mb": { "value": 2687.71328, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2687.438848, "q3": 2688.016384, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.260544, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.047552, "q3": 17.330176, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.1239132248630783, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.096698741071744, "q3": 3.2022921105990445, "n": 5 }, "throughput_ratio_very_large": { "value": 0.606205890576739, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "tiktoken", "load_seconds": { "value": 0.07270533400014756, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.07246958300015649, "q3": 0.07311799999979485, "n": 5 }, "rss_load_delta_mb": { "value": 48.766976, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 48.766976, "q3": 48.771072, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 48.693248, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 48.693248, "q3": 48.697344, "n": 5 }, "latency_small_p50_us": { "value": 15.000000075815478, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.791500007049763, "q3": 15.499999960866262, "n": 5 }, "latency_small_p95_us": { "value": 20.708999727503397, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 20.667000171670225, "q3": 22.290999936558364, "n": 5 }, "encode_mbps_1t_medium": { "value": 19.71889776783207, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.84703351082206, "q3": 20.008905249603483, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 6252922.754686019, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5976451.933860082, "q3": 6344885.013588539, "n": 5 }, "encode_mbps_1t_large": { "value": 18.81460321902255, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.027495725761874, "q3": 18.899841092636482, "n": 5 }, "latency_large_p50_ms": { "value": 56.49562500002503, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 54.90954200001852, "q3": 57.40870799991171, "n": 5 }, "decode_mbps": { "value": 191.80648101361635, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 188.95918319043187, "q3": 193.53821773446285, "n": 5 }, "decode_tokens_per_s": { "value": 62893818.0489996, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 61960182.07238229, "q3": 63461658.7895946, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 51.44433590424662, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 43.07919857589943, "q3": 55.588225891751016, "n": 5 }, "batch_mbps_4t": { "value": 46.55900518851056, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 43.79388111042827, "q3": 51.848860675754004, "n": 5 }, "batch_tokens_per_s_4t": { "value": 14653022.39268892, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13782784.189130792, "q3": 16317842.562166993, "n": 5 }, "batch_cpu_utilization": { "value": 3.962187900134452, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.9566213079728563, "q3": 3.9655026879536712, "n": 5 }, "cpu_cores_used_batch": { "value": 3.962187900134452, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.9566213079728563, "q3": 3.9655026879536712, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.08492953673085811, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.0763106702134939, "q3": 0.09054924083925046, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 13.284179687500002, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 13.259521484375, "q3": 13.323242187500002, "n": 5 }, "rss_peak_large_mb": { "value": 43.13088, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 43.13088, "q3": 43.159552, "n": 5 }, "rss_peak_very_large_mb": { "value": 262.979584, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 262.569984, "q3": 263.63904, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.386944, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 18.243584, "q3": 18.640896, "n": 5 }, "encode_mbps_1t_very_large": { "value": 18.907998534329895, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.853043727580005, "q3": 19.029553106435056, "n": 5 }, "throughput_ratio_very_large": { "value": 0.9588770506825685, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.4991845819315666, "provenance": "measured" }, "chars_per_token": { "value": 2.496933207516206, "provenance": "measured" }, "tokens_per_word": { "value": 2.638307084483412, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3835575023850155, "provenance": "measured" }, "chars_per_token": { "value": 3.3700468941053168, "provenance": "measured" }, "tokens_per_word": { "value": 2.5058671021397667, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.906320266761342, "provenance": "measured" }, "chars_per_token": { "value": 3.9056937396013893, "provenance": "measured" }, "tokens_per_word": { "value": 2.6981911756419166, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.898756126652567, "provenance": "measured" }, "chars_per_token": { "value": 4.871613427016408, "provenance": "measured" }, "tokens_per_word": { "value": 2.5058143915609694, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.2041138711510815, "provenance": "measured" }, "chars_per_token": { "value": 4.195805890809401, "provenance": "measured" }, "tokens_per_word": { "value": 2.5444260133554506, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.529356108850865, "provenance": "measured" }, "chars_per_token": { "value": 3.526539494465025, "provenance": "measured" }, "tokens_per_word": { "value": 2.381430255130338, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.5330461002029208, "provenance": "measured" }, "chars_per_token": { "value": 3.4782074366229385, "provenance": "measured" }, "tokens_per_word": { "value": 3.410233918128655, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.58033269781605, "provenance": "measured" }, "chars_per_token": { "value": 4.577856891145829, "provenance": "measured" }, "tokens_per_word": { "value": 2.155144882380433, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.0296317223957985, "provenance": "measured" }, "chars_per_token": { "value": 4.014691218506947, "provenance": "measured" }, "tokens_per_word": { "value": 2.500966354246763, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.622225178652881, "provenance": "measured" }, "chars_per_token": { "value": 4.606868633115402, "provenance": "measured" }, "tokens_per_word": { "value": 2.8094831268688596, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.5982147389182564, "provenance": "measured" }, "chars_per_token": { "value": 3.5691009158345843, "provenance": "measured" }, "tokens_per_word": { "value": 2.6920238931834155, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.9700037434286553, "provenance": "measured" }, "chars_per_token": { "value": 3.964364186780814, "provenance": "measured" }, "tokens_per_word": { "value": 2.8848925930273506, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.7680147631011405, "provenance": "measured" }, "chars_per_token": { "value": 3.7676590257242593, "provenance": "measured" }, "tokens_per_word": { "value": 2.8703837644656227, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.192633839755959, "provenance": "measured" }, "chars_per_token": { "value": 4.181174084738439, "provenance": "measured" }, "tokens_per_word": { "value": 2.8353197348229826, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.061808942770659, "provenance": "measured" }, "chars_per_token": { "value": 3.04743495240037, "provenance": "measured" }, "tokens_per_word": { "value": 2.931877227267502, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.008885906257327, "provenance": "measured" }, "chars_per_token": { "value": 3.9233418769556674, "provenance": "measured" }, "tokens_per_word": { "value": 2.6047554862895415, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.181788452767188, "provenance": "measured" }, "chars_per_token": { "value": 4.180377105188423, "provenance": "measured" }, "tokens_per_word": { "value": 2.4158212058212056, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.391684579782802, "provenance": "measured" }, "chars_per_token": { "value": 3.3855159327540454, "provenance": "measured" }, "tokens_per_word": { "value": 2.576658006783575, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.38154162554607, "provenance": "measured" }, "chars_per_token": { "value": 3.368228904271251, "provenance": "measured" }, "tokens_per_word": { "value": 2.665152295467922, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.5891010564826225, "provenance": "measured" }, "chars_per_token": { "value": 4.571900548773659, "provenance": "measured" }, "tokens_per_word": { "value": 2.303168393448261, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.9882348064580757, "provenance": "measured" }, "chars_per_token": { "value": 3.902831798492563, "provenance": "measured" }, "tokens_per_word": { "value": 3.1381216667389324, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.691402879543303, "provenance": "measured" }, "chars_per_token": { "value": 4.672324855766316, "provenance": "measured" }, "tokens_per_word": { "value": 1.308924303518502, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 2.6149734826433932, "provenance": "measured" }, "chars_per_token": { "value": 1.4501670799737807, "provenance": "measured" }, "tokens_per_word": { "value": 4.2195466326254305, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 2.196971614321757, "provenance": "measured" }, "chars_per_token": { "value": 0.8252805745153713, "provenance": "measured" }, "tokens_per_word": { "value": 8.679984896852366, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.1416925145135974, "provenance": "measured" }, "chars_per_token": { "value": 3.0623953256077687, "provenance": "measured" }, "tokens_per_word": { "value": 2.8140500671013284, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.7144822394086647, "provenance": "measured" }, "chars_per_token": { "value": 3.6318527137088497, "provenance": "measured" }, "tokens_per_word": { "value": 1.7186176719676296, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.5578734952334563, "provenance": "measured" }, "chars_per_token": { "value": 3.4445271727785123, "provenance": "measured" }, "tokens_per_word": { "value": 1.8634353807652908, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.5934109084805104, "provenance": "measured" }, "chars_per_token": { "value": 0.6190904403918325, "provenance": "measured" }, "tokens_per_word": { "value": 10.28841053475606, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 2.6125250349050635, "provenance": "measured" }, "chars_per_token": { "value": 1.037111795558298, "provenance": "measured" }, "tokens_per_word": { "value": 5.330388003121864, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.2901503787494595, "provenance": "measured" }, "chars_per_token": { "value": 3.2825113304542324, "provenance": "measured" }, "tokens_per_word": { "value": 2.2736066608408585, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 2.61405910089203, "provenance": "measured" }, "chars_per_token": { "value": 0.9172087891499172, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 2.2852846448467967, "provenance": "measured" }, "chars_per_token": { "value": 0.9862639275766016, "provenance": "measured" }, "tokens_per_word": { "value": 4.542956005931784, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 2.58627559262375, "provenance": "measured" }, "chars_per_token": { "value": 0.9827154631832656, "provenance": "measured" }, "tokens_per_word": { "value": 7.141272298393042, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 3.6319247650555564, "provenance": "measured" }, "chars_per_token": { "value": 1.9982535686512277, "provenance": "measured" }, "tokens_per_word": { "value": 3.7032752477180653, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.048291946213615, "provenance": "measured" }, "chars_per_token": { "value": 0.7803884572247618, "provenance": "measured" }, "tokens_per_word": { "value": 12.264226834839606, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.5739708331387838, "provenance": "measured" }, "chars_per_token": { "value": 0.5996171265816874, "provenance": "measured" }, "tokens_per_word": { "value": 13.314061917195076, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 2.9863491182433544, "provenance": "measured" }, "chars_per_token": { "value": 1.0677369735684685, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.818322696749038, "provenance": "measured" }, "chars_per_token": { "value": 2.5763084664989253, "provenance": "measured" }, "tokens_per_word": { "value": 3.029378607124228, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 2.026399984297407, "provenance": "measured" }, "chars_per_token": { "value": 1.1445148880208844, "provenance": "measured" }, "tokens_per_word": { "value": 4.180300966572033, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 2.6935937669083434, "provenance": "measured" }, "chars_per_token": { "value": 2.0622272120621865, "provenance": "measured" }, "tokens_per_word": { "value": 2.2277535030887448, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 2.248724607495485, "provenance": "measured" }, "chars_per_token": { "value": 0.8446020467797712, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 88199, "provenance": "measured" }, "tokens_per_request_mean": { "value": 440.995, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.440995, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 69668, "provenance": "measured" }, "tokens_per_request_mean": { "value": 427.41104294478527, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.42741104294478527, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 47553, "provenance": "measured" }, "tokens_per_request_mean": { "value": 250.27894736842106, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.25027894736842105, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 224357, "provenance": "measured" }, "tokens_per_request_mean": { "value": 534.1833333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5341833333333332, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 148252, "provenance": "measured" }, "tokens_per_request_mean": { "value": 988.3466666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9883466666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 520245, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2601.225, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.601225, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 325029, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1625.145, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.625145, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 410578, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2052.89, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.0528899999999997, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8322, "provenance": "measured" }, "tokens_per_request_mean": { "value": 332.88, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33288, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 762523, "provenance": "measured" }, "tokens_per_request_mean": { "value": 127087.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 127.08716666666668, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 100277, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 205367296, "provenance": "estimated", "note": "100277 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 410734592, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 410734592, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 821469184, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 100277, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 410734592, "provenance": "estimated", "note": "100277 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 821469184, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 821469184, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1642938368, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "olmo-2-0425-1b", "label": "OLMo-2-0425-1B", "provider": "Ai2", "family": "published", "metrics": { "bpt_overall": { "value": 3.7358904392533057, "provenance": "measured" }, "bpt_english": { "value": 4.691402879543303, "provenance": "measured" }, "bpt_code": { "value": 3.8723065053366117, "provenance": "measured" }, "bpt_languages": { "value": 2.6439619328800035, "provenance": "measured" }, "parity_mean_abs_log": { "value": 1.1498212691865821, "provenance": "measured" }, "byte_fallback_share": { "value": 0.22386126267534295, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.435593639061192, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.40110510014516, "q3": 4.51535460133397, "n": 5 }, "batch_mbps_4t": { "value": 13.75398452083423, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.332119780560065, "q3": 14.253268715757482, "n": 5 }, "latency_small_p95_us": { "value": 76.08300029460224, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 73.45799986069323, "q3": 76.33300003817567, "n": 5 }, "decode_mbps": { "value": 37.77913287054666, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.37934585873857, "q3": 38.08236068556311, "n": 5 }, "rss_load_delta_mb": { "value": 86.867968, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 86.867968, "q3": 86.867968, "n": 5 }, "rss_peak_large_mb": { "value": 194.605056, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 193.957888, "q3": 194.633728, "n": 5 }, "load_seconds": { "value": 0.14398874999960753, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.14174250000002075, "q3": 0.14775191599983373, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6161436080151416, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2650.468352, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2650.435584, "q3": 2651.242496, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.358272, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.096704, "q3": 18.436096, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.7329626684601744, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.6602789453083155, "q3": 2.9277326613638865, "n": 5 }, "embedding_params_h2048": { "value": 205520896, "provenance": "estimated", "note": "100352 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.2605544695244901, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2501393004720122, "q3": 0.2676125985325825, "n": 5 }, "cpu_cores_used_batch": { "value": 3.567843218423323, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.565302665999192, "q3": 3.5761472590705963, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 152.235595703125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 152.233642578125, "q3": 152.28173828125, "n": 5 }, "workload_tokens_total": { "value": 2604726, "provenance": "measured" }, "tokens_chat_en": { "value": 88199, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 69668, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 47553, "provenance": "measured" }, "tokens_code": { "value": 224357, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 148252, "provenance": "measured" }, "tokens_rag": { "value": 520245, "provenance": "measured" }, "tokens_json": { "value": 325029, "provenance": "measured" }, "tokens_logs": { "value": 410578, "provenance": "measured" }, "tokens_tech_docs": { "value": 8322, "provenance": "measured" }, "tokens_long_context": { "value": 762523, "provenance": "measured" } }, "parity": { "de": { "value": 1.5819281180756375, "provenance": "measured" }, "fr": { "value": 1.5913546400270402, "provenance": "measured" }, "es": { "value": 1.5378750891951778, "provenance": "measured" }, "ru": { "value": 2.508806850189657, "provenance": "measured" }, "zh": { "value": 1.896984264092838, "provenance": "measured" }, "ja": { "value": 2.2897434934465015, "provenance": "measured" }, "ko": { "value": 2.3872009614301275, "provenance": "measured" }, "ar": { "value": 3.0752243962894807, "provenance": "measured" }, "ur": { "value": 4.410147594546888, "provenance": "measured" }, "hi": { "value": 4.804859728846659, "provenance": "measured" }, "mr": { "value": 5.066849438539828, "provenance": "measured" }, "gu": { "value": 7.692605250309835, "provenance": "measured" }, "bn": { "value": 5.84868742254103, "provenance": "measured" }, "ta": { "value": 7.664738798963458, "provenance": "measured" }, "te": { "value": 8.425996169301836, "provenance": "measured" }, "th": { "value": 4.362564314417696, "provenance": "measured" }, "vi": { "value": 2.447966349945544, "provenance": "measured" }, "tr": { "value": 1.9202688999887332, "provenance": "measured" }, "id": { "value": 1.5519209824614113, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.520916334661354, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 41.95418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5819281180756375, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 42.20418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5913546400270402, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 40.78585657370518, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5378750891951778, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 66.53585657370517, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.508806850189657, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 50.309760956175296, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.896984264092838, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 60.72609561752988, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.2897434934465015, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 63.310756972111555, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.3872009614301275, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 81.55776892430279, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.0752243962894807, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 116.96115537848605, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.410147594546888, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 127.4292828685259, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.804859728846659, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 134.37749003984064, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.066849438539828, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 204.01494023904382, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.692605250309835, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 155.1125498007968, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.84868742254103, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 203.27589641434264, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.664738798963458, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 223.46513944223108, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.425996169301836, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 115.699203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.362564314417696, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 64.92231075697211, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.447966349945544, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 50.927290836653384, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9202688999887332, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 41.158366533864545, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5519209824614113, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 100,352 embedding rows for 100,278 tokenizer IDs (74 unused rows, usually padding)" ], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>", "<|im_start|>user", "<|im_end|>", "<|fim_prefix|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 100278, "provenance": "measured" }, "used_share": { "value": 0.7895849538283571, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 79178, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 21100, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 99497 entries whose text is valid UTF-8; 773 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.14398874999960753, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.14174250000002075, "q3": 0.14775191599983373, "n": 5 }, "rss_load_delta_mb": { "value": 86.867968, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 86.867968, "q3": 86.867968, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 86.79424, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 86.79424, "q3": 86.79424, "n": 5 }, "latency_small_p50_us": { "value": 58.91600017093879, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.66649996985507, "q3": 59.708499975386076, "n": 5 }, "latency_small_p95_us": { "value": 76.08300029460224, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 73.45799986069323, "q3": 76.33300003817567, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.435593639061192, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.40110510014516, "q3": 4.51535460133397, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1406540.3007196372, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1395603.879612179, "q3": 1431832.7456525739, "n": 5 }, "encode_mbps_1t_large": { "value": 3.7135839808471243, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.7127955840985427, "q3": 3.722075335453537, "n": 5 }, "latency_large_p50_ms": { "value": 278.30120799990254, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 276.26287499970203, "q3": 280.42658300000767, "n": 5 }, "decode_mbps": { "value": 37.77913287054666, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 36.37934585873857, "q3": 38.08236068556311, "n": 5 }, "decode_tokens_per_s": { "value": 12387870.817777343, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 11928877.205242625, "q3": 12487299.96597013, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.439123104702026, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.397821463857238, "q3": 4.5219804800253085, "n": 5 }, "batch_mbps_4t": { "value": 13.75398452083423, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.332119780560065, "q3": 14.253268715757482, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4328645.819567779, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4195876.799677967, "q3": 4485780.244130859, "n": 5 }, "batch_cpu_utilization": { "value": 3.567843218423323, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.565302665999192, "q3": 3.5761472590705963, "n": 5 }, "cpu_cores_used_batch": { "value": 3.567843218423323, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.565302665999192, "q3": 3.5761472590705963, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.2605544695244901, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2501393004720122, "q3": 0.2676125985325825, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 152.235595703125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 152.233642578125, "q3": 152.28173828125, "n": 5 }, "rss_peak_large_mb": { "value": 194.605056, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 193.957888, "q3": 194.633728, "n": 5 }, "rss_peak_very_large_mb": { "value": 2650.468352, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2650.435584, "q3": 2651.242496, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.358272, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.096704, "q3": 18.436096, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.7329626684601744, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.6602789453083155, "q3": 2.9277326613638865, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6161436080151416, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.4991845819315666, "provenance": "measured" }, "chars_per_token": { "value": 2.496933207516206, "provenance": "measured" }, "tokens_per_word": { "value": 2.638307084483412, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3835575023850155, "provenance": "measured" }, "chars_per_token": { "value": 3.3700468941053168, "provenance": "measured" }, "tokens_per_word": { "value": 2.5058671021397667, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.906320266761342, "provenance": "measured" }, "chars_per_token": { "value": 3.9056937396013893, "provenance": "measured" }, "tokens_per_word": { "value": 2.6981911756419166, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.898756126652567, "provenance": "measured" }, "chars_per_token": { "value": 4.871613427016408, "provenance": "measured" }, "tokens_per_word": { "value": 2.5058143915609694, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.2041138711510815, "provenance": "measured" }, "chars_per_token": { "value": 4.195805890809401, "provenance": "measured" }, "tokens_per_word": { "value": 2.5444260133554506, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.529356108850865, "provenance": "measured" }, "chars_per_token": { "value": 3.526539494465025, "provenance": "measured" }, "tokens_per_word": { "value": 2.381430255130338, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.5330461002029208, "provenance": "measured" }, "chars_per_token": { "value": 3.4782074366229385, "provenance": "measured" }, "tokens_per_word": { "value": 3.410233918128655, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.58033269781605, "provenance": "measured" }, "chars_per_token": { "value": 4.577856891145829, "provenance": "measured" }, "tokens_per_word": { "value": 2.155144882380433, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.0296317223957985, "provenance": "measured" }, "chars_per_token": { "value": 4.014691218506947, "provenance": "measured" }, "tokens_per_word": { "value": 2.500966354246763, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.622225178652881, "provenance": "measured" }, "chars_per_token": { "value": 4.606868633115402, "provenance": "measured" }, "tokens_per_word": { "value": 2.8094831268688596, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.5982147389182564, "provenance": "measured" }, "chars_per_token": { "value": 3.5691009158345843, "provenance": "measured" }, "tokens_per_word": { "value": 2.6920238931834155, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.9700037434286553, "provenance": "measured" }, "chars_per_token": { "value": 3.964364186780814, "provenance": "measured" }, "tokens_per_word": { "value": 2.8848925930273506, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.7680147631011405, "provenance": "measured" }, "chars_per_token": { "value": 3.7676590257242593, "provenance": "measured" }, "tokens_per_word": { "value": 2.8703837644656227, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.192633839755959, "provenance": "measured" }, "chars_per_token": { "value": 4.181174084738439, "provenance": "measured" }, "tokens_per_word": { "value": 2.8353197348229826, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.061808942770659, "provenance": "measured" }, "chars_per_token": { "value": 3.04743495240037, "provenance": "measured" }, "tokens_per_word": { "value": 2.931877227267502, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.008885906257327, "provenance": "measured" }, "chars_per_token": { "value": 3.9233418769556674, "provenance": "measured" }, "tokens_per_word": { "value": 2.6047554862895415, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.181788452767188, "provenance": "measured" }, "chars_per_token": { "value": 4.180377105188423, "provenance": "measured" }, "tokens_per_word": { "value": 2.4158212058212056, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.391684579782802, "provenance": "measured" }, "chars_per_token": { "value": 3.3855159327540454, "provenance": "measured" }, "tokens_per_word": { "value": 2.576658006783575, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.38154162554607, "provenance": "measured" }, "chars_per_token": { "value": 3.368228904271251, "provenance": "measured" }, "tokens_per_word": { "value": 2.665152295467922, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.5891010564826225, "provenance": "measured" }, "chars_per_token": { "value": 4.571900548773659, "provenance": "measured" }, "tokens_per_word": { "value": 2.303168393448261, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.9882348064580757, "provenance": "measured" }, "chars_per_token": { "value": 3.902831798492563, "provenance": "measured" }, "tokens_per_word": { "value": 3.1381216667389324, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.691402879543303, "provenance": "measured" }, "chars_per_token": { "value": 4.672324855766316, "provenance": "measured" }, "tokens_per_word": { "value": 1.308924303518502, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 2.6149734826433932, "provenance": "measured" }, "chars_per_token": { "value": 1.4501670799737807, "provenance": "measured" }, "tokens_per_word": { "value": 4.2195466326254305, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 2.196971614321757, "provenance": "measured" }, "chars_per_token": { "value": 0.8252805745153713, "provenance": "measured" }, "tokens_per_word": { "value": 8.679984896852366, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.1416925145135974, "provenance": "measured" }, "chars_per_token": { "value": 3.0623953256077687, "provenance": "measured" }, "tokens_per_word": { "value": 2.8140500671013284, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.7144822394086647, "provenance": "measured" }, "chars_per_token": { "value": 3.6318527137088497, "provenance": "measured" }, "tokens_per_word": { "value": 1.7186176719676296, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.5578734952334563, "provenance": "measured" }, "chars_per_token": { "value": 3.4445271727785123, "provenance": "measured" }, "tokens_per_word": { "value": 1.8634353807652908, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.5934109084805104, "provenance": "measured" }, "chars_per_token": { "value": 0.6190904403918325, "provenance": "measured" }, "tokens_per_word": { "value": 10.28841053475606, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 2.6125250349050635, "provenance": "measured" }, "chars_per_token": { "value": 1.037111795558298, "provenance": "measured" }, "tokens_per_word": { "value": 5.330388003121864, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.2901503787494595, "provenance": "measured" }, "chars_per_token": { "value": 3.2825113304542324, "provenance": "measured" }, "tokens_per_word": { "value": 2.2736066608408585, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 2.61405910089203, "provenance": "measured" }, "chars_per_token": { "value": 0.9172087891499172, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 2.2852846448467967, "provenance": "measured" }, "chars_per_token": { "value": 0.9862639275766016, "provenance": "measured" }, "tokens_per_word": { "value": 4.542956005931784, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 2.58627559262375, "provenance": "measured" }, "chars_per_token": { "value": 0.9827154631832656, "provenance": "measured" }, "tokens_per_word": { "value": 7.141272298393042, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 3.6319247650555564, "provenance": "measured" }, "chars_per_token": { "value": 1.9982535686512277, "provenance": "measured" }, "tokens_per_word": { "value": 3.7032752477180653, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.048291946213615, "provenance": "measured" }, "chars_per_token": { "value": 0.7803884572247618, "provenance": "measured" }, "tokens_per_word": { "value": 12.264226834839606, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.5739708331387838, "provenance": "measured" }, "chars_per_token": { "value": 0.5996171265816874, "provenance": "measured" }, "tokens_per_word": { "value": 13.314061917195076, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 2.9863491182433544, "provenance": "measured" }, "chars_per_token": { "value": 1.0677369735684685, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.818322696749038, "provenance": "measured" }, "chars_per_token": { "value": 2.5763084664989253, "provenance": "measured" }, "tokens_per_word": { "value": 3.029378607124228, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 2.026399984297407, "provenance": "measured" }, "chars_per_token": { "value": 1.1445148880208844, "provenance": "measured" }, "tokens_per_word": { "value": 4.180300966572033, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 2.6935937669083434, "provenance": "measured" }, "chars_per_token": { "value": 2.0622272120621865, "provenance": "measured" }, "tokens_per_word": { "value": 2.2277535030887448, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 2.248724607495485, "provenance": "measured" }, "chars_per_token": { "value": 0.8446020467797712, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 88199, "provenance": "measured" }, "tokens_per_request_mean": { "value": 440.995, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.440995, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 69668, "provenance": "measured" }, "tokens_per_request_mean": { "value": 427.41104294478527, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.42741104294478527, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 47553, "provenance": "measured" }, "tokens_per_request_mean": { "value": 250.27894736842106, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.25027894736842105, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 224357, "provenance": "measured" }, "tokens_per_request_mean": { "value": 534.1833333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5341833333333332, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 148252, "provenance": "measured" }, "tokens_per_request_mean": { "value": 988.3466666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9883466666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 520245, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2601.225, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.601225, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 325029, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1625.145, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.625145, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 410578, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2052.89, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.0528899999999997, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8322, "provenance": "measured" }, "tokens_per_request_mean": { "value": 332.88, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33288, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 762523, "provenance": "measured" }, "tokens_per_request_mean": { "value": 127087.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 127.08716666666668, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 100352, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 205520896, "provenance": "estimated", "note": "100352 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 411041792, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 411041792, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 822083584, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 100352, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 411041792, "provenance": "estimated", "note": "100352 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 822083584, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 822083584, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1644167168, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 411041792, "provenance": "estimated", "note": "100352 rows x 2048 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 822083584, "provenance": "estimated", "note": "100352 rows x 2048 hidden x 2 matrices from the published config" } } } }, { "id": "olmo-3-1025-7b", "label": "Olmo-3-1025-7B", "provider": "Ai2", "family": "published", "metrics": { "bpt_overall": { "value": 3.7358904392533057, "provenance": "measured" }, "bpt_english": { "value": 4.691402879543303, "provenance": "measured" }, "bpt_code": { "value": 3.8723065053366117, "provenance": "measured" }, "bpt_languages": { "value": 2.6439619328800035, "provenance": "measured" }, "parity_mean_abs_log": { "value": 1.1498212691865821, "provenance": "measured" }, "byte_fallback_share": { "value": 0.22386126267534295, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.507090268894989, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.4993050260991225, "q3": 4.51790430822814, "n": 5 }, "batch_mbps_4t": { "value": 13.589889003424451, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.767880264876666, "q3": 14.080795932050558, "n": 5 }, "latency_small_p95_us": { "value": 75.49999997991108, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 74.9169998925936, "q3": 83.41700004166341, "n": 5 }, "decode_mbps": { "value": 36.036205083998205, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.90468893793548, "q3": 37.24511055215399, "n": 5 }, "rss_load_delta_mb": { "value": 86.867968, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 86.867968, "q3": 86.867968, "n": 5 }, "rss_peak_large_mb": { "value": 194.383872, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 194.269184, "q3": 194.494464, "n": 5 }, "load_seconds": { "value": 0.1444498329997259, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.14382495900008507, "q3": 0.1445847079999112, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6282700086684136, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2651.15648, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2650.46016, "q3": 2651.508736, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.702912, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.506304, "q3": 18.100224, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.8316696423079772, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.8075927377037675, "q3": 2.8504854497291743, "n": 5 }, "embedding_params_h2048": { "value": 205369344, "provenance": "estimated", "note": "100278 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.2631638051953532, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25372403747963546, "q3": 0.2785353768360899, "n": 5 }, "cpu_cores_used_batch": { "value": 3.572636394806694, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5721148528877453, "q3": 3.5763669023236653, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 152.250244140625, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 152.23095703125003, "q3": 152.260498046875, "n": 5 }, "workload_tokens_total": { "value": 2604726, "provenance": "measured" }, "tokens_chat_en": { "value": 88199, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 69668, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 47553, "provenance": "measured" }, "tokens_code": { "value": 224357, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 148252, "provenance": "measured" }, "tokens_rag": { "value": 520245, "provenance": "measured" }, "tokens_json": { "value": 325029, "provenance": "measured" }, "tokens_logs": { "value": 410578, "provenance": "measured" }, "tokens_tech_docs": { "value": 8322, "provenance": "measured" }, "tokens_long_context": { "value": 762523, "provenance": "measured" } }, "parity": { "de": { "value": 1.5819281180756375, "provenance": "measured" }, "fr": { "value": 1.5913546400270402, "provenance": "measured" }, "es": { "value": 1.5378750891951778, "provenance": "measured" }, "ru": { "value": 2.508806850189657, "provenance": "measured" }, "zh": { "value": 1.896984264092838, "provenance": "measured" }, "ja": { "value": 2.2897434934465015, "provenance": "measured" }, "ko": { "value": 2.3872009614301275, "provenance": "measured" }, "ar": { "value": 3.0752243962894807, "provenance": "measured" }, "ur": { "value": 4.410147594546888, "provenance": "measured" }, "hi": { "value": 4.804859728846659, "provenance": "measured" }, "mr": { "value": 5.066849438539828, "provenance": "measured" }, "gu": { "value": 7.692605250309835, "provenance": "measured" }, "bn": { "value": 5.84868742254103, "provenance": "measured" }, "ta": { "value": 7.664738798963458, "provenance": "measured" }, "te": { "value": 8.425996169301836, "provenance": "measured" }, "th": { "value": 4.362564314417696, "provenance": "measured" }, "vi": { "value": 2.447966349945544, "provenance": "measured" }, "tr": { "value": 1.9202688999887332, "provenance": "measured" }, "id": { "value": 1.5519209824614113, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.520916334661354, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 41.95418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5819281180756375, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 42.20418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5913546400270402, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 40.78585657370518, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5378750891951778, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 66.53585657370517, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.508806850189657, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 50.309760956175296, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.896984264092838, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 60.72609561752988, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.2897434934465015, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 63.310756972111555, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.3872009614301275, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 81.55776892430279, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.0752243962894807, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 116.96115537848605, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.410147594546888, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 127.4292828685259, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.804859728846659, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 134.37749003984064, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.066849438539828, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 204.01494023904382, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.692605250309835, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 155.1125498007968, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.84868742254103, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 203.27589641434264, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.664738798963458, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 223.46513944223108, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.425996169301836, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 115.699203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.362564314417696, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 64.92231075697211, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.447966349945544, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 50.927290836653384, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9202688999887332, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 41.158366533864545, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5519209824614113, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>", "<|im_start|>user", "<|im_end|>", "<|fim_prefix|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 100278, "provenance": "measured" }, "used_share": { "value": 0.7895849538283571, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 79178, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 21100, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 99497 entries whose text is valid UTF-8; 773 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.1444498329997259, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.14382495900008507, "q3": 0.1445847079999112, "n": 5 }, "rss_load_delta_mb": { "value": 86.867968, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 86.867968, "q3": 86.867968, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 86.79424, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 86.79424, "q3": 86.79424, "n": 5 }, "latency_small_p50_us": { "value": 58.625000065148924, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 58.43750000167347, "q3": 63.625000166211976, "n": 5 }, "latency_small_p95_us": { "value": 75.49999997991108, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 74.9169998925936, "q3": 83.41700004166341, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.507090268894989, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.4993050260991225, "q3": 4.51790430822814, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1429212.1005755304, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1426743.3762886971, "q3": 1432641.2655021136, "n": 5 }, "encode_mbps_1t_large": { "value": 3.767134561371253, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.7464540098183874, "q3": 3.7882228406060423, "n": 5 }, "latency_large_p50_ms": { "value": 275.57591599997977, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 275.47341700005745, "q3": 278.45529199998964, "n": 5 }, "decode_mbps": { "value": 36.036205083998205, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.90468893793548, "q3": 37.24511055215399, "n": 5 }, "decode_tokens_per_s": { "value": 11816360.499145605, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 11773236.03058092, "q3": 12212763.58287265, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.422921552613208, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.365668280494425, "q3": 4.431870107482766, "n": 5 }, "batch_mbps_4t": { "value": 13.589889003424451, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.767880264876666, "q3": 14.080795932050558, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4277001.776027546, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4018299.675238266, "q3": 4431499.712329235, "n": 5 }, "batch_cpu_utilization": { "value": 3.572636394806694, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5721148528877453, "q3": 3.5763669023236653, "n": 5 }, "cpu_cores_used_batch": { "value": 3.572636394806694, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5721148528877453, "q3": 3.5763669023236653, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.2631638051953532, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25372403747963546, "q3": 0.2785353768360899, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 152.250244140625, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 152.23095703125003, "q3": 152.260498046875, "n": 5 }, "rss_peak_large_mb": { "value": 194.383872, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 194.269184, "q3": 194.494464, "n": 5 }, "rss_peak_very_large_mb": { "value": 2651.15648, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2650.46016, "q3": 2651.508736, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.702912, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.506304, "q3": 18.100224, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.8316696423079772, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.8075927377037675, "q3": 2.8504854497291743, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6282700086684136, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.4991845819315666, "provenance": "measured" }, "chars_per_token": { "value": 2.496933207516206, "provenance": "measured" }, "tokens_per_word": { "value": 2.638307084483412, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3835575023850155, "provenance": "measured" }, "chars_per_token": { "value": 3.3700468941053168, "provenance": "measured" }, "tokens_per_word": { "value": 2.5058671021397667, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.906320266761342, "provenance": "measured" }, "chars_per_token": { "value": 3.9056937396013893, "provenance": "measured" }, "tokens_per_word": { "value": 2.6981911756419166, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.898756126652567, "provenance": "measured" }, "chars_per_token": { "value": 4.871613427016408, "provenance": "measured" }, "tokens_per_word": { "value": 2.5058143915609694, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.2041138711510815, "provenance": "measured" }, "chars_per_token": { "value": 4.195805890809401, "provenance": "measured" }, "tokens_per_word": { "value": 2.5444260133554506, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.529356108850865, "provenance": "measured" }, "chars_per_token": { "value": 3.526539494465025, "provenance": "measured" }, "tokens_per_word": { "value": 2.381430255130338, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.5330461002029208, "provenance": "measured" }, "chars_per_token": { "value": 3.4782074366229385, "provenance": "measured" }, "tokens_per_word": { "value": 3.410233918128655, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.58033269781605, "provenance": "measured" }, "chars_per_token": { "value": 4.577856891145829, "provenance": "measured" }, "tokens_per_word": { "value": 2.155144882380433, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.0296317223957985, "provenance": "measured" }, "chars_per_token": { "value": 4.014691218506947, "provenance": "measured" }, "tokens_per_word": { "value": 2.500966354246763, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.622225178652881, "provenance": "measured" }, "chars_per_token": { "value": 4.606868633115402, "provenance": "measured" }, "tokens_per_word": { "value": 2.8094831268688596, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.5982147389182564, "provenance": "measured" }, "chars_per_token": { "value": 3.5691009158345843, "provenance": "measured" }, "tokens_per_word": { "value": 2.6920238931834155, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.9700037434286553, "provenance": "measured" }, "chars_per_token": { "value": 3.964364186780814, "provenance": "measured" }, "tokens_per_word": { "value": 2.8848925930273506, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.7680147631011405, "provenance": "measured" }, "chars_per_token": { "value": 3.7676590257242593, "provenance": "measured" }, "tokens_per_word": { "value": 2.8703837644656227, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.192633839755959, "provenance": "measured" }, "chars_per_token": { "value": 4.181174084738439, "provenance": "measured" }, "tokens_per_word": { "value": 2.8353197348229826, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.061808942770659, "provenance": "measured" }, "chars_per_token": { "value": 3.04743495240037, "provenance": "measured" }, "tokens_per_word": { "value": 2.931877227267502, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.008885906257327, "provenance": "measured" }, "chars_per_token": { "value": 3.9233418769556674, "provenance": "measured" }, "tokens_per_word": { "value": 2.6047554862895415, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.181788452767188, "provenance": "measured" }, "chars_per_token": { "value": 4.180377105188423, "provenance": "measured" }, "tokens_per_word": { "value": 2.4158212058212056, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.391684579782802, "provenance": "measured" }, "chars_per_token": { "value": 3.3855159327540454, "provenance": "measured" }, "tokens_per_word": { "value": 2.576658006783575, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.38154162554607, "provenance": "measured" }, "chars_per_token": { "value": 3.368228904271251, "provenance": "measured" }, "tokens_per_word": { "value": 2.665152295467922, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.5891010564826225, "provenance": "measured" }, "chars_per_token": { "value": 4.571900548773659, "provenance": "measured" }, "tokens_per_word": { "value": 2.303168393448261, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.9882348064580757, "provenance": "measured" }, "chars_per_token": { "value": 3.902831798492563, "provenance": "measured" }, "tokens_per_word": { "value": 3.1381216667389324, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.691402879543303, "provenance": "measured" }, "chars_per_token": { "value": 4.672324855766316, "provenance": "measured" }, "tokens_per_word": { "value": 1.308924303518502, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 2.6149734826433932, "provenance": "measured" }, "chars_per_token": { "value": 1.4501670799737807, "provenance": "measured" }, "tokens_per_word": { "value": 4.2195466326254305, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 2.196971614321757, "provenance": "measured" }, "chars_per_token": { "value": 0.8252805745153713, "provenance": "measured" }, "tokens_per_word": { "value": 8.679984896852366, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.1416925145135974, "provenance": "measured" }, "chars_per_token": { "value": 3.0623953256077687, "provenance": "measured" }, "tokens_per_word": { "value": 2.8140500671013284, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.7144822394086647, "provenance": "measured" }, "chars_per_token": { "value": 3.6318527137088497, "provenance": "measured" }, "tokens_per_word": { "value": 1.7186176719676296, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.5578734952334563, "provenance": "measured" }, "chars_per_token": { "value": 3.4445271727785123, "provenance": "measured" }, "tokens_per_word": { "value": 1.8634353807652908, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.5934109084805104, "provenance": "measured" }, "chars_per_token": { "value": 0.6190904403918325, "provenance": "measured" }, "tokens_per_word": { "value": 10.28841053475606, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 2.6125250349050635, "provenance": "measured" }, "chars_per_token": { "value": 1.037111795558298, "provenance": "measured" }, "tokens_per_word": { "value": 5.330388003121864, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.2901503787494595, "provenance": "measured" }, "chars_per_token": { "value": 3.2825113304542324, "provenance": "measured" }, "tokens_per_word": { "value": 2.2736066608408585, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 2.61405910089203, "provenance": "measured" }, "chars_per_token": { "value": 0.9172087891499172, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 2.2852846448467967, "provenance": "measured" }, "chars_per_token": { "value": 0.9862639275766016, "provenance": "measured" }, "tokens_per_word": { "value": 4.542956005931784, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 2.58627559262375, "provenance": "measured" }, "chars_per_token": { "value": 0.9827154631832656, "provenance": "measured" }, "tokens_per_word": { "value": 7.141272298393042, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 3.6319247650555564, "provenance": "measured" }, "chars_per_token": { "value": 1.9982535686512277, "provenance": "measured" }, "tokens_per_word": { "value": 3.7032752477180653, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.048291946213615, "provenance": "measured" }, "chars_per_token": { "value": 0.7803884572247618, "provenance": "measured" }, "tokens_per_word": { "value": 12.264226834839606, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.5739708331387838, "provenance": "measured" }, "chars_per_token": { "value": 0.5996171265816874, "provenance": "measured" }, "tokens_per_word": { "value": 13.314061917195076, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 2.9863491182433544, "provenance": "measured" }, "chars_per_token": { "value": 1.0677369735684685, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.818322696749038, "provenance": "measured" }, "chars_per_token": { "value": 2.5763084664989253, "provenance": "measured" }, "tokens_per_word": { "value": 3.029378607124228, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 2.026399984297407, "provenance": "measured" }, "chars_per_token": { "value": 1.1445148880208844, "provenance": "measured" }, "tokens_per_word": { "value": 4.180300966572033, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 2.6935937669083434, "provenance": "measured" }, "chars_per_token": { "value": 2.0622272120621865, "provenance": "measured" }, "tokens_per_word": { "value": 2.2277535030887448, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 2.248724607495485, "provenance": "measured" }, "chars_per_token": { "value": 0.8446020467797712, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 88199, "provenance": "measured" }, "tokens_per_request_mean": { "value": 440.995, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.440995, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 69668, "provenance": "measured" }, "tokens_per_request_mean": { "value": 427.41104294478527, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.42741104294478527, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 47553, "provenance": "measured" }, "tokens_per_request_mean": { "value": 250.27894736842106, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.25027894736842105, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 224357, "provenance": "measured" }, "tokens_per_request_mean": { "value": 534.1833333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5341833333333332, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 148252, "provenance": "measured" }, "tokens_per_request_mean": { "value": 988.3466666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9883466666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 520245, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2601.225, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.601225, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 325029, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1625.145, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.625145, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 410578, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2052.89, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.0528899999999997, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8322, "provenance": "measured" }, "tokens_per_request_mean": { "value": 332.88, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33288, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 762523, "provenance": "measured" }, "tokens_per_request_mean": { "value": 127087.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 127.08716666666668, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 100278, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 205369344, "provenance": "estimated", "note": "100278 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 410738688, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 410738688, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 821477376, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 100278, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 410738688, "provenance": "estimated", "note": "100278 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 821477376, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 821477376, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1642954752, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 821477376, "provenance": "estimated", "note": "100278 rows x 4096 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 1642954752, "provenance": "estimated", "note": "100278 rows x 4096 hidden x 2 matrices from the published config" } } } }, { "id": "phi-4", "label": "phi-4", "provider": "Microsoft", "family": "published", "metrics": { "bpt_overall": { "value": 3.7358904392533057, "provenance": "measured" }, "bpt_english": { "value": 4.691402879543303, "provenance": "measured" }, "bpt_code": { "value": 3.8723065053366117, "provenance": "measured" }, "bpt_languages": { "value": 2.6439619328800035, "provenance": "measured" }, "parity_mean_abs_log": { "value": 1.1498212691865821, "provenance": "measured" }, "byte_fallback_share": { "value": 0.22386126267534295, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.711554578029423, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.710282680081395, "q3": 4.769314018810956, "n": 5 }, "batch_mbps_4t": { "value": 13.640116466760247, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.387135037687267, "q3": 13.663721551040405, "n": 5 }, "latency_small_p95_us": { "value": 75.16599998780293, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 74.66699997848991, "q3": 78.99999991423101, "n": 5 }, "decode_mbps": { "value": 37.89404495834557, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.87140522427601, "q3": 38.44439263491666, "n": 5 }, "rss_load_delta_mb": { "value": 66.707456, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 65.425408, "q3": 66.78528, "n": 5 }, "rss_peak_large_mb": { "value": 192.458752, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 192.442368, "q3": 192.651264, "n": 5 }, "load_seconds": { "value": 0.10351358299999447, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.09887375000016618, "q3": 0.10645037500012222, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6011794543223464, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2644.963328, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2644.496384, "q3": 2645.098496, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.6128, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.46944, "q3": 17.678336, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.832489810229682, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.7939203820232565, "q3": 2.8582129042085946, "n": 5 }, "embedding_params_h2048": { "value": 205520896, "provenance": "estimated", "note": "100352 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.261536785219296, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2591859034836682, "q3": 0.26556404114186094, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5561461005325987, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5551416799200295, "q3": 3.567392210733258, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 154.921630859375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 154.908203125, "q3": 154.935791015625, "n": 5 }, "workload_tokens_total": { "value": 2604726, "provenance": "measured" }, "tokens_chat_en": { "value": 88199, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 69668, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 47553, "provenance": "measured" }, "tokens_code": { "value": 224357, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 148252, "provenance": "measured" }, "tokens_rag": { "value": 520245, "provenance": "measured" }, "tokens_json": { "value": 325029, "provenance": "measured" }, "tokens_logs": { "value": 410578, "provenance": "measured" }, "tokens_tech_docs": { "value": 8322, "provenance": "measured" }, "tokens_long_context": { "value": 762523, "provenance": "measured" } }, "parity": { "de": { "value": 1.5819281180756375, "provenance": "measured" }, "fr": { "value": 1.5913546400270402, "provenance": "measured" }, "es": { "value": 1.5378750891951778, "provenance": "measured" }, "ru": { "value": 2.508806850189657, "provenance": "measured" }, "zh": { "value": 1.896984264092838, "provenance": "measured" }, "ja": { "value": 2.2897434934465015, "provenance": "measured" }, "ko": { "value": 2.3872009614301275, "provenance": "measured" }, "ar": { "value": 3.0752243962894807, "provenance": "measured" }, "ur": { "value": 4.410147594546888, "provenance": "measured" }, "hi": { "value": 4.804859728846659, "provenance": "measured" }, "mr": { "value": 5.066849438539828, "provenance": "measured" }, "gu": { "value": 7.692605250309835, "provenance": "measured" }, "bn": { "value": 5.84868742254103, "provenance": "measured" }, "ta": { "value": 7.664738798963458, "provenance": "measured" }, "te": { "value": 8.425996169301836, "provenance": "measured" }, "th": { "value": 4.362564314417696, "provenance": "measured" }, "vi": { "value": 2.447966349945544, "provenance": "measured" }, "tr": { "value": 1.9202688999887332, "provenance": "measured" }, "id": { "value": 1.5519209824614113, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.520916334661354, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 41.95418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5819281180756375, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 42.20418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5913546400270402, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 40.78585657370518, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5378750891951778, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 66.53585657370517, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.508806850189657, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 50.309760956175296, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.896984264092838, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 60.72609561752988, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.2897434934465015, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 63.310756972111555, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.3872009614301275, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 81.55776892430279, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.0752243962894807, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 116.96115537848605, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.410147594546888, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 127.4292828685259, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.804859728846659, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 134.37749003984064, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.066849438539828, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 204.01494023904382, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.692605250309835, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 155.1125498007968, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.84868742254103, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 203.27589641434264, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.664738798963458, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 223.46513944223108, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.425996169301836, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 115.699203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.362564314417696, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 64.92231075697211, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.447966349945544, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 50.927290836653384, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9202688999887332, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 41.158366533864545, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5519209824614113, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>", "<|im_start|>user", "<|im_end|>", "<|fim_prefix|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 100352, "provenance": "measured" }, "used_share": { "value": 0.7890027104591837, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 79178, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 21174, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 99483 entries whose text is valid UTF-8; 773 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.10351358299999447, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.09887375000016618, "q3": 0.10645037500012222, "n": 5 }, "rss_load_delta_mb": { "value": 66.707456, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 65.425408, "q3": 66.78528, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 66.637824, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 64.847872, "q3": 66.715648, "n": 5 }, "latency_small_p50_us": { "value": 57.83349999433085, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.20800027120276, "q3": 59.79150000712252, "n": 5 }, "latency_small_p95_us": { "value": 75.16599998780293, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 74.66699997848991, "q3": 78.99999991423101, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.711554578029423, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.710282680081395, "q3": 4.769314018810956, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1494048.3579648004, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1493645.0352378155, "q3": 1512364.0531833214, "n": 5 }, "encode_mbps_1t_large": { "value": 3.886881726300308, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.8647398497998218, "q3": 3.908295374908761, "n": 5 }, "latency_large_p50_ms": { "value": 270.71316699993986, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 266.9781249999801, "q3": 271.6310829999884, "n": 5 }, "decode_mbps": { "value": 37.89404495834557, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.87140522427601, "q3": 38.44439263491666, "n": 5 }, "decode_tokens_per_s": { "value": 12425550.774697786, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12418127.150075074, "q3": 12606011.134801546, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.353792270481319, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.281769762233379, "q3": 4.360565450747259, "n": 5 }, "batch_mbps_4t": { "value": 13.640116466760247, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.387135037687267, "q3": 13.663721551040405, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4292809.333384228, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4213191.168646129, "q3": 4300238.3114549145, "n": 5 }, "batch_cpu_utilization": { "value": 3.5561461005325987, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5551416799200295, "q3": 3.567392210733258, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5561461005325987, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5551416799200295, "q3": 3.567392210733258, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.261536785219296, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2591859034836682, "q3": 0.26556404114186094, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 154.921630859375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 154.908203125, "q3": 154.935791015625, "n": 5 }, "rss_peak_large_mb": { "value": 192.458752, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 192.442368, "q3": 192.651264, "n": 5 }, "rss_peak_very_large_mb": { "value": 2644.963328, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2644.496384, "q3": 2645.098496, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.6128, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.46944, "q3": 17.678336, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.832489810229682, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.7939203820232565, "q3": 2.8582129042085946, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6011794543223464, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.4991845819315666, "provenance": "measured" }, "chars_per_token": { "value": 2.496933207516206, "provenance": "measured" }, "tokens_per_word": { "value": 2.638307084483412, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3835575023850155, "provenance": "measured" }, "chars_per_token": { "value": 3.3700468941053168, "provenance": "measured" }, "tokens_per_word": { "value": 2.5058671021397667, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.906320266761342, "provenance": "measured" }, "chars_per_token": { "value": 3.9056937396013893, "provenance": "measured" }, "tokens_per_word": { "value": 2.6981911756419166, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.898756126652567, "provenance": "measured" }, "chars_per_token": { "value": 4.871613427016408, "provenance": "measured" }, "tokens_per_word": { "value": 2.5058143915609694, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.2041138711510815, "provenance": "measured" }, "chars_per_token": { "value": 4.195805890809401, "provenance": "measured" }, "tokens_per_word": { "value": 2.5444260133554506, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.529356108850865, "provenance": "measured" }, "chars_per_token": { "value": 3.526539494465025, "provenance": "measured" }, "tokens_per_word": { "value": 2.381430255130338, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.5330461002029208, "provenance": "measured" }, "chars_per_token": { "value": 3.4782074366229385, "provenance": "measured" }, "tokens_per_word": { "value": 3.410233918128655, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.58033269781605, "provenance": "measured" }, "chars_per_token": { "value": 4.577856891145829, "provenance": "measured" }, "tokens_per_word": { "value": 2.155144882380433, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.0296317223957985, "provenance": "measured" }, "chars_per_token": { "value": 4.014691218506947, "provenance": "measured" }, "tokens_per_word": { "value": 2.500966354246763, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.622225178652881, "provenance": "measured" }, "chars_per_token": { "value": 4.606868633115402, "provenance": "measured" }, "tokens_per_word": { "value": 2.8094831268688596, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.5982147389182564, "provenance": "measured" }, "chars_per_token": { "value": 3.5691009158345843, "provenance": "measured" }, "tokens_per_word": { "value": 2.6920238931834155, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.9700037434286553, "provenance": "measured" }, "chars_per_token": { "value": 3.964364186780814, "provenance": "measured" }, "tokens_per_word": { "value": 2.8848925930273506, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.7680147631011405, "provenance": "measured" }, "chars_per_token": { "value": 3.7676590257242593, "provenance": "measured" }, "tokens_per_word": { "value": 2.8703837644656227, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.192633839755959, "provenance": "measured" }, "chars_per_token": { "value": 4.181174084738439, "provenance": "measured" }, "tokens_per_word": { "value": 2.8353197348229826, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.061808942770659, "provenance": "measured" }, "chars_per_token": { "value": 3.04743495240037, "provenance": "measured" }, "tokens_per_word": { "value": 2.931877227267502, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.008885906257327, "provenance": "measured" }, "chars_per_token": { "value": 3.9233418769556674, "provenance": "measured" }, "tokens_per_word": { "value": 2.6047554862895415, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.181788452767188, "provenance": "measured" }, "chars_per_token": { "value": 4.180377105188423, "provenance": "measured" }, "tokens_per_word": { "value": 2.4158212058212056, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.391684579782802, "provenance": "measured" }, "chars_per_token": { "value": 3.3855159327540454, "provenance": "measured" }, "tokens_per_word": { "value": 2.576658006783575, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.38154162554607, "provenance": "measured" }, "chars_per_token": { "value": 3.368228904271251, "provenance": "measured" }, "tokens_per_word": { "value": 2.665152295467922, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.5891010564826225, "provenance": "measured" }, "chars_per_token": { "value": 4.571900548773659, "provenance": "measured" }, "tokens_per_word": { "value": 2.303168393448261, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.9882348064580757, "provenance": "measured" }, "chars_per_token": { "value": 3.902831798492563, "provenance": "measured" }, "tokens_per_word": { "value": 3.1381216667389324, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.691402879543303, "provenance": "measured" }, "chars_per_token": { "value": 4.672324855766316, "provenance": "measured" }, "tokens_per_word": { "value": 1.308924303518502, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 2.6149734826433932, "provenance": "measured" }, "chars_per_token": { "value": 1.4501670799737807, "provenance": "measured" }, "tokens_per_word": { "value": 4.2195466326254305, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 2.196971614321757, "provenance": "measured" }, "chars_per_token": { "value": 0.8252805745153713, "provenance": "measured" }, "tokens_per_word": { "value": 8.679984896852366, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.1416925145135974, "provenance": "measured" }, "chars_per_token": { "value": 3.0623953256077687, "provenance": "measured" }, "tokens_per_word": { "value": 2.8140500671013284, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.7144822394086647, "provenance": "measured" }, "chars_per_token": { "value": 3.6318527137088497, "provenance": "measured" }, "tokens_per_word": { "value": 1.7186176719676296, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.5578734952334563, "provenance": "measured" }, "chars_per_token": { "value": 3.4445271727785123, "provenance": "measured" }, "tokens_per_word": { "value": 1.8634353807652908, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.5934109084805104, "provenance": "measured" }, "chars_per_token": { "value": 0.6190904403918325, "provenance": "measured" }, "tokens_per_word": { "value": 10.28841053475606, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 2.6125250349050635, "provenance": "measured" }, "chars_per_token": { "value": 1.037111795558298, "provenance": "measured" }, "tokens_per_word": { "value": 5.330388003121864, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.2901503787494595, "provenance": "measured" }, "chars_per_token": { "value": 3.2825113304542324, "provenance": "measured" }, "tokens_per_word": { "value": 2.2736066608408585, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 2.61405910089203, "provenance": "measured" }, "chars_per_token": { "value": 0.9172087891499172, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 2.2852846448467967, "provenance": "measured" }, "chars_per_token": { "value": 0.9862639275766016, "provenance": "measured" }, "tokens_per_word": { "value": 4.542956005931784, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 2.58627559262375, "provenance": "measured" }, "chars_per_token": { "value": 0.9827154631832656, "provenance": "measured" }, "tokens_per_word": { "value": 7.141272298393042, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 3.6319247650555564, "provenance": "measured" }, "chars_per_token": { "value": 1.9982535686512277, "provenance": "measured" }, "tokens_per_word": { "value": 3.7032752477180653, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.048291946213615, "provenance": "measured" }, "chars_per_token": { "value": 0.7803884572247618, "provenance": "measured" }, "tokens_per_word": { "value": 12.264226834839606, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.5739708331387838, "provenance": "measured" }, "chars_per_token": { "value": 0.5996171265816874, "provenance": "measured" }, "tokens_per_word": { "value": 13.314061917195076, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 2.9863491182433544, "provenance": "measured" }, "chars_per_token": { "value": 1.0677369735684685, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.818322696749038, "provenance": "measured" }, "chars_per_token": { "value": 2.5763084664989253, "provenance": "measured" }, "tokens_per_word": { "value": 3.029378607124228, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 2.026399984297407, "provenance": "measured" }, "chars_per_token": { "value": 1.1445148880208844, "provenance": "measured" }, "tokens_per_word": { "value": 4.180300966572033, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 2.6935937669083434, "provenance": "measured" }, "chars_per_token": { "value": 2.0622272120621865, "provenance": "measured" }, "tokens_per_word": { "value": 2.2277535030887448, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 2.248724607495485, "provenance": "measured" }, "chars_per_token": { "value": 0.8446020467797712, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 88199, "provenance": "measured" }, "tokens_per_request_mean": { "value": 440.995, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.440995, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 69668, "provenance": "measured" }, "tokens_per_request_mean": { "value": 427.41104294478527, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.42741104294478527, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 47553, "provenance": "measured" }, "tokens_per_request_mean": { "value": 250.27894736842106, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.25027894736842105, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 224357, "provenance": "measured" }, "tokens_per_request_mean": { "value": 534.1833333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5341833333333332, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 148252, "provenance": "measured" }, "tokens_per_request_mean": { "value": 988.3466666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9883466666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 520245, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2601.225, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.601225, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 325029, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1625.145, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.625145, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 410578, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2052.89, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.0528899999999997, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8322, "provenance": "measured" }, "tokens_per_request_mean": { "value": 332.88, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33288, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 762523, "provenance": "measured" }, "tokens_per_request_mean": { "value": 127087.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 127.08716666666668, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 100352, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 205520896, "provenance": "estimated", "note": "100352 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 411041792, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 411041792, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 822083584, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 100352, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 411041792, "provenance": "estimated", "note": "100352 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 822083584, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 822083584, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1644167168, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 1027604480, "provenance": "estimated", "note": "100352 rows x 5120 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 2055208960, "provenance": "estimated", "note": "100352 rows x 5120 hidden x 2 matrices from the published config" } } } }, { "id": "ernie-4-5-21b-a3b-pt", "label": "ERNIE-4.5-21B-A3B-PT", "provider": "Baidu", "family": "published", "metrics": { "bpt_overall": { "value": 3.8853175469495205, "provenance": "measured" }, "bpt_english": { "value": 4.401133231240429, "provenance": "measured" }, "bpt_code": { "value": 3.3734449314095594, "provenance": "measured" }, "bpt_languages": { "value": 3.8813744781985733, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.6915169267054722, "provenance": "measured" }, "byte_fallback_share": { "value": 0.035451642326111935, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 6.455317587551054, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.135972573213207, "q3": 6.509016394963281, "n": 5 }, "batch_mbps_4t": { "value": 20.487115783013383, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 20.014627070426474, "q3": 20.547415605708625, "n": 5 }, "latency_small_p95_us": { "value": 58.832999911828665, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.95800007035723, "q3": 59.58400015515508, "n": 5 }, "decode_mbps": { "value": 19.625329580720596, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.770204291667856, "q3": 20.175473635126412, "n": 5 }, "rss_load_delta_mb": { "value": 108.675072, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 108.675072, "q3": 108.675072, "n": 5 }, "rss_peak_large_mb": { "value": 165.261312, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 164.163584, "q3": 165.322752, "n": 5 }, "load_seconds": { "value": 0.22060670800010485, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.2186067920001733, "q3": 0.22601158299994495, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5726661978322217, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 1883.11552, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 1880.662016, "q3": 1904.549888, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.698816, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.93696, "q3": 17.92, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.6967421786623325, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.676949571180051, "q3": 3.709258997173759, "n": 5 }, "embedding_params_h2048": { "value": 211812352, "provenance": "estimated", "note": "103424 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.17094200498739798, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.1705714907665901, "q3": 0.17323995612005327, "n": 5 }, "cpu_cores_used_batch": { "value": 3.4673331154399127, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.4499990535951177, "q3": 3.5124164209491826, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 104.678466796875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 104.53198242187501, "q3": 105.84741210937501, "n": 5 }, "workload_tokens_total": { "value": 2675788, "provenance": "measured" }, "tokens_chat_en": { "value": 97857, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 66085, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 33845, "provenance": "measured" }, "tokens_code": { "value": 279541, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 148859, "provenance": "measured" }, "tokens_rag": { "value": 350437, "provenance": "measured" }, "tokens_json": { "value": 426135, "provenance": "measured" }, "tokens_logs": { "value": 575770, "provenance": "measured" }, "tokens_tech_docs": { "value": 9503, "provenance": "measured" }, "tokens_long_context": { "value": 687756, "provenance": "measured" } }, "parity": { "de": { "value": 1.5689263934193731, "provenance": "measured" }, "fr": { "value": 1.5892781165792087, "provenance": "measured" }, "es": { "value": 1.5179761735923982, "provenance": "measured" }, "ru": { "value": 1.9774145511274996, "provenance": "measured" }, "zh": { "value": 0.945575095731102, "provenance": "measured" }, "ja": { "value": 1.7032335838888102, "provenance": "measured" }, "ko": { "value": 2.1532052191178557, "provenance": "measured" }, "ar": { "value": 1.901786980570132, "provenance": "measured" }, "ur": { "value": 2.103531413983832, "provenance": "measured" }, "hi": { "value": 2.3532832222379803, "provenance": "measured" }, "mr": { "value": 2.506382073464757, "provenance": "measured" }, "gu": { "value": 2.9389093745568005, "provenance": "measured" }, "bn": { "value": 2.590164515671536, "provenance": "measured" }, "ta": { "value": 2.9251879166075736, "provenance": "measured" }, "te": { "value": 3.3076868529286627, "provenance": "measured" }, "th": { "value": 2.6834491561480642, "provenance": "measured" }, "vi": { "value": 1.6951141682030917, "provenance": "measured" }, "tr": { "value": 1.619486597645724, "provenance": "measured" }, "id": { "value": 1.3677137994610693, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 28.09163346613546, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 44.07370517928287, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5689263934193731, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 44.645418326693225, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5892781165792087, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 42.64243027888446, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5179761735923982, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 55.548804780876495, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9774145511274996, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 26.562749003984063, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 0.945575095731102, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 47.84661354581673, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7032335838888102, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 60.48705179282869, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.1532052191178557, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 53.42430278884462, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.901786980570132, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 59.091633466135455, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.103531413983832, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 66.10756972111554, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.3532832222379803, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 70.40836653386454, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.506382073464757, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 82.55876494023904, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.9389093745568005, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 72.76195219123505, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.590164515671536, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 82.17330677290836, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.9251879166075736, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 92.91832669322709, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.3076868529286627, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 75.38247011952191, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.6834491561480642, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 47.61852589641434, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6951141682030917, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 45.49402390438247, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.619486597645724, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 38.42131474103586, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3677137994610693, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 103,424 embedding rows for 101,304 tokenizer IDs (2,120 unused rows, usually padding)" ], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "", "", "" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 101304, "provenance": "measured" }, "used_share": { "value": 0.6918877833056938, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 70091, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 31213, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 191, "provenance": "measured", "note": "of 100157 entries whose text is valid UTF-8; 128 are byte pieces" }, "unreachable_examples": [ "\u0001", "\u0003", "\u0004", "\u0005", "\u0006", "\u0007", "\b", "\f", "\r", "\u000f", "\u0011", "\u001b" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.22060670800010485, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.2186067920001733, "q3": 0.22601158299994495, "n": 5 }, "rss_load_delta_mb": { "value": 108.675072, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 108.675072, "q3": 108.675072, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 108.531712, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 108.531712, "q3": 108.531712, "n": 5 }, "latency_small_p50_us": { "value": 44.81250005028414, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 44.58299986254133, "q3": 45.70900000544498, "n": 5 }, "latency_small_p95_us": { "value": 58.832999911828665, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 57.95800007035723, "q3": 59.58400015515508, "n": 5 }, "encode_mbps_1t_medium": { "value": 6.455317587551054, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.135972573213207, "q3": 6.509016394963281, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1937413.8402322484, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1841569.8415139723, "q3": 1953530.2917117306, "n": 5 }, "encode_mbps_1t_large": { "value": 5.028549221038671, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.01535528811276, "q3": 5.135152346781292, "n": 5 }, "latency_large_p50_ms": { "value": 205.18795899988618, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 204.20370800002274, "q3": 209.37125000000378, "n": 5 }, "decode_mbps": { "value": 19.625329580720596, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.770204291667856, "q3": 20.175473635126412, "n": 5 }, "decode_tokens_per_s": { "value": 5724202.631027231, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5474784.632245583, "q3": 5884665.467114703, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 6.360118289377386, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.308745123768129, "q3": 6.400232864976438, "n": 5 }, "batch_mbps_4t": { "value": 20.487115783013383, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 20.014627070426474, "q3": 20.547415605708625, "n": 5 }, "batch_tokens_per_s_4t": { "value": 6114329.2226829035, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5973316.13068128, "q3": 6132325.556180163, "n": 5 }, "batch_cpu_utilization": { "value": 3.4673331154399127, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.4499990535951177, "q3": 3.5124164209491826, "n": 5 }, "cpu_cores_used_batch": { "value": 3.4673331154399127, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.4499990535951177, "q3": 3.5124164209491826, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.17094200498739798, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.1705714907665901, "q3": 0.17323995612005327, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 104.678466796875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 104.53198242187501, "q3": 105.84741210937501, "n": 5 }, "rss_peak_large_mb": { "value": 165.261312, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 164.163584, "q3": 165.322752, "n": 5 }, "rss_peak_very_large_mb": { "value": 1883.11552, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 1880.662016, "q3": 1904.549888, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.698816, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.93696, "q3": 17.92, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.6967421786623325, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.676949571180051, "q3": 3.709258997173759, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5726661978322217, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.1549908019952593, "provenance": "measured" }, "chars_per_token": { "value": 2.1530494923408923, "provenance": "measured" }, "tokens_per_word": { "value": 3.0596958380689507, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 2.879113905959697, "provenance": "measured" }, "chars_per_token": { "value": 2.867617550378734, "provenance": "measured" }, "tokens_per_word": { "value": 2.944914897557198, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.4025127486826867, "provenance": "measured" }, "chars_per_token": { "value": 3.4019670262371227, "provenance": "measured" }, "tokens_per_word": { "value": 3.0977103251375873, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.267835624094029, "provenance": "measured" }, "chars_per_token": { "value": 4.2441886864947, "provenance": "measured" }, "tokens_per_word": { "value": 2.876252668592239, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.7009292846053667, "provenance": "measured" }, "chars_per_token": { "value": 3.693615674963763, "provenance": "measured" }, "tokens_per_word": { "value": 2.8903704648887754, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 2.966495176455475, "provenance": "measured" }, "chars_per_token": { "value": 2.9641277551095313, "provenance": "measured" }, "tokens_per_word": { "value": 2.8332813366611203, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.0619295312403243, "provenance": "measured" }, "chars_per_token": { "value": 3.0144033686296363, "provenance": "measured" }, "tokens_per_word": { "value": 3.934941520467836, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.077297405992359, "provenance": "measured" }, "chars_per_token": { "value": 4.0750935049266035, "provenance": "measured" }, "tokens_per_word": { "value": 2.4210352079763204, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.5089361948452265, "provenance": "measured" }, "chars_per_token": { "value": 3.4959262528762247, "provenance": "measured" }, "tokens_per_word": { "value": 2.872087948627361, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 3.9023619267674334, "provenance": "measured" }, "chars_per_token": { "value": 3.889397002663586, "provenance": "measured" }, "tokens_per_word": { "value": 3.3277445536095684, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.0118417538663236, "provenance": "measured" }, "chars_per_token": { "value": 2.9874723834033357, "provenance": "measured" }, "tokens_per_word": { "value": 3.216131802920278, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.4633602635278082, "provenance": "measured" }, "chars_per_token": { "value": 3.4584404151698873, "provenance": "measured" }, "tokens_per_word": { "value": 3.306913957037211, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.5350289242407387, "provenance": "measured" }, "chars_per_token": { "value": 3.534695183001446, "provenance": "measured" }, "tokens_per_word": { "value": 3.059564329475834, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.486809028463464, "provenance": "measured" }, "chars_per_token": { "value": 3.4772785092752216, "provenance": "measured" }, "tokens_per_word": { "value": 3.4092654256964536, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.6333758236718396, "provenance": "measured" }, "chars_per_token": { "value": 2.6210131585159404, "provenance": "measured" }, "tokens_per_word": { "value": 3.408874583285435, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.506942234105007, "provenance": "measured" }, "chars_per_token": { "value": 3.432108982112165, "provenance": "measured" }, "tokens_per_word": { "value": 2.9775704477486205, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.606680026720107, "provenance": "measured" }, "chars_per_token": { "value": 3.605462777406665, "provenance": "measured" }, "tokens_per_word": { "value": 2.801039501039501, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 2.985148118248094, "provenance": "measured" }, "chars_per_token": { "value": 2.979718861889825, "provenance": "measured" }, "tokens_per_word": { "value": 2.927563686223569, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.7870131790750188, "provenance": "measured" }, "chars_per_token": { "value": 2.776041044542657, "provenance": "measured" }, "tokens_per_word": { "value": 3.2336852560329605, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.046687515430829, "provenance": "measured" }, "chars_per_token": { "value": 4.031520039502921, "provenance": "measured" }, "tokens_per_word": { "value": 2.611882550191307, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.8570540916136618, "provenance": "measured" }, "chars_per_token": { "value": 3.774460152867604, "provenance": "measured" }, "tokens_per_word": { "value": 3.2448510601396174, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.401133231240429, "provenance": "measured" }, "chars_per_token": { "value": 4.383235615838985, "provenance": "measured" }, "tokens_per_word": { "value": 1.3952522961683216, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 3.952334765137869, "provenance": "measured" }, "chars_per_token": { "value": 2.191817929887765, "provenance": "measured" }, "tokens_per_word": { "value": 2.791768210127268, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 4.825559377388646, "provenance": "measured" }, "chars_per_token": { "value": 1.8126954346466542, "provenance": "measured" }, "tokens_per_word": { "value": 3.9518072289156625, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 2.870668121491607, "provenance": "measured" }, "chars_per_token": { "value": 2.7982116633040994, "provenance": "measured" }, "tokens_per_word": { "value": 3.0797290585736476, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.5593995120543562, "provenance": "measured" }, "chars_per_token": { "value": 3.480219837876125, "provenance": "measured" }, "tokens_per_word": { "value": 1.793497694551614, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.361440555394851, "provenance": "measured" }, "chars_per_token": { "value": 3.254352170825992, "provenance": "measured" }, "tokens_per_word": { "value": 1.9723291969761803, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 3.724199051402229, "provenance": "measured" }, "chars_per_token": { "value": 1.4469689008456113, "provenance": "measured" }, "tokens_per_word": { "value": 4.401930549558996, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 5.2307120496686315, "provenance": "measured" }, "chars_per_token": { "value": 2.0764712656915814, "provenance": "measured" }, "tokens_per_word": { "value": 2.662309064555692, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.546429598941251, "provenance": "measured" }, "chars_per_token": { "value": 3.538195523332758, "provenance": "measured" }, "tokens_per_word": { "value": 2.109306728808622, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.329958829730106, "provenance": "measured" }, "chars_per_token": { "value": 1.168400326179916, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 2.4828680323062664, "provenance": "measured" }, "chars_per_token": { "value": 1.0715353042425617, "provenance": "measured" }, "tokens_per_word": { "value": 4.181433514582303, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 5.01615631788106, "provenance": "measured" }, "chars_per_token": { "value": 1.906005065115767, "provenance": "measured" }, "tokens_per_word": { "value": 3.6819622585876455, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 4.446406816036403, "provenance": "measured" }, "chars_per_token": { "value": 2.4463745431367645, "provenance": "measured" }, "tokens_per_word": { "value": 3.024918240835652, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 4.886842028999507, "provenance": "measured" }, "chars_per_token": { "value": 1.8618611076227551, "provenance": "measured" }, "tokens_per_word": { "value": 5.140480683285332, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 3.6725109671421516, "provenance": "measured" }, "chars_per_token": { "value": 1.39907324017315, "provenance": "measured" }, "tokens_per_word": { "value": 5.706162708773675, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 4.424844573308428, "provenance": "measured" }, "chars_per_token": { "value": 1.582055535420558, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.1555420120771895, "provenance": "measured" }, "chars_per_token": { "value": 2.884570177675234, "provenance": "measured" }, "tokens_per_word": { "value": 2.7056418367518633, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 3.9206073126642464, "provenance": "measured" }, "chars_per_token": { "value": 2.2143670914918503, "provenance": "measured" }, "tokens_per_word": { "value": 2.1606249076915502, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 3.4735758766854863, "provenance": "measured" }, "chars_per_token": { "value": 2.659384939209108, "provenance": "measured" }, "tokens_per_word": { "value": 1.7275174526643564, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.866059287462605, "provenance": "measured" }, "chars_per_token": { "value": 1.452059347898347, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 97857, "provenance": "measured" }, "tokens_per_request_mean": { "value": 489.285, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.489285, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 66085, "provenance": "measured" }, "tokens_per_request_mean": { "value": 405.42944785276075, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.40542944785276075, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 33845, "provenance": "measured" }, "tokens_per_request_mean": { "value": 178.1315789473684, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.1781315789473684, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 279541, "provenance": "measured" }, "tokens_per_request_mean": { "value": 665.5738095238095, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6655738095238095, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 148859, "provenance": "measured" }, "tokens_per_request_mean": { "value": 992.3933333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9923933333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 350437, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1752.185, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.752185, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 426135, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2130.675, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.130675, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 575770, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2878.85, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.87885, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9503, "provenance": "measured" }, "tokens_per_request_mean": { "value": 380.12, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.38012, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 687756, "provenance": "measured" }, "tokens_per_request_mean": { "value": 114626.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 114.626, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 103424, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 211812352, "provenance": "estimated", "note": "103424 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 423624704, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 423624704, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 847249408, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 103424, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 423624704, "provenance": "estimated", "note": "103424 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 847249408, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 847249408, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1694498816, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 264765440, "provenance": "estimated", "note": "103424 rows x 2560 hidden x 1 matrix from the published config" }, "bf16_bytes": { "value": 529530880, "provenance": "estimated", "note": "103424 rows x 2560 hidden x 1 matrix from the published config" } } } }, { "id": "hy3", "label": "Hy3", "provider": "Tencent", "family": "published", "metrics": { "bpt_overall": { "value": 3.9654122045978304, "provenance": "measured" }, "bpt_english": { "value": 4.706494199226563, "provenance": "measured" }, "bpt_code": { "value": 3.695589352616236, "provenance": "measured" }, "bpt_languages": { "value": 3.4941530619506898, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.8561972944417939, "provenance": "measured" }, "byte_fallback_share": { "value": 0.08917222864291212, "provenance": "measured" }, "unicode_probe_failures": { "value": 2, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.135631116558927, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.12265591811356, "q3": 4.220216549664736, "n": 5 }, "batch_mbps_4t": { "value": 14.15198592956625, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.997522586239256, "q3": 14.201477918105386, "n": 5 }, "latency_small_p95_us": { "value": 89.50000005825132, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 88.20899984129937, "q3": 94.00000004688991, "n": 5 }, "decode_mbps": { "value": 35.86769057103556, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.409047448975976, "q3": 36.776853590940746, "n": 5 }, "rss_load_delta_mb": { "value": 93.503488, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 93.503488, "q3": 93.503488, "n": 5 }, "rss_peak_large_mb": { "value": 199.356416, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 199.35232, "q3": 199.417856, "n": 5 }, "load_seconds": { "value": 0.1811083749998943, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.18062112500001604, "q3": 0.18120783299991672, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6947220270881639, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2540.376064, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2539.077632, "q3": 2541.166592, "n": 5 }, "rss_peak_chunked_mb": { "value": 16.986112, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.900096, "q3": 17.637376, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.8731140325847044, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.847153082745879, "q3": 2.925417588254483, "n": 5 }, "embedding_params_h2048": { "value": 247463936, "provenance": "estimated", "note": "120832 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.2556965369318513, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25480100835182923, "q3": 0.2561917480236261, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6059402850343796, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.586049779368823, "q3": 3.621703444562264, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 143.6865234375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 143.66381835937503, "q3": 143.7314453125, "n": 5 }, "workload_tokens_total": { "value": 2357313, "provenance": "measured" }, "tokens_chat_en": { "value": 88615, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 64276, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 36821, "provenance": "measured" }, "tokens_code": { "value": 235462, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 136638, "provenance": "measured" }, "tokens_rag": { "value": 382925, "provenance": "measured" }, "tokens_json": { "value": 325834, "provenance": "measured" }, "tokens_logs": { "value": 421768, "provenance": "measured" }, "tokens_tech_docs": { "value": 8458, "provenance": "measured" }, "tokens_long_context": { "value": 656516, "provenance": "measured" } }, "parity": { "de": { "value": 1.6878811688554645, "provenance": "measured" }, "fr": { "value": 1.6593706738504135, "provenance": "measured" }, "es": { "value": 1.6714184158341752, "provenance": "measured" }, "ru": { "value": 2.125565907135107, "provenance": "measured" }, "zh": { "value": 0.9288734238784749, "provenance": "measured" }, "ja": { "value": 1.6333295917985557, "provenance": "measured" }, "ko": { "value": 1.710218131477532, "provenance": "measured" }, "ar": { "value": 2.026564896920717, "provenance": "measured" }, "ur": { "value": 2.9962210498746584, "provenance": "measured" }, "hi": { "value": 3.3971265012908294, "provenance": "measured" }, "mr": { "value": 3.7214427358102293, "provenance": "measured" }, "gu": { "value": 5.160249934523141, "provenance": "measured" }, "bn": { "value": 3.933774834437086, "provenance": "measured" }, "ta": { "value": 3.8765667676881055, "provenance": "measured" }, "te": { "value": 5.4471508212668835, "provenance": "measured" }, "th": { "value": 2.466232648632469, "provenance": "measured" }, "vi": { "value": 1.7596812212369515, "provenance": "measured" }, "tr": { "value": 1.7881917162420025, "provenance": "measured" }, "id": { "value": 1.519998503386089, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.62051792828685, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 44.93227091633466, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6878811688554645, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 44.17330677290837, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6593706738504135, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 44.49402390438247, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6714184158341752, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 56.58366533864542, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.125565907135107, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 24.727091633466134, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 0.9288734238784749, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 43.4800796812749, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6333295917985557, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 45.526892430278885, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.710218131477532, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 53.94820717131474, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.026564896920717, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 79.7609561752988, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.9962210498746584, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 90.43326693227091, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.3971265012908294, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 99.06673306772909, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.7214427358102293, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 137.36852589641435, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.160249934523141, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 104.7191235059761, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.933774834437086, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 103.19621513944223, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.8765667676881055, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 145.00597609561754, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.4471508212668835, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 65.65239043824701, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.466232648632469, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 46.84362549800797, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7596812212369515, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 47.602589641434264, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7881917162420025, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 40.463147410358566, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.519998503386089, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 120,832 embedding rows for 120,818 tokenizer IDs (14 unused rows, usually padding)" ], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 2, "provenance": "measured" }, "unicode_failed_probes": [ "whitespace (NFC)", "whitespace (NFD)" ], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 120818, "provenance": "measured" }, "used_share": { "value": 0.6446969822377461, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 77891, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 42927, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 119034 entries whose text is valid UTF-8; 998 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.1811083749998943, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.18062112500001604, "q3": 0.18120783299991672, "n": 5 }, "rss_load_delta_mb": { "value": 93.503488, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 93.503488, "q3": 93.503488, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 93.42976, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 93.42976, "q3": 93.42976, "n": 5 }, "latency_small_p50_us": { "value": 60.37499997546547, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 60.291500005860144, "q3": 61.31299997491624, "n": 5 }, "latency_small_p95_us": { "value": 89.50000005825132, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 88.20899984129937, "q3": 94.00000004688991, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.135631116558927, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.12265591811356, "q3": 4.220216549664736, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1154423.0456117128, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1150801.1393815507, "q3": 1178034.2842711322, "n": 5 }, "encode_mbps_1t_large": { "value": 3.5580858941432716, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.497370378718342, "q3": 3.6006839281370056, "n": 5 }, "latency_large_p50_ms": { "value": 297.8369159999943, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 288.1038330000365, "q3": 300.8306670003549, "n": 5 }, "decode_mbps": { "value": 35.86769057103556, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.409047448975976, "q3": 36.776853590940746, "n": 5 }, "decode_tokens_per_s": { "value": 10054114.145947512, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9925551.357877614, "q3": 10308962.691640686, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.447920020046137, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.366505887730721, "q3": 4.4949638684044855, "n": 5 }, "batch_mbps_4t": { "value": 14.15198592956625, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.997522586239256, "q3": 14.201477918105386, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3932654.274903203, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3889730.9049624754, "q3": 3946407.4598816633, "n": 5 }, "batch_cpu_utilization": { "value": 3.6059402850343796, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.586049779368823, "q3": 3.621703444562264, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6059402850343796, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.586049779368823, "q3": 3.621703444562264, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.2556965369318513, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25480100835182923, "q3": 0.2561917480236261, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 143.6865234375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 143.66381835937503, "q3": 143.7314453125, "n": 5 }, "rss_peak_large_mb": { "value": 199.356416, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 199.35232, "q3": 199.417856, "n": 5 }, "rss_peak_very_large_mb": { "value": 2540.376064, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2539.077632, "q3": 2541.166592, "n": 5 }, "rss_peak_chunked_mb": { "value": 16.986112, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.900096, "q3": 17.637376, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.8731140325847044, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.847153082745879, "q3": 2.925417588254483, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6947220270881639, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.399036094756119, "provenance": "measured" }, "chars_per_token": { "value": 2.396874938463659, "provenance": "measured" }, "tokens_per_word": { "value": 2.7484440114737243, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.2430053381584965, "provenance": "measured" }, "chars_per_token": { "value": 3.230055957294732, "provenance": "measured" }, "tokens_per_word": { "value": 2.614471624101416, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.695691403883137, "provenance": "measured" }, "chars_per_token": { "value": 3.6950986590795334, "provenance": "measured" }, "tokens_per_word": { "value": 2.8519694209132402, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.627337366622168, "provenance": "measured" }, "chars_per_token": { "value": 4.601698525861335, "provenance": "measured" }, "tokens_per_word": { "value": 2.6527941730503577, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.017933028724798, "provenance": "measured" }, "chars_per_token": { "value": 4.0099929705722595, "provenance": "measured" }, "tokens_per_word": { "value": 2.6623282718727403, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.3502597689714255, "provenance": "measured" }, "chars_per_token": { "value": 3.347586083015531, "provenance": "measured" }, "tokens_per_word": { "value": 2.5087354409317806, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.457984251087458, "provenance": "measured" }, "chars_per_token": { "value": 3.404310670377778, "provenance": "measured" }, "tokens_per_word": { "value": 3.484259259259259, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.375118675665878, "provenance": "measured" }, "chars_per_token": { "value": 4.372753793305829, "provenance": "measured" }, "tokens_per_word": { "value": 2.2562315002336812, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.852537417776224, "provenance": "measured" }, "chars_per_token": { "value": 3.8382535193364897, "provenance": "measured" }, "tokens_per_word": { "value": 2.6159313368939503, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.299151493724589, "provenance": "measured" }, "chars_per_token": { "value": 4.284868304755171, "provenance": "measured" }, "tokens_per_word": { "value": 3.020610850064075, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.3714652606976396, "provenance": "measured" }, "chars_per_token": { "value": 3.344186109714503, "provenance": "measured" }, "tokens_per_word": { "value": 2.8730772233934565, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.7478450593857073, "provenance": "measured" }, "chars_per_token": { "value": 3.7425210884562787, "provenance": "measured" }, "tokens_per_word": { "value": 3.0558985796454983, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.6363909451775562, "provenance": "measured" }, "chars_per_token": { "value": 3.6360476343739943, "provenance": "measured" }, "tokens_per_word": { "value": 2.9742809734513274, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.002550739081244, "provenance": "measured" }, "chars_per_token": { "value": 3.9916105395150123, "provenance": "measured" }, "tokens_per_word": { "value": 2.9699704617549476, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.9757941588927506, "provenance": "measured" }, "chars_per_token": { "value": 2.9618239741479178, "provenance": "measured" }, "tokens_per_word": { "value": 3.0166226002988847, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.8975796118260213, "provenance": "measured" }, "chars_per_token": { "value": 3.8144107034769004, "provenance": "measured" }, "tokens_per_word": { "value": 2.679141569436839, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.98056932213803, "provenance": "measured" }, "chars_per_token": { "value": 3.979225885725988, "provenance": "measured" }, "tokens_per_word": { "value": 2.537941787941788, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.2041924644123405, "provenance": "measured" }, "chars_per_token": { "value": 3.1983648198126686, "provenance": "measured" }, "tokens_per_word": { "value": 2.727430179692574, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.2687694298789842, "provenance": "measured" }, "chars_per_token": { "value": 3.255900679128474, "provenance": "measured" }, "tokens_per_word": { "value": 2.7570997645673927, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.293124372462566, "provenance": "measured" }, "chars_per_token": { "value": 4.27703322128607, "provenance": "measured" }, "tokens_per_word": { "value": 2.4619534843729847, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.9110402016178205, "provenance": "measured" }, "chars_per_token": { "value": 3.8272902185548014, "provenance": "measured" }, "tokens_per_word": { "value": 3.2000607032909856, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.706494199226563, "provenance": "measured" }, "chars_per_token": { "value": 4.687354805202097, "provenance": "measured" }, "tokens_per_word": { "value": 1.3047272527479237, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 3.9332869090382587, "provenance": "measured" }, "chars_per_token": { "value": 2.1812546970125277, "provenance": "measured" }, "tokens_per_word": { "value": 2.8052879965958764, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 3.2624745282963468, "provenance": "measured" }, "chars_per_token": { "value": 1.2255310152859584, "provenance": "measured" }, "tokens_per_word": { "value": 5.845158411423472, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.0657234674845455, "provenance": "measured" }, "chars_per_token": { "value": 2.9883437583592585, "provenance": "measured" }, "tokens_per_word": { "value": 2.883782612830619, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.4793281405263325, "provenance": "measured" }, "chars_per_token": { "value": 3.4019296727249415, "provenance": "measured" }, "tokens_per_word": { "value": 1.8347722781594054, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.443227044681639, "provenance": "measured" }, "chars_per_token": { "value": 3.333533115593118, "provenance": "measured" }, "tokens_per_word": { "value": 1.925480738060969, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 2.3423546881816137, "provenance": "measured" }, "chars_per_token": { "value": 0.9100787422392407, "provenance": "measured" }, "tokens_per_word": { "value": 6.998797261456856, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 3.6923787563005335, "provenance": "measured" }, "chars_per_token": { "value": 1.465788656822314, "provenance": "measured" }, "tokens_per_word": { "value": 3.771490690154978, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.3625065467877095, "provenance": "measured" }, "chars_per_token": { "value": 3.354699502444134, "provenance": "measured" }, "tokens_per_word": { "value": 2.2246820079619383, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.7223095051060486, "provenance": "measured" }, "chars_per_token": { "value": 1.3060664897953134, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.1227407623273047, "provenance": "measured" }, "chars_per_token": { "value": 1.3476861956786783, "provenance": "measured" }, "tokens_per_word": { "value": 3.3246267918932277, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 3.4610321439918224, "provenance": "measured" }, "chars_per_token": { "value": 1.3150995261972842, "provenance": "measured" }, "tokens_per_word": { "value": 5.336355594869527, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 4.369170881760668, "provenance": "measured" }, "chars_per_token": { "value": 2.403880000634246, "provenance": "measured" }, "tokens_per_word": { "value": 3.0783911748913946, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 4.0135270223096695, "provenance": "measured" }, "chars_per_token": { "value": 1.5291326838246961, "provenance": "measured" }, "tokens_per_word": { "value": 6.25901281159996, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 2.412364149881206, "provenance": "measured" }, "chars_per_token": { "value": 0.9190099520051906, "provenance": "measured" }, "tokens_per_word": { "value": 8.686891292635417, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 5.1233426138885525, "provenance": "measured" }, "chars_per_token": { "value": 1.8317959891861357, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.020843131768362, "provenance": "measured" }, "chars_per_token": { "value": 2.761438122510701, "provenance": "measured" }, "tokens_per_word": { "value": 2.826285945045515, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 2.970940672698391, "provenance": "measured" }, "chars_per_token": { "value": 1.6779934157515481, "provenance": "measured" }, "tokens_per_word": { "value": 2.8512726258266734, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 3.578292902066487, "provenance": "measured" }, "chars_per_token": { "value": 2.7395567535190177, "provenance": "measured" }, "tokens_per_word": { "value": 1.6769624830495706, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.013064309967613, "provenance": "measured" }, "chars_per_token": { "value": 1.5072731977760874, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 88615, "provenance": "measured" }, "tokens_per_request_mean": { "value": 443.075, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.443075, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 64276, "provenance": "measured" }, "tokens_per_request_mean": { "value": 394.3312883435583, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3943312883435583, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 36821, "provenance": "measured" }, "tokens_per_request_mean": { "value": 193.79473684210527, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.19379473684210524, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 235462, "provenance": "measured" }, "tokens_per_request_mean": { "value": 560.6238095238095, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5606238095238096, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 136638, "provenance": "measured" }, "tokens_per_request_mean": { "value": 910.92, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.91092, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 382925, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1914.625, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.914625, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 325834, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1629.17, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.62917, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 421768, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2108.84, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.10884, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8458, "provenance": "measured" }, "tokens_per_request_mean": { "value": 338.32, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33832, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 656516, "provenance": "measured" }, "tokens_per_request_mean": { "value": 109419.33333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 109.41933333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 120832, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 247463936, "provenance": "estimated", "note": "120832 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 494927872, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 494927872, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 989855744, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 120832, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 494927872, "provenance": "estimated", "note": "120832 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 989855744, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 989855744, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1979711488, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 989855744, "provenance": "estimated", "note": "120832 rows x 4096 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 1979711488, "provenance": "estimated", "note": "120832 rows x 4096 hidden x 2 matrices from the published config" } } } }, { "id": "lfm3-1-7b", "label": "LFM3-1.7B", "provider": "Liquid AI", "family": "published", "metrics": { "bpt_overall": { "value": 4.213314950355037, "provenance": "measured" }, "bpt_english": { "value": 4.628380539137757, "provenance": "measured" }, "bpt_code": { "value": 3.593696640402843, "provenance": "measured" }, "bpt_languages": { "value": 4.417867671524511, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.5953536933417769, "provenance": "measured" }, "byte_fallback_share": { "value": 0.01021076196166285, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.983365371128134, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.806476507486226, "q3": 5.34534948371401, "n": 5 }, "batch_mbps_4t": { "value": 16.76986979818239, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.38202464429069, "q3": 16.853742072768885, "n": 5 }, "latency_small_p95_us": { "value": 64.41699997594696, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 64.37499996536644, "q3": 66.83300011900428, "n": 5 }, "decode_mbps": { "value": 37.9316281183514, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.66446508870162, "q3": 39.84568250166923, "n": 5 }, "rss_load_delta_mb": { "value": 168.439808, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 168.439808, "q3": 168.443904, "n": 5 }, "rss_peak_large_mb": { "value": 257.810432, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 256.344064, "q3": 257.859584, "n": 5 }, "load_seconds": { "value": 0.3344739170001958, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.32873833299981925, "q3": 0.3375459169999431, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6312771478600606, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2572.292096, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2570.715136, "q3": 2572.816384, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.012288, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.012288, "q3": 0.012288, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.1458846782303604, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0318225899610045, "q3": 3.3269232047234265, "n": 5 }, "embedding_params_h2048": { "value": 262144000, "provenance": "estimated", "note": "128000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.2140475436027763, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.21235734069974735, "q3": 0.2172071668036425, "n": 5 }, "cpu_cores_used_batch": { "value": 3.579015847412648, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.55829315949383, "q3": 3.5883161772007197, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 140.4375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 140.343505859375, "q3": 140.46875, "n": 5 }, "workload_tokens_total": { "value": 2217620, "provenance": "measured" }, "tokens_chat_en": { "value": 89962, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 57513, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 30569, "provenance": "measured" }, "tokens_code": { "value": 243454, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 129918, "provenance": "measured" }, "tokens_rag": { "value": 304589, "provenance": "measured" }, "tokens_json": { "value": 324359, "provenance": "measured" }, "tokens_logs": { "value": 431342, "provenance": "measured" }, "tokens_tech_docs": { "value": 8563, "provenance": "measured" }, "tokens_long_context": { "value": 597351, "provenance": "measured" } }, "parity": { "de": { "value": 1.3561334714412547, "provenance": "measured" }, "fr": { "value": 1.3894643385617047, "provenance": "measured" }, "es": { "value": 1.3825836046167506, "provenance": "measured" }, "ru": { "value": 1.6612163361941403, "provenance": "measured" }, "zh": { "value": 1.2347957975732464, "provenance": "measured" }, "ja": { "value": 1.313739271973957, "provenance": "measured" }, "ko": { "value": 1.387984610831607, "provenance": "measured" }, "ar": { "value": 1.3560224918614976, "provenance": "measured" }, "ur": { "value": 2.0575984018940514, "provenance": "measured" }, "hi": { "value": 2.5295205682154482, "provenance": "measured" }, "mr": { "value": 2.7226620301864455, "provenance": "measured" }, "gu": { "value": 2.7024637466706127, "provenance": "measured" }, "bn": { "value": 2.869303048239124, "provenance": "measured" }, "ta": { "value": 3.1950281148268718, "provenance": "measured" }, "te": { "value": 3.0483870967741935, "provenance": "measured" }, "th": { "value": 2.073061556673572, "provenance": "measured" }, "vi": { "value": 1.4082198875406926, "provenance": "measured" }, "tr": { "value": 1.5172758212488902, "provenance": "measured" }, "id": { "value": 1.3054897898786624, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.92430278884462, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 36.51294820717131, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3561334714412547, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 37.41035856573705, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3894643385617047, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 37.22509960159363, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3825836046167506, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 44.72709163346614, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6612163361941403, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 33.24601593625498, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2347957975732464, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 35.37151394422311, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.313739271973957, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 37.37051792828685, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.387984610831607, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 36.50996015936255, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3560224918614976, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 55.39940239043825, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0575984018940514, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 68.10557768924303, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.5295205682154482, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 73.30577689243027, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.7226620301864455, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 72.76195219123505, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.7024637466706127, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 77.25398406374502, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.869303048239124, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 86.02390438247012, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.1950281148268718, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 82.07569721115537, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.0483870967741935, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 55.81573705179283, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.073061556673572, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 37.91533864541832, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4082198875406926, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 40.851593625498005, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5172758212488902, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 35.14940239043825, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3054897898786624, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 128,000 embedding rows for 125,033 tokenizer IDs (2,967 unused rows, usually padding)" ], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>", "<|im_start|>user", "<|im_end|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 125033, "provenance": "measured" }, "used_share": { "value": 0.7143554101717147, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 89318, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 35715, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 45, "provenance": "measured", "note": "of 123353 entries whose text is valid UTF-8; 1556 are byte pieces" }, "unreachable_examples": [ " n't", " e-mail", " long-term", " so-called", " African-American", " T-shirt", " and/or", " health-care", " full-time", " o'clock", " self-esteem", " short-term" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.3344739170001958, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.32873833299981925, "q3": 0.3375459169999431, "n": 5 }, "rss_load_delta_mb": { "value": 168.439808, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 168.439808, "q3": 168.443904, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 168.349696, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 168.349696, "q3": 168.353792, "n": 5 }, "latency_small_p50_us": { "value": 48.16600016965822, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 47.99999987881165, "q3": 49.541999942448456, "n": 5 }, "latency_small_p95_us": { "value": 64.41699997594696, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 64.37499996536644, "q3": 66.83300011900428, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.983365371128134, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.806476507486226, "q3": 5.34534948371401, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1284840.909129582, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1239234.3698833706, "q3": 1378169.8067054818, "n": 5 }, "encode_mbps_1t_large": { "value": 4.296459787527361, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.228788868697586, "q3": 4.351793434053933, "n": 5 }, "latency_large_p50_ms": { "value": 238.96045800006505, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 238.4087499999623, "q3": 242.00387500013676, "n": 5 }, "decode_mbps": { "value": 37.9316281183514, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.66446508870162, "q3": 39.84568250166923, "n": 5 }, "decode_tokens_per_s": { "value": 9569526.842332961, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9502125.997435516, "q3": 10052411.32969783, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.2915600815806405, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.138940081685252, "q3": 5.331643050204444, "n": 5 }, "batch_mbps_4t": { "value": 16.76986979818239, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.38202464429069, "q3": 16.853742072768885, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4297049.953909084, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4197669.933628019, "q3": 4318541.078049189, "n": 5 }, "batch_cpu_utilization": { "value": 3.579015847412648, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.55829315949383, "q3": 3.5883161772007197, "n": 5 }, "cpu_cores_used_batch": { "value": 3.579015847412648, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.55829315949383, "q3": 3.5883161772007197, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.2140475436027763, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.21235734069974735, "q3": 0.2172071668036425, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 140.4375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 140.343505859375, "q3": 140.46875, "n": 5 }, "rss_peak_large_mb": { "value": 257.810432, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 256.344064, "q3": 257.859584, "n": 5 }, "rss_peak_very_large_mb": { "value": 2572.292096, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2570.715136, "q3": 2572.816384, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.012288, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.012288, "q3": 0.012288, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.1458846782303604, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0318225899610045, "q3": 3.3269232047234265, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6312771478600606, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.299345566412977, "provenance": "measured" }, "chars_per_token": { "value": 2.2972742156941384, "provenance": "measured" }, "tokens_per_word": { "value": 2.8676056719164364, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.143712313204579, "provenance": "measured" }, "chars_per_token": { "value": 3.1311594112433054, "provenance": "measured" }, "tokens_per_word": { "value": 2.6970487718648464, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.536894984652039, "provenance": "measured" }, "chars_per_token": { "value": 3.536327708899296, "provenance": "measured" }, "tokens_per_word": { "value": 2.980014650913804, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.5342239539846, "provenance": "measured" }, "chars_per_token": { "value": 4.509101029780128, "provenance": "measured" }, "tokens_per_word": { "value": 2.707271128971493, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.8427811817195594, "provenance": "measured" }, "chars_per_token": { "value": 3.8351872507525173, "provenance": "measured" }, "tokens_per_word": { "value": 2.783675726255795, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.2392588040399724, "provenance": "measured" }, "chars_per_token": { "value": 3.236673702773473, "provenance": "measured" }, "tokens_per_word": { "value": 2.5947032723239047, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.3660381211708645, "provenance": "measured" }, "chars_per_token": { "value": 3.313791695030633, "provenance": "measured" }, "tokens_per_word": { "value": 3.5794346978557505, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.239512244078683, "provenance": "measured" }, "chars_per_token": { "value": 4.237220661046434, "provenance": "measured" }, "tokens_per_word": { "value": 2.3284000623150023, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.7407684534507895, "provenance": "measured" }, "chars_per_token": { "value": 3.7268989563164836, "provenance": "measured" }, "tokens_per_word": { "value": 2.6940917309171013, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.153624128979369, "provenance": "measured" }, "chars_per_token": { "value": 4.139824429566881, "provenance": "measured" }, "tokens_per_word": { "value": 3.126441691584793, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.2977497474613218, "provenance": "measured" }, "chars_per_token": { "value": 3.271067042373332, "provenance": "measured" }, "tokens_per_word": { "value": 2.9372999141094716, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.7026169586204802, "provenance": "measured" }, "chars_per_token": { "value": 3.6973572361030995, "provenance": "measured" }, "tokens_per_word": { "value": 3.0932269045662637, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.5753757005928226, "provenance": "measured" }, "chars_per_token": { "value": 3.5750381502239788, "provenance": "measured" }, "tokens_per_word": { "value": 3.025038291354663, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.8632438367181203, "provenance": "measured" }, "chars_per_token": { "value": 3.8526844056698053, "provenance": "measured" }, "tokens_per_word": { "value": 3.07706631245034, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.8605296897322248, "provenance": "measured" }, "chars_per_token": { "value": 2.8471006263965712, "provenance": "measured" }, "tokens_per_word": { "value": 3.1381768019312566, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.811879665721029, "provenance": "measured" }, "chars_per_token": { "value": 3.7305394745946443, "provenance": "measured" }, "tokens_per_word": { "value": 2.739374921022703, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.826174597050417, "provenance": "measured" }, "chars_per_token": { "value": 3.82488326863568, "provenance": "measured" }, "tokens_per_word": { "value": 2.6403534303534304, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.094674367197782, "provenance": "measured" }, "chars_per_token": { "value": 3.0890459093058023, "provenance": "measured" }, "tokens_per_word": { "value": 2.8239517933174567, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.205660488455927, "provenance": "measured" }, "chars_per_token": { "value": 3.1930401899914296, "provenance": "measured" }, "tokens_per_word": { "value": 2.8113780164802824, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.16550608247772, "provenance": "measured" }, "chars_per_token": { "value": 4.149893260140287, "provenance": "measured" }, "tokens_per_word": { "value": 2.537380164223378, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.9680585627384266, "provenance": "measured" }, "chars_per_token": { "value": 3.8830876035330104, "provenance": "measured" }, "tokens_per_word": { "value": 3.1540779603694227, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.628380539137757, "provenance": "measured" }, "chars_per_token": { "value": 4.609558801537764, "provenance": "measured" }, "tokens_per_word": { "value": 1.3267472703907988, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 5.7950874626683735, "provenance": "measured" }, "chars_per_token": { "value": 3.213740070295199, "provenance": "measured" }, "tokens_per_word": { "value": 1.9040269235232679, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 4.45464823516205, "provenance": "measured" }, "chars_per_token": { "value": 1.673364658338278, "provenance": "measured" }, "tokens_per_word": { "value": 4.2808498953077265, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.63511112076644, "provenance": "measured" }, "chars_per_token": { "value": 3.5433599096261217, "provenance": "measured" }, "tokens_per_word": { "value": 2.432079718491435, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.077221266461825, "provenance": "measured" }, "chars_per_token": { "value": 3.9865225263126263, "provenance": "measured" }, "tokens_per_word": { "value": 1.565717041498071, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.988221804481954, "provenance": "measured" }, "chars_per_token": { "value": 3.861165495347223, "provenance": "measured" }, "tokens_per_word": { "value": 1.6623617432346571, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 4.302836282397889, "provenance": "measured" }, "chars_per_token": { "value": 1.6717877321072996, "provenance": "measured" }, "tokens_per_word": { "value": 3.809967310183186, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 4.812694467724379, "provenance": "measured" }, "chars_per_token": { "value": 1.9105279889026963, "provenance": "measured" }, "tokens_per_word": { "value": 2.8935500055747574, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.7876733795883752, "provenance": "measured" }, "chars_per_token": { "value": 3.7788791858457116, "provenance": "measured" }, "tokens_per_word": { "value": 1.974961161277794, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.208823529411765, "provenance": "measured" }, "chars_per_token": { "value": 1.4767722473604827, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.706058809412894, "provenance": "measured" }, "chars_per_token": { "value": 1.5994296926834088, "provenance": "measured" }, "tokens_per_word": { "value": 2.801344537815126, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 4.678845202192323, "provenance": "measured" }, "chars_per_token": { "value": 1.7778358745483236, "provenance": "measured" }, "tokens_per_word": { "value": 3.947405277900634, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.529145597559997, "provenance": "measured" }, "chars_per_token": { "value": 3.042088048800064, "provenance": "measured" }, "tokens_per_word": { "value": 2.432566993703324, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 4.760471265773947, "provenance": "measured" }, "chars_per_token": { "value": 1.8137145115605033, "provenance": "measured" }, "tokens_per_word": { "value": 5.276939120071507, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 4.221984720076817, "provenance": "measured" }, "chars_per_token": { "value": 1.6083997829081953, "provenance": "measured" }, "tokens_per_word": { "value": 4.9635293630071695, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 6.369737528594936, "provenance": "measured" }, "chars_per_token": { "value": 2.2774310711562387, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.480474939466482, "provenance": "measured" }, "chars_per_token": { "value": 3.181600554233231, "provenance": "measured" }, "tokens_per_word": { "value": 2.4530463899311807, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 4.156996174753372, "provenance": "measured" }, "chars_per_token": { "value": 2.347880008053151, "provenance": "measured" }, "tokens_per_word": { "value": 2.037760309828183, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.388132717296293, "provenance": "measured" }, "chars_per_token": { "value": 3.359573671027406, "provenance": "measured" }, "tokens_per_word": { "value": 1.3674752649289337, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.585321255175603, "provenance": "measured" }, "chars_per_token": { "value": 1.3466165044802678, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 89962, "provenance": "measured" }, "tokens_per_request_mean": { "value": 449.81, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.44981, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 57513, "provenance": "measured" }, "tokens_per_request_mean": { "value": 352.840490797546, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.352840490797546, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 30569, "provenance": "measured" }, "tokens_per_request_mean": { "value": 160.88947368421051, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.1608894736842105, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 243454, "provenance": "measured" }, "tokens_per_request_mean": { "value": 579.652380952381, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.579652380952381, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 129918, "provenance": "measured" }, "tokens_per_request_mean": { "value": 866.12, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.86612, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 304589, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1522.945, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.522945, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 324359, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1621.795, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.621795, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 431342, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2156.71, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.15671, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8563, "provenance": "measured" }, "tokens_per_request_mean": { "value": 342.52, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.34252, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 597351, "provenance": "measured" }, "tokens_per_request_mean": { "value": 99558.5, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 99.5585, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 128000, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 262144000, "provenance": "estimated", "note": "128000 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 524288000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 524288000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1048576000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 128000, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 524288000, "provenance": "estimated", "note": "128000 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1048576000, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1048576000, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2097152000, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 262144000, "provenance": "estimated", "note": "128000 rows x 2048 hidden x 1 matrix from the published config" }, "bf16_bytes": { "value": 524288000, "provenance": "estimated", "note": "128000 rows x 2048 hidden x 1 matrix from the published config" } } } }, { "id": "llama-3-2-1b", "label": "Llama-3.2-1B", "provider": "Meta", "family": "published", "metrics": { "bpt_overall": { "value": 4.080155649986431, "provenance": "measured" }, "bpt_english": { "value": 4.692945970975816, "provenance": "measured" }, "bpt_code": { "value": 3.890744681590178, "provenance": "measured" }, "bpt_languages": { "value": 3.6567762973932996, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.8644443457955898, "provenance": "measured" }, "byte_fallback_share": { "value": 0.18432713640698287, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.979777107529496, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.972462633823275, "q3": 4.98280743289368, "n": 5 }, "batch_mbps_4t": { "value": 16.618196262248443, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.467326111026395, "q3": 16.677035908702184, "n": 5 }, "latency_small_p95_us": { "value": 67.41700008205953, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 66.62499981757719, "q3": 67.95799993142282, "n": 5 }, "decode_mbps": { "value": 39.064492844360885, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 38.48001422683172, "q3": 39.0731060065189, "n": 5 }, "rss_load_delta_mb": { "value": 179.073024, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 179.068928, "q3": 179.073024, "n": 5 }, "rss_peak_large_mb": { "value": 239.906816, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 239.53408, "q3": 240.164864, "n": 5 }, "load_seconds": { "value": 0.3169502499999908, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.3169401669999843, "q3": 0.3193151250000028, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6091104358829695, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2636.156928, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2636.136448, "q3": 2637.656064, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.997824, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.739776, "q3": 18.235392, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.0332342045673246, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0260119679495894, "q3": 3.0477949419910066, "n": 5 }, "embedding_params_h2048": { "value": 262668288, "provenance": "estimated", "note": "128256 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.21540275777978535, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.21429717577084959, "q3": 0.21770666956592852, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5796053042140357, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.571813812405879, "q3": 3.58504672428761, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 144.03564453125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 144.034423828125, "q3": 144.125, "n": 5 }, "workload_tokens_total": { "value": 2369272, "provenance": "measured" }, "tokens_chat_en": { "value": 88175, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 60394, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 40321, "provenance": "measured" }, "tokens_code": { "value": 222746, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 138670, "provenance": "measured" }, "tokens_rag": { "value": 412580, "provenance": "measured" }, "tokens_json": { "value": 322854, "provenance": "measured" }, "tokens_logs": { "value": 410572, "provenance": "measured" }, "tokens_tech_docs": { "value": 8305, "provenance": "measured" }, "tokens_long_context": { "value": 664655, "provenance": "measured" } }, "parity": { "de": { "value": 1.577466376136449, "provenance": "measured" }, "fr": { "value": 1.5858065970395974, "provenance": "measured" }, "es": { "value": 1.5353144488691863, "provenance": "measured" }, "ru": { "value": 1.652039972950635, "provenance": "measured" }, "zh": { "value": 1.3015252836426479, "provenance": "measured" }, "ja": { "value": 1.529153204598392, "provenance": "measured" }, "ko": { "value": 1.5006010970020287, "provenance": "measured" }, "ar": { "value": 1.6841610939965437, "provenance": "measured" }, "ur": { "value": 3.1193553234653244, "provenance": "measured" }, "hi": { "value": 2.5478247802239085, "provenance": "measured" }, "mr": { "value": 2.80114959801638, "provenance": "measured" }, "gu": { "value": 7.6952062514088215, "provenance": "measured" }, "bn": { "value": 5.807423547975055, "provenance": "measured" }, "ta": { "value": 7.66733037793974, "provenance": "measured" }, "te": { "value": 8.428732436696972, "provenance": "measured" }, "th": { "value": 2.1982117364189646, "provenance": "measured" }, "vi": { "value": 1.3878578405590203, "provenance": "measured" }, "tr": { "value": 1.407393493124953, "provenance": "measured" }, "id": { "value": 1.5418889473288753, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.51195219123506, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 41.82171314741036, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.577466376136449, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 42.04282868525896, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5858065970395974, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 40.70418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5353144488691863, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 43.798804780876495, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.652039972950635, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 34.50597609561753, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3015252836426479, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 40.54083665338646, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.529153204598392, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 39.78386454183267, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5006010970020287, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 44.6503984063745, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6841610939965437, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 82.70019920318725, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.1193553234653244, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 67.54780876494024, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.5478247802239085, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 74.26394422310757, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.80114959801638, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 204.01494023904382, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.6952062514088215, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 153.96613545816734, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.807423547975055, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 203.27589641434264, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.66733037793974, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 223.4621513944223, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.428732436696972, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 58.278884462151396, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.1982117364189646, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 36.79482071713147, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3878578405590203, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 37.31274900398407, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.407393493124953, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 40.87848605577689, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5418889473288753, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|begin_of_text|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 128256, "provenance": "measured" }, "used_share": { "value": 0.774996881237525, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 99398, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 28858, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 126648 entries whose text is valid UTF-8; 1352 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.3169502499999908, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.3169401669999843, "q3": 0.3193151250000028, "n": 5 }, "rss_load_delta_mb": { "value": 179.073024, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 179.068928, "q3": 179.073024, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 179.003392, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 178.999296, "q3": 179.003392, "n": 5 }, "latency_small_p50_us": { "value": 50.1670001540333, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 50.10450001918798, "q3": 50.771000019267376, "n": 5 }, "latency_small_p95_us": { "value": 67.41700008205953, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 66.62499981757719, "q3": 67.95799993142282, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.979777107529496, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.972462633823275, "q3": 4.98280743289368, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1378719.6649051998, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1376694.5524314002, "q3": 1379558.6520888521, "n": 5 }, "encode_mbps_1t_large": { "value": 4.039969658804392, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.034910027318532, "q3": 4.0939816149147745, "n": 5 }, "latency_large_p50_ms": { "value": 257.44954099991446, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 251.61479100006545, "q3": 257.71491699970284, "n": 5 }, "decode_mbps": { "value": 39.064492844360885, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 38.48001422683172, "q3": 39.0731060065189, "n": 5 }, "decode_tokens_per_s": { "value": 11220981.945248764, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 11053094.855537692, "q3": 11223456.011338694, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.0042838634082125, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.981346887315098, "q3": 5.176442608793633, "n": 5 }, "batch_mbps_4t": { "value": 16.618196262248443, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.467326111026395, "q3": 16.677035908702184, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4601665.912195805, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4559889.174149725, "q3": 4617958.91963771, "n": 5 }, "batch_cpu_utilization": { "value": 3.5796053042140357, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.571813812405879, "q3": 3.58504672428761, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5796053042140357, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.571813812405879, "q3": 3.58504672428761, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.21540275777978535, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.21429717577084959, "q3": 0.21770666956592852, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 144.03564453125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 144.034423828125, "q3": 144.125, "n": 5 }, "rss_peak_large_mb": { "value": 239.906816, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 239.53408, "q3": 240.164864, "n": 5 }, "rss_peak_very_large_mb": { "value": 2636.156928, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2636.136448, "q3": 2637.656064, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.997824, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.739776, "q3": 18.235392, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.0332342045673246, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0260119679495894, "q3": 3.0477949419910066, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6091104358829695, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.499787119443946, "provenance": "measured" }, "chars_per_token": { "value": 2.4975352022365285, "provenance": "measured" }, "tokens_per_word": { "value": 2.6376711587378905, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.389706278268361, "provenance": "measured" }, "chars_per_token": { "value": 3.3761711178117597, "provenance": "measured" }, "tokens_per_word": { "value": 2.50132157107022, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.9067010129843003, "provenance": "measured" }, "chars_per_token": { "value": 3.9060744247571972, "provenance": "measured" }, "tokens_per_word": { "value": 2.6979282105223614, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.921595875375103, "provenance": "measured" }, "chars_per_token": { "value": 4.894326626789922, "provenance": "measured" }, "tokens_per_word": { "value": 2.4941856084390306, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.208475568942436, "provenance": "measured" }, "chars_per_token": { "value": 4.200158969210174, "provenance": "measured" }, "tokens_per_word": { "value": 2.54178894985326, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.5306923340153045, "provenance": "measured" }, "chars_per_token": { "value": 3.527874653250526, "provenance": "measured" }, "tokens_per_word": { "value": 2.3805289794786466, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.5691569221885375, "provenance": "measured" }, "chars_per_token": { "value": 3.5137577594918437, "provenance": "measured" }, "tokens_per_word": { "value": 3.375730994152047, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.582900125667893, "provenance": "measured" }, "chars_per_token": { "value": 4.580422931226211, "provenance": "measured" }, "tokens_per_word": { "value": 2.1539375292101575, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.0385246379466855, "provenance": "measured" }, "chars_per_token": { "value": 4.023551162151583, "provenance": "measured" }, "tokens_per_word": { "value": 2.4954591741308008, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.634821717835839, "provenance": "measured" }, "chars_per_token": { "value": 4.619423322470604, "provenance": "measured" }, "tokens_per_word": { "value": 2.8018475010679196, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.632456609104951, "provenance": "measured" }, "chars_per_token": { "value": 3.603065728696187, "provenance": "measured" }, "tokens_per_word": { "value": 2.666647146091981, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.973301840690666, "provenance": "measured" }, "chars_per_token": { "value": 3.967657598957485, "provenance": "measured" }, "tokens_per_word": { "value": 2.8824979457682827, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.7697748910045377, "provenance": "measured" }, "chars_per_token": { "value": 3.7694189874544, "provenance": "measured" }, "tokens_per_word": { "value": 2.8690435670524166, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.202333704273097, "provenance": "measured" }, "chars_per_token": { "value": 4.190847436548931, "provenance": "measured" }, "tokens_per_word": { "value": 2.8287752242662445, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.075338095950725, "provenance": "measured" }, "chars_per_token": { "value": 3.0609005915202308, "provenance": "measured" }, "tokens_per_word": { "value": 2.9189791930106908, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.0796738334444145, "provenance": "measured" }, "chars_per_token": { "value": 3.9926192885882847, "provenance": "measured" }, "tokens_per_word": { "value": 2.5595594119877005, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.182076372931244, "provenance": "measured" }, "chars_per_token": { "value": 4.180664928179839, "provenance": "measured" }, "tokens_per_word": { "value": 2.415654885654886, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.3949887860947574, "provenance": "measured" }, "chars_per_token": { "value": 3.3888141295206053, "provenance": "measured" }, "tokens_per_word": { "value": 2.574150248971639, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.3850458379677923, "provenance": "measured" }, "chars_per_token": { "value": 3.371719321031579, "provenance": "measured" }, "tokens_per_word": { "value": 2.6623933195997647, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.600608158682634, "provenance": "measured" }, "chars_per_token": { "value": 4.583364520958084, "provenance": "measured" }, "tokens_per_word": { "value": 2.2974076780877866, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.127678590580513, "provenance": "measured" }, "chars_per_token": { "value": 4.039289570209998, "provenance": "measured" }, "tokens_per_word": { "value": 3.032107704982006, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.692945970975816, "provenance": "measured" }, "chars_per_token": { "value": 4.6738616720747785, "provenance": "measured" }, "tokens_per_word": { "value": 1.3084939150394828, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 4.782134893624155, "provenance": "measured" }, "chars_per_token": { "value": 2.65199423273593, "provenance": "measured" }, "tokens_per_word": { "value": 2.307338207419442, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 2.21287197728006, "provenance": "measured" }, "chars_per_token": { "value": 0.8312534603695575, "provenance": "measured" }, "tokens_per_word": { "value": 8.617615762194076, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.1510283865573436, "provenance": "measured" }, "chars_per_token": { "value": 3.0714955576563105, "provenance": "measured" }, "tokens_per_word": { "value": 2.8057125949721557, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.7222267945077707, "provenance": "measured" }, "chars_per_token": { "value": 3.639424989369436, "provenance": "measured" }, "tokens_per_word": { "value": 1.715041874470688, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.563850030320139, "provenance": "measured" }, "chars_per_token": { "value": 3.4503133081014017, "provenance": "measured" }, "tokens_per_word": { "value": 1.860310421286031, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.593778763847872, "provenance": "measured" }, "chars_per_token": { "value": 0.619233363814892, "provenance": "measured" }, "tokens_per_word": { "value": 10.286035897119595, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 4.8863600798090845, "provenance": "measured" }, "chars_per_token": { "value": 1.939771526935566, "provenance": "measured" }, "tokens_per_word": { "value": 2.8499275281525254, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.3213556289642017, "provenance": "measured" }, "chars_per_token": { "value": 3.3136441285357847, "provenance": "measured" }, "tokens_per_word": { "value": 2.252245363627537, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.858298382264829, "provenance": "measured" }, "chars_per_token": { "value": 1.3537816287966078, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.583854916762793, "provenance": "measured" }, "chars_per_token": { "value": 1.5466899644390446, "provenance": "measured" }, "tokens_per_word": { "value": 2.8968660405338604, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 4.655243209096652, "provenance": "measured" }, "chars_per_token": { "value": 1.7688677492475195, "provenance": "measured" }, "tokens_per_word": { "value": 3.9674185463659146, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.47846790067002, "provenance": "measured" }, "chars_per_token": { "value": 3.014205618625365, "provenance": "measured" }, "tokens_per_word": { "value": 2.4550690681895837, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.048549075506386, "provenance": "measured" }, "chars_per_token": { "value": 0.780486422132774, "provenance": "measured" }, "tokens_per_word": { "value": 12.262687456549806, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.574005129993027, "provenance": "measured" }, "chars_per_token": { "value": 0.5996301922502242, "provenance": "measured" }, "tokens_per_word": { "value": 13.31377180985536, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 6.2810063714432705, "provenance": "measured" }, "chars_per_token": { "value": 2.2457062012742885, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.7997274184127363, "provenance": "measured" }, "chars_per_token": { "value": 3.473438272252093, "provenance": "measured" }, "tokens_per_word": { "value": 2.246941831703993, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 2.929181444176479, "provenance": "measured" }, "chars_per_token": { "value": 1.6544077174067244, "provenance": "measured" }, "tokens_per_word": { "value": 2.8919211644813494, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.549959252686657, "provenance": "measured" }, "chars_per_token": { "value": 3.4834687768951307, "provenance": "measured" }, "tokens_per_word": { "value": 1.3188388328059866, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.4868499945492206, "provenance": "measured" }, "chars_per_token": { "value": 1.3096315273084052, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 88175, "provenance": "measured" }, "tokens_per_request_mean": { "value": 440.875, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.440875, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 60394, "provenance": "measured" }, "tokens_per_request_mean": { "value": 370.5153374233129, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3705153374233129, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 40321, "provenance": "measured" }, "tokens_per_request_mean": { "value": 212.2157894736842, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.2122157894736842, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 222746, "provenance": "measured" }, "tokens_per_request_mean": { "value": 530.347619047619, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.530347619047619, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 138670, "provenance": "measured" }, "tokens_per_request_mean": { "value": 924.4666666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9244666666666668, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 412580, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2062.9, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.0629, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 322854, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1614.27, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.61427, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 410572, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2052.86, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.0528600000000004, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8305, "provenance": "measured" }, "tokens_per_request_mean": { "value": 332.2, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3322, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 664655, "provenance": "measured" }, "tokens_per_request_mean": { "value": 110775.83333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 110.77583333333332, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 128256, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 262668288, "provenance": "estimated", "note": "128256 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 525336576, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 525336576, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1050673152, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 128256, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 525336576, "provenance": "estimated", "note": "128256 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1050673152, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1050673152, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2101346304, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 262668288, "provenance": "estimated", "note": "128256 rows x 2048 hidden x 1 matrix from the published config" }, "bf16_bytes": { "value": 525336576, "provenance": "estimated", "note": "128256 rows x 2048 hidden x 1 matrix from the published config" } } } }, { "id": "deepseek-v3", "label": "DeepSeek-V3", "provider": "DeepSeek", "family": "published", "metrics": { "bpt_overall": { "value": 4.201591017457798, "provenance": "measured" }, "bpt_english": { "value": 4.7316667019467555, "provenance": "measured" }, "bpt_code": { "value": 3.610433106937808, "provenance": "measured" }, "bpt_languages": { "value": 4.26267324348883, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.6862448561887082, "provenance": "measured" }, "byte_fallback_share": { "value": 0.03423042314128435, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.593484509872189, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.561213481913486, "q3": 4.6270812985391325, "n": 5 }, "batch_mbps_4t": { "value": 14.352636211190546, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.279768807509049, "q3": 14.46610265055963, "n": 5 }, "latency_small_p95_us": { "value": 89.08300003440672, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 87.54200007388135, "q3": 93.16700015915558, "n": 5 }, "decode_mbps": { "value": 40.39197598096321, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.18262663395477, "q3": 40.96878937743569, "n": 5 }, "rss_load_delta_mb": { "value": 83.18976, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 83.18976, "q3": 83.18976, "n": 5 }, "rss_peak_large_mb": { "value": 199.45472, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 198.688768, "q3": 199.745536, "n": 5 }, "load_seconds": { "value": 0.14764133300013782, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.14453125, "q3": 0.15512591700002076, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6314681330915962, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2510.675968, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2510.630912, "q3": 2511.2576, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.928192, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.748544, "q3": 18.108416, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.9006390878341572, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.8903821960867466, "q3": 2.904593397858615, "n": 5 }, "embedding_params_h2048": { "value": 264765440, "provenance": "estimated", "note": "129280 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.24990969609965827, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.24932316779644886, "q3": 0.2519885336100984, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6064276215178492, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.596928755710072, "q3": 3.6096504361907678, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 144.1591796875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 144.15673828125003, "q3": 144.19384765625, "n": 5 }, "workload_tokens_total": { "value": 2252673, "provenance": "measured" }, "tokens_chat_en": { "value": 88424, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 57084, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 31914, "provenance": "measured" }, "tokens_code": { "value": 241458, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 132588, "provenance": "measured" }, "tokens_rag": { "value": 325542, "provenance": "measured" }, "tokens_json": { "value": 326337, "provenance": "measured" }, "tokens_logs": { "value": 427183, "provenance": "measured" }, "tokens_tech_docs": { "value": 8595, "provenance": "measured" }, "tokens_long_context": { "value": 613548, "provenance": "measured" } }, "parity": { "de": { "value": 1.5118101337351666, "provenance": "measured" }, "fr": { "value": 1.5323413072141647, "provenance": "measured" }, "es": { "value": 1.488415897532492, "provenance": "measured" }, "ru": { "value": 1.6201544546995668, "provenance": "measured" }, "zh": { "value": 0.95792051233754, "provenance": "measured" }, "ja": { "value": 1.4427952533433792, "provenance": "measured" }, "ko": { "value": 1.6690525522697306, "provenance": "measured" }, "ar": { "value": 1.6436617065360708, "provenance": "measured" }, "ur": { "value": 2.3978527029572425, "provenance": "measured" }, "hi": { "value": 2.8304388773780373, "provenance": "measured" }, "mr": { "value": 3.0215859860614054, "provenance": "measured" }, "gu": { "value": 3.882765115841025, "provenance": "measured" }, "bn": { "value": 2.0414013938594837, "provenance": "measured" }, "ta": { "value": 3.1009606328875496, "provenance": "measured" }, "te": { "value": 3.829346392917687, "provenance": "measured" }, "th": { "value": 1.7891128272744397, "provenance": "measured" }, "vi": { "value": 2.1093614616688643, "provenance": "measured" }, "tr": { "value": 2.0325485025428516, "provenance": "measured" }, "id": { "value": 1.345526464494255, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.439243027888445, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 39.971115537848604, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5118101337351666, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 40.51394422310757, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5323413072141647, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 39.352589641434264, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.488415897532492, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 42.83565737051793, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6201544546995668, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 25.326693227091635, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 0.95792051233754, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 38.146414342629484, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4427952533433792, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 44.12848605577689, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6690525522697306, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 43.45717131474104, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6436617065360708, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 63.397410358565736, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.3978527029572425, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 74.83466135458167, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.8304388773780373, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 79.88844621513944, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.0215859860614054, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 102.65737051792829, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.882765115841025, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 53.973107569721115, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0414013938594837, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 81.98705179282868, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.1009606328875496, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 101.24501992031873, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.829346392917687, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 47.30278884462152, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7891128272744397, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 55.7699203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.1093614616688643, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 53.7390438247012, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0325485025428516, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 35.57470119521912, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.345526464494255, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 129,280 embedding rows for 128,815 tokenizer IDs (465 unused rows, usually padding)" ], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 128815, "provenance": "measured" }, "used_share": { "value": 0.70105965920118, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 90307, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 38508, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 126541 entries whose text is valid UTF-8; 1470 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.14764133300013782, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.14453125, "q3": 0.15512591700002076, "n": 5 }, "rss_load_delta_mb": { "value": 83.18976, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 83.18976, "q3": 83.18976, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 83.116032, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 83.116032, "q3": 83.116032, "n": 5 }, "latency_small_p50_us": { "value": 60.312499954306986, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 60.166499906699755, "q3": 60.54149980627699, "n": 5 }, "latency_small_p95_us": { "value": 89.08300003440672, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 87.54200007388135, "q3": 93.16700015915558, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.593484509872189, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.561213481913486, "q3": 4.6270812985391325, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1211079.7095810394, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1202571.4002389458, "q3": 1219937.5579038474, "n": 5 }, "encode_mbps_1t_large": { "value": 3.765439984527799, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.721916910997946, "q3": 3.7756776515819834, "n": 5 }, "latency_large_p50_ms": { "value": 275.2391250000983, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 273.37929100008296, "q3": 279.85499999999774, "n": 5 }, "decode_mbps": { "value": 40.39197598096321, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.18262663395477, "q3": 40.96878937743569, "n": 5 }, "decode_tokens_per_s": { "value": 10546126.46140393, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10491466.479203409, "q3": 10696729.319424486, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.319676131675932, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.239847941838689, "q3": 4.368685421771682, "n": 5 }, "batch_mbps_4t": { "value": 14.352636211190546, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.279768807509049, "q3": 14.46610265055963, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3762913.4110619803, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3743809.343592577, "q3": 3792661.561863328, "n": 5 }, "batch_cpu_utilization": { "value": 3.6064276215178492, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.596928755710072, "q3": 3.6096504361907678, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6064276215178492, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.596928755710072, "q3": 3.6096504361907678, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.24990969609965827, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.24932316779644886, "q3": 0.2519885336100984, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 144.1591796875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 144.15673828125003, "q3": 144.19384765625, "n": 5 }, "rss_peak_large_mb": { "value": 199.45472, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 198.688768, "q3": 199.745536, "n": 5 }, "rss_peak_very_large_mb": { "value": 2510.675968, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2510.630912, "q3": 2511.2576, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.928192, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.748544, "q3": 18.108416, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.9006390878341572, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.8903821960867466, "q3": 2.904593397858615, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6314681330915962, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.341999913494394, "provenance": "measured" }, "chars_per_token": { "value": 2.3398901378803245, "provenance": "measured" }, "tokens_per_word": { "value": 2.8153785787736103, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.1621679318377316, "provenance": "measured" }, "chars_per_token": { "value": 3.14954133624668, "provenance": "measured" }, "tokens_per_word": { "value": 2.681307766124009, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.569790891336018, "provenance": "measured" }, "chars_per_token": { "value": 3.5692183394723616, "provenance": "measured" }, "tokens_per_word": { "value": 2.9525535791431095, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.580328944285648, "provenance": "measured" }, "chars_per_token": { "value": 4.554950564640833, "provenance": "measured" }, "tokens_per_word": { "value": 2.6800200929298, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.9267803331928115, "provenance": "measured" }, "chars_per_token": { "value": 3.919020406888691, "provenance": "measured" }, "tokens_per_word": { "value": 2.724129131045043, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.2676563142254538, "provenance": "measured" }, "chars_per_token": { "value": 3.265048550231464, "provenance": "measured" }, "tokens_per_word": { "value": 2.572154048807543, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.4194362612892966, "provenance": "measured" }, "chars_per_token": { "value": 3.3663610085335325, "provenance": "measured" }, "tokens_per_word": { "value": 3.5235380116959063, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.258495400512454, "provenance": "measured" }, "chars_per_token": { "value": 4.25619355651699, "provenance": "measured" }, "tokens_per_word": { "value": 2.318020719738277, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.757521753616563, "provenance": "measured" }, "chars_per_token": { "value": 3.7435901409432897, "provenance": "measured" }, "tokens_per_word": { "value": 2.682079843720776, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.174800446313621, "provenance": "measured" }, "chars_per_token": { "value": 4.160930392241009, "provenance": "measured" }, "tokens_per_word": { "value": 3.1105830841520716, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.3282738175086526, "provenance": "measured" }, "chars_per_token": { "value": 3.301344136506318, "provenance": "measured" }, "tokens_per_word": { "value": 2.9103615210431797, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.708781559561188, "provenance": "measured" }, "chars_per_token": { "value": 3.7035130799698943, "provenance": "measured" }, "tokens_per_word": { "value": 3.0880854560394413, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.576658459373901, "provenance": "measured" }, "chars_per_token": { "value": 3.5763207879001055, "provenance": "measured" }, "tokens_per_word": { "value": 3.0239533696392105, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.878219487098306, "provenance": "measured" }, "chars_per_token": { "value": 3.8676191230014245, "provenance": "measured" }, "tokens_per_word": { "value": 3.065184295600803, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.9140737534238395, "provenance": "measured" }, "chars_per_token": { "value": 2.900393321740169, "provenance": "measured" }, "tokens_per_word": { "value": 3.0805150017243363, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.8218109501822974, "provenance": "measured" }, "chars_per_token": { "value": 3.7402588393084257, "provenance": "measured" }, "tokens_per_word": { "value": 2.7322564340171014, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.83417497790683, "provenance": "measured" }, "chars_per_token": { "value": 3.832880949375079, "provenance": "measured" }, "tokens_per_word": { "value": 2.6348440748440747, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.102603025761244, "provenance": "measured" }, "chars_per_token": { "value": 3.096960147573114, "provenance": "measured" }, "tokens_per_word": { "value": 2.816735224074475, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.163439624505291, "provenance": "measured" }, "chars_per_token": { "value": 3.150985544486697, "provenance": "measured" }, "tokens_per_word": { "value": 2.8489000882872277, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.227643604536192, "provenance": "measured" }, "chars_per_token": { "value": 4.2117978832248575, "provenance": "measured" }, "tokens_per_word": { "value": 2.5000859808262756, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.8044377957322295, "provenance": "measured" }, "chars_per_token": { "value": 3.7229705684648944, "provenance": "measured" }, "tokens_per_word": { "value": 3.2897281359753716, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.7316667019467555, "provenance": "measured" }, "chars_per_token": { "value": 4.712424941611453, "provenance": "measured" }, "tokens_per_word": { "value": 1.2977860938735268, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 4.895101294823281, "provenance": "measured" }, "chars_per_token": { "value": 2.7146411992346042, "provenance": "measured" }, "tokens_per_word": { "value": 2.25409075084136, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 6.271173622007247, "provenance": "measured" }, "chars_per_token": { "value": 2.3557326530381877, "provenance": "measured" }, "tokens_per_word": { "value": 3.0408471492808844, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.3487393780745047, "provenance": "measured" }, "chars_per_token": { "value": 3.264216269007393, "provenance": "measured" }, "tokens_per_word": { "value": 2.640062134773282, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.9034710434707556, "provenance": "measured" }, "chars_per_token": { "value": 3.816637417647228, "provenance": "measured" }, "tokens_per_word": { "value": 1.6354098052131363, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.727261461302386, "provenance": "measured" }, "chars_per_token": { "value": 3.6085187966088106, "provenance": "measured" }, "tokens_per_word": { "value": 1.778750275541033, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 3.0665009143298856, "provenance": "measured" }, "chars_per_token": { "value": 1.1914324116965003, "provenance": "measured" }, "tokens_per_word": { "value": 5.346049466477518, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 4.559734594042056, "provenance": "measured" }, "chars_per_token": { "value": 1.8101087908878504, "provenance": "measured" }, "tokens_per_word": { "value": 3.0540751477310737, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.8642114634666265, "provenance": "measured" }, "chars_per_token": { "value": 3.8552395641324657, "provenance": "measured" }, "tokens_per_word": { "value": 1.9358432857558987, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.093102661473019, "provenance": "measured" }, "chars_per_token": { "value": 1.4361686523135095, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.289625531754932, "provenance": "measured" }, "chars_per_token": { "value": 1.419708920952554, "provenance": "measured" }, "tokens_per_word": { "value": 3.155966386554622, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 4.312952395706921, "provenance": "measured" }, "chars_per_token": { "value": 1.6388064069129937, "provenance": "measured" }, "tokens_per_word": { "value": 4.2822865988500665, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.569827377102199, "provenance": "measured" }, "chars_per_token": { "value": 3.0644708117723156, "provenance": "measured" }, "tokens_per_word": { "value": 2.414799629033045, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 4.982134943223786, "provenance": "measured" }, "chars_per_token": { "value": 1.8981672066890554, "provenance": "measured" }, "tokens_per_word": { "value": 5.042159102194855, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 3.4580942545872344, "provenance": "measured" }, "chars_per_token": { "value": 1.317389431066673, "provenance": "measured" }, "tokens_per_word": { "value": 6.0599693315098015, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 7.59500651999665, "provenance": "measured" }, "chars_per_token": { "value": 2.715512806709017, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.691491223882497, "provenance": "measured" }, "chars_per_token": { "value": 2.4603682309320347, "provenance": "measured" }, "tokens_per_word": { "value": 3.172132388820649, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 3.760326063202063, "provenance": "measured" }, "chars_per_token": { "value": 2.12383991141804, "provenance": "measured" }, "tokens_per_word": { "value": 2.252720022318132, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 3.033421686012857, "provenance": "measured" }, "chars_per_token": { "value": 2.3224009586773504, "provenance": "measured" }, "tokens_per_word": { "value": 1.9781829139671538, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.568615197828882, "provenance": "measured" }, "chars_per_token": { "value": 1.7159334380802742, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 88424, "provenance": "measured" }, "tokens_per_request_mean": { "value": 442.12, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.44212, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 57084, "provenance": "measured" }, "tokens_per_request_mean": { "value": 350.2085889570552, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.35020858895705526, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 31914, "provenance": "measured" }, "tokens_per_request_mean": { "value": 167.96842105263158, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.16796842105263157, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 241458, "provenance": "measured" }, "tokens_per_request_mean": { "value": 574.9, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5749, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 132588, "provenance": "measured" }, "tokens_per_request_mean": { "value": 883.92, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.88392, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 325542, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1627.71, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.62771, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 326337, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1631.685, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.631685, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 427183, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2135.915, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.135915, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8595, "provenance": "measured" }, "tokens_per_request_mean": { "value": 343.8, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3438, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 613548, "provenance": "measured" }, "tokens_per_request_mean": { "value": 102258.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 102.258, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 129280, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 264765440, "provenance": "estimated", "note": "129280 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 529530880, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 529530880, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1059061760, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 129280, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 529530880, "provenance": "estimated", "note": "129280 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1059061760, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1059061760, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2118123520, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 1853358080, "provenance": "estimated", "note": "129280 rows x 7168 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 3706716160, "provenance": "estimated", "note": "129280 rows x 7168 hidden x 2 matrices from the published config" } } } }, { "id": "mistral-nemo-base-2407", "label": "Mistral-Nemo-Base-2407", "provider": "Mistral", "family": "published", "metrics": { "bpt_overall": { "value": 4.311997871616665, "provenance": "measured" }, "bpt_english": { "value": 4.551519580449641, "provenance": "measured" }, "bpt_code": { "value": 3.5793110392051806, "provenance": "measured" }, "bpt_languages": { "value": 4.8051629951951735, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.5139063938443821, "provenance": "measured" }, "byte_fallback_share": { "value": 0.017356979048135063, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 5.466480170437732, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.3116203562655855, "q3": 5.517293344807472, "n": 5 }, "batch_mbps_4t": { "value": 17.23201412522955, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.968407182053173, "q3": 17.504269329655944, "n": 5 }, "latency_small_p95_us": { "value": 68.7090000610624, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 68.37499995526741, "q3": 69.25000002411252, "n": 5 }, "decode_mbps": { "value": 42.946988438882826, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 42.55410848886436, "q3": 43.65821985876021, "n": 5 }, "rss_load_delta_mb": { "value": 79.0528, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 79.048704, "q3": 79.0528, "n": 5 }, "rss_peak_large_mb": { "value": 194.605056, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 194.51904, "q3": 194.658304, "n": 5 }, "load_seconds": { "value": 0.20511329200007822, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.19820354200010115, "q3": 0.20675441699995645, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5588042485080023, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2571.370496, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2570.027008, "q3": 2572.787712, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.547264, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.53088, "q3": 17.887232, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.0546923436253532, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0006101020101994, "q3": 3.258999833668066, "n": 5 }, "embedding_params_h2048": { "value": 268435456, "provenance": "estimated", "note": "131072 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.20493418127205593, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2048500168448068, "q3": 0.20958659053243178, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5563506080525533, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.542723693726835, "q3": 3.5791808168173396, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 149.272705078125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 149.192626953125, "q3": 149.357421875, "n": 5 }, "workload_tokens_total": { "value": 2392453, "provenance": "measured" }, "tokens_chat_en": { "value": 90659, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 57645, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 28794, "provenance": "measured" }, "tokens_code": { "value": 243869, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 131769, "provenance": "measured" }, "tokens_rag": { "value": 286296, "provenance": "measured" }, "tokens_json": { "value": 395902, "provenance": "measured" }, "tokens_logs": { "value": 565520, "provenance": "measured" }, "tokens_tech_docs": { "value": 8784, "provenance": "measured" }, "tokens_long_context": { "value": 583215, "provenance": "measured" } }, "parity": { "de": { "value": 1.3059999267855182, "provenance": "measured" }, "fr": { "value": 1.3094044001903578, "provenance": "measured" }, "es": { "value": 1.3110883332723213, "provenance": "measured" }, "ru": { "value": 1.5416041292967748, "provenance": "measured" }, "zh": { "value": 1.4341618772193139, "provenance": "measured" }, "ja": { "value": 1.5813229856865687, "provenance": "measured" }, "ko": { "value": 1.2957132920891752, "provenance": "measured" }, "ar": { "value": 1.2687703627777573, "provenance": "measured" }, "ur": { "value": 1.787714609949848, "provenance": "measured" }, "hi": { "value": 1.8192334443752973, "provenance": "measured" }, "mr": { "value": 2.1581432807409304, "provenance": "measured" }, "gu": { "value": 2.7485082549328257, "provenance": "measured" }, "bn": { "value": 2.0725921587289964, "provenance": "measured" }, "ta": { "value": 2.202438042244756, "provenance": "measured" }, "te": { "value": 2.308379397444815, "provenance": "measured" }, "th": { "value": 2.374309038327781, "provenance": "measured" }, "vi": { "value": 1.4283047186733535, "provenance": "measured" }, "tr": { "value": 1.491452209246989, "provenance": "measured" }, "id": { "value": 1.3132847677270565, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 27.20816733067729, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 35.53386454183267, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3059999267855182, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 35.626494023904385, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3094044001903578, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 35.67231075697211, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3110883332723213, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 41.94422310756972, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5416041292967748, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 39.02091633466136, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4341618772193139, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 43.02490039840637, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5813229856865687, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 35.25398406374502, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2957132920891752, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 34.52091633466136, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2687703627777573, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 48.64043824701195, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.787714609949848, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 49.49800796812749, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8192334443752973, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 58.71912350597609, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.1581432807409304, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 74.78187250996017, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.7485082549328257, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 56.39143426294821, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0725921587289964, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 59.92430278884462, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.202438042244756, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 62.80677290836653, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.308379397444815, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 64.60059760956176, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.374309038327781, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 38.86155378486056, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4283047186733535, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 40.5796812749004, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.491452209246989, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 35.73207171314741, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3132847677270565, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "", "", "[INST] hi [/INST]", "", "" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 131072, "provenance": "measured" }, "used_share": { "value": 0.7638702392578125, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 100122, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 30950, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 0, "provenance": "measured", "note": "of 128637 entries whose text is valid UTF-8; 1435 are byte pieces" }, "unreachable_examples": [], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.20511329200007822, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.19820354200010115, "q3": 0.20675441699995645, "n": 5 }, "rss_load_delta_mb": { "value": 79.0528, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 79.048704, "q3": 79.0528, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 78.987264, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 78.983168, "q3": 78.987264, "n": 5 }, "latency_small_p50_us": { "value": 48.41700001634308, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 48.333999984606635, "q3": 48.584000069240574, "n": 5 }, "latency_small_p95_us": { "value": 68.7090000610624, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 68.37499995526741, "q3": 69.25000002411252, "n": 5 }, "encode_mbps_1t_medium": { "value": 5.466480170437732, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.3116203562655855, "q3": 5.517293344807472, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1399364.0034984457, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1359721.446901811, "q3": 1412371.6656319408, "n": 5 }, "encode_mbps_1t_large": { "value": 4.321500782636393, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.229681990285337, "q3": 4.332682857593838, "n": 5 }, "latency_large_p50_ms": { "value": 239.1791249999642, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 237.51629200023672, "q3": 248.2090009998501, "n": 5 }, "decode_mbps": { "value": 42.946988438882826, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 42.55410848886436, "q3": 43.65821985876021, "n": 5 }, "decode_tokens_per_s": { "value": 10702076.570766361, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10604173.750078477, "q3": 10879310.25796412, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.284905176103051, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.21074043176681, "q3": 5.296860629701709, "n": 5 }, "batch_mbps_4t": { "value": 17.23201412522955, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.968407182053173, "q3": 17.504269329655944, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4375167.231335036, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4308238.05803259, "q3": 4444292.177514298, "n": 5 }, "batch_cpu_utilization": { "value": 3.5563506080525533, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.542723693726835, "q3": 3.5791808168173396, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5563506080525533, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.542723693726835, "q3": 3.5791808168173396, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.20493418127205593, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2048500168448068, "q3": 0.20958659053243178, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 149.272705078125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 149.192626953125, "q3": 149.357421875, "n": 5 }, "rss_peak_large_mb": { "value": 194.605056, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 194.51904, "q3": 194.658304, "n": 5 }, "rss_peak_very_large_mb": { "value": 2571.370496, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2570.027008, "q3": 2572.787712, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.547264, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.53088, "q3": 17.887232, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.0546923436253532, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.0006101020101994, "q3": 3.258999833668066, "n": 5 }, "throughput_ratio_very_large": { "value": 0.5588042485080023, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.1209632492470534, "provenance": "measured" }, "chars_per_token": { "value": 2.119052593095524, "provenance": "measured" }, "tokens_per_word": { "value": 3.108783893489203, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.1070181499623675, "provenance": "measured" }, "chars_per_token": { "value": 3.094611768480018, "provenance": "measured" }, "tokens_per_word": { "value": 2.728901159952188, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.627037683407666, "provenance": "measured" }, "chars_per_token": { "value": 3.626455949841639, "provenance": "measured" }, "tokens_per_word": { "value": 2.9059524033133606, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.549418225821465, "provenance": "measured" }, "chars_per_token": { "value": 4.524211114213907, "provenance": "measured" }, "tokens_per_word": { "value": 2.6982293105613464, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.946126810286665, "provenance": "measured" }, "chars_per_token": { "value": 3.938328652346508, "provenance": "measured" }, "tokens_per_word": { "value": 2.710773680404917, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.28697019609438, "provenance": "measured" }, "chars_per_token": { "value": 3.2843470185927024, "provenance": "measured" }, "tokens_per_word": { "value": 2.5570403494176372, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.310798052908919, "provenance": "measured" }, "chars_per_token": { "value": 3.2594090431265945, "provenance": "measured" }, "tokens_per_word": { "value": 3.6391569200779728, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.236960882647944, "provenance": "measured" }, "chars_per_token": { "value": 4.234670678702775, "provenance": "measured" }, "tokens_per_word": { "value": 2.3298021498675805, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.7661302848622285, "provenance": "measured" }, "chars_per_token": { "value": 3.7521667546829858, "provenance": "measured" }, "tokens_per_word": { "value": 2.6759492092520625, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.198004591510883, "provenance": "measured" }, "chars_per_token": { "value": 4.184057445669998, "provenance": "measured" }, "tokens_per_word": { "value": 3.093389577103802, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.33240881892725, "provenance": "measured" }, "chars_per_token": { "value": 3.3054456808612085, "provenance": "measured" }, "tokens_per_word": { "value": 2.9067502147263213, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.7443068869973675, "provenance": "measured" }, "chars_per_token": { "value": 3.7389879421900543, "provenance": "measured" }, "tokens_per_word": { "value": 3.058786242516727, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.506866511701695, "provenance": "measured" }, "chars_per_token": { "value": 3.5065354292690665, "provenance": "measured" }, "tokens_per_word": { "value": 3.0841346153846154, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.873980308867335, "provenance": "measured" }, "chars_per_token": { "value": 3.863391531745577, "provenance": "measured" }, "tokens_per_word": { "value": 3.068538433077982, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.7891334319125076, "provenance": "measured" }, "chars_per_token": { "value": 2.7760395459643257, "provenance": "measured" }, "tokens_per_word": { "value": 3.2185078744683295, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.820044224264791, "provenance": "measured" }, "chars_per_token": { "value": 3.7385298128558553, "provenance": "measured" }, "tokens_per_word": { "value": 2.733520070763658, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.8824854784713843, "provenance": "measured" }, "chars_per_token": { "value": 3.8811751452152863, "provenance": "measured" }, "tokens_per_word": { "value": 2.602058212058212, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.049149266978466, "provenance": "measured" }, "chars_per_token": { "value": 3.0436036081403475, "provenance": "measured" }, "tokens_per_word": { "value": 2.8661145991195784, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.8206734173425976, "provenance": "measured" }, "chars_per_token": { "value": 2.809568766514498, "provenance": "measured" }, "tokens_per_word": { "value": 3.195096380223661, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.205263067870827, "provenance": "measured" }, "chars_per_token": { "value": 4.189501231527093, "provenance": "measured" }, "tokens_per_word": { "value": 2.5133915136924467, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.9917922832250037, "provenance": "measured" }, "chars_per_token": { "value": 3.906313096390541, "provenance": "measured" }, "tokens_per_word": { "value": 3.1353249794042406, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.551519580449641, "provenance": "measured" }, "chars_per_token": { "value": 4.533010405047998, "provenance": "measured" }, "tokens_per_word": { "value": 1.3491518905042872, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 6.087067861715749, "provenance": "measured" }, "chars_per_token": { "value": 3.375661545027742, "provenance": "measured" }, "tokens_per_word": { "value": 1.8126958338168737, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 6.483259618863125, "provenance": "measured" }, "chars_per_token": { "value": 2.435401617439404, "provenance": "measured" }, "tokens_per_word": { "value": 2.941372326914496, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.3112317793775996, "provenance": "measured" }, "chars_per_token": { "value": 3.227655372486316, "provenance": "measured" }, "tokens_per_word": { "value": 2.669967136199846, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.198172901389276, "provenance": "measured" }, "chars_per_token": { "value": 4.104783563847891, "provenance": "measured" }, "tokens_per_word": { "value": 1.52060788557448, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 4.134807291077003, "provenance": "measured" }, "chars_per_token": { "value": 4.003081078458329, "provenance": "measured" }, "tokens_per_word": { "value": 1.6034283787813954, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 4.141006465685129, "provenance": "measured" }, "chars_per_token": { "value": 1.608911739502999, "provenance": "measured" }, "tokens_per_word": { "value": 3.958860173934497, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 6.619774482913967, "provenance": "measured" }, "chars_per_token": { "value": 2.6278968080454743, "provenance": "measured" }, "tokens_per_word": { "value": 2.103662615676218, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.8351409708351225, "provenance": "measured" }, "chars_per_token": { "value": 3.8262365671688228, "provenance": "measured" }, "tokens_per_word": { "value": 1.950517040489368, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.6896886087665557, "provenance": "measured" }, "chars_per_token": { "value": 1.2946206227824668, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.800729646975367, "provenance": "measured" }, "chars_per_token": { "value": 1.6402869365684876, "provenance": "measured" }, "tokens_per_word": { "value": 2.7315669797330697, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 5.850318136711225, "provenance": "measured" }, "chars_per_token": { "value": 2.2229642169167008, "provenance": "measured" }, "tokens_per_word": { "value": 3.156973315642046, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.612826675900841, "provenance": "measured" }, "chars_per_token": { "value": 3.0881286537795614, "provenance": "measured" }, "tokens_per_word": { "value": 2.396300092741739, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 6.657818196175461, "provenance": "measured" }, "chars_per_token": { "value": 2.536593711751313, "provenance": "measured" }, "tokens_per_word": { "value": 3.773115503029099, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 5.37429585698191, "provenance": "measured" }, "chars_per_token": { "value": 2.0473821822587843, "provenance": "measured" }, "tokens_per_word": { "value": 3.8992913092129804, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 5.613055241193946, "provenance": "measured" }, "chars_per_token": { "value": 2.006887466402603, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.578966672544525, "provenance": "measured" }, "chars_per_token": { "value": 3.271634632339975, "provenance": "measured" }, "tokens_per_word": { "value": 2.385539533240228, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 4.873304192668259, "provenance": "measured" }, "chars_per_token": { "value": 2.7524522530517452, "provenance": "measured" }, "tokens_per_word": { "value": 1.7382378522080182, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.280904915119421, "provenance": "measured" }, "chars_per_token": { "value": 3.2774795949809743, "provenance": "measured" }, "tokens_per_word": { "value": 1.4017276882125458, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.1557273938138226, "provenance": "measured" }, "chars_per_token": { "value": 1.1852646638054363, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 90659, "provenance": "measured" }, "tokens_per_request_mean": { "value": 453.295, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.453295, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 57645, "provenance": "measured" }, "tokens_per_request_mean": { "value": 353.6503067484663, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.35365030674846626, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 28794, "provenance": "measured" }, "tokens_per_request_mean": { "value": 151.54736842105262, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.15154736842105263, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 243869, "provenance": "measured" }, "tokens_per_request_mean": { "value": 580.6404761904762, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5806404761904762, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 131769, "provenance": "measured" }, "tokens_per_request_mean": { "value": 878.46, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.87846, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 286296, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1431.48, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.43148, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 395902, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1979.51, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.97951, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 565520, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2827.6, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.8276, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8784, "provenance": "measured" }, "tokens_per_request_mean": { "value": 351.36, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.35136, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 583215, "provenance": "measured" }, "tokens_per_request_mean": { "value": 97202.5, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 97.2025, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 131072, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 268435456, "provenance": "estimated", "note": "131072 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 536870912, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 536870912, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1073741824, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 131072, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 536870912, "provenance": "estimated", "note": "131072 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1073741824, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1073741824, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2147483648, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 1342177280, "provenance": "estimated", "note": "131072 rows x 5120 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 2684354560, "provenance": "estimated", "note": "131072 rows x 5120 hidden x 2 matrices from the published config" } } } }, { "id": "qwen2-5-0-5b", "label": "Qwen2.5-0.5B", "provider": "Alibaba (Qwen)", "family": "published", "metrics": { "bpt_overall": { "value": 3.90295741033963, "provenance": "measured" }, "bpt_english": { "value": 4.573123077727439, "provenance": "measured" }, "bpt_code": { "value": 3.792531069307929, "provenance": "measured" }, "bpt_languages": { "value": 3.343218083983523, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.8940304667578681, "provenance": "measured" }, "byte_fallback_share": { "value": 0.1461490583653184, "provenance": "measured" }, "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.307057689547919, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.279288412295699, "q3": 4.338851432013901, "n": 5 }, "batch_mbps_4t": { "value": 13.201951882847291, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.856043004832054, "q3": 13.433439100652608, "n": 5 }, "latency_small_p95_us": { "value": 87.00000000771979, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 84.79200005240273, "q3": 88.00000023256871, "n": 5 }, "decode_mbps": { "value": 37.897640077907525, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.72617964936915, "q3": 38.63514698416962, "n": 5 }, "rss_load_delta_mb": { "value": 97.73056, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 97.120256, "q3": 98.414592, "n": 5 }, "rss_peak_large_mb": { "value": 209.358848, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 209.129472, "q3": 209.678336, "n": 5 }, "load_seconds": { "value": 0.15903862499999377, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.15626587500037203, "q3": 0.16179104200000438, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6309158041274441, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2646.319104, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2645.385216, "q3": 2646.511616, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.010112, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.649664, "q3": 18.116608, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.7173907656244167, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.680889794757971, "q3": 2.7295215901550276, "n": 5 }, "embedding_params_h2048": { "value": 311164928, "provenance": "estimated", "note": "151936 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.27472890351871687, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.26922698192324535, "q3": 0.28245581784815166, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6269577650814964, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.616644265918417, "q3": 3.631264141220847, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 152.4541015625, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 152.400390625, "q3": 152.5302734375, "n": 5 }, "workload_tokens_total": { "value": 2617712, "provenance": "measured" }, "tokens_chat_en": { "value": 89204, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 60663, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 40814, "provenance": "measured" }, "tokens_code": { "value": 228407, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 141655, "provenance": "measured" }, "tokens_rag": { "value": 421262, "provenance": "measured" }, "tokens_json": { "value": 396209, "provenance": "measured" }, "tokens_logs": { "value": 549999, "provenance": "measured" }, "tokens_tech_docs": { "value": 8625, "provenance": "measured" }, "tokens_long_context": { "value": 680874, "provenance": "measured" } }, "parity": { "de": { "value": 1.5562857670297212, "provenance": "measured" }, "fr": { "value": 1.5639652944434188, "provenance": "measured" }, "es": { "value": 1.5159313273029351, "provenance": "measured" }, "ru": { "value": 1.7856008860993169, "provenance": "measured" }, "zh": { "value": 1.0225955325826104, "provenance": "measured" }, "ja": { "value": 1.4196787889976001, "provenance": "measured" }, "ko": { "value": 1.654125899944619, "provenance": "measured" }, "ar": { "value": 1.6565995938711464, "provenance": "measured" }, "ur": { "value": 3.1986339302196787, "provenance": "measured" }, "hi": { "value": 4.456821118700388, "provenance": "measured" }, "mr": { "value": 4.628539782167251, "provenance": "measured" }, "gu": { "value": 6.763411482370316, "provenance": "measured" }, "bn": { "value": 5.06361454679712, "provenance": "measured" }, "ta": { "value": 6.144360347055565, "provenance": "measured" }, "te": { "value": 7.116411297766291, "provenance": "measured" }, "th": { "value": 2.56012553073657, "provenance": "measured" }, "vi": { "value": 1.4333394868008122, "provenance": "measured" }, "tr": { "value": 1.6258076426066088, "provenance": "measured" }, "id": { "value": 1.542551227616762, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.977091633466134, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 41.98406374501992, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5562857670297212, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 42.191235059760956, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5639652944434188, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 40.895418326693225, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5159313273029351, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 48.1703187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7856008860993169, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 27.586653386454184, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0225955325826104, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 38.298804780876495, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4196787889976001, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 44.623505976095615, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.654125899944619, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 44.690239043824704, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6565995938711464, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 86.2898406374502, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.1986339302196787, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 120.2320717131474, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.456821118700388, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 124.86454183266932, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.628539782167251, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 182.45717131474103, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.763411482370316, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 136.601593625498, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.06361454679712, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 165.75697211155378, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.144360347055565, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 191.9800796812749, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.116411297766291, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 69.06474103585657, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.56012553073657, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 38.667330677290835, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4333394868008122, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 43.85956175298805, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6258076426066088, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 41.613545816733065, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.542551227616762, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 151,936 embedding rows for 151,665 tokenizer IDs (271 unused rows, usually padding)" ], "roundtrip_failures": { "value": 733, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 733, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [ "code_cs" ], "roundtrip_example": " \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\",", "byte_fallback_sets": [], "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "unicode_failed_probes": [ "latin_accents (NFD)", "vietnamese (NFD)", "devanagari_nukta_precomposed (NFC)", "tamil (NFD)", "telugu (NFD)", "japanese (NFD)", "korean (NFD)" ], "nfd_normalized_to_nfc": [ "japanese", "korean", "latin_accents", "tamil", "telugu", "vietnamese" ], "special_strings_parsed_by_default": [ "<|endoftext|>", "<|im_start|>user", "<|im_end|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 151665, "provenance": "measured" }, "used_share": { "value": 0.6676161276497544, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 101254, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 50411, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 1909, "provenance": "measured", "note": "of 150203 entries whose text is valid UTF-8; 1448 are byte pieces" }, "unreachable_examples": [ "１０", "２０", "珊󠄁", "ที่", "ร์", "ติ", "ด้", "เป็น", "ว่า", "มี", "ไม่", "ได้" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.15903862499999377, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.15626587500037203, "q3": 0.16179104200000438, "n": 5 }, "rss_load_delta_mb": { "value": 97.73056, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 97.120256, "q3": 98.414592, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 97.595392, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 96.985088, "q3": 98.279424, "n": 5 }, "latency_small_p50_us": { "value": 64.33399998684308, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 62.542000250687124, "q3": 65.12499999189458, "n": 5 }, "latency_small_p95_us": { "value": 87.00000000771979, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 84.79200005240273, "q3": 88.00000023256871, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.307057689547919, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.279288412295699, "q3": 4.338851432013901, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1238977.3061184417, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1230989.1372099302, "q3": 1248123.1611850034, "n": 5 }, "encode_mbps_1t_large": { "value": 3.534128354652424, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.492332540319072, "q3": 3.5625654653583143, "n": 5 }, "latency_large_p50_ms": { "value": 291.97424999983923, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 289.94987500004754, "q3": 295.99454200001674, "n": 5 }, "decode_mbps": { "value": 37.897640077907525, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.72617964936915, "q3": 38.63514698416962, "n": 5 }, "decode_tokens_per_s": { "value": 11237257.795237085, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 11186417.029606119, "q3": 11455940.41543185, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.147797926499399, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.1350001770813085, "q3": 4.194648489422929, "n": 5 }, "batch_mbps_4t": { "value": 13.201951882847291, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 12.856043004832054, "q3": 13.433439100652608, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3786057.7774363845, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3686857.9765648604, "q3": 3852443.7171160784, "n": 5 }, "batch_cpu_utilization": { "value": 3.6269577650814964, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.616644265918417, "q3": 3.631264141220847, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6269577650814964, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.616644265918417, "q3": 3.631264141220847, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.27472890351871687, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.26922698192324535, "q3": 0.28245581784815166, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 152.4541015625, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 152.400390625, "q3": 152.5302734375, "n": 5 }, "rss_peak_large_mb": { "value": 209.358848, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 209.129472, "q3": 209.678336, "n": 5 }, "rss_peak_very_large_mb": { "value": 2646.319104, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2645.385216, "q3": 2646.511616, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.010112, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.649664, "q3": 18.116608, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.7173907656244167, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.680889794757971, "q3": 2.7295215901550276, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6309158041274441, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.235181609279755, "provenance": "measured" }, "chars_per_token": { "value": 2.233168060250524, "provenance": "measured" }, "tokens_per_word": { "value": 2.949924230123938, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3063878676470586, "provenance": "measured" }, "chars_per_token": { "value": 3.293185399159664, "provenance": "measured" }, "tokens_per_word": { "value": 2.5643529352346, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.884417031822178, "provenance": "measured" }, "chars_per_token": { "value": 3.8837940176797567, "provenance": "measured" }, "tokens_per_word": { "value": 2.7134055861304684, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.867677230444396, "provenance": "measured" }, "chars_per_token": { "value": 4.840706730673466, "provenance": "measured" }, "tokens_per_word": { "value": 2.5218133869144794, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.171306547248829, "provenance": "measured" }, "chars_per_token": { "value": 4.163063399261931, "provenance": "measured" }, "tokens_per_word": { "value": 2.5644379226744927, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.4807418998262967, "provenance": "measured" }, "chars_per_token": { "value": 3.4779640821717224, "provenance": "measured" }, "tokens_per_word": { "value": 2.4146907931225736, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.430253208463406, "provenance": "measured" }, "chars_per_token": { "value": 3.3770100589663543, "provenance": "measured" }, "tokens_per_word": { "value": 3.5124269005847952, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.536959965631126, "provenance": "measured" }, "chars_per_token": { "value": 4.534507603218502, "provenance": "measured" }, "tokens_per_word": { "value": 2.1757477800280416, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.996159931439119, "provenance": "measured" }, "chars_per_token": { "value": 3.98134352956688, "provenance": "measured" }, "tokens_per_word": { "value": 2.5219144204991792, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.5345775921540845, "provenance": "measured" }, "chars_per_token": { "value": 4.519512240598139, "provenance": "measured" }, "tokens_per_word": { "value": 2.863786843229389, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.5410068862168624, "provenance": "measured" }, "chars_per_token": { "value": 3.5123559424840334, "provenance": "measured" }, "tokens_per_word": { "value": 2.7355157335831968, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.9322752516907005, "provenance": "measured" }, "chars_per_token": { "value": 3.926689289946075, "provenance": "measured" }, "tokens_per_word": { "value": 2.912571898110107, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.6985290908297324, "provenance": "measured" }, "chars_per_token": { "value": 3.69817991357863, "provenance": "measured" }, "tokens_per_word": { "value": 2.9243107556160655, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.125421509731034, "provenance": "measured" }, "chars_per_token": { "value": 4.114145466639163, "provenance": "measured" }, "tokens_per_word": { "value": 2.8815134256479307, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.9306092425936905, "provenance": "measured" }, "chars_per_token": { "value": 2.91685118328317, "provenance": "measured" }, "tokens_per_word": { "value": 3.0631336935279916, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.008594204773301, "provenance": "measured" }, "chars_per_token": { "value": 3.9230563999741284, "provenance": "measured" }, "tokens_per_word": { "value": 2.604945031801525, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.125690900909315, "provenance": "measured" }, "chars_per_token": { "value": 4.124298486173491, "provenance": "measured" }, "tokens_per_word": { "value": 2.4486694386694388, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.281622394298451, "provenance": "measured" }, "chars_per_token": { "value": 3.2756539235412476, "provenance": "measured" }, "tokens_per_word": { "value": 2.6630764234682833, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.016406158619543, "provenance": "measured" }, "chars_per_token": { "value": 3.004530931611251, "provenance": "measured" }, "tokens_per_word": { "value": 2.987768540317834, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.551044019103328, "provenance": "measured" }, "chars_per_token": { "value": 4.533986153789197, "provenance": "measured" }, "tokens_per_word": { "value": 2.322428098534027, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.9882899127443054, "provenance": "measured" }, "chars_per_token": { "value": 3.9028857247473177, "provenance": "measured" }, "tokens_per_word": { "value": 3.1380783072453715, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.573123077727439, "provenance": "measured" }, "chars_per_token": { "value": 4.554526049705735, "provenance": "measured" }, "tokens_per_word": { "value": 1.3427784781341285, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 4.705727083006821, "provenance": "measured" }, "chars_per_token": { "value": 2.609621301482294, "provenance": "measured" }, "tokens_per_word": { "value": 2.344802908978376, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 2.5105949341347134, "provenance": "measured" }, "chars_per_token": { "value": 0.9430914883521251, "provenance": "measured" }, "tokens_per_word": { "value": 7.595681872790307, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 2.9070253892473565, "provenance": "measured" }, "chars_per_token": { "value": 2.8336512635554674, "provenance": "measured" }, "tokens_per_word": { "value": 3.041211839421766, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.6416297061724583, "provenance": "measured" }, "chars_per_token": { "value": 3.560620802104232, "provenance": "measured" }, "tokens_per_word": { "value": 1.7529994354003953, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.5205150273694357, "provenance": "measured" }, "chars_per_token": { "value": 3.4083588666643716, "provenance": "measured" }, "tokens_per_word": { "value": 1.8832095019514787, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.7835714980724258, "provenance": "measured" }, "chars_per_token": { "value": 0.6929738326348882, "provenance": "measured" }, "tokens_per_word": { "value": 9.191482143958552, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 2.7882476378633894, "provenance": "measured" }, "chars_per_token": { "value": 1.1068695899676864, "provenance": "measured" }, "tokens_per_word": { "value": 4.994453116289441, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.247230022707755, "provenance": "measured" }, "chars_per_token": { "value": 3.2396906265970506, "provenance": "measured" }, "tokens_per_word": { "value": 2.303658122147781, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.9299345116541087, "provenance": "measured" }, "chars_per_token": { "value": 1.3789169776776284, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.1728978457261987, "provenance": "measured" }, "chars_per_token": { "value": 1.3693325679064567, "provenance": "measured" }, "tokens_per_word": { "value": 3.2720711814137418, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 2.780771898670958, "provenance": "measured" }, "chars_per_token": { "value": 1.0566188507533087, "provenance": "measured" }, "tokens_per_word": { "value": 6.6417882942650746, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 4.8192963892508285, "provenance": "measured" }, "chars_per_token": { "value": 2.6515351586753297, "provenance": "measured" }, "tokens_per_word": { "value": 2.790867379313711, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.506172778344744, "provenance": "measured" }, "chars_per_token": { "value": 0.9548386457538617, "provenance": "measured" }, "tokens_per_word": { "value": 10.02353759062469, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.8284600530841608, "provenance": "measured" }, "chars_per_token": { "value": 0.6965668867658439, "provenance": "measured" }, "tokens_per_word": { "value": 11.460980562808238, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 5.05601879504639, "provenance": "measured" }, "chars_per_token": { "value": 1.807725082626528, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.256330941715547, "provenance": "measured" }, "chars_per_token": { "value": 2.976704188112077, "provenance": "measured" }, "tokens_per_word": { "value": 2.621897662835894, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 2.75405349168494, "provenance": "measured" }, "chars_per_token": { "value": 1.5554950888593668, "provenance": "measured" }, "tokens_per_word": { "value": 3.0758160066954394, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.23201875872231, "provenance": "measured" }, "chars_per_token": { "value": 3.2400521390469037, "provenance": "measured" }, "tokens_per_word": { "value": 1.417919742855708, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.0806468332123975, "provenance": "measured" }, "chars_per_token": { "value": 1.5326566250169442, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 89204, "provenance": "measured" }, "tokens_per_request_mean": { "value": 446.02, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.44602, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 60663, "provenance": "measured" }, "tokens_per_request_mean": { "value": 372.16564417177915, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.37216564417177916, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 40814, "provenance": "measured" }, "tokens_per_request_mean": { "value": 214.81052631578947, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.21481052631578948, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 228407, "provenance": "measured" }, "tokens_per_request_mean": { "value": 543.8261904761905, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5438261904761905, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 141655, "provenance": "measured" }, "tokens_per_request_mean": { "value": 944.3666666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9443666666666666, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 421262, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2106.31, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.10631, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 396209, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1981.045, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.981045, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 549999, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2749.995, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.749995, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8625, "provenance": "measured" }, "tokens_per_request_mean": { "value": 345.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.345, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 680874, "provenance": "measured" }, "tokens_per_request_mean": { "value": 113479.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 113.479, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 151936, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 311164928, "provenance": "estimated", "note": "151936 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 622329856, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 622329856, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1244659712, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 151936, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 622329856, "provenance": "estimated", "note": "151936 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1244659712, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1244659712, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2489319424, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 136134656, "provenance": "estimated", "note": "151936 rows x 896 hidden x 1 matrix from the published config" }, "bf16_bytes": { "value": 272269312, "provenance": "estimated", "note": "151936 rows x 896 hidden x 1 matrix from the published config" } } } }, { "id": "qwen3-0-6b", "label": "Qwen3-0.6B", "provider": "Alibaba (Qwen)", "family": "published", "metrics": { "bpt_overall": { "value": 3.90295741033963, "provenance": "measured" }, "bpt_english": { "value": 4.573123077727439, "provenance": "measured" }, "bpt_code": { "value": 3.792531069307929, "provenance": "measured" }, "bpt_languages": { "value": 3.343218083983523, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.8940304667578681, "provenance": "measured" }, "byte_fallback_share": { "value": 0.1461490583653184, "provenance": "measured" }, "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 3.9961037276744547, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.958106841385716, "q3": 4.0564512397559715, "n": 5 }, "batch_mbps_4t": { "value": 13.319798801502683, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.294485897770201, "q3": 13.450353306550952, "n": 5 }, "latency_small_p95_us": { "value": 89.20800019041053, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 88.83299994977278, "q3": 90.04200001072604, "n": 5 }, "decode_mbps": { "value": 35.16551105439843, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 34.96042948717199, "q3": 35.7962132429691, "n": 5 }, "rss_load_delta_mb": { "value": 135.909376, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 135.909376, "q3": 135.909376, "n": 5 }, "rss_peak_large_mb": { "value": 221.73696, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 221.73696, "q3": 222.216192, "n": 5 }, "load_seconds": { "value": 0.23505091699996683, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.2295157910002672, "q3": 0.23862354199991387, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6769796761910788, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2646.818816, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2646.306816, "q3": 2647.539712, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.489344, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.190912, "q3": 18.546688, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.7052810075870153, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.6686198954638405, "q3": 2.7190636414398757, "n": 5 }, "embedding_params_h2048": { "value": 311164928, "provenance": "estimated", "note": "151936 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.27075577032982207, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.26633940294756697, "q3": 0.2731784183334442, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6064123851390995, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5831378564697354, "q3": 3.6237878116382185, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 147.55810546875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 147.4853515625, "q3": 147.573974609375, "n": 5 }, "workload_tokens_total": { "value": 2617712, "provenance": "measured" }, "tokens_chat_en": { "value": 89204, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 60663, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 40814, "provenance": "measured" }, "tokens_code": { "value": 228407, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 141655, "provenance": "measured" }, "tokens_rag": { "value": 421262, "provenance": "measured" }, "tokens_json": { "value": 396209, "provenance": "measured" }, "tokens_logs": { "value": 549999, "provenance": "measured" }, "tokens_tech_docs": { "value": 8625, "provenance": "measured" }, "tokens_long_context": { "value": 680874, "provenance": "measured" } }, "parity": { "de": { "value": 1.5562857670297212, "provenance": "measured" }, "fr": { "value": 1.5639652944434188, "provenance": "measured" }, "es": { "value": 1.5159313273029351, "provenance": "measured" }, "ru": { "value": 1.7856008860993169, "provenance": "measured" }, "zh": { "value": 1.0225955325826104, "provenance": "measured" }, "ja": { "value": 1.4196787889976001, "provenance": "measured" }, "ko": { "value": 1.654125899944619, "provenance": "measured" }, "ar": { "value": 1.6565995938711464, "provenance": "measured" }, "ur": { "value": 3.1986339302196787, "provenance": "measured" }, "hi": { "value": 4.456821118700388, "provenance": "measured" }, "mr": { "value": 4.628539782167251, "provenance": "measured" }, "gu": { "value": 6.763411482370316, "provenance": "measured" }, "bn": { "value": 5.06361454679712, "provenance": "measured" }, "ta": { "value": 6.144360347055565, "provenance": "measured" }, "te": { "value": 7.116411297766291, "provenance": "measured" }, "th": { "value": 2.56012553073657, "provenance": "measured" }, "vi": { "value": 1.4333394868008122, "provenance": "measured" }, "tr": { "value": 1.6258076426066088, "provenance": "measured" }, "id": { "value": 1.542551227616762, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.977091633466134, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 41.98406374501992, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5562857670297212, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 42.191235059760956, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5639652944434188, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 40.895418326693225, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5159313273029351, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 48.1703187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7856008860993169, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 27.586653386454184, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0225955325826104, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 38.298804780876495, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4196787889976001, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 44.623505976095615, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.654125899944619, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 44.690239043824704, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6565995938711464, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 86.2898406374502, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.1986339302196787, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 120.2320717131474, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.456821118700388, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 124.86454183266932, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.628539782167251, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 182.45717131474103, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.763411482370316, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 136.601593625498, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.06361454679712, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 165.75697211155378, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.144360347055565, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 191.9800796812749, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.116411297766291, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 69.06474103585657, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.56012553073657, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 38.667330677290835, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4333394868008122, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 43.85956175298805, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6258076426066088, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 41.613545816733065, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.542551227616762, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 151,936 embedding rows for 151,669 tokenizer IDs (267 unused rows, usually padding)" ], "roundtrip_failures": { "value": 733, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 733, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [ "code_cs" ], "roundtrip_example": " \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\",", "byte_fallback_sets": [], "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "unicode_failed_probes": [ "latin_accents (NFD)", "vietnamese (NFD)", "devanagari_nukta_precomposed (NFC)", "tamil (NFD)", "telugu (NFD)", "japanese (NFD)", "korean (NFD)" ], "nfd_normalized_to_nfc": [ "japanese", "korean", "latin_accents", "tamil", "telugu", "vietnamese" ], "special_strings_parsed_by_default": [ "<|endoftext|>", "<|im_start|>user", "<|im_end|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 151669, "provenance": "measured" }, "used_share": { "value": 0.6675985204623226, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 101254, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 50415, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 1909, "provenance": "measured", "note": "of 150207 entries whose text is valid UTF-8; 1448 are byte pieces" }, "unreachable_examples": [ "１０", "２０", "珊󠄁", "ที่", "ร์", "ติ", "ด้", "เป็น", "ว่า", "มี", "ไม่", "ได้" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.23505091699996683, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.2295157910002672, "q3": 0.23862354199991387, "n": 5 }, "rss_load_delta_mb": { "value": 135.909376, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 135.909376, "q3": 135.909376, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 135.774208, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 135.774208, "q3": 135.774208, "n": 5 }, "latency_small_p50_us": { "value": 66.54200001321442, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 66.02100000918654, "q3": 66.91699991279165, "n": 5 }, "latency_small_p95_us": { "value": 89.20800019041053, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 88.83299994977278, "q3": 90.04200001072604, "n": 5 }, "encode_mbps_1t_medium": { "value": 3.9961037276744547, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.958106841385716, "q3": 4.0564512397559715, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1149527.6330983245, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1138597.3685863926, "q3": 1166887.3258025001, "n": 5 }, "encode_mbps_1t_large": { "value": 3.370124123652971, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.327448614227616, "q3": 3.4136120389239673, "n": 5 }, "latency_large_p50_ms": { "value": 310.47816699992836, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 304.8828760001925, "q3": 311.73970899999404, "n": 5 }, "decode_mbps": { "value": 35.16551105439843, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 34.96042948717199, "q3": 35.7962132429691, "n": 5 }, "decode_tokens_per_s": { "value": 10427137.742803564, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10366327.770593233, "q3": 10614151.052086653, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.143487684616957, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.136289485634495, "q3": 4.164935923663185, "n": 5 }, "batch_mbps_4t": { "value": 13.319798801502683, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.294485897770201, "q3": 13.450353306550952, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3819853.934768381, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3812594.696369673, "q3": 3857294.375667091, "n": 5 }, "batch_cpu_utilization": { "value": 3.6064123851390995, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5831378564697354, "q3": 3.6237878116382185, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6064123851390995, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5831378564697354, "q3": 3.6237878116382185, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.27075577032982207, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.26633940294756697, "q3": 0.2731784183334442, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 147.55810546875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 147.4853515625, "q3": 147.573974609375, "n": 5 }, "rss_peak_large_mb": { "value": 221.73696, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 221.73696, "q3": 222.216192, "n": 5 }, "rss_peak_very_large_mb": { "value": 2646.818816, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2646.306816, "q3": 2647.539712, "n": 5 }, "rss_peak_chunked_mb": { "value": 18.489344, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.190912, "q3": 18.546688, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.7052810075870153, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.6686198954638405, "q3": 2.7190636414398757, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6769796761910788, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.235181609279755, "provenance": "measured" }, "chars_per_token": { "value": 2.233168060250524, "provenance": "measured" }, "tokens_per_word": { "value": 2.949924230123938, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3063878676470586, "provenance": "measured" }, "chars_per_token": { "value": 3.293185399159664, "provenance": "measured" }, "tokens_per_word": { "value": 2.5643529352346, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.884417031822178, "provenance": "measured" }, "chars_per_token": { "value": 3.8837940176797567, "provenance": "measured" }, "tokens_per_word": { "value": 2.7134055861304684, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.867677230444396, "provenance": "measured" }, "chars_per_token": { "value": 4.840706730673466, "provenance": "measured" }, "tokens_per_word": { "value": 2.5218133869144794, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.171306547248829, "provenance": "measured" }, "chars_per_token": { "value": 4.163063399261931, "provenance": "measured" }, "tokens_per_word": { "value": 2.5644379226744927, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.4807418998262967, "provenance": "measured" }, "chars_per_token": { "value": 3.4779640821717224, "provenance": "measured" }, "tokens_per_word": { "value": 2.4146907931225736, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.430253208463406, "provenance": "measured" }, "chars_per_token": { "value": 3.3770100589663543, "provenance": "measured" }, "tokens_per_word": { "value": 3.5124269005847952, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.536959965631126, "provenance": "measured" }, "chars_per_token": { "value": 4.534507603218502, "provenance": "measured" }, "tokens_per_word": { "value": 2.1757477800280416, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.996159931439119, "provenance": "measured" }, "chars_per_token": { "value": 3.98134352956688, "provenance": "measured" }, "tokens_per_word": { "value": 2.5219144204991792, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.5345775921540845, "provenance": "measured" }, "chars_per_token": { "value": 4.519512240598139, "provenance": "measured" }, "tokens_per_word": { "value": 2.863786843229389, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.5410068862168624, "provenance": "measured" }, "chars_per_token": { "value": 3.5123559424840334, "provenance": "measured" }, "tokens_per_word": { "value": 2.7355157335831968, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.9322752516907005, "provenance": "measured" }, "chars_per_token": { "value": 3.926689289946075, "provenance": "measured" }, "tokens_per_word": { "value": 2.912571898110107, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.6985290908297324, "provenance": "measured" }, "chars_per_token": { "value": 3.69817991357863, "provenance": "measured" }, "tokens_per_word": { "value": 2.9243107556160655, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.125421509731034, "provenance": "measured" }, "chars_per_token": { "value": 4.114145466639163, "provenance": "measured" }, "tokens_per_word": { "value": 2.8815134256479307, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.9306092425936905, "provenance": "measured" }, "chars_per_token": { "value": 2.91685118328317, "provenance": "measured" }, "tokens_per_word": { "value": 3.0631336935279916, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.008594204773301, "provenance": "measured" }, "chars_per_token": { "value": 3.9230563999741284, "provenance": "measured" }, "tokens_per_word": { "value": 2.604945031801525, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.125690900909315, "provenance": "measured" }, "chars_per_token": { "value": 4.124298486173491, "provenance": "measured" }, "tokens_per_word": { "value": 2.4486694386694388, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.281622394298451, "provenance": "measured" }, "chars_per_token": { "value": 3.2756539235412476, "provenance": "measured" }, "tokens_per_word": { "value": 2.6630764234682833, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.016406158619543, "provenance": "measured" }, "chars_per_token": { "value": 3.004530931611251, "provenance": "measured" }, "tokens_per_word": { "value": 2.987768540317834, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.551044019103328, "provenance": "measured" }, "chars_per_token": { "value": 4.533986153789197, "provenance": "measured" }, "tokens_per_word": { "value": 2.322428098534027, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.9882899127443054, "provenance": "measured" }, "chars_per_token": { "value": 3.9028857247473177, "provenance": "measured" }, "tokens_per_word": { "value": 3.1380783072453715, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.573123077727439, "provenance": "measured" }, "chars_per_token": { "value": 4.554526049705735, "provenance": "measured" }, "tokens_per_word": { "value": 1.3427784781341285, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 4.705727083006821, "provenance": "measured" }, "chars_per_token": { "value": 2.609621301482294, "provenance": "measured" }, "tokens_per_word": { "value": 2.344802908978376, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 2.5105949341347134, "provenance": "measured" }, "chars_per_token": { "value": 0.9430914883521251, "provenance": "measured" }, "tokens_per_word": { "value": 7.595681872790307, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 2.9070253892473565, "provenance": "measured" }, "chars_per_token": { "value": 2.8336512635554674, "provenance": "measured" }, "tokens_per_word": { "value": 3.041211839421766, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.6416297061724583, "provenance": "measured" }, "chars_per_token": { "value": 3.560620802104232, "provenance": "measured" }, "tokens_per_word": { "value": 1.7529994354003953, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.5205150273694357, "provenance": "measured" }, "chars_per_token": { "value": 3.4083588666643716, "provenance": "measured" }, "tokens_per_word": { "value": 1.8832095019514787, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.7835714980724258, "provenance": "measured" }, "chars_per_token": { "value": 0.6929738326348882, "provenance": "measured" }, "tokens_per_word": { "value": 9.191482143958552, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 2.7882476378633894, "provenance": "measured" }, "chars_per_token": { "value": 1.1068695899676864, "provenance": "measured" }, "tokens_per_word": { "value": 4.994453116289441, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.247230022707755, "provenance": "measured" }, "chars_per_token": { "value": 3.2396906265970506, "provenance": "measured" }, "tokens_per_word": { "value": 2.303658122147781, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.9299345116541087, "provenance": "measured" }, "chars_per_token": { "value": 1.3789169776776284, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.1728978457261987, "provenance": "measured" }, "chars_per_token": { "value": 1.3693325679064567, "provenance": "measured" }, "tokens_per_word": { "value": 3.2720711814137418, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 2.780771898670958, "provenance": "measured" }, "chars_per_token": { "value": 1.0566188507533087, "provenance": "measured" }, "tokens_per_word": { "value": 6.6417882942650746, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 4.8192963892508285, "provenance": "measured" }, "chars_per_token": { "value": 2.6515351586753297, "provenance": "measured" }, "tokens_per_word": { "value": 2.790867379313711, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.506172778344744, "provenance": "measured" }, "chars_per_token": { "value": 0.9548386457538617, "provenance": "measured" }, "tokens_per_word": { "value": 10.02353759062469, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.8284600530841608, "provenance": "measured" }, "chars_per_token": { "value": 0.6965668867658439, "provenance": "measured" }, "tokens_per_word": { "value": 11.460980562808238, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 5.05601879504639, "provenance": "measured" }, "chars_per_token": { "value": 1.807725082626528, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.256330941715547, "provenance": "measured" }, "chars_per_token": { "value": 2.976704188112077, "provenance": "measured" }, "tokens_per_word": { "value": 2.621897662835894, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 2.75405349168494, "provenance": "measured" }, "chars_per_token": { "value": 1.5554950888593668, "provenance": "measured" }, "tokens_per_word": { "value": 3.0758160066954394, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.23201875872231, "provenance": "measured" }, "chars_per_token": { "value": 3.2400521390469037, "provenance": "measured" }, "tokens_per_word": { "value": 1.417919742855708, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.0806468332123975, "provenance": "measured" }, "chars_per_token": { "value": 1.5326566250169442, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 89204, "provenance": "measured" }, "tokens_per_request_mean": { "value": 446.02, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.44602, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 60663, "provenance": "measured" }, "tokens_per_request_mean": { "value": 372.16564417177915, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.37216564417177916, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 40814, "provenance": "measured" }, "tokens_per_request_mean": { "value": 214.81052631578947, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.21481052631578948, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 228407, "provenance": "measured" }, "tokens_per_request_mean": { "value": 543.8261904761905, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5438261904761905, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 141655, "provenance": "measured" }, "tokens_per_request_mean": { "value": 944.3666666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9443666666666666, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 421262, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2106.31, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.10631, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 396209, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1981.045, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.981045, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 549999, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2749.995, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.749995, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8625, "provenance": "measured" }, "tokens_per_request_mean": { "value": 345.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.345, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 680874, "provenance": "measured" }, "tokens_per_request_mean": { "value": 113479.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 113.479, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 151936, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 311164928, "provenance": "estimated", "note": "151936 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 622329856, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 622329856, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1244659712, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 151936, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 622329856, "provenance": "estimated", "note": "151936 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1244659712, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1244659712, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2489319424, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 155582464, "provenance": "estimated", "note": "151936 rows x 1024 hidden x 1 matrix from the published config" }, "bf16_bytes": { "value": 311164928, "provenance": "estimated", "note": "151936 rows x 1024 hidden x 1 matrix from the published config" } } } }, { "id": "glm-5-3-flash", "label": "GLM-5.3-Flash", "provider": "Z.ai (GLM)", "family": "published", "metrics": { "bpt_overall": { "value": 3.95786121193051, "provenance": "measured" }, "bpt_english": { "value": 4.685469802799975, "provenance": "measured" }, "bpt_code": { "value": 3.8796119999852894, "provenance": "measured" }, "bpt_languages": { "value": 3.308501833006265, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.9329646447008982, "provenance": "measured" }, "byte_fallback_share": { "value": 0.17656260475094415, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.882575809711876, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.783285403986078, "q3": 5.046999136402947, "n": 5 }, "batch_mbps_4t": { "value": 16.669548809795145, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.343995737264507, "q3": 16.85284761351425, "n": 5 }, "latency_small_p95_us": { "value": 65.16599978567683, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 64.16600001557526, "q3": 66.0420000144768, "n": 5 }, "decode_mbps": { "value": 34.27865346085365, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 34.14832150286263, "q3": 36.42605225564783, "n": 5 }, "rss_load_delta_mb": { "value": 175.747072, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 175.747072, "q3": 175.775744, "n": 5 }, "rss_peak_large_mb": { "value": 268.546048, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 268.230656, "q3": 268.648448, "n": 5 }, "load_seconds": { "value": 0.38684520800006794, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.3731782919999205, "q3": 0.39260695799998757, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6358257878685136, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2628.591616, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2628.23936, "q3": 2629.791744, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.02048, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.02048, "q3": 0.02048, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.1044676110377996, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.9248470245029297, "q3": 3.189589033755548, "n": 5 }, "embedding_params_h2048": { "value": 317194240, "provenance": "estimated", "note": "154880 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.2136376890689483, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.21029328560995722, "q3": 0.21722981728689644, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5612438855466726, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5504032077437833, "q3": 3.5617379185266214, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 142.818603515625, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 142.7998046875, "q3": 142.95751953125, "n": 5 }, "workload_tokens_total": { "value": 2461706, "provenance": "measured" }, "tokens_chat_en": { "value": 88272, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 57233, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 42375, "provenance": "measured" }, "tokens_code": { "value": 223243, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 141802, "provenance": "measured" }, "tokens_rag": { "value": 440214, "provenance": "measured" }, "tokens_json": { "value": 340744, "provenance": "measured" }, "tokens_logs": { "value": 431637, "provenance": "measured" }, "tokens_tech_docs": { "value": 8344, "provenance": "measured" }, "tokens_long_context": { "value": 687842, "provenance": "measured" } }, "parity": { "de": { "value": 1.4439566212615857, "provenance": "measured" }, "fr": { "value": 1.49637885098878, "provenance": "measured" }, "es": { "value": 1.4349881796690307, "provenance": "measured" }, "ru": { "value": 1.4553641787684342, "provenance": "measured" }, "zh": { "value": 0.965814852339675, "provenance": "measured" }, "ja": { "value": 1.512589590603775, "provenance": "measured" }, "ko": { "value": 1.9461518255844497, "provenance": "measured" }, "ar": { "value": 1.796390108446846, "provenance": "measured" }, "ur": { "value": 2.7846073023377986, "provenance": "measured" }, "hi": { "value": 4.3526961612068, "provenance": "measured" }, "mr": { "value": 4.5902660512589595, "provenance": "measured" }, "gu": { "value": 7.687530488948929, "provenance": "measured" }, "bn": { "value": 4.7488461105482385, "provenance": "measured" }, "ta": { "value": 7.659649517805546, "provenance": "measured" }, "te": { "value": 8.420391009043492, "provenance": "measured" }, "th": { "value": 4.183834290217269, "provenance": "measured" }, "vi": { "value": 1.455401703628654, "provenance": "measured" }, "tr": { "value": 1.6472663139329806, "provenance": "measured" }, "id": { "value": 1.5382565949941835, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.542828685258964, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 38.32669322709163, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4439566212615857, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 39.71812749003984, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.49637885098878, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 38.08864541832669, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4349881796690307, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 38.62948207171315, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4553641787684342, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 25.63545816733068, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 0.965814852339675, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 40.148406374501995, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.512589590603775, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 51.65637450199203, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9461518255844497, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 47.68127490039841, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.796390108446846, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 73.91135458167331, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.7846073023377986, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 115.53286852589642, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.3526961612068, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 121.83864541832669, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.5902660512589595, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 204.04880478087648, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.687530488948929, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 126.04780876494024, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.7488461105482385, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 203.30876494023903, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.659649517805546, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 223.50099601593627, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 8.420391009043492, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 111.050796812749, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.183834290217269, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 38.6304780876494, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.455401703628654, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 43.723107569721115, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6472663139329806, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 40.8296812749004, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5382565949941835, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 154,880 embedding rows for 154,856 tokenizer IDs (24 unused rows, usually padding)" ], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 154856, "provenance": "measured" }, "used_share": { "value": 0.6918427442268946, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 107136, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 47720, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 267, "provenance": "measured", "note": "of 153743 entries whose text is valid UTF-8; 1095 are byte pieces" }, "unreachable_examples": [ " 1", " 2", " 3", " 4", " 201", " 0", " 5", "2020", "2021", " 19", " 6", " 200" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.38684520800006794, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.3731782919999205, "q3": 0.39260695799998757, "n": 5 }, "rss_load_delta_mb": { "value": 175.747072, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 175.747072, "q3": 175.775744, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 175.616, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 175.616, "q3": 175.644672, "n": 5 }, "latency_small_p50_us": { "value": 48.875000175030436, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 48.66699998729018, "q3": 49.68749999534339, "n": 5 }, "latency_small_p95_us": { "value": 65.16599978567683, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 64.16600001557526, "q3": 66.0420000144768, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.882575809711876, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.783285403986078, "q3": 5.046999136402947, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1402547.2612847683, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1374025.5358574695, "q3": 1449778.7832374198, "n": 5 }, "encode_mbps_1t_large": { "value": 4.120283491064377, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.070834567420788, "q3": 4.213508681578031, "n": 5 }, "latency_large_p50_ms": { "value": 254.51995899993562, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 249.8657920000369, "q3": 255.07737499998484, "n": 5 }, "decode_mbps": { "value": 34.27865346085365, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 34.14832150286263, "q3": 36.42605225564783, "n": 5 }, "decode_tokens_per_s": { "value": 10199051.262607966, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10160273.125589706, "q3": 10837974.562625568, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.264415473646825, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.248950428236241, "q3": 5.353072635225408, "n": 5 }, "batch_mbps_4t": { "value": 16.669548809795145, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.343995737264507, "q3": 16.85284761351425, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4791286.402291238, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4697713.503142124, "q3": 4843971.515477956, "n": 5 }, "batch_cpu_utilization": { "value": 3.5612438855466726, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5504032077437833, "q3": 3.5617379185266214, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5612438855466726, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5504032077437833, "q3": 3.5617379185266214, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.2136376890689483, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.21029328560995722, "q3": 0.21722981728689644, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 142.818603515625, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 142.7998046875, "q3": 142.95751953125, "n": 5 }, "rss_peak_large_mb": { "value": 268.546048, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 268.230656, "q3": 268.648448, "n": 5 }, "rss_peak_very_large_mb": { "value": 2628.591616, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2628.23936, "q3": 2629.791744, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.02048, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.02048, "q3": 0.02048, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.1044676110377996, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.9248470245029297, "q3": 3.189589033755548, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6358257878685136, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.4550918415669996, "provenance": "measured" }, "chars_per_token": { "value": 2.4528801878142414, "provenance": "measured" }, "tokens_per_word": { "value": 2.685690317692266, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3859915691244393, "provenance": "measured" }, "chars_per_token": { "value": 3.372471241570805, "provenance": "measured" }, "tokens_per_word": { "value": 2.5040657250122056, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.9063746545350755, "provenance": "measured" }, "chars_per_token": { "value": 3.905748118651973, "provenance": "measured" }, "tokens_per_word": { "value": 2.698153609196266, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.915853634004546, "provenance": "measured" }, "chars_per_token": { "value": 4.888616201645511, "provenance": "measured" }, "tokens_per_word": { "value": 2.497099083260078, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.208968587351262, "provenance": "measured" }, "chars_per_token": { "value": 4.200651013338242, "provenance": "measured" }, "tokens_per_word": { "value": 2.5414912168772066, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.525404753076996, "provenance": "measured" }, "chars_per_token": { "value": 3.5225912920837787, "provenance": "measured" }, "tokens_per_word": { "value": 2.3840994176372714, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.568822039219649, "provenance": "measured" }, "chars_per_token": { "value": 3.5134280744552626, "provenance": "measured" }, "tokens_per_word": { "value": 3.3760477582846002, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.579918865930015, "provenance": "measured" }, "chars_per_token": { "value": 4.577443282948293, "provenance": "measured" }, "tokens_per_word": { "value": 2.1553396167627357, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.037448693292038, "provenance": "measured" }, "chars_per_token": { "value": 4.022479206733771, "provenance": "measured" }, "tokens_per_word": { "value": 2.496124192107068, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.621873812238692, "provenance": "measured" }, "chars_per_token": { "value": 4.606518434055492, "provenance": "measured" }, "tokens_per_word": { "value": 2.8096967108073474, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.6240998524707497, "provenance": "measured" }, "chars_per_token": { "value": 3.5947765881304683, "provenance": "measured" }, "tokens_per_word": { "value": 2.6727961271179823, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.9683244667870565, "provenance": "measured" }, "chars_per_token": { "value": 3.962687295622041, "provenance": "measured" }, "tokens_per_word": { "value": 2.886113393590797, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.7613523710882593, "provenance": "measured" }, "chars_per_token": { "value": 3.760997262706222, "provenance": "measured" }, "tokens_per_word": { "value": 2.8754680054458817, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.19499399613441, "provenance": "measured" }, "chars_per_token": { "value": 4.183527790085167, "provenance": "measured" }, "tokens_per_word": { "value": 2.8337245482832034, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.0574453423541943, "provenance": "measured" }, "chars_per_token": { "value": 3.043091837373927, "provenance": "measured" }, "tokens_per_word": { "value": 2.936061616277733, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.06639055195604, "provenance": "measured" }, "chars_per_token": { "value": 3.9796194537849585, "provenance": "measured" }, "tokens_per_word": { "value": 2.5679204751274165, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.176469071500399, "provenance": "measured" }, "chars_per_token": { "value": 4.17505951920515, "provenance": "measured" }, "tokens_per_word": { "value": 2.418898128898129, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.3728858406155346, "provenance": "measured" }, "chars_per_token": { "value": 3.3667513839083663, "provenance": "measured" }, "tokens_per_word": { "value": 2.5910189795771092, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.3195038175696276, "provenance": "measured" }, "chars_per_token": { "value": 3.306435331657713, "provenance": "measured" }, "tokens_per_word": { "value": 2.7149610064743968, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.595749254624134, "provenance": "measured" }, "chars_per_token": { "value": 4.578523828661688, "provenance": "measured" }, "tokens_per_word": { "value": 2.299836636430076, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.128888984250954, "provenance": "measured" }, "chars_per_token": { "value": 4.040474044829708, "provenance": "measured" }, "tokens_per_word": { "value": 3.031218835364003, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.685469802799975, "provenance": "measured" }, "chars_per_token": { "value": 4.66641590642839, "provenance": "measured" }, "tokens_per_word": { "value": 1.3105817570228107, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 4.404842794490086, "provenance": "measured" }, "chars_per_token": { "value": 2.4427620606584717, "provenance": "measured" }, "tokens_per_word": { "value": 2.50497079416657, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 2.7218109576558085, "provenance": "measured" }, "chars_per_token": { "value": 1.0224336519476955, "provenance": "measured" }, "tokens_per_word": { "value": 7.006247211066488, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.3603198701882913, "provenance": "measured" }, "chars_per_token": { "value": 3.27550446629394, "provenance": "measured" }, "tokens_per_word": { "value": 2.6309638286855535, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.9360839546285282, "provenance": "measured" }, "chars_per_token": { "value": 3.848524846972818, "provenance": "measured" }, "tokens_per_word": { "value": 1.6218594146984098, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.727614715016841, "provenance": "measured" }, "chars_per_token": { "value": 3.608860796407273, "provenance": "measured" }, "tokens_per_word": { "value": 1.7785817092620686, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.5934968854355893, "provenance": "measured" }, "chars_per_token": { "value": 0.6191238451530903, "provenance": "measured" }, "tokens_per_word": { "value": 10.287855424659224, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 2.8877168239802553, "provenance": "measured" }, "chars_per_token": { "value": 1.1463565479252524, "provenance": "measured" }, "tokens_per_word": { "value": 4.822416099899654, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.3239350026695718, "provenance": "measured" }, "chars_per_token": { "value": 3.3162175134692027, "provenance": "measured" }, "tokens_per_word": { "value": 2.250497621128265, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.8217780347876373, "provenance": "measured" }, "chars_per_token": { "value": 1.340967540669274, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 2.863647767185148, "provenance": "measured" }, "chars_per_token": { "value": 1.2358690198302755, "provenance": "measured" }, "tokens_per_word": { "value": 3.625427582797825, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 2.8584002144759886, "provenance": "measured" }, "chars_per_token": { "value": 1.0861155318262972, "provenance": "measured" }, "tokens_per_word": { "value": 6.46141088014153, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.990618953410005, "provenance": "measured" }, "chars_per_token": { "value": 3.2959866947844425, "provenance": "measured" }, "tokens_per_word": { "value": 2.2451798701615657, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.0479022281235832, "provenance": "measured" }, "chars_per_token": { "value": 0.7802399766824276, "provenance": "measured" }, "tokens_per_word": { "value": 12.266560730956401, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.573534903405696, "provenance": "measured" }, "chars_per_token": { "value": 0.5994510555666202, "provenance": "measured" }, "tokens_per_word": { "value": 13.317750424800034, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 3.1578897626828626, "provenance": "measured" }, "chars_per_token": { "value": 1.1290694840304618, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.2606834860616654, "provenance": "measured" }, "chars_per_token": { "value": 2.9806829719691033, "provenance": "measured" }, "tokens_per_word": { "value": 2.618397805858895, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 3.210247767357397, "provenance": "measured" }, "chars_per_token": { "value": 1.8131545560834847, "provenance": "measured" }, "tokens_per_word": { "value": 2.6387252408224886, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.291977095891198, "provenance": "measured" }, "chars_per_token": { "value": 3.2859565051835995, "provenance": "measured" }, "tokens_per_word": { "value": 1.3981115966048918, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.829033609672882, "provenance": "measured" }, "chars_per_token": { "value": 1.4381528147726932, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 88272, "provenance": "measured" }, "tokens_per_request_mean": { "value": 441.36, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.44136, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 57233, "provenance": "measured" }, "tokens_per_request_mean": { "value": 351.1226993865031, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3511226993865031, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 42375, "provenance": "measured" }, "tokens_per_request_mean": { "value": 223.02631578947367, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.22302631578947368, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 223243, "provenance": "measured" }, "tokens_per_request_mean": { "value": 531.5309523809524, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5315309523809524, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 141802, "provenance": "measured" }, "tokens_per_request_mean": { "value": 945.3466666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9453466666666668, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 440214, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2201.07, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.20107, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 340744, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1703.72, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.70372, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 431637, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2158.185, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.158185, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8344, "provenance": "measured" }, "tokens_per_request_mean": { "value": 333.76, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33376, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 687842, "provenance": "measured" }, "tokens_per_request_mean": { "value": 114640.33333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 114.64033333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 154880, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 317194240, "provenance": "estimated", "note": "154880 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 634388480, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 634388480, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1268776960, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 154880, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 634388480, "provenance": "estimated", "note": "154880 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1268776960, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1268776960, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2537553920, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 1268776960, "provenance": "estimated", "note": "154880 rows x 4096 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 2537553920, "provenance": "estimated", "note": "154880 rows x 4096 hidden x 2 matrices from the published config" } } } }, { "id": "seed-oss-36b-instruct", "label": "Seed-OSS-36B-Instruct", "provider": "ByteDance", "family": "published", "metrics": { "bpt_overall": { "value": 4.109204885574355, "provenance": "measured" }, "bpt_english": { "value": 4.588916388188734, "provenance": "measured" }, "bpt_code": { "value": 3.617158737376367, "provenance": "measured" }, "bpt_languages": { "value": 4.121539531157966, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.6584681815441675, "provenance": "measured" }, "byte_fallback_share": { "value": 0.016971644957749936, "provenance": "measured" }, "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.236921381376127, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.188703953896487, "q3": 4.247458323581646, "n": 5 }, "batch_mbps_4t": { "value": 13.967031663098133, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.93647462267473, "q3": 14.032971454237483, "n": 5 }, "latency_small_p95_us": { "value": 81.83299996744609, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 81.33300002555188, "q3": 82.5830002213479, "n": 5 }, "decode_mbps": { "value": 38.05299281016757, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.25448327934852, "q3": 38.3355817570048, "n": 5 }, "rss_load_delta_mb": { "value": 136.736768, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 136.732672, "q3": 136.736768, "n": 5 }, "rss_peak_large_mb": { "value": 231.493632, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 229.392384, "q3": 233.213952, "n": 5 }, "load_seconds": { "value": 0.24087216699990677, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.2376469589999033, "q3": 0.24242649999996502, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6685284967015641, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2654.588928, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2654.584832, "q3": 2654.691328, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.6128, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.924672, "q3": 17.92, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.8325026817340966, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.830224021697409, "q3": 2.857073570005019, "n": 5 }, "embedding_params_h2048": { "value": 317718528, "provenance": "estimated", "note": "155136 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.25809653445221953, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25703100803472245, "q3": 0.25914680159678755, "n": 5 }, "cpu_cores_used_batch": { "value": 3.604842468830048, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.6003777834383697, "q3": 3.6069087986051453, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 147.872314453125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 147.799560546875, "q3": 147.91015625000003, "n": 5 }, "workload_tokens_total": { "value": 2548124, "provenance": "measured" }, "tokens_chat_en": { "value": 92216, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 58407, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 31858, "provenance": "measured" }, "tokens_code": { "value": 257309, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 145766, "provenance": "measured" }, "tokens_rag": { "value": 325417, "provenance": "measured" }, "tokens_json": { "value": 422606, "provenance": "measured" }, "tokens_logs": { "value": 569727, "provenance": "measured" }, "tokens_tech_docs": { "value": 9069, "provenance": "measured" }, "tokens_long_context": { "value": 635749, "provenance": "measured" } }, "parity": { "de": { "value": 1.3979652020053082, "provenance": "measured" }, "fr": { "value": 1.4515260984960188, "provenance": "measured" }, "es": { "value": 1.4252801533470953, "provenance": "measured" }, "ru": { "value": 1.516809200825715, "provenance": "measured" }, "zh": { "value": 1.003096431731053, "provenance": "measured" }, "ja": { "value": 2.146785608964907, "provenance": "measured" }, "ko": { "value": 1.7215054556178118, "provenance": "measured" }, "ar": { "value": 1.6963653789442643, "provenance": "measured" }, "ur": { "value": 2.265076673547626, "provenance": "measured" }, "hi": { "value": 2.7048437039221467, "provenance": "measured" }, "mr": { "value": 2.7693158360365673, "provenance": "measured" }, "gu": { "value": 3.0769684458861692, "provenance": "measured" }, "bn": { "value": 2.770384842229431, "provenance": "measured" }, "ta": { "value": 3.0001105868475375, "provenance": "measured" }, "te": { "value": 2.9624741964022414, "provenance": "measured" }, "th": { "value": 2.6141256266588027, "provenance": "measured" }, "vi": { "value": 1.462953406074904, "provenance": "measured" }, "tr": { "value": 1.528900029489826, "provenance": "measured" }, "id": { "value": 1.3083161309348275, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 27.0199203187251, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 37.77290836653386, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3979652020053082, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 39.22011952191235, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4515260984960188, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 38.5109561752988, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4252801533470953, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 40.98406374501992, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.516809200825715, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 27.10358565737052, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.003096431731053, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 58.00597609561753, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.146785608964907, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 46.514940239043824, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7215054556178118, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 45.83565737051793, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6963653789442643, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 61.20219123505976, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.265076673547626, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 73.08466135458167, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.7048437039221467, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 74.82669322709164, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.7693158360365673, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 83.13944223107569, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.0769684458861692, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 74.85557768924303, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.770384842229431, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 81.06274900398407, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.0001105868475375, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 80.04581673306772, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.9624741964022414, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 70.63346613545816, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.6141256266588027, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 39.528884462151396, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.462953406074904, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 41.310756972111555, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.528900029489826, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 35.35059760956175, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3083161309348275, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 155,136 embedding rows for 155,121 tokenizer IDs (15 unused rows, usually padding)" ], "roundtrip_failures": { "value": 733, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 733, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [ "code_cs" ], "roundtrip_example": " \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\",", "byte_fallback_sets": [], "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "unicode_failed_probes": [ "latin_accents (NFD)", "vietnamese (NFD)", "devanagari_nukta_precomposed (NFC)", "tamil (NFD)", "telugu (NFD)", "japanese (NFD)", "korean (NFD)" ], "nfd_normalized_to_nfc": [ "japanese", "korean", "latin_accents", "tamil", "telugu", "vietnamese" ], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 155121, "provenance": "measured" }, "used_share": { "value": 0.5945036455412227, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 92220, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 62901, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 954, "provenance": "measured", "note": "of 153433 entries whose text is valid UTF-8; 1566 are byte pieces" }, "unreachable_examples": [ ".\n", ";\n", ");\n", ">\n", ",\n", ")\n", " {\n", "?\n", ":\n", " }\n", ";\r\n", "\"\n" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.24087216699990677, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.2376469589999033, "q3": 0.24242649999996502, "n": 5 }, "rss_load_delta_mb": { "value": 136.736768, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 136.732672, "q3": 136.736768, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 136.245248, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 136.241152, "q3": 136.245248, "n": 5 }, "latency_small_p50_us": { "value": 60.87499997420309, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 59.7920002292085, "q3": 60.958000176469795, "n": 5 }, "latency_small_p95_us": { "value": 81.83299996744609, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 81.33300002555188, "q3": 82.5830002213479, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.236921381376127, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.188703953896487, "q3": 4.247458323581646, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1176330.3184721833, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1162943.3290245265, "q3": 1179255.7738829977, "n": 5 }, "encode_mbps_1t_large": { "value": 3.5832983067200725, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.573083872348928, "q3": 3.5844148058790375, "n": 5 }, "latency_large_p50_ms": { "value": 288.25741699995433, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 286.66162500007886, "q3": 290.97766699987915, "n": 5 }, "decode_mbps": { "value": 38.05299281016757, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.25448327934852, "q3": 38.3355817570048, "n": 5 }, "decode_tokens_per_s": { "value": 10326131.234594325, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10109446.196220456, "q3": 10402815.099252334, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.320678113351477, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.312024187877408, "q3": 4.362579658182125, "n": 5 }, "batch_mbps_4t": { "value": 13.967031663098133, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.93647462267473, "q3": 14.032971454237483, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3844450.2154439176, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3836039.3359189956, "q3": 3862600.259803147, "n": 5 }, "batch_cpu_utilization": { "value": 3.604842468830048, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.6003777834383697, "q3": 3.6069087986051453, "n": 5 }, "cpu_cores_used_batch": { "value": 3.604842468830048, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.6003777834383697, "q3": 3.6069087986051453, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.25809653445221953, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25703100803472245, "q3": 0.25914680159678755, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 147.872314453125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 147.799560546875, "q3": 147.91015625000003, "n": 5 }, "rss_peak_large_mb": { "value": 231.493632, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 229.392384, "q3": 233.213952, "n": 5 }, "rss_peak_very_large_mb": { "value": 2654.588928, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2654.584832, "q3": 2654.691328, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.6128, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 16.924672, "q3": 17.92, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.8325026817340966, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.830224021697409, "q3": 2.857073570005019, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6685284967015641, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.1789155522170507, "provenance": "measured" }, "chars_per_token": { "value": 2.1769526901047604, "provenance": "measured" }, "tokens_per_word": { "value": 3.026100016236402, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.1687534605124075, "provenance": "measured" }, "chars_per_token": { "value": 3.156100568782403, "provenance": "measured" }, "tokens_per_word": { "value": 2.6757352817387496, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.6621417896323756, "provenance": "measured" }, "chars_per_token": { "value": 3.661554425786578, "provenance": "measured" }, "tokens_per_word": { "value": 2.8780968838633334, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.608966258652232, "provenance": "measured" }, "chars_per_token": { "value": 4.5834292072952225, "provenance": "measured" }, "tokens_per_word": { "value": 2.66336807735778, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.9723983794413336, "provenance": "measured" }, "chars_per_token": { "value": 3.9645483048103425, "provenance": "measured" }, "tokens_per_word": { "value": 2.692845901918251, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.297385509808588, "provenance": "measured" }, "chars_per_token": { "value": 3.2947540203311463, "provenance": "measured" }, "tokens_per_word": { "value": 2.5489635330005544, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.393411797000995, "provenance": "measured" }, "chars_per_token": { "value": 3.3407404865662422, "provenance": "measured" }, "tokens_per_word": { "value": 3.5505604288499026, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.328014138982096, "provenance": "measured" }, "chars_per_token": { "value": 4.325674718031471, "provenance": "measured" }, "tokens_per_word": { "value": 2.2807875058420315, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.8077373053699444, "provenance": "measured" }, "chars_per_token": { "value": 3.7936195105098265, "provenance": "measured" }, "tokens_per_word": { "value": 2.646709200108065, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.234477835428492, "provenance": "measured" }, "chars_per_token": { "value": 4.220409513528572, "provenance": "measured" }, "tokens_per_word": { "value": 3.0667449807774454, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.341699608735765, "provenance": "measured" }, "chars_per_token": { "value": 3.314661297165523, "provenance": "measured" }, "tokens_per_word": { "value": 2.898668696806434, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.753150436214245, "provenance": "measured" }, "chars_per_token": { "value": 3.7478189287747536, "provenance": "measured" }, "tokens_per_word": { "value": 3.051578823805611, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.576180461281995, "provenance": "measured" }, "chars_per_token": { "value": 3.575842834935886, "provenance": "measured" }, "tokens_per_word": { "value": 3.0243575561606537, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.894951996168438, "provenance": "measured" }, "chars_per_token": { "value": 3.8843058969890043, "provenance": "measured" }, "tokens_per_word": { "value": 3.0520164249625465, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.8244417277324056, "provenance": "measured" }, "chars_per_token": { "value": 2.811182083203727, "provenance": "measured" }, "tokens_per_word": { "value": 3.1782733647545696, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.790953367637951, "provenance": "measured" }, "chars_per_token": { "value": 3.710059714501984, "provenance": "measured" }, "tokens_per_word": { "value": 2.754496440756497, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.902503272645502, "provenance": "measured" }, "chars_per_token": { "value": 3.901186183412707, "provenance": "measured" }, "tokens_per_word": { "value": 2.5887110187110185, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.1020069673915827, "provenance": "measured" }, "chars_per_token": { "value": 3.09636517328825, "provenance": "measured" }, "tokens_per_word": { "value": 2.817276466767699, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.9117412852541626, "provenance": "measured" }, "chars_per_token": { "value": 2.900278111220718, "provenance": "measured" }, "tokens_per_word": { "value": 3.095166274278988, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.219372388168565, "provenance": "measured" }, "chars_per_token": { "value": 4.203557668379999, "provenance": "measured" }, "tokens_per_word": { "value": 2.504986887923993, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.991129946627582, "provenance": "measured" }, "chars_per_token": { "value": 3.9056649428942785, "provenance": "measured" }, "tokens_per_word": { "value": 3.135845293326974, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.588916388188734, "provenance": "measured" }, "chars_per_token": { "value": 4.570255135209115, "provenance": "measured" }, "tokens_per_word": { "value": 1.3381571436856516, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 4.632850947717196, "provenance": "measured" }, "chars_per_token": { "value": 2.5692069060728615, "provenance": "measured" }, "tokens_per_word": { "value": 2.381687362191018, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 4.552439873806695, "provenance": "measured" }, "chars_per_token": { "value": 1.71009956160118, "provenance": "measured" }, "tokens_per_word": { "value": 4.188892321422442, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.2216959516960673, "provenance": "measured" }, "chars_per_token": { "value": 3.1403794538852243, "provenance": "measured" }, "tokens_per_word": { "value": 2.7441695814356515, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.885279231424542, "provenance": "measured" }, "chars_per_token": { "value": 3.798850286712436, "provenance": "measured" }, "tokens_per_word": { "value": 1.6430671873529687, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.7992138563393048, "provenance": "measured" }, "chars_per_token": { "value": 3.6781789405636753, "provenance": "measured" }, "tokens_per_word": { "value": 1.7450629530218746, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 3.795895487750016, "provenance": "measured" }, "chars_per_token": { "value": 1.4748252297541433, "provenance": "measured" }, "tokens_per_word": { "value": 4.3187873928329115, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 4.588673457204001, "provenance": "measured" }, "chars_per_token": { "value": 1.8215968478190987, "provenance": "measured" }, "tokens_per_word": { "value": 3.034814360575315, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.8145381401825778, "provenance": "measured" }, "chars_per_token": { "value": 3.8056815720253754, "provenance": "measured" }, "tokens_per_word": { "value": 1.9610520438877561, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 2.9086376428894063, "provenance": "measured" }, "chars_per_token": { "value": 1.0205691254647162, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.0803487096452478, "provenance": "measured" }, "chars_per_token": { "value": 1.3293909901030758, "provenance": "measured" }, "tokens_per_word": { "value": 3.37038062283737, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 4.599440110142267, "provenance": "measured" }, "chars_per_token": { "value": 1.7476640660853602, "provenance": "measured" }, "tokens_per_word": { "value": 4.015553589856996, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.703857419348162, "provenance": "measured" }, "chars_per_token": { "value": 3.138212980883677, "provenance": "measured" }, "tokens_per_word": { "value": 2.3580563284033778, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 5.059518332566559, "provenance": "measured" }, "chars_per_token": { "value": 1.9276498709819374, "provenance": "measured" }, "tokens_per_word": { "value": 4.965041215612275, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 4.3388565201348905, "provenance": "measured" }, "chars_per_token": { "value": 1.6529230558010624, "provenance": "measured" }, "tokens_per_word": { "value": 4.8298313233039085, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 4.989068762278978, "provenance": "measured" }, "chars_per_token": { "value": 1.7837878192534382, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.4382360514439823, "provenance": "measured" }, "chars_per_token": { "value": 3.1429887923674262, "provenance": "measured" }, "tokens_per_word": { "value": 2.4831821776513943, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 3.840336572083265, "provenance": "measured" }, "chars_per_token": { "value": 2.169030011754728, "provenance": "measured" }, "tokens_per_word": { "value": 2.2057863038876215, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.145416695579765, "provenance": "measured" }, "chars_per_token": { "value": 3.1737492193463326, "provenance": "measured" }, "tokens_per_word": { "value": 1.4475415599417407, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.9149473297684345, "provenance": "measured" }, "chars_per_token": { "value": 1.4704212853525502, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 92216, "provenance": "measured" }, "tokens_per_request_mean": { "value": 461.08, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.46108, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 58407, "provenance": "measured" }, "tokens_per_request_mean": { "value": 358.32515337423314, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.35832515337423315, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 31858, "provenance": "measured" }, "tokens_per_request_mean": { "value": 167.67368421052632, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.1676736842105263, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 257309, "provenance": "measured" }, "tokens_per_request_mean": { "value": 612.6404761904762, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6126404761904762, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 145766, "provenance": "measured" }, "tokens_per_request_mean": { "value": 971.7733333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9717733333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 325417, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1627.085, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.627085, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 422606, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2113.03, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.11303, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 569727, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2848.635, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.848635, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9069, "provenance": "measured" }, "tokens_per_request_mean": { "value": 362.76, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.36276, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 635749, "provenance": "measured" }, "tokens_per_request_mean": { "value": 105958.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 105.95816666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 155136, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 317718528, "provenance": "estimated", "note": "155136 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 635437056, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 635437056, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1270874112, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 155136, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 635437056, "provenance": "estimated", "note": "155136 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1270874112, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1270874112, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2541748224, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 1588592640, "provenance": "estimated", "note": "155136 rows x 5120 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 3177185280, "provenance": "estimated", "note": "155136 rows x 5120 hidden x 2 matrices from the published config" } } } }, { "id": "ling-3-0-tiny", "label": "Ling-3.0-tiny", "provider": "inclusionAI (Ant Group)", "family": "published", "metrics": { "bpt_overall": { "value": 3.9197020959245976, "provenance": "measured" }, "bpt_english": { "value": 4.617364480575817, "provenance": "measured" }, "bpt_code": { "value": 3.668718343481931, "provenance": "measured" }, "bpt_languages": { "value": 3.4730234637160464, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.8697444892017941, "provenance": "measured" }, "byte_fallback_share": { "value": 0.14186577700897782, "provenance": "measured" }, "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.229124727384775, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.186029504625122, "q3": 4.309772640935677, "n": 5 }, "batch_mbps_4t": { "value": 14.130000644302076, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.545959144013002, "q3": 14.310461923491644, "n": 5 }, "latency_small_p95_us": { "value": 82.08300005208002, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 80.7499999382344, "q3": 82.66600002571067, "n": 5 }, "decode_mbps": { "value": 34.64389579633848, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 34.567945515610404, "q3": 34.84015594000904, "n": 5 }, "rss_load_delta_mb": { "value": 103.542784, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 103.542784, "q3": 103.54688, "n": 5 }, "rss_peak_large_mb": { "value": 232.214528, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 231.739392, "q3": 232.460288, "n": 5 }, "load_seconds": { "value": 0.2463905419999719, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.24160091700014164, "q3": 0.24775787499993385, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6608930515710159, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2665.668608, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2665.443328, "q3": 2667.6224, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.42848, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.32608, "q3": 18.305024, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.7949991465557646, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.656495133019671, "q3": 2.796124742398232, "n": 5 }, "embedding_params_h2048": { "value": 321912832, "provenance": "estimated", "note": "157184 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.25372713701944916, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25231183945528207, "q3": 0.2657993677415632, "n": 5 }, "cpu_cores_used_batch": { "value": 3.592977491594083, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.586836403021493, "q3": 3.6005073759317026, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 148.587646484375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 148.55419921875, "q3": 148.701171875, "n": 5 }, "workload_tokens_total": { "value": 2726875, "provenance": "measured" }, "tokens_chat_en": { "value": 93408, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 62360, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 39810, "provenance": "measured" }, "tokens_code": { "value": 256964, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 152168, "provenance": "measured" }, "tokens_rag": { "value": 415967, "provenance": "measured" }, "tokens_json": { "value": 420136, "provenance": "measured" }, "tokens_logs": { "value": 562034, "provenance": "measured" }, "tokens_tech_docs": { "value": 9042, "provenance": "measured" }, "tokens_long_context": { "value": 714986, "provenance": "measured" } }, "parity": { "de": { "value": 1.557815806080404, "provenance": "measured" }, "fr": { "value": 1.5782694235123798, "provenance": "measured" }, "es": { "value": 1.5828724154571439, "provenance": "measured" }, "ru": { "value": 1.7620178922751402, "provenance": "measured" }, "zh": { "value": 0.881621441033446, "provenance": "measured" }, "ja": { "value": 1.5135305690634395, "provenance": "measured" }, "ko": { "value": 1.501985968298749, "provenance": "measured" }, "ar": { "value": 1.8212999740153681, "provenance": "measured" }, "ur": { "value": 3.60024499795835, "provenance": "measured" }, "hi": { "value": 3.8340695645718106, "provenance": "measured" }, "mr": { "value": 4.270537139463232, "provenance": "measured" }, "gu": { "value": 6.9767623148594975, "provenance": "measured" }, "bn": { "value": 2.571067968372991, "provenance": "measured" }, "ta": { "value": 7.0051969263892495, "provenance": "measured" }, "te": { "value": 6.80893871338951, "provenance": "measured" }, "th": { "value": 2.2906195478674043, "provenance": "measured" }, "vi": { "value": 1.598240469208211, "provenance": "measured" }, "tr": { "value": 1.6484650506700322, "provenance": "measured" }, "id": { "value": 1.5326849548981032, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.83167330677291, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 41.798804780876495, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.557815806080404, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 42.34760956175299, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5782694235123798, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 42.471115537848604, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5828724154571439, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 47.27788844621514, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7620178922751402, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 23.655378486055778, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 0.881621441033446, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 40.6105577689243, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5135305690634395, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 40.300796812749006, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.501985968298749, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 48.86852589641434, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8212999740153681, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 96.60059760956176, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.60024499795835, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 102.87450199203187, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.8340695645718106, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 114.58565737051792, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.270537139463232, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 187.19820717131475, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.9767623148594975, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 68.98605577689243, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.571067968372991, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 187.96115537848607, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.0051969263892495, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 182.69521912350598, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.80893871338951, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 61.461155378486055, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.2906195478674043, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 42.883466135458164, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.598240469208211, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 44.23107569721115, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6484650506700322, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 41.124501992031874, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5326849548981032, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 157,184 embedding rows for 157,153 tokenizer IDs (31 unused rows, usually padding)" ], "roundtrip_failures": { "value": 733, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 733, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [ "code_cs" ], "roundtrip_example": " \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\",", "byte_fallback_sets": [], "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "unicode_failed_probes": [ "latin_accents (NFD)", "vietnamese (NFD)", "devanagari_nukta_precomposed (NFC)", "tamil (NFD)", "telugu (NFD)", "japanese (NFD)", "korean (NFD)" ], "nfd_normalized_to_nfc": [ "japanese", "korean", "latin_accents", "tamil", "telugu", "vietnamese" ], "special_strings_parsed_by_default": [ "<|endoftext|>", "[CLS]" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 157153, "provenance": "measured" }, "used_share": { "value": 0.6360171298034399, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 99952, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 57201, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 487, "provenance": "measured", "note": "of 154690 entries whose text is valid UTF-8; 2211 are byte pieces" }, "unreachable_examples": [ "tiv", " ра", "cion", "tig", "ере", " tod", "ciones", "forma", "tions", " parti", "cip", "ски" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.2463905419999719, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.24160091700014164, "q3": 0.24775787499993385, "n": 5 }, "rss_load_delta_mb": { "value": 103.542784, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 103.542784, "q3": 103.54688, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 103.40352, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 103.40352, "q3": 103.407616, "n": 5 }, "latency_small_p50_us": { "value": 61.29200028226478, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 60.541499976807245, "q3": 62.08299998888833, "n": 5 }, "latency_small_p95_us": { "value": 82.08300005208002, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 80.7499999382344, "q3": 82.66600002571067, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.229124727384775, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.186029504625122, "q3": 4.309772640935677, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1277111.916236651, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1264098.0123990925, "q3": 1301465.9890187916, "n": 5 }, "encode_mbps_1t_large": { "value": 3.5616588071884467, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.4902895167718397, "q3": 3.5922621305761613, "n": 5 }, "latency_large_p50_ms": { "value": 290.688541000236, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 285.02687500008506, "q3": 302.0147080000015, "n": 5 }, "decode_mbps": { "value": 34.64389579633848, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 34.567945515610404, "q3": 34.84015594000904, "n": 5 }, "decode_tokens_per_s": { "value": 10729037.181077695, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10705515.76793654, "q3": 10789817.942888778, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.482832739614926, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.473135286428638, "q3": 4.494126811129551, "n": 5 }, "batch_mbps_4t": { "value": 14.130000644302076, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.545959144013002, "q3": 14.310461923491644, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4266975.038939523, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4090606.2923150696, "q3": 4321470.703390004, "n": 5 }, "batch_cpu_utilization": { "value": 3.592977491594083, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.586836403021493, "q3": 3.6005073759317026, "n": 5 }, "cpu_cores_used_batch": { "value": 3.592977491594083, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.586836403021493, "q3": 3.6005073759317026, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.25372713701944916, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25231183945528207, "q3": 0.2657993677415632, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 148.587646484375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 148.55419921875, "q3": 148.701171875, "n": 5 }, "rss_peak_large_mb": { "value": 232.214528, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 231.739392, "q3": 232.460288, "n": 5 }, "rss_peak_very_large_mb": { "value": 2665.668608, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2665.443328, "q3": 2667.6224, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.42848, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.32608, "q3": 18.305024, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.7949991465557646, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.656495133019671, "q3": 2.796124742398232, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6608930515710159, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.180641321663176, "provenance": "measured" }, "chars_per_token": { "value": 2.17867690490254, "provenance": "measured" }, "tokens_per_word": { "value": 3.023705146939438, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.173585642809432, "provenance": "measured" }, "chars_per_token": { "value": 3.160913456085296, "provenance": "measured" }, "tokens_per_word": { "value": 2.6716611390764156, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.729117793653431, "provenance": "measured" }, "chars_per_token": { "value": 3.7285196876557567, "provenance": "measured" }, "tokens_per_word": { "value": 2.8264054546479085, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.693143844824275, "provenance": "measured" }, "chars_per_token": { "value": 4.6671403879393125, "provenance": "measured" }, "tokens_per_word": { "value": 2.6155971367575033, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.064260954581815, "provenance": "measured" }, "chars_per_token": { "value": 4.056229345270319, "provenance": "measured" }, "tokens_per_word": { "value": 2.6319807749564035, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.269881727825653, "provenance": "measured" }, "chars_per_token": { "value": 3.2672721878329356, "provenance": "measured" }, "tokens_per_word": { "value": 2.570403494176373, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.382062050285902, "provenance": "measured" }, "chars_per_token": { "value": 3.3295669065141857, "provenance": "measured" }, "tokens_per_word": { "value": 3.562475633528265, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.402821085179749, "provenance": "measured" }, "chars_per_token": { "value": 4.40044122883969, "provenance": "measured" }, "tokens_per_word": { "value": 2.242035363763826, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.836596808519055, "provenance": "measured" }, "chars_per_token": { "value": 3.8223720124368072, "provenance": "measured" }, "tokens_per_word": { "value": 2.626800224443567, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.336715406562054, "provenance": "measured" }, "chars_per_token": { "value": 4.322307417974322, "provenance": "measured" }, "tokens_per_word": { "value": 2.9944468175993166, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.341744617518907, "provenance": "measured" }, "chars_per_token": { "value": 3.314705941774249, "provenance": "measured" }, "tokens_per_word": { "value": 2.8986296556570625, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.7711383559314173, "provenance": "measured" }, "chars_per_token": { "value": 3.7657812958983317, "provenance": "measured" }, "tokens_per_word": { "value": 3.0370231247799038, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.685365729901346, "provenance": "measured" }, "chars_per_token": { "value": 3.685017795400016, "provenance": "measured" }, "tokens_per_word": { "value": 2.934755786249149, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.9890113245027723, "provenance": "measured" }, "chars_per_token": { "value": 3.978108132311876, "provenance": "measured" }, "tokens_per_word": { "value": 2.9800510702306653, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.8767369553364865, "provenance": "measured" }, "chars_per_token": { "value": 2.863231805254704, "provenance": "measured" }, "tokens_per_word": { "value": 3.1204966088056096, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.909288023338327, "provenance": "measured" }, "chars_per_token": { "value": 3.8258692738311124, "provenance": "measured" }, "tokens_per_word": { "value": 2.671117476096205, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.017129063192355, "provenance": "measured" }, "chars_per_token": { "value": 4.015773287920373, "provenance": "measured" }, "tokens_per_word": { "value": 2.514844074844075, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.199261602678837, "provenance": "measured" }, "chars_per_token": { "value": 3.1934429261140354, "provenance": "measured" }, "tokens_per_word": { "value": 2.731633831276611, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.92871018451552, "provenance": "measured" }, "chars_per_token": { "value": 2.917180205973593, "provenance": "measured" }, "tokens_per_word": { "value": 3.0772329311359625, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.361322997232668, "provenance": "measured" }, "chars_per_token": { "value": 4.344976229333712, "provenance": "measured" }, "tokens_per_word": { "value": 2.423455569408022, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.8945497230673745, "provenance": "measured" }, "chars_per_token": { "value": 3.811152862761501, "provenance": "measured" }, "tokens_per_word": { "value": 3.213610545028834, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.617364480575817, "provenance": "measured" }, "chars_per_token": { "value": 4.598587540796982, "provenance": "measured" }, "tokens_per_word": { "value": 1.3299126097719551, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 4.266770379123879, "provenance": "measured" }, "chars_per_token": { "value": 2.366192232036678, "provenance": "measured" }, "tokens_per_word": { "value": 2.586031488143592, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 4.986822853552354, "provenance": "measured" }, "chars_per_token": { "value": 1.873273192406086, "provenance": "measured" }, "tokens_per_word": { "value": 3.8240140047368962, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.036901386978159, "provenance": "measured" }, "chars_per_token": { "value": 2.960249155144814, "provenance": "measured" }, "tokens_per_word": { "value": 2.9111515010619975, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.504626789531122, "provenance": "measured" }, "chars_per_token": { "value": 3.4266655473688967, "provenance": "measured" }, "tokens_per_word": { "value": 1.8215277124306013, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.505934626540226, "provenance": "measured" }, "chars_per_token": { "value": 3.394242966559014, "provenance": "measured" }, "tokens_per_word": { "value": 1.8910413506048935, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.7315567629545663, "provenance": "measured" }, "chars_per_token": { "value": 0.6727644659865275, "provenance": "measured" }, "tokens_per_word": { "value": 9.46758773823475, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 3.1944002199502552, "provenance": "measured" }, "chars_per_token": { "value": 1.2681027372297777, "provenance": "measured" }, "tokens_per_word": { "value": 4.359432489686698, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.294476723985867, "provenance": "measured" }, "chars_per_token": { "value": 3.286827630812322, "provenance": "measured" }, "tokens_per_word": { "value": 2.270620934071269, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.8205570937414426, "provenance": "measured" }, "chars_per_token": { "value": 1.3405391426050446, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.45604797040684, "provenance": "measured" }, "chars_per_token": { "value": 1.4915321173984875, "provenance": "measured" }, "tokens_per_word": { "value": 3.0039940682155213, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 3.0151688367951093, "provenance": "measured" }, "chars_per_token": { "value": 1.145683409949698, "provenance": "measured" }, "tokens_per_word": { "value": 6.12546071060003, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 4.8449717355183, "provenance": "measured" }, "chars_per_token": { "value": 2.6656615119519635, "provenance": "measured" }, "tokens_per_word": { "value": 2.7760775125689463, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.2108444265748326, "provenance": "measured" }, "chars_per_token": { "value": 0.8423200971951508, "provenance": "measured" }, "tokens_per_word": { "value": 11.362498758565895, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.9147527823113537, "provenance": "measured" }, "chars_per_token": { "value": 0.7294408112723844, "provenance": "measured" }, "tokens_per_word": { "value": 10.944465166397281, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 5.776802125607381, "provenance": "measured" }, "chars_per_token": { "value": 2.065433401881745, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.2018577297079, "provenance": "measured" }, "chars_per_token": { "value": 2.92690868476018, "provenance": "measured" }, "tokens_per_word": { "value": 2.6665040130571587, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 2.493334299376902, "provenance": "measured" }, "chars_per_token": { "value": 1.408240351639859, "provenance": "measured" }, "tokens_per_word": { "value": 3.3974432610729113, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 3.829805238931699, "provenance": "measured" }, "chars_per_token": { "value": 2.932115702691267, "provenance": "measured" }, "tokens_per_word": { "value": 1.566832404198684, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.9018138290166897, "provenance": "measured" }, "chars_per_token": { "value": 1.4654884529464238, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 93408, "provenance": "measured" }, "tokens_per_request_mean": { "value": 467.04, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.46704, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 62360, "provenance": "measured" }, "tokens_per_request_mean": { "value": 382.5766871165644, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.38257668711656445, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 39810, "provenance": "measured" }, "tokens_per_request_mean": { "value": 209.52631578947367, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.20952631578947367, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 256964, "provenance": "measured" }, "tokens_per_request_mean": { "value": 611.8190476190476, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6118190476190477, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 152168, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1014.4533333333334, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.0144533333333334, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 415967, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2079.835, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.079835, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 420136, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2100.68, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.10068, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 562034, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2810.17, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.81017, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9042, "provenance": "measured" }, "tokens_per_request_mean": { "value": 361.68, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.36168, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 714986, "provenance": "measured" }, "tokens_per_request_mean": { "value": 119164.33333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 119.16433333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 157184, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 321912832, "provenance": "estimated", "note": "157184 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 643825664, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 643825664, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1287651328, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 157184, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 643825664, "provenance": "estimated", "note": "157184 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1287651328, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1287651328, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2575302656, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 482869248, "provenance": "estimated", "note": "157184 rows x 1536 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 965738496, "provenance": "estimated", "note": "157184 rows x 1536 hidden x 2 matrices from the published config" } } } }, { "id": "kimi-k3", "label": "Kimi-K3", "provider": "Moonshot AI (Kimi)", "family": "published", "metrics": { "bpt_overall": { "value": 4.0411548086607345, "provenance": "measured" }, "bpt_english": { "value": 4.693471510327113, "provenance": "measured" }, "bpt_code": { "value": 3.837333893356698, "provenance": "measured" }, "bpt_languages": { "value": 3.592659022298393, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.8384546893437685, "provenance": "measured" }, "byte_fallback_share": { "value": 0.058563459613022874, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.938964989355018, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.767492476666688, "q3": 4.9664928780057656, "n": 5 }, "batch_mbps_4t": { "value": 16.933337146737262, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.351862609405714, "q3": 17.09634579773465, "n": 5 }, "latency_small_p95_us": { "value": 64.8340001134784, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 63.41700009215856, "q3": 71.45800009311642, "n": 5 }, "decode_mbps": { "value": 36.614670171051586, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.2603091461058, "q3": 37.82018196408981, "n": 5 }, "rss_load_delta_mb": { "value": 188.178432, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 188.174336, "q3": 188.178432, "n": 5 }, "rss_peak_large_mb": { "value": 260.07552, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 260.07552, "q3": 260.07552, "n": 5 }, "load_seconds": { "value": 0.4387297920002311, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.4382846260000406, "q3": 0.441998125000282, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6713862051862792, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2572.242944, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2571.79648, "q3": 2572.603392, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.0, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.0, "q3": 0.0, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.3159529617509573, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.3129666324242, "q3": 3.317849229675027, "n": 5 }, "embedding_params_h2048": { "value": 335544320, "provenance": "estimated", "note": "163840 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.21131684902689948, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.20789614920326327, "q3": 0.2180292601326937, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5651845065201857, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5542644567964254, "q3": 3.5686436916357844, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 138.894287109375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 138.86767578125, "q3": 138.91552734375, "n": 5 }, "workload_tokens_total": { "value": 2297892, "provenance": "measured" }, "tokens_chat_en": { "value": 87699, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 65407, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 35210, "provenance": "measured" }, "tokens_code": { "value": 225688, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 133875, "provenance": "measured" }, "tokens_rag": { "value": 371138, "provenance": "measured" }, "tokens_json": { "value": 326493, "provenance": "measured" }, "tokens_logs": { "value": 410951, "provenance": "measured" }, "tokens_tech_docs": { "value": 8191, "provenance": "measured" }, "tokens_long_context": { "value": 633240, "provenance": "measured" } }, "parity": { "de": { "value": 1.7202915429988352, "provenance": "measured" }, "fr": { "value": 1.7203291129729121, "provenance": "measured" }, "es": { "value": 1.6885824848780855, "provenance": "measured" }, "ru": { "value": 2.0846827215689223, "provenance": "measured" }, "zh": { "value": 0.88999511590337, "provenance": "measured" }, "ja": { "value": 1.6606304241650074, "provenance": "measured" }, "ko": { "value": 1.9963181425404817, "provenance": "measured" }, "ar": { "value": 2.0892662584062816, "provenance": "measured" }, "ur": { "value": 2.2042303790810385, "provenance": "measured" }, "hi": { "value": 2.7742795957470787, "provenance": "measured" }, "mr": { "value": 2.9024683472968404, "provenance": "measured" }, "gu": { "value": 3.7784498628695946, "provenance": "measured" }, "bn": { "value": 2.9851222902656196, "provenance": "measured" }, "ta": { "value": 4.190066498854116, "provenance": "measured" }, "te": { "value": 4.000901679377841, "provenance": "measured" }, "th": { "value": 3.548446481571928, "provenance": "measured" }, "vi": { "value": 2.7955066311004244, "provenance": "measured" }, "tr": { "value": 1.948867265281587, "provenance": "measured" }, "id": { "value": 1.5761355524664689, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.510956175298805, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 45.60657370517928, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7202915429988352, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 45.60756972111554, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7203291129729121, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 44.76593625498008, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6885824848780855, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 55.266932270916335, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0846827215689223, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 23.594621513944222, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 0.88999511590337, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 44.02490039840637, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6606304241650074, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 52.92430278884462, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9963181425404817, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 55.38844621513944, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0892662584062816, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 58.43625498007968, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.2042303790810385, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 73.5488047808765, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.7742795957470787, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 76.94721115537848, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.9024683472968404, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 100.1703187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.7784498628695946, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 79.13844621513944, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.9851222902656196, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 111.08266932270917, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.190066498854116, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 106.06772908366534, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.000901679377841, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 94.07270916334662, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.548446481571928, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 74.11155378486056, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.7955066311004244, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 51.66633466135458, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.948867265281587, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 41.78486055776892, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5761355524664689, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 163,840 embedding rows for 163,584 tokenizer IDs (256 unused rows, usually padding)", "the earlier BPT benchmark listed a vocabulary of 163,840; the tokenizer's own files give 163,584" ], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (converted from tiktoken)" } }, "vocabulary": { "size": { "value": 163584, "provenance": "measured" }, "used_share": { "value": 0.5580924784820032, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 91295, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 72289, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 2, "provenance": "measured", "note": "of 162412 entries whose text is valid UTF-8; 1172 are byte pieces" }, "unreachable_examples": [ " I'", "⭐️⭐️" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (converted from tiktoken)", "load_seconds": { "value": 0.4387297920002311, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.4382846260000406, "q3": 0.441998125000282, "n": 5 }, "rss_load_delta_mb": { "value": 188.178432, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 188.174336, "q3": 188.178432, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 188.112896, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 188.1088, "q3": 188.112896, "n": 5 }, "latency_small_p50_us": { "value": 47.125000037340214, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 46.874999952706276, "q3": 49.24999996092083, "n": 5 }, "latency_small_p95_us": { "value": 64.8340001134784, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 63.41700009215856, "q3": 71.45800009311642, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.938964989355018, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.767492476666688, "q3": 4.9664928780057656, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1346476.3250427046, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1299728.943915678, "q3": 1353981.0654947085, "n": 5 }, "encode_mbps_1t_large": { "value": 4.183181658678838, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.161220798044554, "q3": 4.231672728624415, "n": 5 }, "latency_large_p50_ms": { "value": 245.3072499999962, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 244.1529999996419, "q3": 251.7798760000005, "n": 5 }, "decode_mbps": { "value": 36.614670171051586, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 35.2603091461058, "q3": 37.82018196408981, "n": 5 }, "decode_tokens_per_s": { "value": 9850065.388366895, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9485715.672997372, "q3": 10174371.736953428, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.293959269106383, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.289596655148852, "q3": 5.321573927870741, "n": 5 }, "batch_mbps_4t": { "value": 16.933337146737262, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16.351862609405714, "q3": 17.09634579773465, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4586013.98583115, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4428534.66929129, "q3": 4630161.217224904, "n": 5 }, "batch_cpu_utilization": { "value": 3.5651845065201857, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5542644567964254, "q3": 3.5686436916357844, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5651845065201857, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5542644567964254, "q3": 3.5686436916357844, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.21131684902689948, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.20789614920326327, "q3": 0.2180292601326937, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 138.894287109375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 138.86767578125, "q3": 138.91552734375, "n": 5 }, "rss_peak_large_mb": { "value": 260.07552, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 260.07552, "q3": 260.07552, "n": 5 }, "rss_peak_very_large_mb": { "value": 2572.242944, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2571.79648, "q3": 2572.603392, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.0, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.0, "q3": 0.0, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.3159529617509573, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.3129666324242, "q3": 3.317849229675027, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6713862051862792, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "tiktoken", "load_seconds": { "value": 0.0925006669999675, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.09022929200000362, "q3": 0.09309383400000115, "n": 5 }, "rss_load_delta_mb": { "value": 61.571072, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 61.571072, "q3": 61.571072, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 60.70272, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 60.70272, "q3": 60.70272, "n": 5 }, "latency_small_p50_us": { "value": 11.999999969702912, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 11.95799995912239, "q3": 12.167000022600405, "n": 5 }, "latency_small_p95_us": { "value": 18.459000102666323, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.917000150191598, "q3": 19.083000097452896, "n": 5 }, "encode_mbps_1t_medium": { "value": 24.208677865641175, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 23.100220077421366, "q3": 24.27460189874134, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 6599846.663607951, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6297655.380137834, "q3": 6617819.082933058, "n": 5 }, "encode_mbps_1t_large": { "value": 21.84440613916368, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 21.82323148162718, "q3": 22.002742858614003, "n": 5 }, "latency_large_p50_ms": { "value": 47.536540999772114, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 46.991499999876396, "q3": 48.16237500017451, "n": 5 }, "decode_mbps": { "value": 188.09905815436682, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 178.80151925666618, "q3": 188.40322955806735, "n": 5 }, "decode_tokens_per_s": { "value": 50602340.91021795, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 48101120.343013644, "q3": 50684168.99174168, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 61.02043570207256, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 59.29361619410512, "q3": 64.3858744505348, "n": 5 }, "batch_mbps_4t": { "value": 67.95394005965954, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 66.90219298158598, "q3": 68.00159234623783, "n": 5 }, "batch_tokens_per_s_4t": { "value": 18403798.188473266, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18118956.118194647, "q3": 18416703.739860512, "n": 5 }, "batch_cpu_utilization": { "value": 3.9018225002706273, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.882584066074877, "q3": 3.9089593406459127, "n": 5 }, "cpu_cores_used_batch": { "value": 3.9018225002706273, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.882584066074877, "q3": 3.9089593406459127, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.057708901234594386, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.05748335010661224, "q3": 0.05772177624612826, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 12.383544921875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 12.382568359375, "q3": 12.418701171875002, "n": 5 }, "rss_peak_large_mb": { "value": 55.697408, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 55.697408, "q3": 55.709696, "n": 5 }, "rss_peak_very_large_mb": { "value": 255.65184, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 255.63136, "q3": 256.237568, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.780736, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.477632, "q3": 18.563072, "n": 5 }, "encode_mbps_1t_very_large": { "value": 23.57508482550199, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 23.207410106862486, "q3": 23.8564988059844, "n": 5 }, "throughput_ratio_very_large": { "value": 0.973827854472035, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.518038360581196, "provenance": "measured" }, "chars_per_token": { "value": 2.5157700018601576, "provenance": "measured" }, "tokens_per_word": { "value": 2.618552795367213, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3599682435903424, "provenance": "measured" }, "chars_per_token": { "value": 3.346551827661434, "provenance": "measured" }, "tokens_per_word": { "value": 2.523459990908938, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.8593574833043323, "provenance": "measured" }, "chars_per_token": { "value": 3.858738488414479, "provenance": "measured" }, "tokens_per_word": { "value": 2.7310242491406678, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.746101108974733, "provenance": "measured" }, "chars_per_token": { "value": 4.719804230029715, "provenance": "measured" }, "tokens_per_word": { "value": 2.5864121562225293, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.160507204420255, "provenance": "measured" }, "chars_per_token": { "value": 4.152285397607901, "provenance": "measured" }, "tokens_per_word": { "value": 2.571094381353409, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.5245848021223245, "provenance": "measured" }, "chars_per_token": { "value": 3.521771995493695, "provenance": "measured" }, "tokens_per_word": { "value": 2.384654048807543, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.589521901360396, "provenance": "measured" }, "chars_per_token": { "value": 3.533806640823497, "provenance": "measured" }, "tokens_per_word": { "value": 3.356578947368421, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.474011703338894, "provenance": "measured" }, "chars_per_token": { "value": 4.47159336634275, "provenance": "measured" }, "tokens_per_word": { "value": 2.206360024926001, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.960844209941683, "provenance": "measured" }, "chars_per_token": { "value": 3.946158746753353, "provenance": "measured" }, "tokens_per_word": { "value": 2.5444003408217126, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.485319611965623, "provenance": "measured" }, "chars_per_token": { "value": 4.470417911548818, "provenance": "measured" }, "tokens_per_word": { "value": 2.8952370781717214, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.587133944901072, "provenance": "measured" }, "chars_per_token": { "value": 3.5581097785777795, "provenance": "measured" }, "tokens_per_word": { "value": 2.7003396579995314, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.8735449651426848, "provenance": "measured" }, "chars_per_token": { "value": 3.8680424322307094, "provenance": "measured" }, "tokens_per_word": { "value": 2.9567320107993895, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.8056108624380602, "provenance": "measured" }, "chars_per_token": { "value": 3.8052515756223895, "provenance": "measured" }, "tokens_per_word": { "value": 2.8420268890401634, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.131770345822055, "provenance": "measured" }, "chars_per_token": { "value": 4.120476949412668, "provenance": "measured" }, "tokens_per_word": { "value": 2.877085721564132, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.0720473339260566, "provenance": "measured" }, "chars_per_token": { "value": 3.0576252783307236, "provenance": "measured" }, "tokens_per_word": { "value": 2.92210598919416, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.035938135938136, "provenance": "measured" }, "chars_per_token": { "value": 3.94981684981685, "provenance": "measured" }, "tokens_per_word": { "value": 2.5872962385746177, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.177653977096874, "provenance": "measured" }, "chars_per_token": { "value": 4.176244024897692, "provenance": "measured" }, "tokens_per_word": { "value": 2.4182120582120583, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.405371014798411, "provenance": "measured" }, "chars_per_token": { "value": 3.399177475482442, "provenance": "measured" }, "tokens_per_word": { "value": 2.566302229919896, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.4105633900853367, "provenance": "measured" }, "chars_per_token": { "value": 3.397136413626049, "provenance": "measured" }, "tokens_per_word": { "value": 2.6424735138316655, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.383679982882971, "provenance": "measured" }, "chars_per_token": { "value": 4.367249418288475, "provenance": "measured" }, "tokens_per_word": { "value": 2.411095825630884, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.022443177859223, "provenance": "measured" }, "chars_per_token": { "value": 3.9363076408534123, "provenance": "measured" }, "tokens_per_word": { "value": 3.111433898452066, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.693471510327113, "provenance": "measured" }, "chars_per_token": { "value": 4.674385074271689, "provenance": "measured" }, "tokens_per_word": { "value": 1.3083473998125825, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 3.831502028315826, "provenance": "measured" }, "chars_per_token": { "value": 2.124808586089246, "provenance": "measured" }, "tokens_per_word": { "value": 2.8798112258713395, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 4.315944438402138, "provenance": "measured" }, "chars_per_token": { "value": 1.6212613228507948, "provenance": "measured" }, "tokens_per_word": { "value": 4.4184258401125875, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.0520123884185066, "provenance": "measured" }, "chars_per_token": { "value": 2.974978750725029, "provenance": "measured" }, "tokens_per_word": { "value": 2.8967379244026925, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.470850440632874, "provenance": "measured" }, "chars_per_token": { "value": 3.39364056124733, "provenance": "measured" }, "tokens_per_word": { "value": 1.8392537875223487, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.368283058518172, "provenance": "measured" }, "chars_per_token": { "value": 3.2609766862759306, "provenance": "measured" }, "tokens_per_word": { "value": 1.968322506191569, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 3.192851265233556, "provenance": "measured" }, "chars_per_token": { "value": 1.2405235117814175, "provenance": "measured" }, "tokens_per_word": { "value": 5.1344908406834024, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 4.679226374449752, "provenance": "measured" }, "chars_per_token": { "value": 1.8575442540039337, "provenance": "measured" }, "tokens_per_word": { "value": 2.9760842903333704, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.327667845476878, "provenance": "measured" }, "chars_per_token": { "value": 3.3199416893880085, "provenance": "measured" }, "tokens_per_word": { "value": 2.2479731041848723, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.633503573598773, "provenance": "measured" }, "chars_per_token": { "value": 1.2749066813275847, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 2.781738232099812, "provenance": "measured" }, "chars_per_token": { "value": 1.2005191915445947, "provenance": "measured" }, "tokens_per_word": { "value": 3.732179930795848, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 4.521460989452219, "provenance": "measured" }, "chars_per_token": { "value": 1.7180340885508305, "provenance": "measured" }, "tokens_per_word": { "value": 4.084807607253428, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 4.4950408639337045, "provenance": "measured" }, "chars_per_token": { "value": 2.4731325753250353, "provenance": "measured" }, "tokens_per_word": { "value": 2.992190169375702, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 3.717526455026455, "provenance": "measured" }, "chars_per_token": { "value": 1.416358024691358, "provenance": "measured" }, "tokens_per_word": { "value": 6.757374118581786, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 3.299032432749835, "provenance": "measured" }, "chars_per_token": { "value": 1.2567935225841809, "provenance": "measured" }, "tokens_per_word": { "value": 6.35214886650918, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 3.6763992240206154, "provenance": "measured" }, "chars_per_token": { "value": 1.3144569592031734, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.8012543061567, "provenance": "measured" }, "chars_per_token": { "value": 2.5607057680416925, "provenance": "measured" }, "tokens_per_word": { "value": 3.0478369874308022, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 4.020773940272001, "provenance": "measured" }, "chars_per_token": { "value": 2.2709414092318236, "provenance": "measured" }, "tokens_per_word": { "value": 2.106798825016, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 2.3594957106971894, "provenance": "measured" }, "chars_per_token": { "value": 1.8064402736306617, "provenance": "measured" }, "tokens_per_word": { "value": 2.543197227663101, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.7159578562144655, "provenance": "measured" }, "chars_per_token": { "value": 1.3956825129430217, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 87699, "provenance": "measured" }, "tokens_per_request_mean": { "value": 438.495, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.438495, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 65407, "provenance": "measured" }, "tokens_per_request_mean": { "value": 401.26993865030676, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.4012699386503068, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 35210, "provenance": "measured" }, "tokens_per_request_mean": { "value": 185.31578947368422, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.1853157894736842, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 225688, "provenance": "measured" }, "tokens_per_request_mean": { "value": 537.3523809523809, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5373523809523809, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 133875, "provenance": "measured" }, "tokens_per_request_mean": { "value": 892.5, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8925, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 371138, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1855.69, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.85569, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 326493, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1632.465, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.632465, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 410951, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2054.755, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.054755, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8191, "provenance": "measured" }, "tokens_per_request_mean": { "value": 327.64, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.32764, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 633240, "provenance": "measured" }, "tokens_per_request_mean": { "value": 105540.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 105.54, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 163840, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 335544320, "provenance": "estimated", "note": "163840 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 671088640, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 671088640, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1342177280, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 163840, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 671088640, "provenance": "estimated", "note": "163840 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1342177280, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1342177280, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2684354560, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 2348810240, "provenance": "estimated", "note": "163840 rows x 7168 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 4697620480, "provenance": "estimated", "note": "163840 rows x 7168 hidden x 2 matrices from the published config" } } } }, { "id": "o200k_base", "label": "OpenAI o200k_base (GPT-4o)", "provider": "OpenAI", "family": "published", "metrics": { "bpt_overall": { "value": 4.64036271113042, "provenance": "measured" }, "bpt_english": { "value": 4.7423522728076355, "provenance": "measured" }, "bpt_code": { "value": 3.852379102439795, "provenance": "measured" }, "bpt_languages": { "value": 5.326356758143829, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.4440150070128051, "provenance": "measured" }, "byte_fallback_share": { "value": 0.008041190901993326, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 5.101910820251484, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.0714635495205, "q3": 5.138293990144321, "n": 5 }, "batch_mbps_4t": { "value": 17.73679441869205, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.62775178570308, "q3": 17.788999687840956, "n": 5 }, "latency_small_p95_us": { "value": 65.54100036737509, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 64.70800008173683, "q3": 66.58300026174402, "n": 5 }, "decode_mbps": { "value": 39.120725163970164, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.16271940307519, "q3": 40.102457704134835, "n": 5 }, "rss_load_delta_mb": { "value": 249.864192, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 249.860096, "q3": 249.868288, "n": 5 }, "rss_peak_large_mb": { "value": 321.458176, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 320.503808, "q3": 321.8432, "n": 5 }, "load_seconds": { "value": 0.5957294170000296, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.5924940419999984, "q3": 0.6016672920000019, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6840984851826348, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2471.8336, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2469.187584, "q3": 2474.323968, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.0, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.0, "q3": 0.0, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.490209463670934, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.3747772136124436, "q3": 3.4979244468201665, "n": 5 }, "embedding_params_h2048": { "value": 409638912, "provenance": "estimated", "note": "200019 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.20344163363868564, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2031924783228915, "q3": 0.20352126796928408, "n": 5 }, "cpu_cores_used_batch": { "value": 3.614590933457547, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5862186206606927, "q3": 3.619274973598692, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 129.434814453125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 129.2568359375, "q3": 129.58325195312503, "n": 5 }, "workload_tokens_total": { "value": 2062920, "provenance": "measured" }, "tokens_chat_en": { "value": 87171, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 52950, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 26250, "provenance": "measured" }, "tokens_code": { "value": 224729, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 126331, "provenance": "measured" }, "tokens_rag": { "value": 266142, "provenance": "measured" }, "tokens_json": { "value": 320664, "provenance": "measured" }, "tokens_logs": { "value": 410777, "provenance": "measured" }, "tokens_tech_docs": { "value": 8298, "provenance": "measured" }, "tokens_long_context": { "value": 539608, "provenance": "measured" } }, "parity": { "de": { "value": 1.3078703703703705, "provenance": "measured" }, "fr": { "value": 1.3551533090467516, "provenance": "measured" }, "es": { "value": 1.3088949605343048, "provenance": "measured" }, "ru": { "value": 1.4493776563448695, "provenance": "measured" }, "zh": { "value": 1.268290831815422, "provenance": "measured" }, "ja": { "value": 1.670916818457802, "provenance": "measured" }, "ko": { "value": 1.482695810564663, "provenance": "measured" }, "ar": { "value": 1.4069140862173648, "provenance": "measured" }, "ur": { "value": 1.6862856709168184, "provenance": "measured" }, "hi": { "value": 1.5906952034001214, "provenance": "measured" }, "mr": { "value": 1.8326123254401943, "provenance": "measured" }, "gu": { "value": 1.8099195506982393, "provenance": "measured" }, "bn": { "value": 1.7211596842744383, "provenance": "measured" }, "ta": { "value": 2.0013281724347296, "provenance": "measured" }, "te": { "value": 1.9645567698846387, "provenance": "measured" }, "th": { "value": 1.9492638129933213, "provenance": "measured" }, "vi": { "value": 1.4862629022465088, "provenance": "measured" }, "tr": { "value": 1.4327185792349726, "provenance": "measured" }, "id": { "value": 1.2414238008500305, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.247011952191237, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 34.32768924302789, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3078703703703705, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 35.56872509960159, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3551533090467516, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 34.354581673306775, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3088949605343048, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 38.04183266932271, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4493776563448695, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 33.288844621513945, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.268290831815422, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 43.85657370517928, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.670916818457802, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 38.91633466135458, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.482695810564663, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 36.927290836653384, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4069140862173648, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 44.25996015936255, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6862856709168184, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 41.75099601593625, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5906952034001214, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 48.10059760956175, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8326123254401943, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 47.504980079681275, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8099195506982393, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 45.17529880478088, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7211596842744383, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 52.528884462151396, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0013281724347296, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 51.56374501992032, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9645567698846387, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 51.16235059760956, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9492638129933213, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 39.00996015936255, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4862629022465088, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 37.604581673306775, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4327185792349726, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 32.58366533864542, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2414238008500305, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|endoftext|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "inputs whose IDs differ between the reference and tokenizers (converted from tiktoken)" } }, "vocabulary": { "size": { "value": 200019, "provenance": "measured" }, "used_share": { "value": 0.6547477989590988, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 130962, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 69057, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 12, "provenance": "measured", "note": "of 198436 entries whose text is valid UTF-8; 1581 are byte pieces" }, "unreachable_examples": [ " I'", "\n//", "\r\n//", "\n///", "\n\n//", "\n//\n//", " 亚洲AV", "\r\n\r\n//", "无码AV", "\n\n\n//", " 天天中彩票APP", "亚洲AV" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers (converted from tiktoken)", "load_seconds": { "value": 0.5957294170000296, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.5924940419999984, "q3": 0.6016672920000019, "n": 5 }, "rss_load_delta_mb": { "value": 249.864192, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 249.860096, "q3": 249.868288, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 249.798656, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 249.79456, "q3": 249.802752, "n": 5 }, "latency_small_p50_us": { "value": 48.0829999105481, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 47.54099995807337, "q3": 48.66650010626472, "n": 5 }, "latency_small_p95_us": { "value": 65.54100036737509, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 64.70800008173683, "q3": 66.58300026174402, "n": 5 }, "encode_mbps_1t_medium": { "value": 5.101910820251484, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.0714635495205, "q3": 5.138293990144321, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1204323.402603543, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1197136.2207850802, "q3": 1212911.7736093467, "n": 5 }, "encode_mbps_1t_large": { "value": 4.245682664571319, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.151819548918667, "q3": 4.250193157969837, "n": 5 }, "latency_large_p50_ms": { "value": 244.1465000001699, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 243.4156259998872, "q3": 248.1409590000112, "n": 5 }, "decode_mbps": { "value": 39.120725163970164, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.16271940307519, "q3": 40.102457704134835, "n": 5 }, "decode_tokens_per_s": { "value": 8860731.913739756, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 8417249.231347483, "q3": 9083091.515023565, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.502043996848703, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.428230889104638, "q3": 5.5113063993413585, "n": 5 }, "batch_mbps_4t": { "value": 17.73679441869205, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.62775178570308, "q3": 17.788999687840956, "n": 5 }, "batch_tokens_per_s_4t": { "value": 4155776.328437913, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4130227.360441275, "q3": 4168008.156615395, "n": 5 }, "batch_cpu_utilization": { "value": 3.614590933457547, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5862186206606927, "q3": 3.619274973598692, "n": 5 }, "cpu_cores_used_batch": { "value": 3.614590933457547, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5862186206606927, "q3": 3.619274973598692, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.20344163363868564, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.2031924783228915, "q3": 0.20352126796928408, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 129.434814453125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 129.2568359375, "q3": 129.58325195312503, "n": 5 }, "rss_peak_large_mb": { "value": 321.458176, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 320.503808, "q3": 321.8432, "n": 5 }, "rss_peak_very_large_mb": { "value": 2471.8336, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2469.187584, "q3": 2474.323968, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.0, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.0, "q3": 0.0, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.490209463670934, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.3747772136124436, "q3": 3.4979244468201665, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6840984851826348, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } }, "reference": { "implementation": "tiktoken", "load_seconds": { "value": 0.1457424169998376, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.14254829100013922, "q3": 0.15049458299995422, "n": 5 }, "rss_load_delta_mb": { "value": 91.897856, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 91.897856, "q3": 91.897856, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 91.83232, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 91.83232, "q3": 91.83232, "n": 5 }, "latency_small_p50_us": { "value": 11.95799995912239, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 11.541999811015557, "q3": 12.792000006811577, "n": 5 }, "latency_small_p95_us": { "value": 19.249999922976713, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.49999989644857, "q3": 20.707999965452473, "n": 5 }, "encode_mbps_1t_medium": { "value": 25.11267863118479, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 25.054576960652053, "q3": 25.3858707437138, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 5927933.208387004, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5914218.1114325095, "q3": 5992421.13577676, "n": 5 }, "encode_mbps_1t_large": { "value": 24.106913431770202, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 23.72236971449411, "q3": 25.293628738584722, "n": 5 }, "latency_large_p50_ms": { "value": 43.57687499987151, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 41.09370899993792, "q3": 43.72849999981554, "n": 5 }, "decode_mbps": { "value": 188.6404882960821, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 188.52886841947648, "q3": 191.52183450810887, "n": 5 }, "decode_tokens_per_s": { "value": 42726528.91434578, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 42701247.33281317, "q3": 43379145.55752989, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 65.58447623387329, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 62.142387474542794, "q3": 66.10919166536996, "n": 5 }, "batch_mbps_4t": { "value": 74.47334099824387, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 71.89185807330907, "q3": 74.93683693291769, "n": 5 }, "batch_tokens_per_s_4t": { "value": 17449294.40542106, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 16844446.348976042, "q3": 17557892.68920196, "n": 5 }, "batch_cpu_utilization": { "value": 3.971083304762188, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.9650940792592437, "q3": 3.9816114326064684, "n": 5 }, "cpu_cores_used_batch": { "value": 3.971083304762188, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.9650940792592437, "q3": 3.9816114326064684, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.05307079754258946, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.052912482585950826, "q3": 0.055236899020092406, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 9.112060546875002, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 9.105468750000002, "q3": 9.115478515625002, "n": 5 }, "rss_peak_large_mb": { "value": 76.345344, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 76.345344, "q3": 76.34944, "n": 5 }, "rss_peak_very_large_mb": { "value": 227.688448, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 227.57376, "q3": 227.745792, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.973248, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.575936, "q3": 18.026496, "n": 5 }, "encode_mbps_1t_very_large": { "value": 24.833425471055417, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 24.767575308420252, "q3": 25.149910356498324, "n": 5 }, "throughput_ratio_very_large": { "value": 0.9888799930811604, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.501108590550292, "provenance": "measured" }, "chars_per_token": { "value": 2.4988554829040965, "provenance": "measured" }, "tokens_per_word": { "value": 2.6362775342317475, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3606408605307587, "provenance": "measured" }, "chars_per_token": { "value": 3.347221758829849, "provenance": "measured" }, "tokens_per_word": { "value": 2.5229549319012103, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.8788588887506394, "provenance": "measured" }, "chars_per_token": { "value": 3.8782367660680466, "provenance": "measured" }, "tokens_per_word": { "value": 2.7172937132553203, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.803192012107632, "provenance": "measured" }, "chars_per_token": { "value": 4.776578807712719, "provenance": "measured" }, "tokens_per_word": { "value": 2.5556699736280297, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.1509964926758816, "provenance": "measured" }, "chars_per_token": { "value": 4.142793480503404, "provenance": "measured" }, "tokens_per_word": { "value": 2.576985240951044, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.519315485046193, "provenance": "measured" }, "chars_per_token": { "value": 3.5165068836135887, "provenance": "measured" }, "tokens_per_word": { "value": 2.3882244869661675, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.583382853829988, "provenance": "measured" }, "chars_per_token": { "value": 3.527762881368215, "provenance": "measured" }, "tokens_per_word": { "value": 3.362329434697856, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.48831237825394, "provenance": "measured" }, "chars_per_token": { "value": 4.485886311315743, "provenance": "measured" }, "tokens_per_word": { "value": 2.199330113724879, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.9775752753877573, "provenance": "measured" }, "chars_per_token": { "value": 3.962827779099238, "provenance": "measured" }, "tokens_per_word": { "value": 2.533697707766163, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.517646840286807, "provenance": "measured" }, "chars_per_token": { "value": 4.502637738232344, "provenance": "measured" }, "tokens_per_word": { "value": 2.8745194361384026, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.6174684522464333, "provenance": "measured" }, "chars_per_token": { "value": 3.588198843795791, "provenance": "measured" }, "tokens_per_word": { "value": 2.6776957913640977, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.945122395012009, "provenance": "measured" }, "chars_per_token": { "value": 3.9395181833620416, "provenance": "measured" }, "tokens_per_word": { "value": 2.903087216809485, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.7496183431298076, "provenance": "measured" }, "chars_per_token": { "value": 3.749264342554557, "provenance": "measured" }, "tokens_per_word": { "value": 2.884466473791695, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.163668384624209, "provenance": "measured" }, "chars_per_token": { "value": 4.15228780108516, "provenance": "measured" }, "tokens_per_word": { "value": 2.8550442467140975, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.0580919037884664, "provenance": "measured" }, "chars_per_token": { "value": 3.0437353634562214, "provenance": "measured" }, "tokens_per_word": { "value": 2.9354408552707207, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.052928434217518, "provenance": "measured" }, "chars_per_token": { "value": 3.9664445988474273, "provenance": "measured" }, "tokens_per_word": { "value": 2.576450023166674, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.155184060883321, "provenance": "measured" }, "chars_per_token": { "value": 4.153781692248493, "provenance": "measured" }, "tokens_per_word": { "value": 2.431288981288981, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.358014849117164, "provenance": "measured" }, "chars_per_token": { "value": 3.3519074391165398, "provenance": "measured" }, "tokens_per_word": { "value": 2.6024933246734503, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.355328663092083, "provenance": "measured" }, "chars_per_token": { "value": 3.3421191390869063, "provenance": "measured" }, "tokens_per_word": { "value": 2.685973366686286, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.463423047247311, "provenance": "measured" }, "chars_per_token": { "value": 4.446693595969682, "provenance": "measured" }, "tokens_per_word": { "value": 2.3680194316667382, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.200082940457482, "provenance": "measured" }, "chars_per_token": { "value": 4.110143472440487, "provenance": "measured" }, "tokens_per_word": { "value": 2.9798378354940813, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.7423522728076355, "provenance": "measured" }, "chars_per_token": { "value": 4.723067058609984, "provenance": "measured" }, "tokens_per_word": { "value": 1.2948618941366437, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 5.729954500256125, "provenance": "measured" }, "chars_per_token": { "value": 3.1776197507055977, "provenance": "measured" }, "tokens_per_word": { "value": 1.9256701868399675, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 7.662115381963121, "provenance": "measured" }, "chars_per_token": { "value": 2.878232446522405, "provenance": "measured" }, "tokens_per_word": { "value": 2.4888271032849345, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.7026797371157976, "provenance": "measured" }, "chars_per_token": { "value": 3.609223075391118, "provenance": "measured" }, "tokens_per_word": { "value": 2.387697737575687, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.366059289650456, "provenance": "measured" }, "chars_per_token": { "value": 4.2689352801576765, "provenance": "measured" }, "tokens_per_word": { "value": 1.4621365390044228, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 4.096084981614554, "provenance": "measured" }, "chars_per_token": { "value": 3.9655923798537174, "provenance": "measured" }, "tokens_per_word": { "value": 1.6185863772513323, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 6.493861395814755, "provenance": "measured" }, "chars_per_token": { "value": 2.523070157221564, "provenance": "measured" }, "tokens_per_word": { "value": 2.524486523160427, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 7.621213045733288, "provenance": "measured" }, "chars_per_token": { "value": 3.025444671568478, "provenance": "measured" }, "tokens_per_word": { "value": 1.8272382651354666, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.084143424182786, "provenance": "measured" }, "chars_per_token": { "value": 4.074660888349933, "provenance": "measured" }, "tokens_per_word": { "value": 1.8315977279347508, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.5330102414045355, "provenance": "measured" }, "chars_per_token": { "value": 1.2396460525575375, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.5902262579573194, "provenance": "measured" }, "chars_per_token": { "value": 1.5494396542882345, "provenance": "measured" }, "tokens_per_word": { "value": 2.8917251606524963, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 7.0111650553355815, "provenance": "measured" }, "chars_per_token": { "value": 2.6640549578162385, "provenance": "measured" }, "tokens_per_word": { "value": 2.634269497272593, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 6.078833871981822, "provenance": "measured" }, "chars_per_token": { "value": 3.344521779415171, "provenance": "measured" }, "tokens_per_word": { "value": 2.212598233025821, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 7.374681108503287, "provenance": "measured" }, "chars_per_token": { "value": 2.809714710555855, "provenance": "measured" }, "tokens_per_word": { "value": 3.4063462111431124, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 6.2666815387666075, "provenance": "measured" }, "chars_per_token": { "value": 2.3873438429506244, "provenance": "measured" }, "tokens_per_word": { "value": 3.3440258609971405, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 6.863488940301412, "provenance": "measured" }, "chars_per_token": { "value": 2.4539665722502106, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.7784777606505373, "provenance": "measured" }, "chars_per_token": { "value": 3.454013359247593, "provenance": "measured" }, "tokens_per_word": { "value": 2.2595783345392135, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.253358436800326, "provenance": "measured" }, "chars_per_token": { "value": 2.9671076735192345, "provenance": "measured" }, "tokens_per_word": { "value": 1.6124850255181582, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.252201208618346, "provenance": "measured" }, "chars_per_token": { "value": 3.2555039112825734, "provenance": "measured" }, "tokens_per_word": { "value": 1.4111897945859073, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.442542228082095, "provenance": "measured" }, "chars_per_token": { "value": 1.292989902930908, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 87171, "provenance": "measured" }, "tokens_per_request_mean": { "value": 435.855, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.435855, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 52950, "provenance": "measured" }, "tokens_per_request_mean": { "value": 324.84662576687117, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3248466257668711, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 26250, "provenance": "measured" }, "tokens_per_request_mean": { "value": 138.1578947368421, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.13815789473684212, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 224729, "provenance": "measured" }, "tokens_per_request_mean": { "value": 535.0690476190476, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5350690476190477, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 126331, "provenance": "measured" }, "tokens_per_request_mean": { "value": 842.2066666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8422066666666668, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 266142, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1330.71, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.33071, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 320664, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1603.32, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.60332, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 410777, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2053.885, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.053885, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8298, "provenance": "measured" }, "tokens_per_request_mean": { "value": 331.92, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33192, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 539608, "provenance": "measured" }, "tokens_per_request_mean": { "value": 89934.66666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 89.93466666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 200019, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 409638912, "provenance": "estimated", "note": "200019 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 819277824, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 819277824, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1638555648, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 200019, "provenance": "estimated", "note": "tokenizer vocabulary" }, "params_tied": { "value": 819277824, "provenance": "estimated", "note": "200019 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1638555648, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1638555648, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 3277111296, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" }, "bf16_bytes": { "value": null, "provenance": "unavailable", "note": "no published model config with a hidden size for this tokenizer" } } } }, { "id": "minimax-m2-5", "label": "MiniMax-M2.5", "provider": "MiniMax", "family": "published", "metrics": { "bpt_overall": { "value": 4.160958625799099, "provenance": "measured" }, "bpt_english": { "value": 4.8212380473074985, "provenance": "measured" }, "bpt_code": { "value": 3.7795135935400275, "provenance": "measured" }, "bpt_languages": { "value": 3.8821242365497692, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.8000080000188676, "provenance": "measured" }, "byte_fallback_share": { "value": 0.11145137654431465, "provenance": "measured" }, "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.2863295702025015, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.106062903165659, "q3": 4.314534054925623, "n": 5 }, "batch_mbps_4t": { "value": 13.28344203289287, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.277353389014594, "q3": 13.417999827684188, "n": 5 }, "latency_small_p95_us": { "value": 86.95799988345243, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 82.7090000257158, "q3": 87.5829998676636, "n": 5 }, "decode_mbps": { "value": 38.192575850644786, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.39748293691054, "q3": 38.33750896901568, "n": 5 }, "rss_load_delta_mb": { "value": 73.449472, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 73.445376, "q3": 74.268672, "n": 5 }, "rss_peak_large_mb": { "value": 206.053376, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 205.836288, "q3": 206.176256, "n": 5 }, "load_seconds": { "value": 0.23188758399999188, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.23095666700010042, "q3": 0.23225875100024496, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6551446305305266, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2552.291328, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2552.180736, "q3": 2552.840192, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.670144, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.657856, "q3": 17.895424, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.808165802602389, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.6829537705191093, "q3": 2.8466032585417334, "n": 5 }, "embedding_params_h2048": { "value": 409731072, "provenance": "estimated", "note": "200064 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.2705223511392359, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.27027891805079, "q3": 0.2719390792602399, "n": 5 }, "cpu_cores_used_batch": { "value": 3.612286995831657, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.588588708560855, "q3": 3.6298688609709884, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 145.1064453125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 145.104736328125, "q3": 145.13916015625, "n": 5 }, "workload_tokens_total": { "value": 2291283, "provenance": "measured" }, "tokens_chat_en": { "value": 86679, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 55254, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 36257, "provenance": "measured" }, "tokens_code": { "value": 228767, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 134539, "provenance": "measured" }, "tokens_rag": { "value": 370722, "provenance": "measured" }, "tokens_json": { "value": 321750, "provenance": "measured" }, "tokens_logs": { "value": 415525, "provenance": "measured" }, "tokens_tech_docs": { "value": 8201, "provenance": "measured" }, "tokens_long_context": { "value": 633589, "provenance": "measured" } }, "parity": { "de": { "value": 1.4902562528770906, "provenance": "measured" }, "fr": { "value": 1.2871336504526623, "provenance": "measured" }, "es": { "value": 1.2425579254258095, "provenance": "measured" }, "ru": { "value": 1.7738606720883843, "provenance": "measured" }, "zh": { "value": 0.9291468467086083, "provenance": "measured" }, "ja": { "value": 1.0342949209759091, "provenance": "measured" }, "ko": { "value": 1.3332438238453277, "provenance": "measured" }, "ar": { "value": 1.3773592143624367, "provenance": "measured" }, "ur": { "value": 3.8176308117231854, "provenance": "measured" }, "hi": { "value": 2.971267454350161, "provenance": "measured" }, "mr": { "value": 3.3076952585545496, "provenance": "measured" }, "gu": { "value": 6.728402639251189, "provenance": "measured" }, "bn": { "value": 3.7769295688200093, "provenance": "measured" }, "ta": { "value": 4.051864354764462, "provenance": "measured" }, "te": { "value": 4.2494245818628205, "provenance": "measured" }, "th": { "value": 4.7520331440847015, "provenance": "measured" }, "vi": { "value": 2.7897422126745437, "provenance": "measured" }, "tr": { "value": 1.9588767837962253, "provenance": "measured" }, "id": { "value": 1.0838192419825072, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 25.96414342629482, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 38.69322709163347, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4902562528770906, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 33.419322709163346, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2871336504526623, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 32.26195219123506, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2425579254258095, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 46.05677290836653, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7738606720883843, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 24.124501992031874, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 0.9291468467086083, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 26.85458167330677, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0342949209759091, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 34.616533864541836, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3332438238453277, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 35.76195219123506, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3773592143624367, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 99.12151394422311, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.8176308117231854, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 77.14641434262948, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.971267454350161, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 85.88147410358566, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.3076952585545496, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 174.69721115537848, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.728402639251189, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 98.06474103585657, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.7769295688200093, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 105.20318725099601, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.051864354764462, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 110.33266932270917, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.2494245818628205, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 123.38247011952191, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.7520331440847015, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 72.43326693227091, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.7897422126745437, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 50.8605577689243, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.9588767837962253, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 28.140438247011954, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0838192419825072, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 200,064 embedding rows for 200,054 tokenizer IDs (10 unused rows, usually padding)" ], "roundtrip_failures": { "value": 733, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 733, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [ "code_cs" ], "roundtrip_example": " \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\",", "byte_fallback_sets": [], "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "unicode_failed_probes": [ "latin_accents (NFD)", "vietnamese (NFD)", "devanagari_nukta_precomposed (NFC)", "tamil (NFD)", "telugu (NFD)", "japanese (NFD)", "korean (NFD)" ], "nfd_normalized_to_nfc": [ "japanese", "korean", "latin_accents", "tamil", "telugu", "vietnamese" ], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 200054, "provenance": "measured" }, "used_share": { "value": 0.6708088816019675, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 134198, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 65856, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 4, "provenance": "measured", "note": "of 198945 entries whose text is valid UTF-8; 1059 are byte pieces" }, "unreachable_examples": [ " I'", "\n//", " O'", "ビタミンC" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.23188758399999188, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.23095666700010042, "q3": 0.23225875100024496, "n": 5 }, "rss_load_delta_mb": { "value": 73.449472, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 73.445376, "q3": 74.268672, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 73.1136, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 73.109504, "q3": 74.1376, "n": 5 }, "latency_small_p50_us": { "value": 65.04199996015814, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 64.72899985965341, "q3": 65.08349997602636, "n": 5 }, "latency_small_p95_us": { "value": 86.95799988345243, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 82.7090000257158, "q3": 87.5829998676636, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.2863295702025015, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.106062903165659, "q3": 4.314534054925623, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1156603.0538169728, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1107960.7424451085, "q3": 1164213.6196000783, "n": 5 }, "encode_mbps_1t_large": { "value": 3.599784435402161, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5947278035864523, "q3": 3.600271723922392, "n": 5 }, "latency_large_p50_ms": { "value": 286.4820419999887, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 285.96204200016473, "q3": 289.59770900019066, "n": 5 }, "decode_mbps": { "value": 38.192575850644786, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.39748293691054, "q3": 38.33750896901568, "n": 5 }, "decode_tokens_per_s": { "value": 10436746.175514325, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10219474.029245762, "q3": 10476351.521191401, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.22164900838112, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.1974233020891045, "q3": 4.244943710485491, "n": 5 }, "batch_mbps_4t": { "value": 13.28344203289287, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.277353389014594, "q3": 13.417999827684188, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3577112.9632952376, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3575473.3455747752, "q3": 3613348.182364856, "n": 5 }, "batch_cpu_utilization": { "value": 3.612286995831657, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.588588708560855, "q3": 3.6298688609709884, "n": 5 }, "cpu_cores_used_batch": { "value": 3.612286995831657, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.588588708560855, "q3": 3.6298688609709884, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.2705223511392359, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.27027891805079, "q3": 0.2719390792602399, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 145.1064453125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 145.104736328125, "q3": 145.13916015625, "n": 5 }, "rss_peak_large_mb": { "value": 206.053376, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 205.836288, "q3": 206.176256, "n": 5 }, "rss_peak_very_large_mb": { "value": 2552.291328, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2552.180736, "q3": 2552.840192, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.670144, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.657856, "q3": 17.895424, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.808165802602389, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.6829537705191093, "q3": 2.8466032585417334, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6551446305305266, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.462423510507673, "provenance": "measured" }, "chars_per_token": { "value": 2.460205252067932, "provenance": "measured" }, "tokens_per_word": { "value": 2.677693889700709, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.2763603008144893, "provenance": "measured" }, "chars_per_token": { "value": 3.263277732962086, "provenance": "measured" }, "tokens_per_word": { "value": 2.5878550143941816, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.772109438020973, "provenance": "measured" }, "chars_per_token": { "value": 3.771504436676526, "provenance": "measured" }, "tokens_per_word": { "value": 2.794192227502395, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.713147794557272, "provenance": "measured" }, "chars_per_token": { "value": 4.687033501128276, "provenance": "measured" }, "tokens_per_word": { "value": 2.604495793042823, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.108747682178711, "provenance": "measured" }, "chars_per_token": { "value": 4.100628160201273, "provenance": "measured" }, "tokens_per_word": { "value": 2.603483475819829, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.466459840448346, "provenance": "measured" }, "chars_per_token": { "value": 3.4636934206387786, "provenance": "measured" }, "tokens_per_word": { "value": 2.4246394897393233, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.5719930650870477, "provenance": "measured" }, "chars_per_token": { "value": 3.5165498808061835, "provenance": "measured" }, "tokens_per_word": { "value": 3.3730506822612085, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.418401945487984, "provenance": "measured" }, "chars_per_token": { "value": 4.416013667227418, "provenance": "measured" }, "tokens_per_word": { "value": 2.234129147842343, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.902388386390704, "provenance": "measured" }, "chars_per_token": { "value": 3.8879196575144044, "provenance": "measured" }, "tokens_per_word": { "value": 2.5825141835865253, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.433237937257332, "provenance": "measured" }, "chars_per_token": { "value": 4.418509269217449, "provenance": "measured" }, "tokens_per_word": { "value": 2.9292503203759077, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.4963220269718023, "provenance": "measured" }, "chars_per_token": { "value": 3.468032636726181, "provenance": "measured" }, "tokens_per_word": { "value": 2.770477082845319, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.8494291057436616, "provenance": "measured" }, "chars_per_token": { "value": 3.843960830420339, "provenance": "measured" }, "tokens_per_word": { "value": 2.975255311656298, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.761714154649778, "provenance": "measured" }, "chars_per_token": { "value": 3.76135901211184, "provenance": "measured" }, "tokens_per_word": { "value": 2.8751914567733152, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.081987699241669, "provenance": "measured" }, "chars_per_token": { "value": 4.070830374083847, "provenance": "measured" }, "tokens_per_word": { "value": 2.912173760091223, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.0320877209331223, "provenance": "measured" }, "chars_per_token": { "value": 3.0178532600254715, "provenance": "measured" }, "tokens_per_word": { "value": 2.9606161627773306, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.04565297213496, "provenance": "measured" }, "chars_per_token": { "value": 3.9593243849700133, "provenance": "measured" }, "tokens_per_word": { "value": 2.5810833579040477, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.1159061840912745, "provenance": "measured" }, "chars_per_token": { "value": 4.114517071683283, "provenance": "measured" }, "tokens_per_word": { "value": 2.4544906444906447, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.2975282000313144, "provenance": "measured" }, "chars_per_token": { "value": 3.2915308004928625, "provenance": "measured" }, "tokens_per_word": { "value": 2.6502309302157756, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.277734667665632, "provenance": "measured" }, "chars_per_token": { "value": 3.264830621855935, "provenance": "measured" }, "tokens_per_word": { "value": 2.7495585638610947, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.2970637070698245, "provenance": "measured" }, "chars_per_token": { "value": 4.280957790789128, "provenance": "measured" }, "tokens_per_word": { "value": 2.459696487683247, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.9890891250570073, "provenance": "measured" }, "chars_per_token": { "value": 3.9036678229384045, "provenance": "measured" }, "tokens_per_word": { "value": 3.1374495945887353, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.8212380473074985, "provenance": "measured" }, "chars_per_token": { "value": 4.801632036810698, "provenance": "measured" }, "tokens_per_word": { "value": 1.2736751818467633, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 5.901924270639355, "provenance": "measured" }, "chars_per_token": { "value": 3.272987792261535, "provenance": "measured" }, "tokens_per_word": { "value": 1.8695601717535104, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 3.447795002916827, "provenance": "measured" }, "chars_per_token": { "value": 1.2951456551689897, "provenance": "measured" }, "tokens_per_word": { "value": 5.530978615315965, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.494645079906101, "provenance": "measured" }, "chars_per_token": { "value": 3.4064392705278896, "provenance": "measured" }, "tokens_per_word": { "value": 2.5298363150275276, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.65552944002196, "provenance": "measured" }, "chars_per_token": { "value": 4.551966099368652, "provenance": "measured" }, "tokens_per_word": { "value": 1.371224240143032, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 4.3824257956133055, "provenance": "measured" }, "chars_per_token": { "value": 4.242810981306409, "provenance": "measured" }, "tokens_per_word": { "value": 1.512830487156546, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 1.8604962900741986, "provenance": "measured" }, "chars_per_token": { "value": 0.7228615427691446, "provenance": "measured" }, "tokens_per_word": { "value": 8.811447603774749, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 4.457579029077704, "provenance": "measured" }, "chars_per_token": { "value": 1.7695554028854648, "provenance": "measured" }, "tokens_per_word": { "value": 3.124066228119077, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.622100566200458, "provenance": "measured" }, "chars_per_token": { "value": 4.611369005212044, "provenance": "measured" }, "tokens_per_word": { "value": 1.618421691426352, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 5.334788427224063, "provenance": "measured" }, "chars_per_token": { "value": 1.8718455263828702, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.9977691487741738, "provenance": "measured" }, "chars_per_token": { "value": 1.7253235876351454, "provenance": "measured" }, "tokens_per_word": { "value": 2.59693524468611, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 4.023517604078847, "provenance": "measured" }, "chars_per_token": { "value": 1.5288289373318882, "provenance": "measured" }, "tokens_per_word": { "value": 4.590336134453781, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.363252396477057, "provenance": "measured" }, "chars_per_token": { "value": 2.9508150454965696, "provenance": "measured" }, "tokens_per_word": { "value": 2.507809830624298, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 3.910402881701812, "provenance": "measured" }, "chars_per_token": { "value": 1.4898429286995238, "provenance": "measured" }, "tokens_per_word": { "value": 6.424073890157911, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 3.181644287278356, "provenance": "measured" }, "chars_per_token": { "value": 1.21207354366183, "provenance": "measured" }, "tokens_per_word": { "value": 6.586514153093788, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 2.828950827707595, "provenance": "measured" }, "chars_per_token": { "value": 1.011460909475748, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 2.8323397414428295, "provenance": "measured" }, "chars_per_token": { "value": 2.5891218433920558, "provenance": "measured" }, "tokens_per_word": { "value": 3.01438643132372, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 2.474924485784149, "provenance": "measured" }, "chars_per_token": { "value": 1.3978424509756917, "provenance": "measured" }, "tokens_per_word": { "value": 3.4227152633047244, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 2.43467416554591, "provenance": "measured" }, "chars_per_token": { "value": 1.8639972286750621, "provenance": "measured" }, "tokens_per_word": { "value": 2.4646677715835468, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.555601053980914, "provenance": "measured" }, "chars_per_token": { "value": 1.7110454351232018, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 86679, "provenance": "measured" }, "tokens_per_request_mean": { "value": 433.395, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.433395, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 55254, "provenance": "measured" }, "tokens_per_request_mean": { "value": 338.9815950920245, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3389815950920245, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 36257, "provenance": "measured" }, "tokens_per_request_mean": { "value": 190.82631578947368, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.19082631578947368, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 228767, "provenance": "measured" }, "tokens_per_request_mean": { "value": 544.6833333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5446833333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 134539, "provenance": "measured" }, "tokens_per_request_mean": { "value": 896.9266666666666, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8969266666666666, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 370722, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1853.61, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.85361, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 321750, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1608.75, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.60875, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 415525, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2077.625, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.077625, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8201, "provenance": "measured" }, "tokens_per_request_mean": { "value": 328.04, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.32804, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 633589, "provenance": "measured" }, "tokens_per_request_mean": { "value": 105598.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 105.59816666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 200064, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 409731072, "provenance": "estimated", "note": "200064 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 819462144, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 819462144, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1638924288, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 200064, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 819462144, "provenance": "estimated", "note": "200064 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1638924288, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1638924288, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 3277848576, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 1229193216, "provenance": "estimated", "note": "200064 rows x 3072 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 2458386432, "provenance": "estimated", "note": "200064 rows x 3072 hidden x 2 matrices from the published config" } } } }, { "id": "mistral-large-4-1t-a52b", "label": "Mistral-Large-4-1T-A52B", "provider": "Mistral", "family": "published", "metrics": { "bpt_overall": { "value": 4.584435829919155, "provenance": "measured" }, "bpt_english": { "value": 4.609229190103171, "provenance": "measured" }, "bpt_code": { "value": 3.726840526794806, "provenance": "measured" }, "bpt_languages": { "value": 5.417237772859487, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.4016180425047495, "provenance": "measured" }, "byte_fallback_share": { "value": 0.011690707266487825, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 5.069619965517145, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.944411374013184, "q3": 5.130161389281384, "n": 5 }, "batch_mbps_4t": { "value": 16.036505735404884, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 15.577337956412958, "q3": 17.061922351465657, "n": 5 }, "latency_small_p95_us": { "value": 66.95899992337218, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 66.16700011363719, "q3": 67.08299997626455, "n": 5 }, "decode_mbps": { "value": 38.47395577739818, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.69634192986763, "q3": 38.718842024315094, "n": 5 }, "rss_load_delta_mb": { "value": 236.31872, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 236.31872, "q3": 236.326912, "n": 5 }, "rss_peak_large_mb": { "value": 290.353152, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 289.792, "q3": 290.783232, "n": 5 }, "load_seconds": { "value": 0.5272523330000354, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.5243418330001077, "q3": 0.5360050419999425, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6694503767504908, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2466.238464, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2465.366016, "q3": 2467.115008, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.004096, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.004096, "q3": 0.004096, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.3938589958972627, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.3057099221295534, "q3": 3.455474373080335, "n": 5 }, "embedding_params_h2048": { "value": 419430400, "provenance": "estimated", "note": "204800 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.2225183474875011, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.21084858008889001, "q3": 0.22929775633849928, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5718486431320344, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5684167557161284, "q3": 3.5754822186916937, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 130.468017578125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 130.44677734375, "q3": 130.48876953125, "n": 5 }, "workload_tokens_total": { "value": 2085839, "provenance": "measured" }, "tokens_chat_en": { "value": 89142, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 53059, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 26596, "provenance": "measured" }, "tokens_code": { "value": 233232, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 126785, "provenance": "measured" }, "tokens_rag": { "value": 262621, "provenance": "measured" }, "tokens_json": { "value": 322537, "provenance": "measured" }, "tokens_logs": { "value": 416106, "provenance": "measured" }, "tokens_tech_docs": { "value": 8324, "provenance": "measured" }, "tokens_long_context": { "value": 547437, "provenance": "measured" } }, "parity": { "de": { "value": 1.2441510817748442, "provenance": "measured" }, "fr": { "value": 1.283975064173084, "provenance": "measured" }, "es": { "value": 1.2516318298496516, "provenance": "measured" }, "ru": { "value": 1.4117711771177117, "provenance": "measured" }, "zh": { "value": 1.2504217088375504, "provenance": "measured" }, "ja": { "value": 1.6532086541987532, "provenance": "measured" }, "ko": { "value": 1.396039603960396, "provenance": "measured" }, "ar": { "value": 1.317895122845618, "provenance": "measured" }, "ur": { "value": 1.5635863586358636, "provenance": "measured" }, "hi": { "value": 1.6137513751375137, "provenance": "measured" }, "mr": { "value": 1.5604327099376605, "provenance": "measured" }, "gu": { "value": 1.771983865053172, "provenance": "measured" }, "bn": { "value": 1.5667766776677667, "provenance": "measured" }, "ta": { "value": 1.7282361569490283, "provenance": "measured" }, "te": { "value": 1.8017968463513019, "provenance": "measured" }, "th": { "value": 2.4005867253392004, "provenance": "measured" }, "vi": { "value": 1.4081408140814082, "provenance": "measured" }, "tr": { "value": 1.3223689035570223, "provenance": "measured" }, "id": { "value": 1.2614228089475614, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 27.161354581673308, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 33.79282868525896, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2441510817748442, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 34.874501992031874, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.283975064173084, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 33.99601593625498, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2516318298496516, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 38.34561752988048, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4117711771177117, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 33.963147410358566, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2504217088375504, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 44.90338645418327, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6532086541987532, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 37.918326693227094, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.396039603960396, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 35.79581673306773, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.317895122845618, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 42.46912350597609, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5635863586358636, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 43.831673306772906, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6137513751375137, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 42.383466135458164, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5604327099376605, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 48.12948207171315, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.771983865053172, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 42.55577689243028, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5667766776677667, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 46.941235059760956, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7282361569490283, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 48.939243027888445, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8017968463513019, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 65.20318725099601, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.4005867253392004, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 38.24701195219124, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4081408140814082, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 35.917330677290835, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3223689035570223, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 34.26195219123506, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2614228089475614, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "", "", "[INST] hi [/INST]", "", "" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 204800, "provenance": "measured" }, "used_share": { "value": 0.5979296875, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 122456, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 82344, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 2, "provenance": "measured", "note": "of 201694 entries whose text is valid UTF-8; 2106 are byte pieces" }, "unreachable_examples": [ "\n//", "\n/" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.5272523330000354, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.5243418330001077, "q3": 0.5360050419999425, "n": 5 }, "rss_load_delta_mb": { "value": 236.31872, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 236.31872, "q3": 236.326912, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 236.253184, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 236.253184, "q3": 236.261376, "n": 5 }, "latency_small_p50_us": { "value": 48.37500000576256, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 48.062499899970135, "q3": 48.50000004807953, "n": 5 }, "latency_small_p95_us": { "value": 66.95899992337218, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 66.16700011363719, "q3": 67.08299997626455, "n": 5 }, "encode_mbps_1t_medium": { "value": 5.069619965517145, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.944411374013184, "q3": 5.130161389281384, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1220683.7932471149, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1190535.5573904272, "q3": 1235261.2044360766, "n": 5 }, "encode_mbps_1t_large": { "value": 4.168548514758301, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.1143444338287845, "q3": 4.212060286984122, "n": 5 }, "latency_large_p50_ms": { "value": 248.44079200011038, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 247.64379199996256, "q3": 254.02824999991935, "n": 5 }, "decode_mbps": { "value": 38.47395577739818, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 37.69634192986763, "q3": 38.718842024315094, "n": 5 }, "decode_tokens_per_s": { "value": 8850073.154078033, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 8671200.477816539, "q3": 8906403.758920997, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.51958897999539, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.483732355350463, "q3": 5.541969723896008, "n": 5 }, "batch_mbps_4t": { "value": 16.036505735404884, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 15.577337956412958, "q3": 17.061922351465657, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3833402.1350643956, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3723641.644008394, "q3": 4078520.012374756, "n": 5 }, "batch_cpu_utilization": { "value": 3.5718486431320344, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5684167557161284, "q3": 3.5754822186916937, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5718486431320344, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.5684167557161284, "q3": 3.5754822186916937, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.2225183474875011, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.21084858008889001, "q3": 0.22929775633849928, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 130.468017578125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 130.44677734375, "q3": 130.48876953125, "n": 5 }, "rss_peak_large_mb": { "value": 290.353152, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 289.792, "q3": 290.783232, "n": 5 }, "rss_peak_very_large_mb": { "value": 2466.238464, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2465.366016, "q3": 2467.115008, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.004096, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.004096, "q3": 0.004096, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.3938589958972627, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.3057099221295534, "q3": 3.455474373080335, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6694503767504908, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.3895429515688513, "provenance": "measured" }, "chars_per_token": { "value": 2.3873903471135978, "provenance": "measured" }, "tokens_per_word": { "value": 2.7593629918276776, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.184320841684634, "provenance": "measured" }, "chars_per_token": { "value": 3.171605789110958, "provenance": "measured" }, "tokens_per_word": { "value": 2.66265425343861, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.684311086307081, "provenance": "measured" }, "chars_per_token": { "value": 3.6837201667706245, "provenance": "measured" }, "tokens_per_word": { "value": 2.86077875241834, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.649966224894631, "provenance": "measured" }, "chars_per_token": { "value": 4.6242020036724485, "provenance": "measured" }, "tokens_per_word": { "value": 2.639884465653648, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.0612091656305, "provenance": "measured" }, "chars_per_token": { "value": 4.053183587126778, "provenance": "measured" }, "tokens_per_word": { "value": 2.633958572583046, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.2878839243338533, "provenance": "measured" }, "chars_per_token": { "value": 3.285260017628314, "provenance": "measured" }, "tokens_per_word": { "value": 2.5563297282307267, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.5399511751609, "provenance": "measured" }, "chars_per_token": { "value": 3.4850053335051507, "provenance": "measured" }, "tokens_per_word": { "value": 3.4035818713450294, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.33834856732053, "provenance": "measured" }, "chars_per_token": { "value": 4.3360035603026255, "provenance": "measured" }, "tokens_per_word": { "value": 2.2753544165757904, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.8424030172413794, "provenance": "measured" }, "chars_per_token": { "value": 3.8281566937119678, "provenance": "measured" }, "tokens_per_word": { "value": 2.6228308983977224, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.382825734366553, "provenance": "measured" }, "chars_per_token": { "value": 4.368264552171563, "provenance": "measured" }, "tokens_per_word": { "value": 2.962943186672362, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.4042630553497433, "provenance": "measured" }, "chars_per_token": { "value": 3.376718531324607, "provenance": "measured" }, "tokens_per_word": { "value": 2.8453970484891076, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.833246900193296, "provenance": "measured" }, "chars_per_token": { "value": 3.8278016123709397, "provenance": "measured" }, "tokens_per_word": { "value": 2.9878154712994482, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.687343617414765, "provenance": "measured" }, "chars_per_token": { "value": 3.686995496181545, "provenance": "measured" }, "tokens_per_word": { "value": 2.933181586113002, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.03562449423349, "provenance": "measured" }, "chars_per_token": { "value": 4.0245938939439805, "provenance": "measured" }, "tokens_per_word": { "value": 2.945630219990174, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.0086302552089754, "provenance": "measured" }, "chars_per_token": { "value": 2.994505917889286, "provenance": "measured" }, "tokens_per_word": { "value": 2.983699275778825, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.9525753142911806, "provenance": "measured" }, "chars_per_token": { "value": 3.8682328744190495, "provenance": "measured" }, "tokens_per_word": { "value": 2.64186428541342, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.012617671345995, "provenance": "measured" }, "chars_per_token": { "value": 4.011263418662263, "provenance": "measured" }, "tokens_per_word": { "value": 2.5176715176715176, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.247455786326276, "provenance": "measured" }, "chars_per_token": { "value": 3.2415494562958393, "provenance": "measured" }, "tokens_per_word": { "value": 2.6910947535541605, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.2594844503575584, "provenance": "measured" }, "chars_per_token": { "value": 3.2466522534508564, "provenance": "measured" }, "tokens_per_word": { "value": 2.764953649205415, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.32085237258348, "provenance": "measured" }, "chars_per_token": { "value": 4.304657293497364, "provenance": "measured" }, "tokens_per_word": { "value": 2.4461545075448177, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.140794456877259, "provenance": "measured" }, "chars_per_token": { "value": 4.052124576806105, "provenance": "measured" }, "tokens_per_word": { "value": 3.022503577158219, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.609229190103171, "provenance": "measured" }, "chars_per_token": { "value": 4.590485333235578, "provenance": "measured" }, "tokens_per_word": { "value": 1.3322599058029188, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 5.976616519297657, "provenance": "measured" }, "chars_per_token": { "value": 3.314409336629929, "provenance": "measured" }, "tokens_per_word": { "value": 1.8461955050094774, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 8.24452029383394, "provenance": "measured" }, "chars_per_token": { "value": 3.0970097202641536, "provenance": "measured" }, "tokens_per_word": { "value": 2.3130127346994818, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.8498907116397856, "provenance": "measured" }, "chars_per_token": { "value": 3.75271840415986, "provenance": "measured" }, "tokens_per_word": { "value": 2.2963976625490052, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.447601330863915, "provenance": "measured" }, "chars_per_token": { "value": 4.348663399603363, "provenance": "measured" }, "tokens_per_word": { "value": 1.4353298202691258, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 4.227872593768605, "provenance": "measured" }, "chars_per_token": { "value": 4.093181517496858, "provenance": "measured" }, "tokens_per_word": { "value": 1.5681331932936555, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 6.339666070363744, "provenance": "measured" }, "chars_per_token": { "value": 2.463160405485987, "provenance": "measured" }, "tokens_per_word": { "value": 2.585887867760439, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 7.54866735162577, "provenance": "measured" }, "chars_per_token": { "value": 2.996645714976429, "provenance": "measured" }, "tokens_per_word": { "value": 1.8447987512543205, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 3.9472406704110976, "provenance": "measured" }, "chars_per_token": { "value": 3.938075994440993, "provenance": "measured" }, "tokens_per_word": { "value": 1.895123312943004, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 3.5442786982415537, "provenance": "measured" }, "chars_per_token": { "value": 1.2435998758079538, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.6808319721834715, "provenance": "measured" }, "chars_per_token": { "value": 1.5885425064318712, "provenance": "measured" }, "tokens_per_word": { "value": 2.820543746910529, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 7.726724643044377, "provenance": "measured" }, "chars_per_token": { "value": 2.935948438030037, "provenance": "measured" }, "tokens_per_word": { "value": 2.39031402034498, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 6.190546263325208, "provenance": "measured" }, "chars_per_token": { "value": 3.4059849701762466, "provenance": "measured" }, "tokens_per_word": { "value": 2.172670473959096, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 8.08129522835828, "provenance": "measured" }, "chars_per_token": { "value": 3.0789309733382324, "provenance": "measured" }, "tokens_per_word": { "value": 3.108501340748833, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 6.509597435534328, "provenance": "measured" }, "chars_per_token": { "value": 2.4798846505400567, "provenance": "measured" }, "tokens_per_word": { "value": 3.219238261013718, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 5.459462016063843, "provenance": "measured" }, "chars_per_token": { "value": 1.9519718624769964, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.961215982013209, "provenance": "measured" }, "chars_per_token": { "value": 3.621059534404422, "provenance": "measured" }, "tokens_per_word": { "value": 2.1553398058252426, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.465330495823142, "provenance": "measured" }, "chars_per_token": { "value": 3.086829929379877, "provenance": "measured" }, "tokens_per_word": { "value": 1.5499450251899503, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.380089303389569, "provenance": "measured" }, "chars_per_token": { "value": 3.3534155980320994, "provenance": "measured" }, "tokens_per_word": { "value": 1.3699864396564714, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.3460701045487533, "provenance": "measured" }, "chars_per_token": { "value": 1.2567557848348732, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 89142, "provenance": "measured" }, "tokens_per_request_mean": { "value": 445.71, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.44571, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 53059, "provenance": "measured" }, "tokens_per_request_mean": { "value": 325.5153374233129, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3255153374233129, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 26596, "provenance": "measured" }, "tokens_per_request_mean": { "value": 139.97894736842105, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.13997894736842104, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 233232, "provenance": "measured" }, "tokens_per_request_mean": { "value": 555.3142857142857, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5553142857142856, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 126785, "provenance": "measured" }, "tokens_per_request_mean": { "value": 845.2333333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8452333333333334, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 262621, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1313.105, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.313105, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 322537, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1612.685, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.612685, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 416106, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2080.53, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.0805300000000004, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8324, "provenance": "measured" }, "tokens_per_request_mean": { "value": 332.96, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33296, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 547437, "provenance": "measured" }, "tokens_per_request_mean": { "value": 91239.5, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 91.2395, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 204800, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 419430400, "provenance": "estimated", "note": "204800 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 838860800, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 838860800, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 1677721600, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 204800, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 838860800, "provenance": "estimated", "note": "204800 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1677721600, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1677721600, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 3355443200, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 2516582400, "provenance": "estimated", "note": "204800 rows x 6144 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 5033164800, "provenance": "estimated", "note": "204800 rows x 6144 hidden x 2 matrices from the published config" } } } }, { "id": "qwen3-8-27b", "label": "Qwen3.8-27B", "provider": "Alibaba (Qwen)", "family": "published", "metrics": { "bpt_overall": { "value": 4.456133321321136, "provenance": "measured" }, "bpt_english": { "value": 4.589397945472672, "provenance": "measured" }, "bpt_code": { "value": 3.808701292805579, "provenance": "measured" }, "bpt_languages": { "value": 4.970300725685158, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.47528665820294747, "provenance": "measured" }, "byte_fallback_share": { "value": 0.008170098822102588, "provenance": "measured" }, "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.178264118415175, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.007467155386007, "q3": 4.17966360199197, "n": 5 }, "batch_mbps_4t": { "value": 13.768086388465784, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.390027005597226, "q3": 14.101835965023056, "n": 5 }, "latency_small_p95_us": { "value": 90.45800015883287, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 88.33399988361634, "q3": 100.91700005432358, "n": 5 }, "decode_mbps": { "value": 40.20413132029879, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.108593477309256, "q3": 40.25545147393336, "n": 5 }, "rss_load_delta_mb": { "value": 115.396608, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 115.396608, "q3": 115.396608, "n": 5 }, "rss_peak_large_mb": { "value": 218.599424, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 218.456064, "q3": 218.607616, "n": 5 }, "load_seconds": { "value": 0.30378187499991327, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.29920245799985423, "q3": 0.3053127919997678, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6535449391425532, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2551.861248, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2550.76352, "q3": 2551.967744, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.567744, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.518592, "q3": 17.911808, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.7306833689911594, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.6666099229808524, "q3": 2.7622100457324215, "n": 5 }, "embedding_params_h2048": { "value": 508559360, "provenance": "estimated", "note": "248320 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.2645099591790607, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25799401121222765, "q3": 0.27142193296362976, "n": 5 }, "cpu_cores_used_batch": { "value": 3.638189226073153, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.6343470122944024, "q3": 3.641795968586866, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 142.276611328125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 142.21142578125, "q3": 142.282958984375, "n": 5 }, "workload_tokens_total": { "value": 2402419, "provenance": "measured" }, "tokens_chat_en": { "value": 91532, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 52985, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 28327, "provenance": "measured" }, "tokens_code": { "value": 243819, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 140448, "provenance": "measured" }, "tokens_rag": { "value": 282720, "provenance": "measured" }, "tokens_json": { "value": 417871, "provenance": "measured" }, "tokens_logs": { "value": 555949, "provenance": "measured" }, "tokens_tech_docs": { "value": 8826, "provenance": "measured" }, "tokens_long_context": { "value": 579942, "provenance": "measured" } }, "parity": { "de": { "value": 1.3128541273191867, "provenance": "measured" }, "fr": { "value": 1.378032070510684, "provenance": "measured" }, "es": { "value": 1.3140391808317595, "provenance": "measured" }, "ru": { "value": 1.3925860089619673, "provenance": "measured" }, "zh": { "value": 0.928304262489353, "provenance": "measured" }, "ja": { "value": 1.1626115616783321, "provenance": "measured" }, "ko": { "value": 1.2349738917898012, "provenance": "measured" }, "ar": { "value": 1.3280376254490243, "provenance": "measured" }, "ur": { "value": 2.013887345850461, "provenance": "measured" }, "hi": { "value": 2.0376995148687183, "provenance": "measured" }, "mr": { "value": 2.318223901048032, "provenance": "measured" }, "gu": { "value": 3.1013220753249637, "provenance": "measured" }, "bn": { "value": 2.2128652371958673, "provenance": "measured" }, "ta": { "value": 2.615857497315113, "provenance": "measured" }, "te": { "value": 2.912898566825908, "provenance": "measured" }, "th": { "value": 1.2045328296855904, "provenance": "measured" }, "vi": { "value": 1.311706106728882, "provenance": "measured" }, "tr": { "value": 1.4330259600785098, "provenance": "measured" }, "id": { "value": 1.090619560789542, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.89541832669323, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 35.309760956175296, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3128541273191867, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 37.06274900398407, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.378032070510684, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 35.341633466135455, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3140391808317595, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 37.45418326693227, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3925860089619673, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 24.967131474103585, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 0.928304262489353, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 31.268924302788843, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1626115616783321, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 33.21513944223108, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2349738917898012, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 35.71812749003984, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3280376254490243, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 54.16434262948207, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.013887345850461, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 54.80478087649402, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0376995148687183, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 62.3496015936255, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.318223901048032, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 83.41135458167331, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.1013220753249637, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 59.51593625498008, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.2128652371958673, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 70.35458167330677, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.615857497315113, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 78.34362549800797, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.912898566825908, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 32.396414342629484, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2045328296855904, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 35.278884462151396, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.311706106728882, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 38.54183266932271, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4330259600785098, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 29.33266932270916, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.090619560789542, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 248,320 embedding rows for 248,077 tokenizer IDs (243 unused rows, usually padding)" ], "roundtrip_failures": { "value": 733, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 733, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [ "code_cs" ], "roundtrip_example": " \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\",", "byte_fallback_sets": [], "unicode_probe_failures": { "value": 7, "provenance": "measured" }, "unicode_failed_probes": [ "latin_accents (NFD)", "vietnamese (NFD)", "devanagari_nukta_precomposed (NFC)", "tamil (NFD)", "telugu (NFD)", "japanese (NFD)", "korean (NFD)" ], "nfd_normalized_to_nfc": [ "japanese", "korean", "latin_accents", "tamil", "telugu", "vietnamese" ], "special_strings_parsed_by_default": [ "<|endoftext|>", "<|im_start|>user", "<|im_end|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 248077, "provenance": "measured" }, "used_share": { "value": 0.6124509728834192, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 151935, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 96142, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 203, "provenance": "measured", "note": "of 247112 entries whose text is valid UTF-8; 944 are byte pieces" }, "unreachable_examples": [ "１０", "２０", "俱乐部", "毛泽东", "学一做", "全心全意为", "承担一切因您的行为而", "加拿大", "新加坡", "各种各样的", "阿拉伯", "众所周知" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.30378187499991327, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.29920245799985423, "q3": 0.3053127919997678, "n": 5 }, "rss_load_delta_mb": { "value": 115.396608, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 115.396608, "q3": 115.396608, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 115.26144, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 115.26144, "q3": 115.26144, "n": 5 }, "latency_small_p50_us": { "value": 66.22950013479567, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 65.8335000025545, "q3": 67.81249999221473, "n": 5 }, "latency_small_p95_us": { "value": 90.45800015883287, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 88.33399988361634, "q3": 100.91700005432358, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.178264118415175, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.007467155386007, "q3": 4.17966360199197, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1057334.384053543, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1014113.2049742368, "q3": 1057688.5316286427, "n": 5 }, "encode_mbps_1t_large": { "value": 3.5684038611473166, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.4980311283045378, "q3": 3.572450454259579, "n": 5 }, "latency_large_p50_ms": { "value": 293.8214160003554, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 292.73533399964435, "q3": 297.53108299973974, "n": 5 }, "decode_mbps": { "value": 40.20413132029879, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 40.108593477309256, "q3": 40.25545147393336, "n": 5 }, "decode_tokens_per_s": { "value": 10018346.513276307, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 9994539.72564608, "q3": 10031134.827943018, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.174806676905193, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.104242768427687, "q3": 4.214837223592826, "n": 5 }, "batch_mbps_4t": { "value": 13.768086388465784, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 13.390027005597226, "q3": 14.101835965023056, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3468772.9093545987, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3373523.4964426924, "q3": 3552858.776991244, "n": 5 }, "batch_cpu_utilization": { "value": 3.638189226073153, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.6343470122944024, "q3": 3.641795968586866, "n": 5 }, "cpu_cores_used_batch": { "value": 3.638189226073153, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.6343470122944024, "q3": 3.641795968586866, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.2645099591790607, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25799401121222765, "q3": 0.27142193296362976, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 142.276611328125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 142.21142578125, "q3": 142.282958984375, "n": 5 }, "rss_peak_large_mb": { "value": 218.599424, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 218.456064, "q3": 218.607616, "n": 5 }, "rss_peak_very_large_mb": { "value": 2551.861248, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2550.76352, "q3": 2551.967744, "n": 5 }, "rss_peak_chunked_mb": { "value": 17.567744, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 17.518592, "q3": 17.911808, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.7306833689911594, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.6666099229808524, "q3": 2.7622100457324215, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6535449391425532, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.2372441718467373, "provenance": "measured" }, "chars_per_token": { "value": 2.2352287647712354, "provenance": "measured" }, "tokens_per_word": { "value": 2.9472046327867076, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.316228566913372, "provenance": "measured" }, "chars_per_token": { "value": 3.302986804330076, "provenance": "measured" }, "tokens_per_word": { "value": 2.5567433795181738, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.8863809510617373, "provenance": "measured" }, "chars_per_token": { "value": 3.885757621930104, "provenance": "measured" }, "tokens_per_word": { "value": 2.712034410864216, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.886609276423008, "provenance": "measured" }, "chars_per_token": { "value": 4.859533879240529, "provenance": "measured" }, "tokens_per_word": { "value": 2.512043199799071, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.176155091535556, "provenance": "measured" }, "chars_per_token": { "value": 4.167902362073976, "provenance": "measured" }, "tokens_per_word": { "value": 2.561460592913955, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.4836424764012013, "provenance": "measured" }, "chars_per_token": { "value": 3.48086234393184, "provenance": "measured" }, "tokens_per_word": { "value": 2.412680255130338, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.451587683845343, "provenance": "measured" }, "chars_per_token": { "value": 3.3980133883386037, "provenance": "measured" }, "tokens_per_word": { "value": 3.490716374269006, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.540739546382888, "provenance": "measured" }, "chars_per_token": { "value": 4.538285140993945, "provenance": "measured" }, "tokens_per_word": { "value": 2.1739367502726283, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.001237644185547, "provenance": "measured" }, "chars_per_token": { "value": 3.98640241588145, "provenance": "measured" }, "tokens_per_word": { "value": 2.5187140214883934, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.543557456984326, "provenance": "measured" }, "chars_per_token": { "value": 4.528462271377062, "provenance": "measured" }, "tokens_per_word": { "value": 2.8581268688594617, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.5617467825637568, "provenance": "measured" }, "chars_per_token": { "value": 3.5329280284813986, "provenance": "measured" }, "tokens_per_word": { "value": 2.71958694463965, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.94639895807211, "provenance": "measured" }, "chars_per_token": { "value": 3.9407929330108318, "provenance": "measured" }, "tokens_per_word": { "value": 2.9021481394529873, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.703863974123612, "provenance": "measured" }, "chars_per_token": { "value": 3.7035142932074483, "provenance": "measured" }, "tokens_per_word": { "value": 2.9200987066031314, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.134568284085084, "provenance": "measured" }, "chars_per_token": { "value": 4.123267240051727, "provenance": "measured" }, "tokens_per_word": { "value": 2.8751387448368724, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.9430679822412507, "provenance": "measured" }, "chars_per_token": { "value": 2.929251434041623, "provenance": "measured" }, "tokens_per_word": { "value": 3.050166685825957, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.039193163395818, "provenance": "measured" }, "chars_per_token": { "value": 3.9530024195322238, "provenance": "measured" }, "tokens_per_word": { "value": 2.5852112379427994, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.129863507334568, "provenance": "measured" }, "chars_per_token": { "value": 4.128469684350088, "provenance": "measured" }, "tokens_per_word": { "value": 2.4461954261954264, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.2868998181472735, "provenance": "measured" }, "chars_per_token": { "value": 3.2809217490432374, "provenance": "measured" }, "tokens_per_word": { "value": 2.6588006061918166, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.018933722728477, "provenance": "measured" }, "chars_per_token": { "value": 3.0070485450053295, "provenance": "measured" }, "tokens_per_word": { "value": 2.9852670688640375, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.559441791459966, "provenance": "measured" }, "chars_per_token": { "value": 4.5423524502758585, "provenance": "measured" }, "tokens_per_word": { "value": 2.318150552426809, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.135366299185525, "provenance": "measured" }, "chars_per_token": { "value": 4.046812656251119, "provenance": "measured" }, "tokens_per_word": { "value": 3.026470970819061, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.589397945472672, "provenance": "measured" }, "chars_per_token": { "value": 4.570734734195812, "provenance": "measured" }, "tokens_per_word": { "value": 1.3380167332598722, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 5.860393445991063, "provenance": "measured" }, "chars_per_token": { "value": 3.2499563408495558, "provenance": "measured" }, "tokens_per_word": { "value": 1.882809175660516, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 5.441196058843118, "provenance": "measured" }, "chars_per_token": { "value": 2.043956043956044, "provenance": "measured" }, "tokens_per_word": { "value": 3.504685408299866, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.3573397753584513, "provenance": "measured" }, "chars_per_token": { "value": 3.272599589882703, "provenance": "measured" }, "tokens_per_word": { "value": 2.633299166252787, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.169692453076622, "provenance": "measured" }, "chars_per_token": { "value": 4.07693666976544, "provenance": "measured" }, "tokens_per_word": { "value": 1.530994165804084, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.918713643017543, "provenance": "measured" }, "chars_per_token": { "value": 3.7938717168543117, "provenance": "measured" }, "tokens_per_word": { "value": 1.6918478754165531, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 3.719560577965924, "provenance": "measured" }, "chars_per_token": { "value": 1.445166707483469, "provenance": "measured" }, "tokens_per_word": { "value": 4.407419971627706, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 5.738384847753926, "provenance": "measured" }, "chars_per_token": { "value": 2.2780055821646394, "provenance": "measured" }, "tokens_per_word": { "value": 2.4267755602631285, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.446244742784178, "provenance": "measured" }, "chars_per_token": { "value": 4.435921482625037, "provenance": "measured" }, "tokens_per_word": { "value": 1.6824327604621807, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.627356143938975, "provenance": "measured" }, "chars_per_token": { "value": 1.623625006909513, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 4.0001752234098475, "provenance": "measured" }, "chars_per_token": { "value": 1.7263619810911086, "provenance": "measured" }, "tokens_per_word": { "value": 2.595373208106772, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 5.346018029551395, "provenance": "measured" }, "chars_per_token": { "value": 2.0313436816557315, "provenance": "measured" }, "tokens_per_word": { "value": 3.454776647501106, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 6.040985673101165, "provenance": "measured" }, "chars_per_token": { "value": 3.3236980279960977, "provenance": "measured" }, "tokens_per_word": { "value": 2.226460682383951, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 5.581765621034745, "provenance": "measured" }, "chars_per_token": { "value": 2.126623340799506, "provenance": "measured" }, "tokens_per_word": { "value": 4.500496573641871, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 4.309935987589604, "provenance": "measured" }, "chars_per_token": { "value": 1.6419055412074566, "provenance": "measured" }, "tokens_per_word": { "value": 4.862240457540719, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 10.679052633349594, "provenance": "measured" }, "chars_per_token": { "value": 3.818180288987199, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.631684047037941, "provenance": "measured" }, "chars_per_token": { "value": 3.319825074972981, "provenance": "measured" }, "tokens_per_word": { "value": 2.3509111406505023, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 4.3225895593629104, "provenance": "measured" }, "chars_per_token": { "value": 2.441407493007754, "provenance": "measured" }, "tokens_per_word": { "value": 1.959696079557576, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.624953548866593, "provenance": "measured" }, "chars_per_token": { "value": 3.5408847392543046, "provenance": "measured" }, "tokens_per_word": { "value": 1.297453668826277, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.6196717759844015, "provenance": "measured" }, "chars_per_token": { "value": 1.7351098593583563, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 91532, "provenance": "measured" }, "tokens_per_request_mean": { "value": 457.66, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.45766, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 52985, "provenance": "measured" }, "tokens_per_request_mean": { "value": 325.06134969325154, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.32506134969325157, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 28327, "provenance": "measured" }, "tokens_per_request_mean": { "value": 149.08947368421053, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.14908947368421052, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 243819, "provenance": "measured" }, "tokens_per_request_mean": { "value": 580.5214285714286, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5805214285714286, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 140448, "provenance": "measured" }, "tokens_per_request_mean": { "value": 936.32, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.93632, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 282720, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1413.6, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.4136, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 417871, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2089.355, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.089355, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 555949, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2779.745, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.779745, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8826, "provenance": "measured" }, "tokens_per_request_mean": { "value": 353.04, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.35304, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 579942, "provenance": "measured" }, "tokens_per_request_mean": { "value": 96657.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 96.657, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 248320, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 508559360, "provenance": "estimated", "note": "248320 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1017118720, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1017118720, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2034237440, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 248320, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 1017118720, "provenance": "estimated", "note": "248320 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 2034237440, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 2034237440, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 4068474880, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 2542796800, "provenance": "estimated", "note": "248320 rows x 5120 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 5085593600, "provenance": "estimated", "note": "248320 rows x 5120 hidden x 2 matrices from the published config" } } } }, { "id": "north-micro-vision-instruct", "label": "North-Micro-Vision-Instruct", "provider": "Cohere", "family": "published", "metrics": { "bpt_overall": { "value": 4.639569893118902, "provenance": "measured" }, "bpt_english": { "value": 4.887795562552844, "provenance": "measured" }, "bpt_code": { "value": 3.9196326536411035, "provenance": "measured" }, "bpt_languages": { "value": 5.111281463162758, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.5123200691333425, "provenance": "measured" }, "byte_fallback_share": { "value": 0.05088274291002607, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 4.009868145514843, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.92947061141834, "q3": 4.030443939003501, "n": 5 }, "batch_mbps_4t": { "value": 14.617944898121555, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.150318780627517, "q3": 14.632415683022046, "n": 5 }, "latency_small_p95_us": { "value": 88.54199995766976, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 87.37500002098386, "q3": 91.87499995277904, "n": 5 }, "decode_mbps": { "value": 38.502504064913786, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 34.63283446547719, "q3": 38.52000742646196, "n": 5 }, "rss_load_delta_mb": { "value": 185.212928, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 185.208832, "q3": 185.212928, "n": 5 }, "rss_peak_large_mb": { "value": 254.160896, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 254.1568, "q3": 254.160896, "n": 5 }, "load_seconds": { "value": 0.440281541999866, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.4303102079998098, "q3": 0.44316733400000885, "n": 5 }, "throughput_ratio_very_large": { "value": 0.7279806743707192, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2461.401088, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2461.392896, "q3": 2461.995008, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.008192, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.008192, "q3": 0.008192, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.9191065167095607, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.9163230338947304, "q3": 2.9225136445230726, "n": 5 }, "embedding_params_h2048": { "value": 536870912, "provenance": "estimated", "note": "262144 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.25203025818451374, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25065096296741357, "q3": 0.25860414370324913, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6676290814889603, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.6674417007271884, "q3": 3.683267231248008, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 132.162841796875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 132.162353515625, "q3": 132.1982421875, "n": 5 }, "workload_tokens_total": { "value": 2070986, "provenance": "measured" }, "tokens_chat_en": { "value": 85590, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 51377, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 28350, "provenance": "measured" }, "tokens_code": { "value": 220175, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 126781, "provenance": "measured" }, "tokens_rag": { "value": 280073, "provenance": "measured" }, "tokens_json": { "value": 318389, "provenance": "measured" }, "tokens_logs": { "value": 407196, "provenance": "measured" }, "tokens_tech_docs": { "value": 8020, "provenance": "measured" }, "tokens_long_context": { "value": 545035, "provenance": "measured" } }, "parity": { "de": { "value": 1.3072261072261073, "provenance": "measured" }, "fr": { "value": 1.3728826728826728, "provenance": "measured" }, "es": { "value": 1.3313908313908314, "provenance": "measured" }, "ru": { "value": 1.5052836052836054, "provenance": "measured" }, "zh": { "value": 1.10998445998446, "provenance": "measured" }, "ja": { "value": 1.1920357420357421, "provenance": "measured" }, "ko": { "value": 1.2247863247863249, "provenance": "measured" }, "ar": { "value": 1.3677544677544677, "provenance": "measured" }, "ur": { "value": 2.025213675213675, "provenance": "measured" }, "hi": { "value": 1.6665889665889666, "provenance": "measured" }, "mr": { "value": 2.3786713286713286, "provenance": "measured" }, "gu": { "value": 5.8456099456099455, "provenance": "measured" }, "bn": { "value": 2.0365190365190364, "provenance": "measured" }, "ta": { "value": 2.3086635586635587, "provenance": "measured" }, "te": { "value": 2.4887334887334887, "provenance": "measured" }, "th": { "value": 1.7022144522144522, "provenance": "measured" }, "vi": { "value": 1.3557109557109557, "provenance": "measured" }, "tr": { "value": 1.5466588966588966, "provenance": "measured" }, "id": { "value": 1.0804584304584304, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 25.637450199203187, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 33.51394422310757, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3072261072261073, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 35.19721115537848, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3728826728826728, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 34.133466135458164, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3313908313908314, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 38.591633466135455, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5052836052836054, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 28.457171314741036, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.10998445998446, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 30.560756972111555, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1920357420357421, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 31.400398406374503, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2247863247863249, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 35.06573705179283, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3677544677544677, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 51.92131474103586, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.025213675213675, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 42.72709163346614, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6665889665889666, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 60.983067729083665, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.3786713286713286, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 149.86653386454182, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.8456099456099455, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 52.211155378486055, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.0365190365190364, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 59.18824701195219, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.3086635586635587, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 63.80478087649402, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.4887334887334887, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 43.64043824701195, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.7022144522144522, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 34.756972111553786, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3557109557109557, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 39.65239043824701, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5466588966588966, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 27.70019920318725, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0804584304584304, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the model config has 262,144 embedding rows for 255,033 tokenizer IDs (7,111 unused rows, usually padding)" ], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 255033, "provenance": "measured" }, "used_share": { "value": 0.635204071629946, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 161998, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 93035, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 7, "provenance": "measured", "note": "of 248870 entries whose text is valid UTF-8; 6147 are byte pieces" }, "unreachable_examples": [ " I'", "\n//", " O'", "\n\n//", "\n//\n//", "\r\n//", "\n\n\n//" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.440281541999866, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.4303102079998098, "q3": 0.44316733400000885, "n": 5 }, "rss_load_delta_mb": { "value": 185.212928, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 185.208832, "q3": 185.212928, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 185.131008, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 185.126912, "q3": 185.131008, "n": 5 }, "latency_small_p50_us": { "value": 62.12500011315569, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 62.06249986462353, "q3": 63.18750001810258, "n": 5 }, "latency_small_p95_us": { "value": 88.54199995766976, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 87.37500002098386, "q3": 91.87499995277904, "n": 5 }, "encode_mbps_1t_medium": { "value": 4.009868145514843, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.92947061141834, "q3": 4.030443939003501, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 937364.8608615486, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 918570.7707251249, "q3": 942174.7511374134, "n": 5 }, "encode_mbps_1t_large": { "value": 3.3280284894056953, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.254840011117819, "q3": 3.334069500033872, "n": 5 }, "latency_large_p50_ms": { "value": 313.72695899972314, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 312.1287920000668, "q3": 319.4686249998995, "n": 5 }, "decode_mbps": { "value": 38.502504064913786, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 34.63283446547719, "q3": 38.52000742646196, "n": 5 }, "decode_tokens_per_s": { "value": 8993711.774882454, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 8089804.512573448, "q3": 8997800.345033381, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 4.304562026945707, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.245856696707851, "q3": 4.331441971777405, "n": 5 }, "batch_mbps_4t": { "value": 14.617944898121555, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 14.150318780627517, "q3": 14.632415683022046, "n": 5 }, "batch_tokens_per_s_4t": { "value": 3416747.800678705, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3307446.491936393, "q3": 3420130.151811332, "n": 5 }, "batch_cpu_utilization": { "value": 3.6676290814889603, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.6674417007271884, "q3": 3.683267231248008, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6676290814889603, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.6674417007271884, "q3": 3.683267231248008, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.25203025818451374, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.25065096296741357, "q3": 0.25860414370324913, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 132.162841796875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 132.162353515625, "q3": 132.1982421875, "n": 5 }, "rss_peak_large_mb": { "value": 254.160896, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 254.1568, "q3": 254.160896, "n": 5 }, "rss_peak_very_large_mb": { "value": 2461.401088, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2461.392896, "q3": 2461.995008, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.008192, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.008192, "q3": 0.008192, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.9191065167095607, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.9163230338947304, "q3": 2.9225136445230726, "n": 5 }, "throughput_ratio_very_large": { "value": 0.7279806743707192, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.5615173958065043, "provenance": "measured" }, "chars_per_token": { "value": 2.559209869275205, "provenance": "measured" }, "tokens_per_word": { "value": 2.5741056448557664, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.3870390670710795, "provenance": "measured" }, "chars_per_token": { "value": 3.373514556838587, "provenance": "measured" }, "tokens_per_word": { "value": 2.503291301200357, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.9162165180129254, "provenance": "measured" }, "chars_per_token": { "value": 3.9155884036123556, "provenance": "measured" }, "tokens_per_word": { "value": 2.6913728657563065, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.825576114215754, "provenance": "measured" }, "chars_per_token": { "value": 4.798838885488044, "provenance": "measured" }, "tokens_per_word": { "value": 2.543815145045837, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 4.208299518929095, "provenance": "measured" }, "chars_per_token": { "value": 4.199983267098934, "provenance": "measured" }, "tokens_per_word": { "value": 2.541895283058994, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.5888839550029603, "provenance": "measured" }, "chars_per_token": { "value": 3.5860198342214327, "provenance": "measured" }, "tokens_per_word": { "value": 2.341930116472546, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.704624121551763, "provenance": "measured" }, "chars_per_token": { "value": 3.6471222859882824, "provenance": "measured" }, "tokens_per_word": { "value": 3.25229044834308, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.520855815280886, "provenance": "measured" }, "chars_per_token": { "value": 4.518412157642673, "provenance": "measured" }, "tokens_per_word": { "value": 2.183498208443683, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 4.017779766195246, "provenance": "measured" }, "chars_per_token": { "value": 4.002883205328959, "provenance": "measured" }, "tokens_per_word": { "value": 2.508343897420977, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.5635930346018165, "provenance": "measured" }, "chars_per_token": { "value": 4.54843128424529, "provenance": "measured" }, "tokens_per_word": { "value": 2.845578812473302, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.6622679803682794, "provenance": "measured" }, "chars_per_token": { "value": 3.6326358906232703, "provenance": "measured" }, "tokens_per_word": { "value": 2.644940267041462, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.9608174267458005, "provenance": "measured" }, "chars_per_token": { "value": 3.9551909196456845, "provenance": "measured" }, "tokens_per_word": { "value": 2.8915835191923933, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.9277674343147178, "provenance": "measured" }, "chars_per_token": { "value": 3.927396614726173, "provenance": "measured" }, "tokens_per_word": { "value": 2.7536376786929884, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 4.222660070927181, "provenance": "measured" }, "chars_per_token": { "value": 4.211118245004719, "provenance": "measured" }, "tokens_per_word": { "value": 2.815158517871548, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 3.1448178098521216, "provenance": "measured" }, "chars_per_token": { "value": 3.1300541254550724, "provenance": "measured" }, "tokens_per_word": { "value": 2.854489021726635, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 4.148489336245053, "provenance": "measured" }, "chars_per_token": { "value": 4.059966364617586, "provenance": "measured" }, "tokens_per_word": { "value": 2.517101217303399, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 4.239691486205873, "provenance": "measured" }, "chars_per_token": { "value": 4.238260596436736, "provenance": "measured" }, "tokens_per_word": { "value": 2.382827442827443, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.513050730681365, "provenance": "measured" }, "chars_per_token": { "value": 3.50666134822497, "provenance": "measured" }, "tokens_per_word": { "value": 2.4876416251713933, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 3.4998714285714287, "provenance": "measured" }, "chars_per_token": { "value": 3.4860928571428573, "provenance": "measured" }, "tokens_per_word": { "value": 2.5750441436138907, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.488521118403637, "provenance": "measured" }, "chars_per_token": { "value": 4.471697596509324, "provenance": "measured" }, "tokens_per_word": { "value": 2.3547783844202743, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.20994559747969, "provenance": "measured" }, "chars_per_token": { "value": 4.119794933127197, "provenance": "measured" }, "tokens_per_word": { "value": 2.972856957030742, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.887795562552844, "provenance": "measured" }, "chars_per_token": { "value": 4.86791888976355, "provenance": "measured" }, "tokens_per_word": { "value": 1.2563314418624527, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 6.0129010497913065, "provenance": "measured" }, "chars_per_token": { "value": 3.334531388338463, "provenance": "measured" }, "tokens_per_word": { "value": 1.8350547367606669, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 6.93207142233258, "provenance": "measured" }, "chars_per_token": { "value": 2.603995358297044, "provenance": "measured" }, "tokens_per_word": { "value": 2.7509353653931967, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.8650260551408087, "provenance": "measured" }, "chars_per_token": { "value": 3.767471727400399, "provenance": "measured" }, "tokens_per_word": { "value": 2.2874050278444096, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.426850356449534, "provenance": "measured" }, "chars_per_token": { "value": 4.328374035465505, "provenance": "measured" }, "tokens_per_word": { "value": 1.4420579655594241, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 4.196951414711025, "provenance": "measured" }, "chars_per_token": { "value": 4.063245421786632, "provenance": "measured" }, "tokens_per_word": { "value": 1.5796864667211266, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 2.1494607960082326, "provenance": "measured" }, "chars_per_token": { "value": 0.8351333756555565, "provenance": "measured" }, "tokens_per_word": { "value": 7.626873496576821, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 7.244583973782663, "provenance": "measured" }, "chars_per_token": { "value": 2.875931672515298, "provenance": "measured" }, "tokens_per_word": { "value": 1.9222321329022187, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.727829215344845, "provenance": "measured" }, "chars_per_token": { "value": 4.716852174313264, "provenance": "measured" }, "tokens_per_word": { "value": 1.5822288571705991, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.801829792929389, "provenance": "measured" }, "chars_per_token": { "value": 1.68484350203629, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 4.303933703287786, "provenance": "measured" }, "chars_per_token": { "value": 1.857455511201095, "provenance": "measured" }, "tokens_per_word": { "value": 2.412199703410776, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 5.671476753135044, "provenance": "measured" }, "chars_per_token": { "value": 2.1550092806374215, "provenance": "measured" }, "tokens_per_word": { "value": 3.2565236620964173, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.995833061339948, "provenance": "measured" }, "chars_per_token": { "value": 3.2988554518354114, "provenance": "measured" }, "tokens_per_word": { "value": 2.243227412505491, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 6.611440744419467, "provenance": "measured" }, "chars_per_token": { "value": 2.518924146583721, "provenance": "measured" }, "tokens_per_word": { "value": 3.799582878140828, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 5.178464626602761, "provenance": "measured" }, "chars_per_token": { "value": 1.9727786653557822, "provenance": "measured" }, "tokens_per_word": { "value": 4.0467487255999, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 7.98454301920489, "provenance": "measured" }, "chars_per_token": { "value": 2.8547873879085914, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.5907691654459377, "provenance": "measured" }, "chars_per_token": { "value": 3.282423624822198, "provenance": "measured" }, "tokens_per_word": { "value": 2.377698507512914, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 4.458027463511529, "provenance": "measured" }, "chars_per_token": { "value": 2.5179031004404524, "provenance": "measured" }, "tokens_per_word": { "value": 1.9001591807932783, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.786136614911311, "provenance": "measured" }, "chars_per_token": { "value": 3.664287202166354, "provenance": "measured" }, "tokens_per_word": { "value": 1.2537592285671237, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.176218571743335, "provenance": "measured" }, "chars_per_token": { "value": 1.5685525660799242, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 85590, "provenance": "measured" }, "tokens_per_request_mean": { "value": 427.95, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.42795, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 51377, "provenance": "measured" }, "tokens_per_request_mean": { "value": 315.1963190184049, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3151963190184049, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 28350, "provenance": "measured" }, "tokens_per_request_mean": { "value": 149.21052631578948, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.14921052631578946, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 220175, "provenance": "measured" }, "tokens_per_request_mean": { "value": 524.2261904761905, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.5242261904761905, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 126781, "provenance": "measured" }, "tokens_per_request_mean": { "value": 845.2066666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.8452066666666668, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 280073, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1400.365, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.400365, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 318389, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1591.945, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.591945, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 407196, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2035.98, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.03598, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 8020, "provenance": "measured" }, "tokens_per_request_mean": { "value": 320.8, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3208, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 545035, "provenance": "measured" }, "tokens_per_request_mean": { "value": 90839.16666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 90.83916666666667, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 262144, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 536870912, "provenance": "estimated", "note": "262144 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1073741824, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1073741824, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2147483648, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 262144, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 1073741824, "provenance": "estimated", "note": "262144 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 2147483648, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 2147483648, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 4294967296, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 536870912, "provenance": "estimated", "note": "262144 rows x 2048 hidden x 1 matrix from the published config" }, "bf16_bytes": { "value": 1073741824, "provenance": "estimated", "note": "262144 rows x 2048 hidden x 1 matrix from the published config" } } } }, { "id": "falcon-h1-34b-instruct", "label": "Falcon-H1-34B-Instruct", "provider": "TII (Falcon)", "family": "published", "metrics": { "bpt_overall": { "value": 4.133771333511425, "provenance": "measured" }, "bpt_english": { "value": 4.531476375824414, "provenance": "measured" }, "bpt_code": { "value": 3.249805325148841, "provenance": "measured" }, "bpt_languages": { "value": 4.620032299561019, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.555894460840154, "provenance": "measured" }, "byte_fallback_share": { "value": 0.10179611775253003, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 2.765068889391423, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.7389083970847636, "q3": 2.765128679894381, "n": 5 }, "batch_mbps_4t": { "value": 10.159878525556868, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10.050722230334545, "q3": 10.360540892551512, "n": 5 }, "latency_small_p95_us": { "value": 124.20799998835719, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 123.87500009936048, "q3": 124.4170000518352, "n": 5 }, "decode_mbps": { "value": 32.697688559727155, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 31.341726365224908, "q3": 33.74930397972445, "n": 5 }, "rss_load_delta_mb": { "value": 172.048384, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 172.048384, "q3": 172.048384, "n": 5 }, "rss_peak_large_mb": { "value": 288.256, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 287.137792, "q3": 288.70656, "n": 5 }, "load_seconds": { "value": 0.4538142919998336, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.44709249999982603, "q3": 0.46073687599994173, "n": 5 }, "throughput_ratio_very_large": { "value": 0.807513391314601, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2683.24864, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2683.097088, "q3": 2683.600896, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.0, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.0, "q3": 0.0, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.2328301560909654, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.2216903317471846, "q3": 2.236002216389324, "n": 5 }, "embedding_params_h2048": { "value": 534773760, "provenance": "estimated", "note": "261120 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.3633895709068609, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.3555668092693407, "q3": 0.36451661126835316, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6838644674690686, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.6536249873171274, "q3": 3.6904060767556643, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 145.377685546875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 145.372314453125, "q3": 145.402099609375, "n": 5 }, "workload_tokens_total": { "value": 2715577, "provenance": "measured" }, "tokens_chat_en": { "value": 94544, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 58120, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 32199, "provenance": "measured" }, "tokens_code": { "value": 285645, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 164832, "provenance": "measured" }, "tokens_rag": { "value": 328861, "provenance": "measured" }, "tokens_json": { "value": 458328, "provenance": "measured" }, "tokens_logs": { "value": 609637, "provenance": "measured" }, "tokens_tech_docs": { "value": 9755, "provenance": "measured" }, "tokens_long_context": { "value": 673656, "provenance": "measured" } }, "parity": { "de": { "value": 1.4108231147661356, "provenance": "measured" }, "fr": { "value": 1.4947132682282105, "provenance": "measured" }, "es": { "value": 1.3735957118731184, "provenance": "measured" }, "ru": { "value": 1.273882076510757, "provenance": "measured" }, "zh": { "value": 0.9161465599530069, "provenance": "measured" }, "ja": { "value": 1.1113885013583964, "provenance": "measured" }, "ko": { "value": 1.1610984653792495, "provenance": "measured" }, "ar": { "value": 1.1261840076363903, "provenance": "measured" }, "ur": { "value": 1.5600998604890226, "provenance": "measured" }, "hi": { "value": 1.6932227035758867, "provenance": "measured" }, "mr": { "value": 1.6919010206329392, "provenance": "measured" }, "gu": { "value": 2.398487407298627, "provenance": "measured" }, "bn": { "value": 1.8667303032528086, "provenance": "measured" }, "ta": { "value": 7.273000954548792, "provenance": "measured" }, "te": { "value": 6.518356707540936, "provenance": "measured" }, "th": { "value": 2.470702694764667, "provenance": "measured" }, "vi": { "value": 1.6802261546369044, "provenance": "measured" }, "tr": { "value": 1.5538585799251046, "provenance": "measured" }, "id": { "value": 1.0787135619355313, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 27.12948207171315, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 38.27490039840637, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4108231147661356, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 40.550796812749006, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4947132682282105, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 37.264940239043824, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3735957118731184, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 34.559760956175296, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.273882076510757, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 24.85458167330677, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 0.9161465599530069, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 30.15139442231076, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1113885013583964, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 31.5, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1610984653792495, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 30.552788844621514, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1261840076363903, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 42.32470119521912, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5600998604890226, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 45.93625498007968, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6932227035758867, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 45.9003984063745, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6919010206329392, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 65.06972111553785, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.398487407298627, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 50.64342629482072, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8667303032528086, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 197.31274900398407, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 7.273000954548792, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 176.83964143426294, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 6.518356707540936, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 67.0288844621514, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 2.470702694764667, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 45.58366533864542, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6802261546369044, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 42.155378486055774, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5538585799251046, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 29.264940239043824, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0787135619355313, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "<|im_start|>user", "<|im_end|>", "<|begin_of_text|>" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 261120, "provenance": "measured" }, "used_share": { "value": 0.5549823835784313, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 144917, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 116203, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 11, "provenance": "measured", "note": "of 256194 entries whose text is valid UTF-8; 4082 are byte pieces" }, "unreachable_examples": [ "\n\t", "\n ", "维生素C", "\n\t\t", "\n ", "\n ", "\n\t\t\t\t\t\t\t\t", "兼CEO", "\n\t\t\t", "\n ", " \n\t" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.4538142919998336, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.44709249999982603, "q3": 0.46073687599994173, "n": 5 }, "rss_load_delta_mb": { "value": 172.048384, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 172.048384, "q3": 172.048384, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 171.978752, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 171.978752, "q3": 171.978752, "n": 5 }, "latency_small_p50_us": { "value": 81.54149986694392, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 81.33350002026418, "q3": 81.66650013663457, "n": 5 }, "latency_small_p95_us": { "value": 124.20799998835719, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 123.87500009936048, "q3": 124.4170000518352, "n": 5 }, "encode_mbps_1t_medium": { "value": 2.765068889391423, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.7389083970847636, "q3": 2.765128679894381, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 797780.1726087428, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 790232.3237475413, "q3": 797797.4234186425, "n": 5 }, "encode_mbps_1t_large": { "value": 2.484073072009955, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.4722143326201884, "q3": 2.4873418418737394, "n": 5 }, "latency_large_p50_ms": { "value": 413.0233750001935, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 412.2203750000608, "q3": 427.2502920000534, "n": 5 }, "decode_mbps": { "value": 32.697688559727155, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 31.341726365224908, "q3": 33.74930397972445, "n": 5 }, "decode_tokens_per_s": { "value": 9286548.431102835, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 8901438.377620367, "q3": 9585220.232042544, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 3.1750475444172572, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.17412294443961, "q3": 3.1840942981868454, "n": 5 }, "batch_mbps_4t": { "value": 10.159878525556868, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 10.050722230334545, "q3": 10.360540892551512, "n": 5 }, "batch_tokens_per_s_4t": { "value": 2936918.4811898232, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2905364.645189694, "q3": 2994923.998935279, "n": 5 }, "batch_cpu_utilization": { "value": 3.6838644674690686, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.6536249873171274, "q3": 3.6904060767556643, "n": 5 }, "cpu_cores_used_batch": { "value": 3.6838644674690686, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.6536249873171274, "q3": 3.6904060767556643, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.3633895709068609, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.3555668092693407, "q3": 0.36451661126835316, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 145.377685546875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 145.372314453125, "q3": 145.402099609375, "n": 5 }, "rss_peak_large_mb": { "value": 288.256, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 287.137792, "q3": 288.70656, "n": 5 }, "rss_peak_very_large_mb": { "value": 2683.24864, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 2683.097088, "q3": 2683.600896, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.0, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.0, "q3": 0.0, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.2328301560909654, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.2216903317471846, "q3": 2.236002216389324, "n": 5 }, "throughput_ratio_very_large": { "value": 0.807513391314601, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.1036225815642022, "provenance": "measured" }, "chars_per_token": { "value": 2.101727546642033, "provenance": "measured" }, "tokens_per_word": { "value": 3.1344103480002166, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 2.8604394917843776, "provenance": "measured" }, "chars_per_token": { "value": 2.8490177034878768, "provenance": "measured" }, "tokens_per_word": { "value": 2.9641408104513545, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.249908781846719, "provenance": "measured" }, "chars_per_token": { "value": 3.24938753525654, "provenance": "measured" }, "tokens_per_word": { "value": 3.243167602697271, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.15220844979483, "provenance": "measured" }, "chars_per_token": { "value": 4.129202171493624, "provenance": "measured" }, "tokens_per_word": { "value": 2.9563481100087907, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.4964340330877244, "provenance": "measured" }, "chars_per_token": { "value": 3.4895245377450297, "provenance": "measured" }, "tokens_per_word": { "value": 3.059418995363872, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.009931102973124, "provenance": "measured" }, "chars_per_token": { "value": 3.0075290174414997, "provenance": "measured" }, "tokens_per_word": { "value": 2.792394620077648, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 2.9712948274204405, "provenance": "measured" }, "chars_per_token": { "value": 2.9251754638977023, "provenance": "measured" }, "tokens_per_word": { "value": 4.0549707602339184, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 3.830679594042122, "provenance": "measured" }, "chars_per_token": { "value": 3.828608997271951, "provenance": "measured" }, "tokens_per_word": { "value": 2.5769006075712726, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.3071142147903654, "provenance": "measured" }, "chars_per_token": { "value": 3.294852560080744, "provenance": "measured" }, "tokens_per_word": { "value": 3.0473617489972775, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 3.7206915015681177, "provenance": "measured" }, "chars_per_token": { "value": 3.7083301461026545, "provenance": "measured" }, "tokens_per_word": { "value": 3.490228534814182, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 2.999389506769826, "provenance": "measured" }, "chars_per_token": { "value": 2.9751208897485495, "provenance": "measured" }, "tokens_per_word": { "value": 3.2294838760053097, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.3899574728298636, "provenance": "measured" }, "chars_per_token": { "value": 3.3851418962114685, "provenance": "measured" }, "tokens_per_word": { "value": 3.3785186054701257, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.2958984564920035, "provenance": "measured" }, "chars_per_token": { "value": 3.295587291503251, "provenance": "measured" }, "tokens_per_word": { "value": 3.2815478216473792, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.3699988995343655, "provenance": "measured" }, "chars_per_token": { "value": 3.360787658277909, "provenance": "measured" }, "tokens_per_word": { "value": 3.527436601949403, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.548582916672106, "provenance": "measured" }, "chars_per_token": { "value": 2.536618320909649, "provenance": "measured" }, "tokens_per_word": { "value": 3.522289918381423, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.32260010051935, "provenance": "measured" }, "chars_per_token": { "value": 3.251700452337075, "provenance": "measured" }, "tokens_per_word": { "value": 3.142769891748452, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.50208282283754, "provenance": "measured" }, "chars_per_token": { "value": 3.500900874929732, "provenance": "measured" }, "tokens_per_word": { "value": 2.8846985446985447, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 2.7724345949782223, "provenance": "measured" }, "chars_per_token": { "value": 2.7673922126385797, "provenance": "measured" }, "tokens_per_word": { "value": 3.1521794039113806, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.6336888047988647, "provenance": "measured" }, "chars_per_token": { "value": 2.6233202898239125, "provenance": "measured" }, "tokens_per_word": { "value": 3.4219393761035906, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 3.8116695865180383, "provenance": "measured" }, "chars_per_token": { "value": 3.7973829863102897, "provenance": "measured" }, "tokens_per_word": { "value": 2.7729246378057693, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 3.8972840873034627, "provenance": "measured" }, "chars_per_token": { "value": 3.8138286740432195, "provenance": "measured" }, "tokens_per_word": { "value": 3.211355851363656, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.531476375824414, "provenance": "measured" }, "chars_per_token": { "value": 4.513048707968429, "provenance": "measured" }, "tokens_per_word": { "value": 1.3551193335999097, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 6.73450596151576, "provenance": "measured" }, "chars_per_token": { "value": 3.7347066462046983, "provenance": "measured" }, "tokens_per_word": { "value": 1.638427913813779, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 6.856527534371683, "provenance": "measured" }, "chars_per_token": { "value": 2.5756177029595437, "provenance": "measured" }, "tokens_per_word": { "value": 2.7812446366663233, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.0596244217301467, "provenance": "measured" }, "chars_per_token": { "value": 2.9823986542083416, "provenance": "measured" }, "tokens_per_word": { "value": 2.8895311360730402, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 3.96087461046117, "provenance": "measured" }, "chars_per_token": { "value": 3.872764028871487, "provenance": "measured" }, "tokens_per_word": { "value": 1.6117083843041309, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.5863476632367486, "provenance": "measured" }, "chars_per_token": { "value": 3.4720942140296978, "provenance": "measured" }, "tokens_per_word": { "value": 1.8486404481269694, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 4.693913412038959, "provenance": "measured" }, "chars_per_token": { "value": 1.823733542901041, "provenance": "measured" }, "tokens_per_word": { "value": 3.49253685314254, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 6.782435752976474, "provenance": "measured" }, "chars_per_token": { "value": 2.6924695564817203, "provenance": "measured" }, "tokens_per_word": { "value": 2.0532110603188762, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.366858203614876, "provenance": "measured" }, "chars_per_token": { "value": 4.356719262570941, "provenance": "measured" }, "tokens_per_word": { "value": 1.7130182541994368, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.702213110149975, "provenance": "measured" }, "chars_per_token": { "value": 1.6498904678986688, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 4.180336613483647, "provenance": "measured" }, "chars_per_token": { "value": 1.8041145186459029, "provenance": "measured" }, "tokens_per_word": { "value": 2.4835195254572415, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 7.115498757543486, "provenance": "measured" }, "chars_per_token": { "value": 2.7036989705360313, "provenance": "measured" }, "tokens_per_word": { "value": 2.5956435205661212, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 6.47653128378696, "provenance": "measured" }, "chars_per_token": { "value": 3.5633314530155595, "provenance": "measured" }, "tokens_per_word": { "value": 2.076731585883731, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 2.1021441928111364, "provenance": "measured" }, "chars_per_token": { "value": 0.800905879908581, "provenance": "measured" }, "tokens_per_word": { "value": 11.950044691627769, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 1.9795718625700773, "provenance": "measured" }, "chars_per_token": { "value": 0.7541341726956685, "provenance": "measured" }, "tokens_per_word": { "value": 10.58609971403705, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 5.1316251060906115, "provenance": "measured" }, "chars_per_token": { "value": 1.8347573050963908, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.3283131739826306, "provenance": "measured" }, "chars_per_token": { "value": 3.0425051819589117, "provenance": "measured" }, "tokens_per_word": { "value": 2.565193249314331, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.421476058941531, "provenance": "measured" }, "chars_per_token": { "value": 3.0620608530347746, "provenance": "measured" }, "tokens_per_word": { "value": 1.5624825639594992, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 3.560676139614724, "provenance": "measured" }, "chars_per_token": { "value": 2.726069282853328, "provenance": "measured" }, "tokens_per_word": { "value": 1.6852594043493547, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 3.7411398327387566, "provenance": "measured" }, "chars_per_token": { "value": 1.4051406515000877, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 94544, "provenance": "measured" }, "tokens_per_request_mean": { "value": 472.72, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.47272, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 58120, "provenance": "measured" }, "tokens_per_request_mean": { "value": 356.5644171779141, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3565644171779141, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 32199, "provenance": "measured" }, "tokens_per_request_mean": { "value": 169.46842105263158, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.16946842105263157, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 285645, "provenance": "measured" }, "tokens_per_request_mean": { "value": 680.1071428571429, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6801071428571428, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 164832, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1098.88, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.09888, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 328861, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1644.305, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.644305, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 458328, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2291.64, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.29164, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 609637, "provenance": "measured" }, "tokens_per_request_mean": { "value": 3048.185, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 3.048185, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9755, "provenance": "measured" }, "tokens_per_request_mean": { "value": 390.2, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3902, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 673656, "provenance": "measured" }, "tokens_per_request_mean": { "value": 112276.0, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 112.276, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 261120, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 534773760, "provenance": "estimated", "note": "261120 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1069547520, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1069547520, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2139095040, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 261120, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 1069547520, "provenance": "estimated", "note": "261120 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 2139095040, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 2139095040, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 4278190080, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 2673868800, "provenance": "estimated", "note": "261120 rows x 5120 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 5347737600, "provenance": "estimated", "note": "261120 rows x 5120 hidden x 2 matrices from the published config" } } } }, { "id": "gemma-4-31b-it", "label": "gemma-4-31B-it", "provider": "Google", "family": "published", "metrics": { "bpt_overall": { "value": 4.710001784807477, "provenance": "measured" }, "bpt_english": { "value": 4.659014958348676, "provenance": "measured" }, "bpt_code": { "value": 3.5048973369676704, "provenance": "measured" }, "bpt_languages": { "value": 5.9660930591060835, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.32985771705191436, "provenance": "measured" }, "byte_fallback_share": { "value": 0.00366319321237354, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 6.426287669332646, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.3731937836178565, "q3": 6.496928971519612, "n": 5 }, "batch_mbps_4t": { "value": 20.89084307099784, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 20.777198523991068, "q3": 21.004698820989567, "n": 5 }, "latency_small_p95_us": { "value": 53.33300032361876, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 52.24999995334656, "q3": 54.58300000782401, "n": 5 }, "decode_mbps": { "value": 19.99801044816113, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 19.358661512641568, "q3": 20.0630422976614, "n": 5 }, "rss_load_delta_mb": { "value": 309.850112, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 309.850112, "q3": 309.850112, "n": 5 }, "rss_peak_large_mb": { "value": 373.338112, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 373.338112, "q3": 373.342208, "n": 5 }, "load_seconds": { "value": 0.7190039170000091, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.6862907080000014, "q3": 0.7275295000000597, "n": 5 }, "throughput_ratio_very_large": { "value": 0.3716238670552402, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 1770.811392, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 1770.807296, "q3": 1770.840064, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.069632, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.069632, "q3": 0.069632, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.3881618744868045, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.387348580112849, "q3": 2.394519108270606, "n": 5 }, "embedding_params_h2048": { "value": 536870912, "provenance": "estimated", "note": "262144 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.1690639222864304, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.16802319218744302, "q3": 0.1726202865834512, "n": 5 }, "cpu_cores_used_batch": { "value": 3.551847617761725, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.531887869453192, "q3": 3.5882750340083382, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 83.546630859375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 83.531005859375, "q3": 83.546630859375, "n": 5 }, "workload_tokens_total": { "value": 2419975, "provenance": "measured" }, "tokens_chat_en": { "value": 92377, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 53821, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 25167, "provenance": "measured" }, "tokens_code": { "value": 267604, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 140393, "provenance": "measured" }, "tokens_rag": { "value": 250725, "provenance": "measured" }, "tokens_json": { "value": 422477, "provenance": "measured" }, "tokens_logs": { "value": 574778, "provenance": "measured" }, "tokens_tech_docs": { "value": 9128, "provenance": "measured" }, "tokens_long_context": { "value": 583505, "provenance": "measured" } }, "parity": { "de": { "value": 1.319847156664419, "provenance": "measured" }, "fr": { "value": 1.3843934966659175, "provenance": "measured" }, "es": { "value": 1.2846707125196672, "provenance": "measured" }, "ru": { "value": 1.3923353562598337, "provenance": "measured" }, "zh": { "value": 1.0932793886266576, "provenance": "measured" }, "ja": { "value": 1.2126695137484078, "provenance": "measured" }, "ko": { "value": 1.3735296321270698, "provenance": "measured" }, "ar": { "value": 1.4920206788042256, "provenance": "measured" }, "ur": { "value": 1.546714617517045, "provenance": "measured" }, "hi": { "value": 1.316700382108339, "provenance": "measured" }, "mr": { "value": 1.3961564396493593, "provenance": "measured" }, "gu": { "value": 1.8734921705252117, "provenance": "measured" }, "bn": { "value": 1.2200869109163108, "provenance": "measured" }, "ta": { "value": 1.4811193526635198, "provenance": "measured" }, "te": { "value": 1.802577358207837, "provenance": "measured" }, "th": { "value": 1.6013710946280062, "provenance": "measured" }, "vi": { "value": 1.3662246197647412, "provenance": "measured" }, "tr": { "value": 1.3692215479133887, "provenance": "measured" }, "id": { "value": 1.1418296246347495, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.58764940239044, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 35.091633466135455, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.319847156664419, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 36.80776892430279, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3843934966659175, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 34.15637450199203, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2846707125196672, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 37.01892430278885, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3923353562598337, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 29.067729083665338, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0932793886266576, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 32.24203187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2126695137484078, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 36.51892430278885, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3735296321270698, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 39.669322709163346, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4920206788042256, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 41.123505976095615, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.546714617517045, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 35.00796812749004, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.316700382108339, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 37.12051792828685, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3961564396493593, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 49.81175298804781, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8734921705252117, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 32.439243027888445, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2200869109163108, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 39.37948207171315, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4811193526635198, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 47.92629482071713, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.802577358207837, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 42.57669322709163, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6013710946280062, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 36.32470119521912, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3662246197647412, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 36.40438247011952, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3692215479133887, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 30.358565737051794, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1418296246347495, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "", "", "", "", "" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 262144, "provenance": "measured" }, "used_share": { "value": 0.5986976623535156, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 156945, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 105199, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 6412, "provenance": "measured", "note": "of 261992 entries whose text is valid UTF-8; 128 are byte pieces" }, "unreachable_examples": [ "[multimodal]", "", "", "", "", "", "", "", "", "", "", "" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.7190039170000091, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.6862907080000014, "q3": 0.7275295000000597, "n": 5 }, "rss_load_delta_mb": { "value": 309.850112, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 309.850112, "q3": 309.850112, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 309.764096, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 309.764096, "q3": 309.764096, "n": 5 }, "latency_small_p50_us": { "value": 43.74999980427674, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 43.08350003157102, "q3": 45.312499992178346, "n": 5 }, "latency_small_p95_us": { "value": 53.33300032361876, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 52.24999995334656, "q3": 54.58300000782401, "n": 5 }, "encode_mbps_1t_medium": { "value": 6.426287669332646, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.3731937836178565, "q3": 6.496928971519612, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1649383.6651784077, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1635756.483775258, "q3": 1667514.6010949777, "n": 5 }, "encode_mbps_1t_large": { "value": 4.041897166396821, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.026811353506183, "q3": 4.079074308920921, "n": 5 }, "latency_large_p50_ms": { "value": 253.41445899994142, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 252.841415999967, "q3": 253.57108299976971, "n": 5 }, "decode_mbps": { "value": 19.99801044816113, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 19.358661512641568, "q3": 20.0630422976614, "n": 5 }, "decode_tokens_per_s": { "value": 4913346.544657753, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4756263.774283738, "q3": 4929324.334741578, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 6.425657141548711, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.424836479910537, "q3": 6.506455481792995, "n": 5 }, "batch_mbps_4t": { "value": 20.89084307099784, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 20.777198523991068, "q3": 21.004698820989567, "n": 5 }, "batch_tokens_per_s_4t": { "value": 5342866.820068845, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5313802.043821151, "q3": 5371985.61182069, "n": 5 }, "batch_cpu_utilization": { "value": 3.551847617761725, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.531887869453192, "q3": 3.5882750340083382, "n": 5 }, "cpu_cores_used_batch": { "value": 3.551847617761725, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.531887869453192, "q3": 3.5882750340083382, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.1690639222864304, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.16802319218744302, "q3": 0.1726202865834512, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 83.546630859375, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 83.531005859375, "q3": 83.546630859375, "n": 5 }, "rss_peak_large_mb": { "value": 373.338112, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 373.338112, "q3": 373.342208, "n": 5 }, "rss_peak_very_large_mb": { "value": 1770.811392, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 1770.807296, "q3": 1770.840064, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.069632, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.069632, "q3": 0.069632, "n": 5 }, "encode_mbps_1t_very_large": { "value": 2.3881618744868045, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 2.387348580112849, "q3": 2.394519108270606, "n": 5 }, "throughput_ratio_very_large": { "value": 0.3716238670552402, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.1169737225072436, "provenance": "measured" }, "chars_per_token": { "value": 2.1150666602953123, "provenance": "measured" }, "tokens_per_word": { "value": 3.1146425285490067, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.0855649701937864, "provenance": "measured" }, "chars_per_token": { "value": 3.073244251658794, "provenance": "measured" }, "tokens_per_word": { "value": 2.7478745433424807, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.4607020833076363, "provenance": "measured" }, "chars_per_token": { "value": 3.460147027987123, "provenance": "measured" }, "tokens_per_word": { "value": 3.0456244482428296, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.527419593893582, "provenance": "measured" }, "chars_per_token": { "value": 4.5023343708314805, "provenance": "measured" }, "tokens_per_word": { "value": 2.711339947256059, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.804242928452579, "provenance": "measured" }, "chars_per_token": { "value": 3.7967251550446224, "provenance": "measured" }, "tokens_per_word": { "value": 2.8118752924163157, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.182728090153121, "provenance": "measured" }, "chars_per_token": { "value": 3.180188103410933, "provenance": "measured" }, "tokens_per_word": { "value": 2.6407896561286743, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.1174486489212807, "provenance": "measured" }, "chars_per_token": { "value": 3.069060738648543, "provenance": "measured" }, "tokens_per_word": { "value": 3.8648635477582847, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.180942404922346, "provenance": "measured" }, "chars_per_token": { "value": 4.178682480638058, "provenance": "measured" }, "tokens_per_word": { "value": 2.361018071350678, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.6749723396838387, "provenance": "measured" }, "chars_per_token": { "value": 3.661346792161142, "provenance": "measured" }, "tokens_per_word": { "value": 2.7423263160082296, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.06632781019579, "provenance": "measured" }, "chars_per_token": { "value": 4.052818137738467, "provenance": "measured" }, "tokens_per_word": { "value": 3.193560444254592, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.191714210367207, "provenance": "measured" }, "chars_per_token": { "value": 3.165889458484219, "provenance": "measured" }, "tokens_per_word": { "value": 3.0348832669633796, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.7524287153768645, "provenance": "measured" }, "chars_per_token": { "value": 3.74709823317206, "provenance": "measured" }, "tokens_per_word": { "value": 3.052165747153422, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.6356108548750403, "provenance": "measured" }, "chars_per_token": { "value": 3.635267617719618, "provenance": "measured" }, "tokens_per_word": { "value": 2.9749191626957114, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.586474255630237, "provenance": "measured" }, "chars_per_token": { "value": 3.576671320788553, "provenance": "measured" }, "tokens_per_word": { "value": 3.314524688999278, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.72670642624691, "provenance": "measured" }, "chars_per_token": { "value": 2.7139056105702752, "provenance": "measured" }, "tokens_per_word": { "value": 3.2921945051155306, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.5956763264003713, "provenance": "measured" }, "chars_per_token": { "value": 3.5189496127404056, "provenance": "measured" }, "tokens_per_word": { "value": 2.904089970936355, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.7146766758655487, "provenance": "measured" }, "chars_per_token": { "value": 3.713422977838594, "provenance": "measured" }, "tokens_per_word": { "value": 2.7196049896049894, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.04470285049813, "provenance": "measured" }, "chars_per_token": { "value": 3.0391652786071215, "provenance": "measured" }, "tokens_per_word": { "value": 2.870300209280508, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.8650567185124545, "provenance": "measured" }, "chars_per_token": { "value": 2.853777335984095, "provenance": "measured" }, "tokens_per_word": { "value": 3.145600353148911, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.2418305728088335, "provenance": "measured" }, "chars_per_token": { "value": 4.225931677018633, "provenance": "measured" }, "tokens_per_word": { "value": 2.49172434547096, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.030643877508274, "provenance": "measured" }, "chars_per_token": { "value": 3.9443327328836943, "provenance": "measured" }, "tokens_per_word": { "value": 3.1051034123921433, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.659014958348676, "provenance": "measured" }, "chars_per_token": { "value": 4.640068643049196, "provenance": "measured" }, "tokens_per_word": { "value": 1.3180235095891164, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 5.287460724977524, "provenance": "measured" }, "chars_per_token": { "value": 2.9322291529570963, "provenance": "measured" }, "tokens_per_word": { "value": 2.086824494216858, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 11.371781225692882, "provenance": "measured" }, "chars_per_token": { "value": 4.271748475048103, "provenance": "measured" }, "tokens_per_word": { "value": 1.6769299419901829, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.3938114303562807, "provenance": "measured" }, "chars_per_token": { "value": 3.308150690210491, "provenance": "measured" }, "tokens_per_word": { "value": 2.605000369849841, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.2979109252747945, "provenance": "measured" }, "chars_per_token": { "value": 4.202302892077671, "provenance": "measured" }, "tokens_per_word": { "value": 1.4853204102757127, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.9926129531008416, "provenance": "measured" }, "chars_per_token": { "value": 3.8654167512611077, "provenance": "measured" }, "tokens_per_word": { "value": 1.6605334474397375, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 6.192912147442245, "provenance": "measured" }, "chars_per_token": { "value": 2.4061418735510327, "provenance": "measured" }, "tokens_per_word": { "value": 2.6471658545611545, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 9.056485090183994, "provenance": "measured" }, "chars_per_token": { "value": 3.5952143569292123, "provenance": "measured" }, "tokens_per_word": { "value": 1.5376574868993198, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.3329044957643505, "provenance": "measured" }, "chars_per_token": { "value": 4.3228443882034515, "provenance": "measured" }, "tokens_per_word": { "value": 1.7264418875618992, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.589262382705784, "provenance": "measured" }, "chars_per_token": { "value": 1.610258846747741, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.7147637711714516, "provenance": "measured" }, "chars_per_token": { "value": 1.6031865068696673, "provenance": "measured" }, "tokens_per_word": { "value": 2.7947800296589222, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 8.86181406617504, "provenance": "measured" }, "chars_per_token": { "value": 3.3672520204431713, "provenance": "measured" }, "tokens_per_word": { "value": 2.0841441839893853, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.976780504733914, "provenance": "measured" }, "chars_per_token": { "value": 3.2883729001051982, "provenance": "measured" }, "tokens_per_word": { "value": 2.2503782886708645, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 9.408763739840422, "provenance": "measured" }, "chars_per_token": { "value": 3.5846894935555267, "provenance": "measured" }, "tokens_per_word": { "value": 2.6699275002482867, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 6.741407354078341, "provenance": "measured" }, "chars_per_token": { "value": 2.568194544436445, "provenance": "measured" }, "tokens_per_word": { "value": 3.1085415889593437, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 8.086448687411634, "provenance": "measured" }, "chars_per_token": { "value": 2.8912226623699193, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.841962595593246, "provenance": "measured" }, "chars_per_token": { "value": 3.5120466419323084, "provenance": "measured" }, "tokens_per_word": { "value": 2.222240918039407, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.590034870373178, "provenance": "measured" }, "chars_per_token": { "value": 3.1572632171709514, "provenance": "measured" }, "tokens_per_word": { "value": 1.5153683312273332, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.447028346831825, "provenance": "measured" }, "chars_per_token": { "value": 3.4046644235350167, "provenance": "measured" }, "tokens_per_word": { "value": 1.349364672793933, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.171622811307835, "provenance": "measured" }, "chars_per_token": { "value": 1.566826437118915, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 92377, "provenance": "measured" }, "tokens_per_request_mean": { "value": 461.885, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.461885, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 53821, "provenance": "measured" }, "tokens_per_request_mean": { "value": 330.19018404907973, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.33019018404907974, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 25167, "provenance": "measured" }, "tokens_per_request_mean": { "value": 132.4578947368421, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.1324578947368421, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 267604, "provenance": "measured" }, "tokens_per_request_mean": { "value": 637.152380952381, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.637152380952381, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 140393, "provenance": "measured" }, "tokens_per_request_mean": { "value": 935.9533333333334, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9359533333333334, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 250725, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1253.625, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.253625, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 422477, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2112.385, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.112385, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 574778, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2873.89, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.87389, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9128, "provenance": "measured" }, "tokens_per_request_mean": { "value": 365.12, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.36512, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 583505, "provenance": "measured" }, "tokens_per_request_mean": { "value": 97250.83333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 97.25083333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 262144, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 536870912, "provenance": "estimated", "note": "262144 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1073741824, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1073741824, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2147483648, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 262144, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 1073741824, "provenance": "estimated", "note": "262144 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 2147483648, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 2147483648, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 4294967296, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 1409286144, "provenance": "estimated", "note": "262144 rows x 5376 hidden x 1 matrix from the published config" }, "bf16_bytes": { "value": 2818572288, "provenance": "estimated", "note": "262144 rows x 5376 hidden x 1 matrix from the published config" } } } }, { "id": "sarvam-105b", "label": "sarvam-105b", "provider": "Sarvam", "family": "published", "metrics": { "bpt_overall": { "value": 4.457858902958406, "provenance": "measured" }, "bpt_english": { "value": 4.652786804108452, "provenance": "measured" }, "bpt_code": { "value": 3.4979797668462114, "provenance": "measured" }, "bpt_languages": { "value": 5.222810137920554, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.6278657201181677, "provenance": "measured" }, "byte_fallback_share": { "value": 0.44851763768129654, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 5.616660252990121, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.597913809617128, "q3": 5.63931486359084, "n": 5 }, "batch_mbps_4t": { "value": 17.953576871051627, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.901230339315713, "q3": 18.587922055381327, "n": 5 }, "latency_small_p95_us": { "value": 60.2499999331485, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 60.04200008646876, "q3": 67.29200003974256, "n": 5 }, "decode_mbps": { "value": 17.318860027422122, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 15.063022196315604, "q3": 17.415340016447725, "n": 5 }, "rss_load_delta_mb": { "value": 312.578048, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 312.578048, "q3": 312.578048, "n": 5 }, "rss_peak_large_mb": { "value": 372.129792, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 372.125696, "q3": 377.229312, "n": 5 }, "load_seconds": { "value": 0.693015374999959, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.6841009579998172, "q3": 0.7270837920000304, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6054571881162989, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 2000.072704, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 1993.474048, "q3": 2003.259392, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.012288, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.012288, "q3": 0.012288, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.4006473233799785, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.3947789834237785, "q3": 3.457782406501008, "n": 5 }, "embedding_params_h2048": { "value": 536870912, "provenance": "estimated", "note": "262144 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.1934449024610585, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.1857699650300381, "q3": 0.19419499109597585, "n": 5 }, "cpu_cores_used_batch": { "value": 3.4716281983425694, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.453077630209263, "q3": 3.4730279266476978, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 97.684326171875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 97.28295898437501, "q3": 97.874267578125, "n": 5 }, "workload_tokens_total": { "value": 2764594, "provenance": "measured" }, "tokens_chat_en": { "value": 92483, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 61691, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 36378, "provenance": "measured" }, "tokens_code": { "value": 270349, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 156109, "provenance": "measured" }, "tokens_rag": { "value": 413278, "provenance": "measured" }, "tokens_json": { "value": 422477, "provenance": "measured" }, "tokens_logs": { "value": 574778, "provenance": "measured" }, "tokens_tech_docs": { "value": 9128, "provenance": "measured" }, "tokens_long_context": { "value": 727923, "provenance": "measured" } }, "parity": { "de": { "value": 1.3197498033634218, "provenance": "measured" }, "fr": { "value": 1.3842840555826061, "provenance": "measured" }, "es": { "value": 1.2845799468144874, "provenance": "measured" }, "ru": { "value": 1.3920746095359378, "provenance": "measured" }, "zh": { "value": 1.0932244653357803, "provenance": "measured" }, "ja": { "value": 4.5018914566088615, "provenance": "measured" }, "ko": { "value": 5.766732836435821, "provenance": "measured" }, "ar": { "value": 1.4917412637177423, "provenance": "measured" }, "ur": { "value": 1.5464249597363198, "provenance": "measured" }, "hi": { "value": 1.3164537997677817, "provenance": "measured" }, "mr": { "value": 1.3958949773399754, "provenance": "measured" }, "gu": { "value": 1.5088954642495973, "provenance": "measured" }, "bn": { "value": 1.220008240008989, "provenance": "measured" }, "ta": { "value": 1.4808419791003409, "provenance": "measured" }, "te": { "value": 1.4679576014082925, "provenance": "measured" }, "th": { "value": 13.37874077680812, "provenance": "measured" }, "vi": { "value": 3.9134424510281285, "provenance": "measured" }, "tr": { "value": 1.6132439417206637, "provenance": "measured" }, "id": { "value": 1.14161579085359, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.592629482071715, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 35.09561752988048, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3197498033634218, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 36.81175298804781, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3842840555826061, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 34.16035856573705, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2845799468144874, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 37.01892430278885, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3920746095359378, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 29.07171314741036, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0932244653357803, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 119.71713147410358, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 4.5018914566088615, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 153.35258964143426, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 5.766732836435821, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 39.669322709163346, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4917412637177423, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 41.123505976095615, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5464249597363198, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 35.00796812749004, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3164537997677817, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 37.12051792828685, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3958949773399754, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 40.125498007968126, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.5088954642495973, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 32.44322709163347, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.220008240008989, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 39.37948207171315, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4808419791003409, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 39.036852589641434, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4679576014082925, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 355.77589641434264, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 13.37874077680812, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 104.0687250996016, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 3.9134424510281285, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 42.9003984063745, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6132439417206637, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 30.358565737051794, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.14161579085359, "provenance": "measured" } } }, "correctness": { "metadata_notes": [], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [ { "set": "wiki_ja", "bytes_per_token": { "value": 1.5324050486159628, "provenance": "measured" }, "split_byte_token_share": { "value": 0.7932898682929954, "provenance": "measured" } }, { "set": "wiki_ko", "bytes_per_token": { "value": 1.02577993283464, "provenance": "measured" }, "split_byte_token_share": { "value": 0.8614514203773997, "provenance": "measured" } }, { "set": "wiki_th", "bytes_per_token": { "value": 1.0132615114516, "provenance": "measured" }, "split_byte_token_share": { "value": 0.9758997685739367, "provenance": "measured" } }, { "set": "wiki_vi", "bytes_per_token": { "value": 1.7293469891097306, "provenance": "measured" }, "split_byte_token_share": { "value": 0.5074975538868594, "provenance": "measured" } } ], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "", "", "", "" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 262144, "provenance": "measured" }, "used_share": { "value": 0.5728530883789062, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 150170, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 111974, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 1446, "provenance": "measured", "note": "of 262005 entries whose text is valid UTF-8; 128 are byte pieces" }, "unreachable_examples": [ "\u0001", "\u0002", "\u0003", "\u0005", "\b", "\t", "\n", "\u0012", "\u0013", "\u0014", "\u0017", "\u001a" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.693015374999959, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.6841009579998172, "q3": 0.7270837920000304, "n": 5 }, "rss_load_delta_mb": { "value": 312.578048, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 312.578048, "q3": 312.578048, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 312.446976, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 312.446976, "q3": 312.446976, "n": 5 }, "latency_small_p50_us": { "value": 48.50000004807953, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 48.47899998594585, "q3": 50.41599979449529, "n": 5 }, "latency_small_p95_us": { "value": 60.2499999331485, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 60.04200008646876, "q3": 67.29200003974256, "n": 5 }, "encode_mbps_1t_medium": { "value": 5.616660252990121, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.597913809617128, "q3": 5.63931486359084, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1828927.035758328, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1822822.7183232603, "q3": 1836303.9515670019, "n": 5 }, "encode_mbps_1t_large": { "value": 4.324465840593276, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.201689504609623, "q3": 4.369252744545422, "n": 5 }, "latency_large_p50_ms": { "value": 240.69608399986464, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 235.61766700004227, "q3": 242.291334000015, "n": 5 }, "decode_mbps": { "value": 17.318860027422122, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 15.063022196315604, "q3": 17.415340016447725, "n": 5 }, "decode_tokens_per_s": { "value": 5265305.13753142, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4579482.0231488235, "q3": 5294637.124803166, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.731839037768034, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.68256852735252, "q3": 5.799382924631135, "n": 5 }, "batch_mbps_4t": { "value": 17.953576871051627, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 17.901230339315713, "q3": 18.587922055381327, "n": 5 }, "batch_tokens_per_s_4t": { "value": 5769686.488707927, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5752864.042737598, "q3": 5973544.074608002, "n": 5 }, "batch_cpu_utilization": { "value": 3.4716281983425694, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.453077630209263, "q3": 3.4730279266476978, "n": 5 }, "cpu_cores_used_batch": { "value": 3.4716281983425694, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.453077630209263, "q3": 3.4730279266476978, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.1934449024610585, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.1857699650300381, "q3": 0.19419499109597585, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 97.684326171875, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 97.28295898437501, "q3": 97.874267578125, "n": 5 }, "rss_peak_large_mb": { "value": 372.129792, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 372.125696, "q3": 377.229312, "n": 5 }, "rss_peak_very_large_mb": { "value": 2000.072704, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 1993.474048, "q3": 2003.259392, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.012288, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.012288, "q3": 0.012288, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.4006473233799785, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.3947789834237785, "q3": 3.457782406501008, "n": 5 }, "throughput_ratio_very_large": { "value": 0.6054571881162989, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.117212854790569, "provenance": "measured" }, "chars_per_token": { "value": 2.1153055771578524, "provenance": "measured" }, "tokens_per_word": { "value": 3.1142907398387183, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.078774429793191, "provenance": "measured" }, "chars_per_token": { "value": 3.066480826012801, "provenance": "measured" }, "tokens_per_word": { "value": 2.7539352514352093, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.4607020833076363, "provenance": "measured" }, "chars_per_token": { "value": 3.460147027987123, "provenance": "measured" }, "tokens_per_word": { "value": 3.0456244482428296, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.4913480182688685, "provenance": "measured" }, "chars_per_token": { "value": 4.4664626581755025, "provenance": "measured" }, "tokens_per_word": { "value": 2.7331156599271633, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.80401276582872, "provenance": "measured" }, "chars_per_token": { "value": 3.796495447257767, "provenance": "measured" }, "tokens_per_word": { "value": 2.8120454255454894, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.1836056748577017, "provenance": "measured" }, "chars_per_token": { "value": 3.18106498775612, "provenance": "measured" }, "tokens_per_word": { "value": 2.6400617027176927, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.203445282334344, "provenance": "measured" }, "chars_per_token": { "value": 3.15372256342481, "provenance": "measured" }, "tokens_per_word": { "value": 3.761111111111111, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.188335027142255, "provenance": "measured" }, "chars_per_token": { "value": 4.1860711069247865, "provenance": "measured" }, "tokens_per_word": { "value": 2.356850755569403, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.6863141571393823, "provenance": "measured" }, "chars_per_token": { "value": 3.6726465580150816, "provenance": "measured" }, "tokens_per_word": { "value": 2.73388890043434, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.047851269931094, "provenance": "measured" }, "chars_per_token": { "value": 4.03440298259046, "provenance": "measured" }, "tokens_per_word": { "value": 3.2081375480563863, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.191036886510958, "provenance": "measured" }, "chars_per_token": { "value": 3.165217614980965, "provenance": "measured" }, "tokens_per_word": { "value": 3.035527445928008, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.79097796946031, "provenance": "measured" }, "chars_per_token": { "value": 3.785592726424991, "provenance": "measured" }, "tokens_per_word": { "value": 3.021129240521188, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.6356108548750403, "provenance": "measured" }, "chars_per_token": { "value": 3.635267617719618, "provenance": "measured" }, "tokens_per_word": { "value": 2.9749191626957114, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.579753239755614, "provenance": "measured" }, "chars_per_token": { "value": 3.569968675513018, "provenance": "measured" }, "tokens_per_word": { "value": 3.3207477361088364, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.6956635805665443, "provenance": "measured" }, "chars_per_token": { "value": 2.6830084988573835, "provenance": "measured" }, "tokens_per_word": { "value": 3.3301069088400967, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.457325151663064, "provenance": "measured" }, "chars_per_token": { "value": 3.383550658949864, "provenance": "measured" }, "tokens_per_word": { "value": 3.020302430394676, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.7146766758655487, "provenance": "measured" }, "chars_per_token": { "value": 3.713422977838594, "provenance": "measured" }, "tokens_per_word": { "value": 2.7196049896049894, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.0447219882585137, "provenance": "measured" }, "chars_per_token": { "value": 3.039184381560587, "provenance": "measured" }, "tokens_per_word": { "value": 2.8702821678574004, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.8651404847527995, "provenance": "measured" }, "chars_per_token": { "value": 2.853860772446861, "provenance": "measured" }, "tokens_per_word": { "value": 3.145508387286639, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.191998158536378, "provenance": "measured" }, "chars_per_token": { "value": 4.176286040682705, "provenance": "measured" }, "tokens_per_word": { "value": 2.5213447401229527, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.029068550131908, "provenance": "measured" }, "chars_per_token": { "value": 3.9427911391521615, "provenance": "measured" }, "tokens_per_word": { "value": 3.1063174782118543, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.652786804108452, "provenance": "measured" }, "chars_per_token": { "value": 4.6338658161743105, "provenance": "measured" }, "tokens_per_word": { "value": 1.3197877971130396, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 5.285403251957196, "provenance": "measured" }, "chars_per_token": { "value": 2.9310881549080463, "provenance": "measured" }, "tokens_per_word": { "value": 2.0876368419016673, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 11.371548459727766, "provenance": "measured" }, "chars_per_token": { "value": 4.271661037764814, "provenance": "measured" }, "tokens_per_word": { "value": 1.676964267325713, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.3910877645237254, "provenance": "measured" }, "chars_per_token": { "value": 3.3054957704576498, "provenance": "measured" }, "tokens_per_word": { "value": 2.607092663235869, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.294781906811851, "provenance": "measured" }, "chars_per_token": { "value": 4.199243479361864, "provenance": "measured" }, "tokens_per_word": { "value": 1.4864025595182084, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.9921453500628528, "provenance": "measured" }, "chars_per_token": { "value": 3.86496404506664, "provenance": "measured" }, "tokens_per_word": { "value": 1.6607279469923886, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 7.459215603732548, "provenance": "measured" }, "chars_per_token": { "value": 2.8981407422998666, "provenance": "measured" }, "tokens_per_word": { "value": 2.197773391722692, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 9.055664310313576, "provenance": "measured" }, "chars_per_token": { "value": 3.594888526373029, "provenance": "measured" }, "tokens_per_word": { "value": 1.5377968558367712, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.329982225781749, "provenance": "measured" }, "chars_per_token": { "value": 4.319928903126998, "provenance": "measured" }, "tokens_per_word": { "value": 1.7276070492280804, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 1.5324050486159628, "provenance": "measured" }, "chars_per_token": { "value": 0.5376830916518445, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 1.02577993283464, "provenance": "measured" }, "chars_per_token": { "value": 0.44269747651263697, "provenance": "measured" }, "tokens_per_word": { "value": 10.121028175976273, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 8.86181406617504, "provenance": "measured" }, "chars_per_token": { "value": 3.3672520204431713, "provenance": "measured" }, "tokens_per_word": { "value": 2.0841441839893853, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.9751604649145635, "provenance": "measured" }, "chars_per_token": { "value": 3.287481568219273, "provenance": "measured" }, "tokens_per_word": { "value": 2.2509884316883877, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 9.406664311348296, "provenance": "measured" }, "chars_per_token": { "value": 3.583889622343294, "provenance": "measured" }, "tokens_per_word": { "value": 2.6705233886185322, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 7.568644473715723, "provenance": "measured" }, "chars_per_token": { "value": 2.883337324871273, "provenance": "measured" }, "tokens_per_word": { "value": 2.7687844502465913, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 1.0132615114516, "provenance": "measured" }, "chars_per_token": { "value": 0.3622807437554864, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.182159117990141, "provenance": "measured" }, "chars_per_token": { "value": 2.90890162680146, "provenance": "measured" }, "tokens_per_word": { "value": 2.6830105500496373, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.590034870373178, "provenance": "measured" }, "chars_per_token": { "value": 3.1572632171709514, "provenance": "measured" }, "tokens_per_word": { "value": 1.5153683312273332, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 1.7293469891097306, "provenance": "measured" }, "chars_per_token": { "value": 1.3239956462080904, "provenance": "measured" }, "tokens_per_word": { "value": 3.469901059715735, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.168292961050385, "provenance": "measured" }, "chars_per_token": { "value": 1.5655757733722124, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 92483, "provenance": "measured" }, "tokens_per_request_mean": { "value": 462.415, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.462415, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 61691, "provenance": "measured" }, "tokens_per_request_mean": { "value": 378.4723926380368, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3784723926380368, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 36378, "provenance": "measured" }, "tokens_per_request_mean": { "value": 191.46315789473684, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.19146315789473684, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 270349, "provenance": "measured" }, "tokens_per_request_mean": { "value": 643.6880952380952, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6436880952380952, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 156109, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1040.7266666666667, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.0407266666666668, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 413278, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2066.39, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.98, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.0663899999999997, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 422477, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2112.385, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.112385, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 574778, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2873.89, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.87389, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9128, "provenance": "measured" }, "tokens_per_request_mean": { "value": 365.12, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.36512, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 727923, "provenance": "measured" }, "tokens_per_request_mean": { "value": 121320.5, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 121.3205, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 262144, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 536870912, "provenance": "estimated", "note": "262144 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1073741824, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1073741824, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2147483648, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 262144, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 1073741824, "provenance": "estimated", "note": "262144 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 2147483648, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 2147483648, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 4294967296, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 2147483648, "provenance": "estimated", "note": "262144 rows x 4096 hidden x 2 matrices from the published config" }, "bf16_bytes": { "value": 4294967296, "provenance": "estimated", "note": "262144 rows x 4096 hidden x 2 matrices from the published config" } } } }, { "id": "gemma-3-1b-it", "label": "gemma-3-1b-it", "provider": "Google", "family": "published", "metrics": { "bpt_overall": { "value": 4.712520973696624, "provenance": "measured" }, "bpt_english": { "value": 4.659014958348676, "provenance": "measured" }, "bpt_code": { "value": 3.51232417601049, "provenance": "measured" }, "bpt_languages": { "value": 5.966223786730707, "provenance": "measured" }, "parity_mean_abs_log": { "value": 0.32984617060773985, "provenance": "measured" }, "byte_fallback_share": { "value": 0.0036632293192396174, "provenance": "measured" }, "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "encode_mbps_1t_medium": { "value": 6.103693269272489, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.069012027010557, "q3": 6.156752985263483, "n": 5 }, "batch_mbps_4t": { "value": 19.8543184165146, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.675361294380902, "q3": 20.066319688176616, "n": 5 }, "latency_small_p95_us": { "value": 64.33400000105394, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 61.29199982751743, "q3": 65.6249999906322, "n": 5 }, "decode_mbps": { "value": 19.322156916252645, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.940035095345923, "q3": 19.963442728719404, "n": 5 }, "rss_load_delta_mb": { "value": 312.958976, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 312.95488, "q3": 312.958976, "n": 5 }, "rss_peak_large_mb": { "value": 372.293632, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 371.99872, "q3": 374.054912, "n": 5 }, "load_seconds": { "value": 0.6804287500003738, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.678886625999894, "q3": 0.709873000000016, "n": 5 }, "throughput_ratio_very_large": { "value": 0.591403446797877, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" }, "rss_peak_very_large_mb": { "value": 1918.242816, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 1917.41952, "q3": 1920.2048, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.065536, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.065536, "q3": 0.065536, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.6097452376447525, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5897974403735957, "q3": 3.620644386593567, "n": 5 }, "embedding_params_h2048": { "value": 536870912, "provenance": "estimated", "note": "262144 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "cpu_seconds_per_mb": { "value": 0.1793093319275728, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.1762870421830679, "q3": 0.18460692232147904, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5600645711425387, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.537432145328517, "q3": 3.564383409200697, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 92.78125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 92.732177734375, "q3": 92.898193359375, "n": 5 }, "workload_tokens_total": { "value": 2417824, "provenance": "measured" }, "tokens_chat_en": { "value": 92367, "provenance": "measured" }, "tokens_chat_multilingual": { "value": 53716, "provenance": "measured" }, "tokens_chat_mixed_language": { "value": 25167, "provenance": "measured" }, "tokens_code": { "value": 266710, "provenance": "measured" }, "tokens_agent_tool_calls": { "value": 140393, "provenance": "measured" }, "tokens_rag": { "value": 250723, "provenance": "measured" }, "tokens_json": { "value": 422477, "provenance": "measured" }, "tokens_logs": { "value": 574778, "provenance": "measured" }, "tokens_tech_docs": { "value": 9128, "provenance": "measured" }, "tokens_long_context": { "value": 582365, "provenance": "measured" } }, "parity": { "de": { "value": 1.319847156664419, "provenance": "measured" }, "fr": { "value": 1.3843934966659175, "provenance": "measured" }, "es": { "value": 1.2846707125196672, "provenance": "measured" }, "ru": { "value": 1.3923353562598337, "provenance": "measured" }, "zh": { "value": 1.0932793886266576, "provenance": "measured" }, "ja": { "value": 1.2126695137484078, "provenance": "measured" }, "ko": { "value": 1.3735296321270698, "provenance": "measured" }, "ar": { "value": 1.4920206788042256, "provenance": "measured" }, "ur": { "value": 1.546714617517045, "provenance": "measured" }, "hi": { "value": 1.316700382108339, "provenance": "measured" }, "mr": { "value": 1.3961564396493593, "provenance": "measured" }, "gu": { "value": 1.8734921705252117, "provenance": "measured" }, "bn": { "value": 1.2200869109163108, "provenance": "measured" }, "ta": { "value": 1.4811193526635198, "provenance": "measured" }, "te": { "value": 1.802577358207837, "provenance": "measured" }, "th": { "value": 1.6013710946280062, "provenance": "measured" }, "vi": { "value": 1.3659249269498763, "provenance": "measured" }, "tr": { "value": 1.3692215479133887, "provenance": "measured" }, "id": { "value": 1.1418296246347495, "provenance": "measured" } }, "flores": { "en": { "tokens_per_sentence": { "value": 26.58764940239044, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0, "provenance": "measured" } }, "de": { "tokens_per_sentence": { "value": 35.091633466135455, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.319847156664419, "provenance": "measured" } }, "fr": { "tokens_per_sentence": { "value": 36.80776892430279, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3843934966659175, "provenance": "measured" } }, "es": { "tokens_per_sentence": { "value": 34.15637450199203, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2846707125196672, "provenance": "measured" } }, "ru": { "tokens_per_sentence": { "value": 37.01892430278885, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3923353562598337, "provenance": "measured" } }, "zh": { "tokens_per_sentence": { "value": 29.067729083665338, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.0932793886266576, "provenance": "measured" } }, "ja": { "tokens_per_sentence": { "value": 32.24203187250996, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2126695137484078, "provenance": "measured" } }, "ko": { "tokens_per_sentence": { "value": 36.51892430278885, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3735296321270698, "provenance": "measured" } }, "ar": { "tokens_per_sentence": { "value": 39.669322709163346, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4920206788042256, "provenance": "measured" } }, "ur": { "tokens_per_sentence": { "value": 41.123505976095615, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.546714617517045, "provenance": "measured" } }, "hi": { "tokens_per_sentence": { "value": 35.00796812749004, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.316700382108339, "provenance": "measured" } }, "mr": { "tokens_per_sentence": { "value": 37.12051792828685, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3961564396493593, "provenance": "measured" } }, "gu": { "tokens_per_sentence": { "value": 49.81175298804781, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.8734921705252117, "provenance": "measured" } }, "bn": { "tokens_per_sentence": { "value": 32.439243027888445, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.2200869109163108, "provenance": "measured" } }, "ta": { "tokens_per_sentence": { "value": 39.37948207171315, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.4811193526635198, "provenance": "measured" } }, "te": { "tokens_per_sentence": { "value": 47.92629482071713, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.802577358207837, "provenance": "measured" } }, "th": { "tokens_per_sentence": { "value": 42.57669322709163, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.6013710946280062, "provenance": "measured" } }, "vi": { "tokens_per_sentence": { "value": 36.31673306772908, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3659249269498763, "provenance": "measured" } }, "tr": { "tokens_per_sentence": { "value": 36.40438247011952, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.3692215479133887, "provenance": "measured" } }, "id": { "tokens_per_sentence": { "value": 30.358565737051794, "provenance": "measured", "note": "mean over the same 1004 FLORES-200 sentences" }, "ratio_to_english": { "value": 1.1418296246347495, "provenance": "measured" } } }, "correctness": { "metadata_notes": [ "the tokenizer has 262,145 IDs but the model config has 262,144 embedding rows, so 1 IDs have no embedding" ], "roundtrip_failures": { "value": 0, "provenance": "measured", "note": "lines, sentences and requests that do not decode back exactly" }, "roundtrip_failures_nfc_equivalent": { "value": 0, "provenance": "measured", "note": "failures where the decoded text equals the input normalized to NFC (canonically equivalent, but not byte-identical)" }, "roundtrip_failed_sets": [], "roundtrip_example": null, "byte_fallback_sets": [], "unicode_probe_failures": { "value": 0, "provenance": "measured" }, "unicode_failed_probes": [], "nfd_normalized_to_nfc": [], "special_strings_parsed_by_default": [ "", "", "", "" ], "hf_backend_parity": { "value": 0, "provenance": "measured", "note": "same implementation" } }, "vocabulary": { "size": { "value": 262145, "provenance": "measured" }, "used_share": { "value": 0.598878483282153, "provenance": "measured", "note": "share of vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "used": { "value": 156993, "provenance": "measured", "note": "vocabulary IDs that appear in any test set, FLORES-200 sentence or scenario" }, "unused": { "value": 105152, "provenance": "measured", "note": "vocabulary IDs never produced on any test set, FLORES-200 sentence or scenario" }, "unreachable": { "value": 114, "provenance": "measured", "note": "of 262008 entries whose text is valid UTF-8; 128 are byte pieces" }, "unreachable_examples": [ "\u0001", "\u0002", "\u0003", "\u0005", "\b", "\t", "\n", "\u0012", "\u0013", "\u0014", "\u0017", "\u001a" ], "undertrained": { "value": null, "provenance": "unavailable", "note": "needs the trained model's embedding weights; see evaluation/model_quality/PLAN.md" } }, "runtime": { "hf": { "implementation": "tokenizers", "load_seconds": { "value": 0.6804287500003738, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 0.678886625999894, "q3": 0.709873000000016, "n": 5 }, "rss_load_delta_mb": { "value": 312.958976, "provenance": "measured", "note": "resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes", "q1": 312.95488, "q3": 312.958976, "n": 5 }, "rss_load_delta_at_return_mb": { "value": 312.89344, "provenance": "measured", "note": "resident-memory growth from loading, read when load returned; median of 5 fresh processes", "q1": 312.889344, "q3": 312.89344, "n": 5 }, "latency_small_p50_us": { "value": 49.25000007460767, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 48.66600011155242, "q3": 51.250000012714736, "n": 5 }, "latency_small_p95_us": { "value": 64.33400000105394, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 61.29199982751743, "q3": 65.6249999906322, "n": 5 }, "encode_mbps_1t_medium": { "value": 6.103693269272489, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 6.069012027010557, "q3": 6.156752985263483, "n": 5 }, "encode_tokens_per_s_1t_medium": { "value": 1565278.293482592, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 1556384.367574339, "q3": 1578885.3372895222, "n": 5 }, "encode_mbps_1t_large": { "value": 4.510122046811685, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4.44617554211986, "q3": 4.563805134725821, "n": 5 }, "latency_large_p50_ms": { "value": 228.2536250000362, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 228.08750000012878, "q3": 236.27233400020486, "n": 5 }, "decode_mbps": { "value": 19.322156916252645, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.940035095345923, "q3": 19.963442728719404, "n": 5 }, "decode_tokens_per_s": { "value": 4743406.781588305, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4649599.488513419, "q3": 4900836.383520194, "n": 5 }, "concurrent_mbps_4_python_threads": { "value": 5.916949877952694, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 5.887326306215639, "q3": 5.954349007112568, "n": 5 }, "batch_mbps_4t": { "value": 19.8543184165146, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 18.675361294380902, "q3": 20.066319688176616, "n": 5 }, "batch_tokens_per_s_4t": { "value": 5073343.141357659, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 4772086.059444607, "q3": 5127515.496962219, "n": 5 }, "batch_cpu_utilization": { "value": 3.5600645711425387, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.537432145328517, "q3": 3.564383409200697, "n": 5 }, "cpu_cores_used_batch": { "value": 3.5600645711425387, "provenance": "measured", "note": "CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes", "q1": 3.537432145328517, "q3": 3.564383409200697, "n": 5 }, "cpu_seconds_per_mb": { "value": 0.1793093319275728, "provenance": "measured", "note": "CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes", "q1": 0.1762870421830679, "q3": 0.18460692232147904, "n": 5 }, "ram_working_mb_per_input_mb": { "value": 92.78125, "provenance": "measured", "note": "peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes", "q1": 92.732177734375, "q3": 92.898193359375, "n": 5 }, "rss_peak_large_mb": { "value": 372.293632, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes", "q1": 371.99872, "q3": 374.054912, "n": 5 }, "rss_peak_very_large_mb": { "value": 1918.242816, "provenance": "measured", "note": "peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes", "q1": 1917.41952, "q3": 1920.2048, "n": 5 }, "rss_peak_chunked_mb": { "value": 0.065536, "provenance": "measured", "note": "peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes", "q1": 0.065536, "q3": 0.065536, "n": 5 }, "encode_mbps_1t_very_large": { "value": 3.6097452376447525, "provenance": "measured", "note": "median of 5 fresh processes", "q1": 3.5897974403735957, "q3": 3.620644386593567, "n": 5 }, "throughput_ratio_very_large": { "value": 0.591403446797877, "provenance": "measured", "note": "median 16 MiB throughput / median 16 KiB throughput" } } }, "per_set": { "code_asm": { "bytes_per_token": { "value": 2.117212854790569, "provenance": "measured" }, "chars_per_token": { "value": 2.1153055771578524, "provenance": "measured" }, "tokens_per_word": { "value": 3.1142907398387183, "provenance": "measured" } }, "code_c": { "bytes_per_token": { "value": 3.0855649701937864, "provenance": "measured" }, "chars_per_token": { "value": 3.073244251658794, "provenance": "measured" }, "tokens_per_word": { "value": 2.7478745433424807, "provenance": "measured" } }, "code_cpp": { "bytes_per_token": { "value": 3.4607020833076363, "provenance": "measured" }, "chars_per_token": { "value": 3.460147027987123, "provenance": "measured" }, "tokens_per_word": { "value": 3.0456244482428296, "provenance": "measured" } }, "code_cs": { "bytes_per_token": { "value": 4.52746153347352, "provenance": "measured" }, "chars_per_token": { "value": 4.502376078035405, "provenance": "measured" }, "tokens_per_word": { "value": 2.7113148310938087, "provenance": "measured" } }, "code_dart": { "bytes_per_token": { "value": 3.804242928452579, "provenance": "measured" }, "chars_per_token": { "value": 3.7967251550446224, "provenance": "measured" }, "tokens_per_word": { "value": 2.8118752924163157, "provenance": "measured" } }, "code_go": { "bytes_per_token": { "value": 3.183626575630252, "provenance": "measured" }, "chars_per_token": { "value": 3.1810858718487394, "provenance": "measured" }, "tokens_per_word": { "value": 2.6400443704936216, "provenance": "measured" } }, "code_html": { "bytes_per_token": { "value": 3.2057090251350107, "provenance": "measured" }, "chars_per_token": { "value": 3.155951169228575, "provenance": "measured" }, "tokens_per_word": { "value": 3.7584551656920078, "provenance": "measured" } }, "code_java": { "bytes_per_token": { "value": 4.188335027142255, "provenance": "measured" }, "chars_per_token": { "value": 4.1860711069247865, "provenance": "measured" }, "tokens_per_word": { "value": 2.356850755569403, "provenance": "measured" } }, "code_js": { "bytes_per_token": { "value": 3.686622421906478, "provenance": "measured" }, "chars_per_token": { "value": 3.672953679841266, "provenance": "measured" }, "tokens_per_word": { "value": 2.733660300504998, "provenance": "measured" } }, "code_kotlin": { "bytes_per_token": { "value": 4.066599782626871, "provenance": "measured" }, "chars_per_token": { "value": 4.053089206588078, "provenance": "measured" }, "tokens_per_word": { "value": 3.193346860316104, "provenance": "measured" } }, "code_lua": { "bytes_per_token": { "value": 3.191714210367207, "provenance": "measured" }, "chars_per_token": { "value": 3.165889458484219, "provenance": "measured" }, "tokens_per_word": { "value": 3.0348832669633796, "provenance": "measured" } }, "code_php": { "bytes_per_token": { "value": 3.7934246745773783, "provenance": "measured" }, "chars_per_token": { "value": 3.7880359558949315, "provenance": "measured" }, "tokens_per_word": { "value": 3.019180655006456, "provenance": "measured" } }, "code_ps": { "bytes_per_token": { "value": 3.6356108548750403, "provenance": "measured" }, "chars_per_token": { "value": 3.635267617719618, "provenance": "measured" }, "tokens_per_word": { "value": 2.9749191626957114, "provenance": "measured" } }, "code_py": { "bytes_per_token": { "value": 3.5880763421666897, "provenance": "measured" }, "chars_per_token": { "value": 3.578269028330816, "provenance": "measured" }, "tokens_per_word": { "value": 3.3130447440726387, "provenance": "measured" } }, "code_r": { "bytes_per_token": { "value": 2.7304246882845336, "provenance": "measured" }, "chars_per_token": { "value": 2.71760641682809, "provenance": "measured" }, "tokens_per_word": { "value": 3.2877112311759973, "provenance": "measured" } }, "code_ruby": { "bytes_per_token": { "value": 3.5956763264003713, "provenance": "measured" }, "chars_per_token": { "value": 3.5189496127404056, "provenance": "measured" }, "tokens_per_word": { "value": 2.904089970936355, "provenance": "measured" } }, "code_rust": { "bytes_per_token": { "value": 3.7147902667247665, "provenance": "measured" }, "chars_per_token": { "value": 3.713536530361061, "provenance": "measured" }, "tokens_per_word": { "value": 2.7195218295218293, "provenance": "measured" } }, "code_shell": { "bytes_per_token": { "value": 3.0447219882585137, "provenance": "measured" }, "chars_per_token": { "value": 3.039184381560587, "provenance": "measured" }, "tokens_per_word": { "value": 2.8702821678574004, "provenance": "measured" } }, "code_sql": { "bytes_per_token": { "value": 2.8650567185124545, "provenance": "measured" }, "chars_per_token": { "value": 2.853777335984095, "provenance": "measured" }, "tokens_per_word": { "value": 3.145600353148911, "provenance": "measured" } }, "code_swift": { "bytes_per_token": { "value": 4.2418305728088335, "provenance": "measured" }, "chars_per_token": { "value": 4.225931677018633, "provenance": "measured" }, "tokens_per_word": { "value": 2.49172434547096, "provenance": "measured" } }, "code_ts": { "bytes_per_token": { "value": 4.031403850585549, "provenance": "measured" }, "chars_per_token": { "value": 3.9450764320979896, "provenance": "measured" }, "tokens_per_word": { "value": 3.104518059229068, "provenance": "measured" } }, "en_fineweb": { "bytes_per_token": { "value": 4.659014958348676, "provenance": "measured" }, "chars_per_token": { "value": 4.640068643049196, "provenance": "measured" }, "tokens_per_word": { "value": 1.3180235095891164, "provenance": "measured" } }, "wiki_ar": { "bytes_per_token": { "value": 5.287509731954176, "provenance": "measured" }, "chars_per_token": { "value": 2.9322563304044786, "provenance": "measured" }, "tokens_per_word": { "value": 2.086805152605315, "provenance": "measured" } }, "wiki_bn": { "bytes_per_token": { "value": 11.372246786211415, "provenance": "measured" }, "chars_per_token": { "value": 4.271923360353721, "provenance": "measured" }, "tokens_per_word": { "value": 1.6768612913191225, "provenance": "measured" } }, "wiki_de": { "bytes_per_token": { "value": 3.3938114303562807, "provenance": "measured" }, "chars_per_token": { "value": 3.308150690210491, "provenance": "measured" }, "tokens_per_word": { "value": 2.605000369849841, "provenance": "measured" } }, "wiki_es": { "bytes_per_token": { "value": 4.2979109252747945, "provenance": "measured" }, "chars_per_token": { "value": 4.202302892077671, "provenance": "measured" }, "tokens_per_word": { "value": 1.4853204102757127, "provenance": "measured" } }, "wiki_fr": { "bytes_per_token": { "value": 3.9926129531008416, "provenance": "measured" }, "chars_per_token": { "value": 3.8654167512611077, "provenance": "measured" }, "tokens_per_word": { "value": 1.6605334474397375, "provenance": "measured" } }, "wiki_gu": { "bytes_per_token": { "value": 6.192912147442245, "provenance": "measured" }, "chars_per_token": { "value": 2.4061418735510327, "provenance": "measured" }, "tokens_per_word": { "value": 2.6471658545611545, "provenance": "measured" } }, "wiki_hi": { "bytes_per_token": { "value": 9.056813443793848, "provenance": "measured" }, "chars_per_token": { "value": 3.595344705690408, "provenance": "measured" }, "tokens_per_word": { "value": 1.5376017393243393, "provenance": "measured" } }, "wiki_id": { "bytes_per_token": { "value": 4.3329044957643505, "provenance": "measured" }, "chars_per_token": { "value": 4.3228443882034515, "provenance": "measured" }, "tokens_per_word": { "value": 1.7264418875618992, "provenance": "measured" } }, "wiki_ja": { "bytes_per_token": { "value": 4.589262382705784, "provenance": "measured" }, "chars_per_token": { "value": 1.610258846747741, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_ko": { "bytes_per_token": { "value": 3.7148163346162555, "provenance": "measured" }, "chars_per_token": { "value": 1.6032091917591125, "provenance": "measured" }, "tokens_per_word": { "value": 2.7947404844290658, "provenance": "measured" } }, "wiki_mr": { "bytes_per_token": { "value": 8.86181406617504, "provenance": "measured" }, "chars_per_token": { "value": 3.3672520204431713, "provenance": "measured" }, "tokens_per_word": { "value": 2.0841441839893853, "provenance": "measured" } }, "wiki_ru": { "bytes_per_token": { "value": 5.976780504733914, "provenance": "measured" }, "chars_per_token": { "value": 3.2883729001051982, "provenance": "measured" }, "tokens_per_word": { "value": 2.2503782886708645, "provenance": "measured" } }, "wiki_ta": { "bytes_per_token": { "value": 9.409988839285715, "provenance": "measured" }, "chars_per_token": { "value": 3.58515625, "provenance": "measured" }, "tokens_per_word": { "value": 2.669579898698977, "provenance": "measured" } }, "wiki_te": { "bytes_per_token": { "value": 6.741407354078341, "provenance": "measured" }, "chars_per_token": { "value": 2.568194544436445, "provenance": "measured" }, "tokens_per_word": { "value": 3.1085415889593437, "provenance": "measured" } }, "wiki_th": { "bytes_per_token": { "value": 8.086448687411634, "provenance": "measured" }, "chars_per_token": { "value": 2.8912226623699193, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } }, "wiki_tr": { "bytes_per_token": { "value": 3.841962595593246, "provenance": "measured" }, "chars_per_token": { "value": 3.5120466419323084, "provenance": "measured" }, "tokens_per_word": { "value": 2.222240918039407, "provenance": "measured" } }, "wiki_ur": { "bytes_per_token": { "value": 5.590398111245885, "provenance": "measured" }, "chars_per_token": { "value": 3.1574683763645814, "provenance": "measured" }, "tokens_per_word": { "value": 1.5152698688809754, "provenance": "measured" } }, "wiki_vi": { "bytes_per_token": { "value": 4.447028346831825, "provenance": "measured" }, "chars_per_token": { "value": 3.4046644235350167, "provenance": "measured" }, "tokens_per_word": { "value": 1.349364672793933, "provenance": "measured" } }, "wiki_zh": { "bytes_per_token": { "value": 4.171622811307835, "provenance": "measured" }, "chars_per_token": { "value": 1.566826437118915, "provenance": "measured" }, "tokens_per_word": { "value": null, "provenance": "unavailable", "note": "this language is not written with spaces between words, so whitespace words are not words" } } }, "workloads": { "chat_en": { "tokens": { "value": 92367, "provenance": "measured" }, "tokens_per_request_mean": { "value": 461.835, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.461835, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_multilingual": { "tokens": { "value": 53716, "provenance": "measured" }, "tokens_per_request_mean": { "value": 329.5460122699387, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.3295460122699387, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "chat_mixed_language": { "tokens": { "value": 25167, "provenance": "measured" }, "tokens_per_request_mean": { "value": 132.4578947368421, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.1324578947368421, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "code": { "tokens": { "value": 266710, "provenance": "measured" }, "tokens_per_request_mean": { "value": 635.0238095238095, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.6350238095238094, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "agent_tool_calls": { "tokens": { "value": 140393, "provenance": "measured" }, "tokens_per_request_mean": { "value": 935.9533333333334, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.9359533333333334, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "rag": { "tokens": { "value": 250723, "provenance": "measured" }, "tokens_per_request_mean": { "value": 1253.615, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 1.253615, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "json": { "tokens": { "value": 422477, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2112.385, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.112385, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "logs": { "tokens": { "value": 574778, "provenance": "measured" }, "tokens_per_request_mean": { "value": 2873.89, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 2.87389, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" }, "tech_docs": { "tokens": { "value": 9128, "provenance": "measured" }, "tokens_per_request_mean": { "value": 365.12, "provenance": "measured" }, "context_fit": { "8192": { "value": 1.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 1.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 0.36512, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "corpus" }, "long_context": { "tokens": { "value": 582365, "provenance": "measured" }, "tokens_per_request_mean": { "value": 97060.83333333333, "provenance": "measured" }, "context_fit": { "8192": { "value": 0.0, "provenance": "measured", "note": "share of requests within 8192 tokens" }, "32768": { "value": 0.0, "provenance": "measured", "note": "share of requests within 32768 tokens" }, "131072": { "value": 1.0, "provenance": "measured", "note": "share of requests within 131072 tokens" }, "1048576": { "value": 1.0, "provenance": "measured", "note": "share of requests within 1048576 tokens" } }, "cost_per_1k_requests_usd": { "value": 97.06083333333333, "provenance": "estimated", "note": "tokens x 1.0 USD per million tokens (a unit rate, not a real price)" }, "kind": "constructed" } }, "embedding": { "h2048": { "rows": { "value": 262144, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 536870912, "provenance": "estimated", "note": "262144 rows x 2048 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 1073741824, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 1073741824, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 2147483648, "provenance": "estimated", "note": "2 bytes per parameter" } }, "h4096": { "rows": { "value": 262144, "provenance": "estimated", "note": "model config vocab_size (embedding rows)" }, "params_tied": { "value": 1073741824, "provenance": "estimated", "note": "262144 rows x 4096 hidden; one matrix shared by embedding and LM head" }, "params_untied": { "value": 2147483648, "provenance": "estimated", "note": "separate embedding and LM-head matrices" }, "fp16_bf16_bytes_tied": { "value": 2147483648, "provenance": "estimated", "note": "2 bytes per parameter" }, "fp16_bf16_bytes_untied": { "value": 4294967296, "provenance": "estimated", "note": "2 bytes per parameter" } }, "published_model": { "params": { "value": 603979776, "provenance": "estimated", "note": "262144 rows x 1152 hidden x 2 matrices from the published config (tie_word_embeddings not stated; counted as untied)" }, "bf16_bytes": { "value": 1207959552, "provenance": "estimated", "note": "262144 rows x 1152 hidden x 2 matrices from the published config (tie_word_embeddings not stated; counted as untied)" } } } } ] }