{
 "generated": "2026-11-08",
 "rules": {
  "vocab_slack": 1.02,
  "merge_tolerance": 0.01,
  "byte_fallback": 1.2
 },
 "counts": {
  "sets": 41,
  "code": 21,
  "languages": 19,
  "published": 31,
  "distinct": 19
 },
 "excluded": [
  {
   "name": "Qwen3.8-27B",
   "provider": "Alibaba (Qwen)",
   "reason": "it does not round-trip: 733 inputs do not decode back exactly (all only by NFC normalization)"
  },
  {
   "name": "MiniMax-M2.5",
   "provider": "MiniMax",
   "reason": "it does not round-trip: 733 inputs do not decode back exactly (all only by NFC normalization)"
  },
  {
   "name": "Seed-OSS-36B-Instruct",
   "provider": "ByteDance",
   "reason": "it does not round-trip: 733 inputs do not decode back exactly (all only by NFC normalization)"
  },
  {
   "name": "Ling-3.0-tiny",
   "provider": "inclusionAI (Ant Group)",
   "reason": "it does not round-trip: 733 inputs do not decode back exactly (all only by NFC normalization)"
  },
  {
   "name": "Qwen2.5-0.5B",
   "provider": "Alibaba (Qwen)",
   "reason": "it does not round-trip: 733 inputs do not decode back exactly (all only by NFC normalization)"
  },
  {
   "name": "Qwen3-0.6B",
   "provider": "Alibaba (Qwen)",
   "reason": "it does not round-trip: 733 inputs do not decode back exactly (all only by NFC normalization)"
  },
  {
   "name": "sarvam-1",
   "provider": "Sarvam",
   "reason": "it does not round-trip: 215,393 inputs do not decode back exactly (0 only by NFC normalization)"
  },
  {
   "name": "pythia-160m",
   "provider": "EleutherAI",
   "reason": "it does not round-trip: 733 inputs do not decode back exactly (all only by NFC normalization)"
  }
 ],
 "akshara": [
  {
   "name": "Akshara 32k",
   "vocab": 32000,
   "overall": 3.9282,
   "english": 4.0927,
   "code_avg": 3.4521,
   "languages_avg": 4.24
  },
  {
   "name": "Akshara 49k",
   "vocab": 49000,
   "overall": 4.2021,
   "english": 4.3059,
   "code_avg": 3.5977,
   "languages_avg": 4.7026
  },
  {
   "name": "Akshara 59k",
   "vocab": 59000,
   "overall": 4.3191,
   "english": 4.3932,
   "code_avg": 3.6578,
   "languages_avg": 4.9061
  },
  {
   "name": "Akshara 92k",
   "vocab": 92000,
   "overall": 4.5886,
   "english": 4.5649,
   "code_avg": 3.782,
   "languages_avg": 5.4188
  },
  {
   "name": "Akshara 120k",
   "vocab": 120000,
   "overall": 4.7469,
   "english": 4.6553,
   "code_avg": 3.8468,
   "languages_avg": 5.7387
  },
  {
   "name": "Akshara 151k",
   "vocab": 151000,
   "overall": 4.8772,
   "english": 4.7185,
   "code_avg": 3.8937,
   "languages_avg": 6.0195
  },
  {
   "name": "Akshara 200k",
   "vocab": 200000,
   "overall": 5.0315,
   "english": 4.7918,
   "code_avg": 3.9523,
   "languages_avg": 6.3505
  },
  {
   "name": "Akshara 248k",
   "vocab": 248000,
   "overall": 5.1408,
   "english": 4.8394,
   "code_avg": 3.9914,
   "languages_avg": 6.5915
  },
  {
   "name": "Akshara 448k",
   "vocab": 448000,
   "overall": 5.409,
   "english": 4.9363,
   "code_avg": 4.0776,
   "languages_avg": 7.2132
  }
 ],
 "published": [
  {
   "label": "Phi-3.5-mini-instruct",
   "members": [
    "Phi-3.5-mini-instruct"
   ],
   "provider": "Microsoft",
   "vocab": 32011,
   "overall": 3.1695,
   "english": 4.0442,
   "code_avg": 3.0768,
   "languages_avg": 2.3875,
   "byte_fallback": [
    {
     "id": "wiki_gu",
     "label": "Gujarati",
     "score": 1.0817
    },
    {
     "id": "wiki_te",
     "label": "Telugu",
     "score": 1.066
    }
   ]
  },
  {
   "label": "SmolLM2-135M",
   "members": [
    "SmolLM2-135M"
   ],
   "provider": "Hugging Face",
   "vocab": 49152,
   "overall": 3.2034,
   "english": 4.5429,
   "code_avg": 3.1345,
   "languages_avg": 1.9328,
   "byte_fallback": [
    {
     "id": "wiki_gu",
     "label": "Gujarati",
     "score": 1.0118
    },
    {
     "id": "wiki_ko",
     "label": "Korean",
     "score": 1.1791
    },
    {
     "id": "wiki_te",
     "label": "Telugu",
     "score": 1.0109
    },
    {
     "id": "wiki_zh",
     "label": "Chinese",
     "score": 1.6462
    }
   ]
  },
  {
   "label": "gpt2",
   "members": [
    "gpt2"
   ],
   "provider": "OpenAI",
   "vocab": 50257,
   "overall": 2.9504,
   "english": 4.6591,
   "code_avg": 2.4051,
   "languages_avg": 1.7871,
   "byte_fallback": [
    {
     "id": "wiki_bn",
     "label": "Bengali",
     "score": 1.3305
    },
    {
     "id": "wiki_gu",
     "label": "Gujarati",
     "score": 1.0123
    },
    {
     "id": "wiki_ko",
     "label": "Korean",
     "score": 1.1704
    },
    {
     "id": "wiki_ta",
     "label": "Tamil",
     "score": 1.0205
    },
    {
     "id": "wiki_te",
     "label": "Telugu",
     "score": 1.014
    },
    {
     "id": "wiki_ur",
     "label": "Urdu",
     "score": 1.4178
    },
    {
     "id": "wiki_zh",
     "label": "Chinese",
     "score": 1.4374
    }
   ]
  },
  {
   "label": "OpenAI p50k_base (Codex)",
   "members": [
    "OpenAI p50k_base (Codex)"
   ],
   "provider": "OpenAI",
   "vocab": 50281,
   "overall": 3.2165,
   "english": 4.6591,
   "code_avg": 3.2019,
   "languages_avg": 1.7884,
   "byte_fallback": [
    {
     "id": "wiki_bn",
     "label": "Bengali",
     "score": 1.3396
    },
    {
     "id": "wiki_gu",
     "label": "Gujarati",
     "score": 1.0123
    },
    {
     "id": "wiki_ko",
     "label": "Korean",
     "score": 1.1704
    },
    {
     "id": "wiki_ta",
     "label": "Tamil",
     "score": 1.0207
    },
    {
     "id": "wiki_te",
     "label": "Telugu",
     "score": 1.0141
    },
    {
     "id": "wiki_ur",
     "label": "Urdu",
     "score": 1.4183
    },
    {
     "id": "wiki_zh",
     "label": "Chinese",
     "score": 1.4374
    }
   ]
  },
  {
   "label": "cl100k_base (GPT-4, OLMo-2, Olmo-3, phi-4)",
   "members": [
    "OpenAI cl100k_base (GPT-4)",
    "OLMo-2-0425-1B",
    "Olmo-3-1025-7B",
    "phi-4"
   ],
   "provider": "OpenAI",
   "vocab": 100277,
   "overall": 3.7359,
   "english": 4.6914,
   "code_avg": 3.8723,
   "languages_avg": 2.644,
   "byte_fallback": []
  },
  {
   "label": "ERNIE-4.5-21B-A3B-PT",
   "members": [
    "ERNIE-4.5-21B-A3B-PT"
   ],
   "provider": "Baidu",
   "vocab": 101304,
   "overall": 3.8853,
   "english": 4.4011,
   "code_avg": 3.3734,
   "languages_avg": 3.8814,
   "byte_fallback": []
  },
  {
   "label": "Hy3",
   "members": [
    "Hy3"
   ],
   "provider": "Tencent",
   "vocab": 120818,
   "overall": 3.9654,
   "english": 4.7065,
   "code_avg": 3.6956,
   "languages_avg": 3.4941,
   "byte_fallback": []
  },
  {
   "label": "LFM3-1.7B",
   "members": [
    "LFM3-1.7B"
   ],
   "provider": "Liquid AI",
   "vocab": 125033,
   "overall": 4.2133,
   "english": 4.6284,
   "code_avg": 3.5937,
   "languages_avg": 4.4179,
   "byte_fallback": []
  },
  {
   "label": "Llama-3.2-1B",
   "members": [
    "Llama-3.2-1B"
   ],
   "provider": "Meta",
   "vocab": 128256,
   "overall": 4.0801,
   "english": 4.6929,
   "code_avg": 3.8907,
   "languages_avg": 3.6568,
   "byte_fallback": []
  },
  {
   "label": "DeepSeek-V3",
   "members": [
    "DeepSeek-V3"
   ],
   "provider": "DeepSeek",
   "vocab": 128815,
   "overall": 4.2016,
   "english": 4.7317,
   "code_avg": 3.6104,
   "languages_avg": 4.2627,
   "byte_fallback": []
  },
  {
   "label": "Mistral-Nemo-Base-2407",
   "members": [
    "Mistral-Nemo-Base-2407"
   ],
   "provider": "Mistral",
   "vocab": 131072,
   "overall": 4.312,
   "english": 4.5515,
   "code_avg": 3.5793,
   "languages_avg": 4.8052,
   "byte_fallback": []
  },
  {
   "label": "GLM-5.3-Flash",
   "members": [
    "GLM-5.3-Flash"
   ],
   "provider": "Z.ai (GLM)",
   "vocab": 154856,
   "overall": 3.9579,
   "english": 4.6855,
   "code_avg": 3.8796,
   "languages_avg": 3.3085,
   "byte_fallback": []
  },
  {
   "label": "Kimi-K3",
   "members": [
    "Kimi-K3"
   ],
   "provider": "Moonshot AI (Kimi)",
   "vocab": 163840,
   "overall": 4.0412,
   "english": 4.6935,
   "code_avg": 3.8373,
   "languages_avg": 3.5927,
   "byte_fallback": []
  },
  {
   "label": "OpenAI o200k_base (GPT-4o)",
   "members": [
    "OpenAI o200k_base (GPT-4o)"
   ],
   "provider": "OpenAI",
   "vocab": 200019,
   "overall": 4.6404,
   "english": 4.7424,
   "code_avg": 3.8524,
   "languages_avg": 5.3264,
   "byte_fallback": []
  },
  {
   "label": "Mistral-Large-4-1T-A52B",
   "members": [
    "Mistral-Large-4-1T-A52B"
   ],
   "provider": "Mistral",
   "vocab": 204800,
   "overall": 4.5844,
   "english": 4.6092,
   "code_avg": 3.7268,
   "languages_avg": 5.4172,
   "byte_fallback": []
  },
  {
   "label": "North-Micro-Vision-Instruct",
   "members": [
    "North-Micro-Vision-Instruct"
   ],
   "provider": "Cohere",
   "vocab": 255033,
   "overall": 4.6396,
   "english": 4.8878,
   "code_avg": 3.9196,
   "languages_avg": 5.1113,
   "byte_fallback": []
  },
  {
   "label": "Falcon-H1-34B-Instruct",
   "members": [
    "Falcon-H1-34B-Instruct"
   ],
   "provider": "TII (Falcon)",
   "vocab": 261120,
   "overall": 4.1338,
   "english": 4.5315,
   "code_avg": 3.2498,
   "languages_avg": 4.62,
   "byte_fallback": []
  },
  {
   "label": "sarvam-105b",
   "members": [
    "sarvam-105b"
   ],
   "provider": "Sarvam",
   "vocab": 262144,
   "overall": 4.4579,
   "english": 4.6528,
   "code_avg": 3.498,
   "languages_avg": 5.2228,
   "byte_fallback": [
    {
     "id": "wiki_ja",
     "label": "Japanese",
     "score": 1.5324
    },
    {
     "id": "wiki_ko",
     "label": "Korean",
     "score": 1.0258
    },
    {
     "id": "wiki_th",
     "label": "Thai",
     "score": 1.0133
    },
    {
     "id": "wiki_vi",
     "label": "Vietnamese",
     "score": 1.7293
    }
   ]
  },
  {
   "label": "Gemma 3 / Gemma 4 tokenizer",
   "members": [
    "gemma-4-31B-it",
    "gemma-3-1b-it"
   ],
   "provider": "Google",
   "vocab": 262144,
   "overall": 4.7125,
   "english": 4.659,
   "code_avg": 3.5123,
   "languages_avg": 5.9662,
   "byte_fallback": []
  }
 ],
 "matched": [
  {
   "published": "Phi-3.5-mini-instruct",
   "published_vocab": 32011,
   "provider": "Microsoft",
   "byte_fallback": true,
   "akshara": "Akshara 32k",
   "akshara_vocab": 32000,
   "diff": {
    "overall": 0.7587,
    "english": 0.0485,
    "code_avg": 0.3753,
    "languages_avg": 1.8525
   }
  },
  {
   "published": "SmolLM2-135M",
   "published_vocab": 49152,
   "provider": "Hugging Face",
   "byte_fallback": true,
   "akshara": "Akshara 49k",
   "akshara_vocab": 49000,
   "diff": {
    "overall": 0.9987,
    "english": -0.237,
    "code_avg": 0.4632,
    "languages_avg": 2.7698
   }
  },
  {
   "published": "gpt2",
   "published_vocab": 50257,
   "provider": "OpenAI",
   "byte_fallback": true,
   "akshara": "Akshara 49k",
   "akshara_vocab": 49000,
   "diff": {
    "overall": 1.2517,
    "english": -0.3532,
    "code_avg": 1.1926,
    "languages_avg": 2.9155
   }
  },
  {
   "published": "OpenAI p50k_base (Codex)",
   "published_vocab": 50281,
   "provider": "OpenAI",
   "byte_fallback": true,
   "akshara": "Akshara 49k",
   "akshara_vocab": 49000,
   "diff": {
    "overall": 0.9856,
    "english": -0.3532,
    "code_avg": 0.3958,
    "languages_avg": 2.9142
   }
  },
  {
   "published": "cl100k_base (GPT-4, OLMo-2, Olmo-3, phi-4)",
   "published_vocab": 100277,
   "provider": "OpenAI",
   "byte_fallback": false,
   "akshara": "Akshara 92k",
   "akshara_vocab": 92000,
   "diff": {
    "overall": 0.8527,
    "english": -0.1265,
    "code_avg": -0.0903,
    "languages_avg": 2.7748
   }
  },
  {
   "published": "ERNIE-4.5-21B-A3B-PT",
   "published_vocab": 101304,
   "provider": "Baidu",
   "byte_fallback": false,
   "akshara": "Akshara 92k",
   "akshara_vocab": 92000,
   "diff": {
    "overall": 0.7033,
    "english": 0.1638,
    "code_avg": 0.4086,
    "languages_avg": 1.5374
   }
  },
  {
   "published": "Hy3",
   "published_vocab": 120818,
   "provider": "Tencent",
   "byte_fallback": false,
   "akshara": "Akshara 120k",
   "akshara_vocab": 120000,
   "diff": {
    "overall": 0.7815,
    "english": -0.0512,
    "code_avg": 0.1512,
    "languages_avg": 2.2446
   }
  },
  {
   "published": "LFM3-1.7B",
   "published_vocab": 125033,
   "provider": "Liquid AI",
   "byte_fallback": false,
   "akshara": "Akshara 120k",
   "akshara_vocab": 120000,
   "diff": {
    "overall": 0.5336,
    "english": 0.0269,
    "code_avg": 0.2531,
    "languages_avg": 1.3208
   }
  },
  {
   "published": "Llama-3.2-1B",
   "published_vocab": 128256,
   "provider": "Meta",
   "byte_fallback": false,
   "akshara": "Akshara 120k",
   "akshara_vocab": 120000,
   "diff": {
    "overall": 0.6668,
    "english": -0.0376,
    "code_avg": -0.0439,
    "languages_avg": 2.0819
   }
  },
  {
   "published": "DeepSeek-V3",
   "published_vocab": 128815,
   "provider": "DeepSeek",
   "byte_fallback": false,
   "akshara": "Akshara 120k",
   "akshara_vocab": 120000,
   "diff": {
    "overall": 0.5453,
    "english": -0.0764,
    "code_avg": 0.2364,
    "languages_avg": 1.476
   }
  },
  {
   "published": "Mistral-Nemo-Base-2407",
   "published_vocab": 131072,
   "provider": "Mistral",
   "byte_fallback": false,
   "akshara": "Akshara 120k",
   "akshara_vocab": 120000,
   "diff": {
    "overall": 0.4349,
    "english": 0.1038,
    "code_avg": 0.2675,
    "languages_avg": 0.9335
   }
  },
  {
   "published": "GLM-5.3-Flash",
   "published_vocab": 154856,
   "provider": "Z.ai (GLM)",
   "byte_fallback": false,
   "akshara": "Akshara 151k",
   "akshara_vocab": 151000,
   "diff": {
    "overall": 0.9193,
    "english": 0.033,
    "code_avg": 0.0141,
    "languages_avg": 2.711
   }
  },
  {
   "published": "Kimi-K3",
   "published_vocab": 163840,
   "provider": "Moonshot AI (Kimi)",
   "byte_fallback": false,
   "akshara": "Akshara 151k",
   "akshara_vocab": 151000,
   "diff": {
    "overall": 0.836,
    "english": 0.025,
    "code_avg": 0.0564,
    "languages_avg": 2.4268
   }
  },
  {
   "published": "OpenAI o200k_base (GPT-4o)",
   "published_vocab": 200019,
   "provider": "OpenAI",
   "byte_fallback": false,
   "akshara": "Akshara 200k",
   "akshara_vocab": 200000,
   "diff": {
    "overall": 0.3911,
    "english": 0.0494,
    "code_avg": 0.0999,
    "languages_avg": 1.0241
   }
  },
  {
   "published": "Mistral-Large-4-1T-A52B",
   "published_vocab": 204800,
   "provider": "Mistral",
   "byte_fallback": false,
   "akshara": "Akshara 200k",
   "akshara_vocab": 200000,
   "diff": {
    "overall": 0.4471,
    "english": 0.1826,
    "code_avg": 0.2255,
    "languages_avg": 0.9333
   }
  },
  {
   "published": "North-Micro-Vision-Instruct",
   "published_vocab": 255033,
   "provider": "Cohere",
   "byte_fallback": false,
   "akshara": "Akshara 248k",
   "akshara_vocab": 248000,
   "diff": {
    "overall": 0.5012,
    "english": -0.0484,
    "code_avg": 0.0718,
    "languages_avg": 1.4802
   }
  },
  {
   "published": "Falcon-H1-34B-Instruct",
   "published_vocab": 261120,
   "provider": "TII (Falcon)",
   "byte_fallback": false,
   "akshara": "Akshara 248k",
   "akshara_vocab": 248000,
   "diff": {
    "overall": 1.007,
    "english": 0.3079,
    "code_avg": 0.7416,
    "languages_avg": 1.9715
   }
  },
  {
   "published": "sarvam-105b",
   "published_vocab": 262144,
   "provider": "Sarvam",
   "byte_fallback": true,
   "akshara": "Akshara 248k",
   "akshara_vocab": 248000,
   "diff": {
    "overall": 0.6829,
    "english": 0.1866,
    "code_avg": 0.4934,
    "languages_avg": 1.3687
   }
  },
  {
   "published": "Gemma 3 / Gemma 4 tokenizer",
   "published_vocab": 262144,
   "provider": "Google",
   "byte_fallback": false,
   "akshara": "Akshara 248k",
   "akshara_vocab": 248000,
   "diff": {
    "overall": 0.4283,
    "english": 0.1804,
    "code_avg": 0.4791,
    "languages_avg": 0.6253
   }
  }
 ],
 "tradeoffs": [
  {
   "akshara": "Akshara 49k",
   "published": "SmolLM2-135M",
   "diff": {
    "english": -0.237
   }
  },
  {
   "akshara": "Akshara 49k",
   "published": "gpt2",
   "diff": {
    "english": -0.3532
   }
  },
  {
   "akshara": "Akshara 49k",
   "published": "OpenAI p50k_base (Codex)",
   "diff": {
    "english": -0.3532
   }
  },
  {
   "akshara": "Akshara 92k",
   "published": "cl100k_base (GPT-4, OLMo-2, Olmo-3, phi-4)",
   "diff": {
    "english": -0.1265,
    "code_avg": -0.0903
   }
  },
  {
   "akshara": "Akshara 120k",
   "published": "Hy3",
   "diff": {
    "english": -0.0512
   }
  },
  {
   "akshara": "Akshara 120k",
   "published": "Llama-3.2-1B",
   "diff": {
    "english": -0.0376,
    "code_avg": -0.0439
   }
  },
  {
   "akshara": "Akshara 120k",
   "published": "DeepSeek-V3",
   "diff": {
    "english": -0.0764
   }
  },
  {
   "akshara": "Akshara 248k",
   "published": "North-Micro-Vision-Instruct",
   "diff": {
    "english": -0.0484
   }
  }
 ],
 "level_from": {
  "english": null,
  "code_avg": 151000,
  "languages_avg": 32000
 }
}