{"generated":"2026-11-08","unit":"bytes_per_token","text_unit":"lines","measure":"UTF-8 bytes per token on held-out text; higher is better","ranking":"rank counts lossless tokenizers only, by overall score; a tokenizer that does not decode every input back exactly has rank null and a flag saying why; tokenizers with identical IDs on every input are one row","resources_measured_on":{"cpu":"Apple M4 Pro (container)","logical_cpus":14,"memory_bytes":18829856768,"gpu":null,"os":"Linux-7.0.14-linuxkit-aarch64-with-glibc2.41","python":"3.13.18","libraries":{"tokenizers":"0.23.3","tiktoken":"0.14.0","transformers":"5.22.2","numpy":"2.5.4"},"git_commit":"ef2fc972f4e24836ebc9fea119b9338548969a1d","git_dirty":false,"config_version":"1.3","library":"every tokenizer on the Hugging Face tokenizers library (tiktoken files converted, Akshara exported); resources.native is each tokenizer's own runtime where it differs, never mixed with the library numbers","note":"each value is the median of 5 fresh processes run in one seeded random order over all tokenizers; q1 and q3 give the spread, and values whose q1-q3 ranges overlap are ties"},"query":{"provider":[],"family":null,"q":null,"sets":[],"sort":"rank","order":"asc","offset":0,"limit":null},"total":36,"count":36,"tokenizers":[{"rank":1,"name":"Akshara 448k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":448000,"models":["Akshara 448k"],"overall":5.409,"english":4.9363,"code_avg":4.0776,"languages_avg":7.2132,"bpt":{"code_asm":2.7413,"code_c":3.5702,"code_cpp":3.9995,"code_cs":5.0448,"code_dart":4.2901,"code_go":3.7281,"code_html":3.843,"code_java":4.722,"code_js":4.1326,"code_kotlin":4.7655,"code_lua":3.856,"code_php":4.0435,"code_ps":4.2526,"code_py":4.3606,"code_r":3.2167,"code_ruby":4.2121,"code_rust":4.383,"code_shell":3.6305,"code_sql":3.6156,"code_swift":4.8209,"code_ts":4.4003,"en_fineweb":4.9363,"wiki_ar":7.2141,"wiki_bn":11.6908,"wiki_de":4.1961,"wiki_es":4.8297,"wiki_fr":4.5989,"wiki_gu":8.6161,"wiki_hi":9.1979,"wiki_id":4.9855,"wiki_ja":5.9287,"wiki_ko":4.5465,"wiki_mr":9.9304,"wiki_ru":7.3757,"wiki_ta":10.9733,"wiki_te":9.2259,"wiki_th":12.3622,"wiki_tr":4.9241,"wiki_ur":6.3696,"wiki_vi":4.9869,"wiki_zh":5.0979},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":5.620347250340921,"provenance":"measured","note":"median of 5 fresh processes","q1":5.4628430832448185,"q3":5.638177126563517,"n":5},"batch_mb_per_s_4_threads":{"value":18.300612613557615,"provenance":"measured","note":"median of 5 fresh processes","q1":18.190271144082107,"q3":18.779172937414017,"n":5},"cpu_cores_busy_batch":{"value":3.602732836677368,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.586244613345892,"q3":3.605149734177987,"n":5},"cpu_seconds_per_mb":{"value":0.19461342897082637,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.19197595901550035,"q3":0.19861731913171873,"n":5},"latency_p95_us_256_bytes":{"value":63.45899964799173,"provenance":"measured","note":"median of 5 fresh processes","q1":61.58399992273189,"q3":65.66699994436931,"n":5},"decode_mb_per_s":{"value":43.08721156622042,"provenance":"measured","note":"median of 5 fresh processes","q1":38.015456932945334,"q3":44.60498553642126,"n":5},"load_seconds":{"value":0.6785901250000279,"provenance":"measured","note":"median of 5 fresh processes","q1":0.6715609589996347,"q3":0.6974505419998422,"n":5},"load_ram_mb":{"value":204.791808,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":204.791808,"q3":205.451264,"n":5},"peak_ram_mb_1_mib_input":{"value":257.273856,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":256.765952,"q3":258.854912,"n":5},"peak_ram_mb_16_mib_input":{"value":2394.68544,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2393.84576,"q3":2396.40576,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.57184,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.769024,"q3":18.415616,"n":5},"ram_mb_per_input_mb":{"value":130.41162109375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":130.26196289062503,"q3":130.486083984375,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":36.18286900914715,"provenance":"measured","note":"median of 5 fresh processes","q1":34.1841535617166,"q3":36.61716876583317,"n":5},"batch_mb_per_s_4_threads":{"value":96.03981959090198,"provenance":"measured","note":"median of 5 fresh processes","q1":94.16075364712611,"q3":97.95374011367092,"n":5},"cpu_seconds_per_mb":{"value":0.03312668939821483,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.032212086727031373,"q3":0.03355299533566941,"n":5},"latency_p95_us_256_bytes":{"value":19.458000224403804,"provenance":"measured","note":"median of 5 fresh processes","q1":19.250000150350388,"q3":20.166000012977747,"n":5},"decode_mb_per_s":{"value":112.98578979403972,"provenance":"measured","note":"median of 5 fresh processes","q1":112.44912130252726,"q3":113.2155370183665,"n":5},"load_seconds":{"value":0.8073415829999249,"provenance":"measured","note":"median of 5 fresh processes","q1":0.7976622080000197,"q3":0.8185076249999952,"n":5},"load_ram_mb":{"value":93.540352,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":93.536256,"q3":93.540352,"n":5},"peak_ram_mb_1_mib_input":{"value":88.43264,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":87.703552,"q3":88.604672,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.707008,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.686528,"q3":17.887232,"n":5}}}},{"rank":2,"name":"Akshara 248k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":248000,"models":["Akshara 248k"],"overall":5.1408,"english":4.8394,"code_avg":3.9914,"languages_avg":6.5915,"bpt":{"code_asm":2.6835,"code_cs":4.9354,"code_dart":4.2171,"code_go":3.648,"code_html":3.7873,"code_kotlin":4.6319,"code_lua":3.7525,"code_php":3.9549,"code_ps":4.1292,"code_r":3.1425,"code_ruby":4.1493,"code_shell":3.5268,"code_sql":3.5431,"code_swift":4.7112,"code_ts":4.295,"code_c":3.5042,"code_cpp":3.9276,"code_java":4.6261,"code_js":4.0622,"code_py":4.2687,"code_rust":4.3223,"en_fineweb":4.8394,"wiki_ar":6.5818,"wiki_bn":10.5595,"wiki_de":3.9407,"wiki_es":4.5835,"wiki_fr":4.3493,"wiki_gu":7.7535,"wiki_hi":8.4888,"wiki_id":4.6375,"wiki_ja":5.4672,"wiki_ko":4.1718,"wiki_mr":8.9641,"wiki_ru":6.762,"wiki_ta":9.8155,"wiki_te":8.1998,"wiki_th":10.9216,"wiki_tr":4.4721,"wiki_ur":5.9721,"wiki_vi":4.8656,"wiki_zh":4.7322},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":5.836628228431435,"provenance":"measured","note":"median of 5 fresh processes","q1":5.729098732557851,"q3":5.852981716836457,"n":5},"batch_mb_per_s_4_threads":{"value":18.40640894684117,"provenance":"measured","note":"median of 5 fresh processes","q1":18.07314929157135,"q3":18.44548721846948,"n":5},"cpu_cores_busy_batch":{"value":3.599999563918848,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5902364777768137,"q3":3.600140658904089,"n":5},"cpu_seconds_per_mb":{"value":0.19559169142126065,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.19464037112459176,"q3":0.1991904955711153,"n":5},"latency_p95_us_256_bytes":{"value":61.37500008662755,"provenance":"measured","note":"median of 5 fresh processes","q1":60.58300004951889,"q3":65.91700002900325,"n":5},"decode_mb_per_s":{"value":41.01579788879382,"provenance":"measured","note":"median of 5 fresh processes","q1":40.03215785134015,"q3":43.44499781195673,"n":5},"load_seconds":{"value":0.33785904199999095,"provenance":"measured","note":"median of 5 fresh processes","q1":0.33691641699988395,"q3":0.34502274999999827,"n":5},"load_ram_mb":{"value":119.328768,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":119.328768,"q3":119.328768,"n":5},"peak_ram_mb_1_mib_input":{"value":220.925952,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":220.91776,"q3":220.971008,"n":5},"peak_ram_mb_16_mib_input":{"value":2374.10304,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2373.025792,"q3":2375.737344,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.403328,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.616896,"q3":18.522112,"n":5},"ram_mb_per_input_mb":{"value":131.10302734375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":131.0390625,"q3":131.19409179687503,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":41.148355701987875,"provenance":"measured","note":"median of 5 fresh processes","q1":38.179145933126726,"q3":41.25354307372325,"n":5},"batch_mb_per_s_4_threads":{"value":106.473221629358,"provenance":"measured","note":"median of 5 fresh processes","q1":104.02521484246932,"q3":109.93066708625295,"n":5},"cpu_seconds_per_mb":{"value":0.028916322200501585,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.028385108761659932,"q3":0.02962468626100365,"n":5},"latency_p95_us_256_bytes":{"value":15.874999917286914,"provenance":"measured","note":"median of 5 fresh processes","q1":15.624999832652977,"q3":16.2080000336573,"n":5},"decode_mb_per_s":{"value":110.3553940136642,"provenance":"measured","note":"median of 5 fresh processes","q1":109.80790261679124,"q3":110.50125141473536,"n":5},"load_seconds":{"value":0.4160123340000154,"provenance":"measured","note":"median of 5 fresh processes","q1":0.41419583400011106,"q3":0.4196818330001406,"n":5},"load_ram_mb":{"value":72.159232,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":72.155136,"q3":72.159232,"n":5},"peak_ram_mb_1_mib_input":{"value":63.01696,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":63.01696,"q3":63.021056,"n":5},"peak_ram_mb_16_mib_chunked":{"value":19.304448,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":19.214336,"q3":19.39456,"n":5}}}},{"rank":3,"name":"Akshara 200k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":200000,"models":["Akshara 200k"],"overall":5.0315,"english":4.7918,"code_avg":3.9523,"languages_avg":6.3505,"bpt":{"code_asm":2.6446,"code_cs":4.877,"code_dart":4.1926,"code_go":3.6145,"code_html":3.7629,"code_kotlin":4.5733,"code_lua":3.7102,"code_php":3.9198,"code_ps":4.0742,"code_r":3.1143,"code_ruby":4.1136,"code_shell":3.4859,"code_sql":3.5022,"code_swift":4.6727,"code_ts":4.2449,"code_c":3.4704,"code_cpp":3.8865,"code_java":4.5848,"code_js":4.0293,"code_py":4.2306,"code_rust":4.2934,"en_fineweb":4.7918,"wiki_ar":6.3332,"wiki_bn":10.075,"wiki_de":3.842,"wiki_es":4.4673,"wiki_fr":4.2451,"wiki_gu":7.4366,"wiki_hi":8.1685,"wiki_id":4.4894,"wiki_ja":5.3145,"wiki_ko":4.0399,"wiki_mr":8.577,"wiki_ru":6.5089,"wiki_ta":9.3755,"wiki_te":7.8364,"wiki_th":10.4265,"wiki_tr":4.3145,"wiki_ur":5.7932,"wiki_vi":4.8026,"wiki_zh":4.6132},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":5.734110759561312,"provenance":"measured","note":"median of 5 fresh processes","q1":5.429966938050952,"q3":5.844051229124779,"n":5},"batch_mb_per_s_4_threads":{"value":18.650088442681785,"provenance":"measured","note":"median of 5 fresh processes","q1":18.585399546469404,"q3":18.674387758522915,"n":5},"cpu_cores_busy_batch":{"value":3.5789323405534805,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.572936159223736,"q3":3.6132156532798265,"n":5},"cpu_seconds_per_mb":{"value":0.1927506055427878,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.19189894737243665,"q3":0.19449302377037084,"n":5},"latency_p95_us_256_bytes":{"value":61.70799997562426,"provenance":"measured","note":"median of 5 fresh processes","q1":59.04099998588208,"q3":61.70899996504886,"n":5},"decode_mb_per_s":{"value":45.14105379322546,"provenance":"measured","note":"median of 5 fresh processes","q1":44.88950725625062,"q3":45.64243052736785,"n":5},"load_seconds":{"value":0.2555477079999946,"provenance":"measured","note":"median of 5 fresh processes","q1":0.2500330009997924,"q3":0.2636542090003786,"n":5},"load_ram_mb":{"value":84.148224,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":84.148224,"q3":84.148224,"n":5},"peak_ram_mb_1_mib_input":{"value":191.758336,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":191.75424,"q3":191.758336,"n":5},"peak_ram_mb_16_mib_input":{"value":2359.476224,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2359.369728,"q3":2361.46688,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.129472,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.068032,"q3":17.199104,"n":5},"ram_mb_per_input_mb":{"value":134.06909179687503,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":134.06494140625,"q3":134.18994140625,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":37.12108302421878,"provenance":"measured","note":"median of 5 fresh processes","q1":36.41212202025844,"q3":40.28306496977799,"n":5},"batch_mb_per_s_4_threads":{"value":107.02999409867677,"provenance":"measured","note":"median of 5 fresh processes","q1":105.7416243454292,"q3":107.8570836187658,"n":5},"cpu_seconds_per_mb":{"value":0.029026475076958005,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.02871485211261057,"q3":0.029048648707932997,"n":5},"latency_p95_us_256_bytes":{"value":16.041999970184406,"provenance":"measured","note":"median of 5 fresh processes","q1":15.95799994902336,"q3":16.95800006018544,"n":5},"decode_mb_per_s":{"value":108.66943691211009,"provenance":"measured","note":"median of 5 fresh processes","q1":108.54476329906092,"q3":110.9778271687773,"n":5},"load_seconds":{"value":0.3114650829999732,"provenance":"measured","note":"median of 5 fresh processes","q1":0.3081390420002208,"q3":0.3124225839999326,"n":5},"load_ram_mb":{"value":51.34336,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":51.34336,"q3":51.34336,"n":5},"peak_ram_mb_1_mib_input":{"value":57.91744,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":57.901056,"q3":58.642432,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.096704,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.842752,"q3":17.567744,"n":5}}}},{"rank":4,"name":"Akshara 151k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":151000,"models":["Akshara 151k"],"overall":4.8772,"english":4.7185,"code_avg":3.8937,"languages_avg":6.0195,"bpt":{"code_asm":2.6032,"code_cs":4.813,"code_dart":4.143,"code_go":3.5683,"code_html":3.722,"code_kotlin":4.4921,"code_lua":3.6472,"code_php":3.8753,"code_ps":3.9948,"code_r":3.0438,"code_ruby":4.0555,"code_shell":3.4289,"code_sql":3.4237,"code_swift":4.5994,"code_ts":4.1744,"code_c":3.4318,"code_cpp":3.8389,"code_java":4.515,"code_js":3.983,"code_py":4.1745,"code_rust":4.2397,"en_fineweb":4.7185,"wiki_ar":6.03,"wiki_bn":9.3852,"wiki_de":3.7056,"wiki_es":4.3179,"wiki_fr":4.12,"wiki_gu":6.954,"wiki_hi":7.7859,"wiki_id":4.2931,"wiki_ja":5.0919,"wiki_ko":3.8708,"wiki_mr":8.0431,"wiki_ru":6.1917,"wiki_ta":8.8353,"wiki_te":7.3259,"wiki_th":9.6561,"wiki_tr":4.0739,"wiki_ur":5.548,"wiki_vi":4.7028,"wiki_zh":4.439},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":6.116414017986229,"provenance":"measured","note":"median of 5 fresh processes","q1":6.0974718698952675,"q3":6.128655757761346,"n":5},"batch_mb_per_s_4_threads":{"value":18.45893944644569,"provenance":"measured","note":"median of 5 fresh processes","q1":18.424939107275005,"q3":18.56070840705968,"n":5},"cpu_cores_busy_batch":{"value":3.5890226977657993,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5785923410417104,"q3":3.591094386953844,"n":5},"cpu_seconds_per_mb":{"value":0.19385141902893324,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.192804728276458,"q3":0.1945450006687853,"n":5},"latency_p95_us_256_bytes":{"value":59.750000218627974,"provenance":"measured","note":"median of 5 fresh processes","q1":59.29200005994062,"q3":71.33400004022405,"n":5},"decode_mb_per_s":{"value":46.21559142784512,"provenance":"measured","note":"median of 5 fresh processes","q1":46.12766144661114,"q3":46.52376839601841,"n":5},"load_seconds":{"value":0.199583500000017,"provenance":"measured","note":"median of 5 fresh processes","q1":0.1833465839999917,"q3":0.20041354199975103,"n":5},"load_ram_mb":{"value":93.712384,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":93.36832,"q3":93.71648,"n":5},"peak_ram_mb_1_mib_input":{"value":188.841984,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":188.833792,"q3":188.850176,"n":5},"peak_ram_mb_16_mib_input":{"value":2368.565248,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2368.323584,"q3":2368.7168,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.649664,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.010688,"q3":17.743872,"n":5},"ram_mb_per_input_mb":{"value":136.063720703125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":136.041259765625,"q3":136.068115234375,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":41.239480939779185,"provenance":"measured","note":"median of 5 fresh processes","q1":41.192756585857424,"q3":43.16628297529017,"n":5},"batch_mb_per_s_4_threads":{"value":111.58481435588936,"provenance":"measured","note":"median of 5 fresh processes","q1":103.2065184299624,"q3":115.39791728780762,"n":5},"cpu_seconds_per_mb":{"value":0.027358445804903996,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.026594051601615668,"q3":0.028802831358091977,"n":5},"latency_p95_us_256_bytes":{"value":14.416999874811154,"provenance":"measured","note":"median of 5 fresh processes","q1":14.20800026608049,"q3":15.250000160449416,"n":5},"decode_mb_per_s":{"value":110.06677227775393,"provenance":"measured","note":"median of 5 fresh processes","q1":104.97068436158962,"q3":111.11994214786446,"n":5},"load_seconds":{"value":0.23353245799989963,"provenance":"measured","note":"median of 5 fresh processes","q1":0.23245729200016285,"q3":0.23717933299997185,"n":5},"load_ram_mb":{"value":45.367296,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":45.3632,"q3":45.367296,"n":5},"peak_ram_mb_1_mib_input":{"value":46.661632,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":46.641152,"q3":47.104,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.3296,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.473536,"q3":18.55488,"n":5}}}},{"rank":5,"name":"Akshara 120k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":120000,"models":["Akshara 120k"],"overall":4.7469,"english":4.6553,"code_avg":3.8468,"languages_avg":5.7387,"bpt":{"code_asm":2.573,"code_cs":4.7558,"code_dart":4.1107,"code_go":3.5334,"code_html":3.6805,"code_kotlin":4.4315,"code_lua":3.5823,"code_php":3.837,"code_ps":3.9378,"code_r":3.0069,"code_ruby":4.0117,"code_shell":3.374,"code_sql":3.3728,"code_swift":4.5379,"code_ts":4.1158,"code_c":3.3894,"code_cpp":3.8081,"code_java":4.4626,"code_js":3.9454,"code_py":4.1251,"code_rust":4.1919,"en_fineweb":4.6553,"wiki_ar":5.752,"wiki_bn":8.811,"wiki_de":3.598,"wiki_es":4.1929,"wiki_fr":3.9884,"wiki_gu":6.5997,"wiki_hi":7.3936,"wiki_id":4.1076,"wiki_ja":4.8985,"wiki_ko":3.7331,"wiki_mr":7.591,"wiki_ru":5.9348,"wiki_ta":8.331,"wiki_te":6.9625,"wiki_th":9.0311,"wiki_tr":3.8994,"wiki_ur":5.3396,"wiki_vi":4.5816,"wiki_zh":4.2892},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":5.967619867743156,"provenance":"measured","note":"median of 5 fresh processes","q1":5.956208157401425,"q3":6.016314594826091,"n":5},"batch_mb_per_s_4_threads":{"value":18.09350513047292,"provenance":"measured","note":"median of 5 fresh processes","q1":17.71619448610454,"q3":18.27981824551409,"n":5},"cpu_cores_busy_batch":{"value":3.5869782359774,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.581509474669802,"q3":3.5968366058587065,"n":5},"cpu_seconds_per_mb":{"value":0.19794448056600464,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.1962261433785127,"q3":0.203311929818789,"n":5},"latency_p95_us_256_bytes":{"value":62.000000013995304,"provenance":"measured","note":"median of 5 fresh processes","q1":59.959000054732314,"q3":66.79100010842376,"n":5},"decode_mb_per_s":{"value":46.44357459654735,"provenance":"measured","note":"median of 5 fresh processes","q1":45.12073969902209,"q3":47.21170720911533,"n":5},"load_seconds":{"value":0.1461671250000336,"provenance":"measured","note":"median of 5 fresh processes","q1":0.14010229199993773,"q3":0.15664995900033318,"n":5},"load_ram_mb":{"value":81.068032,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":81.068032,"q3":81.068032,"n":5},"peak_ram_mb_1_mib_input":{"value":187.088896,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":186.998784,"q3":187.138048,"n":5},"peak_ram_mb_16_mib_input":{"value":2378.186752,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2377.9328,"q3":2378.264576,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.088512,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.826368,"q3":17.661952,"n":5},"ram_mb_per_input_mb":{"value":137.31005859375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":137.30590820312503,"q3":137.3251953125,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":40.06823361453194,"provenance":"measured","note":"median of 5 fresh processes","q1":36.20469114167294,"q3":40.9232847662627,"n":5},"batch_mb_per_s_4_threads":{"value":109.3360270854944,"provenance":"measured","note":"median of 5 fresh processes","q1":108.50220877359422,"q3":109.78547342923947,"n":5},"cpu_seconds_per_mb":{"value":0.027261644792260505,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.02709283908548316,"q3":0.02751866817065875,"n":5},"latency_p95_us_256_bytes":{"value":14.457999895967076,"provenance":"measured","note":"median of 5 fresh processes","q1":13.292000005549198,"q3":14.87499997665509,"n":5},"decode_mb_per_s":{"value":107.71359840311378,"provenance":"measured","note":"median of 5 fresh processes","q1":100.94939648822276,"q3":107.86364610800392,"n":5},"load_seconds":{"value":0.2049808749998192,"provenance":"measured","note":"median of 5 fresh processes","q1":0.19523945800006004,"q3":0.2066935839998223,"n":5},"load_ram_mb":{"value":41.598976,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":41.598976,"q3":41.603072,"n":5},"peak_ram_mb_1_mib_input":{"value":44.07296,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":44.068864,"q3":44.089344,"n":5},"peak_ram_mb_16_mib_chunked":{"value":16.994304,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.994304,"q3":18.2272,"n":5}}}},{"rank":6,"name":"gemma-3-1b-it","provider":"Google","family":"Published","version":null,"vocab":262145,"models":["gemma-3-1b-it"],"overall":4.7125,"english":4.659,"code_avg":3.5123,"languages_avg":5.9662,"bpt":{"code_c":3.0856,"code_cpp":3.4607,"code_java":4.1883,"code_js":3.6866,"code_rust":3.7148,"wiki_bn":11.3722,"wiki_gu":6.1929,"wiki_id":4.3329,"wiki_mr":8.8618,"wiki_ta":9.41,"wiki_te":6.7414,"wiki_th":8.0864,"wiki_tr":3.842,"wiki_ur":5.5904,"wiki_vi":4.447,"code_py":3.5881,"en_fineweb":4.659,"wiki_ar":5.2875,"wiki_de":3.3938,"wiki_es":4.2979,"wiki_fr":3.9926,"wiki_hi":9.0568,"wiki_ja":4.5893,"wiki_ko":3.7148,"wiki_ru":5.9768,"wiki_zh":4.1716,"code_asm":2.1172,"code_cs":4.5275,"code_dart":3.8042,"code_go":3.1836,"code_html":3.2057,"code_kotlin":4.0666,"code_lua":3.1917,"code_php":3.7934,"code_ps":3.6356,"code_r":2.7304,"code_ruby":3.5957,"code_shell":3.0447,"code_sql":2.8651,"code_swift":4.2418,"code_ts":4.0314},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":6.103693269272489,"provenance":"measured","note":"median of 5 fresh processes","q1":6.069012027010557,"q3":6.156752985263483,"n":5},"batch_mb_per_s_4_threads":{"value":19.8543184165146,"provenance":"measured","note":"median of 5 fresh processes","q1":18.675361294380902,"q3":20.066319688176616,"n":5},"cpu_cores_busy_batch":{"value":3.5600645711425387,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.537432145328517,"q3":3.564383409200697,"n":5},"cpu_seconds_per_mb":{"value":0.1793093319275728,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.1762870421830679,"q3":0.18460692232147904,"n":5},"latency_p95_us_256_bytes":{"value":64.33400000105394,"provenance":"measured","note":"median of 5 fresh processes","q1":61.29199982751743,"q3":65.6249999906322,"n":5},"decode_mb_per_s":{"value":19.322156916252645,"provenance":"measured","note":"median of 5 fresh processes","q1":18.940035095345923,"q3":19.963442728719404,"n":5},"load_seconds":{"value":0.6804287500003738,"provenance":"measured","note":"median of 5 fresh processes","q1":0.678886625999894,"q3":0.709873000000016,"n":5},"load_ram_mb":{"value":312.958976,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":312.95488,"q3":312.958976,"n":5},"peak_ram_mb_1_mib_input":{"value":372.293632,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":371.99872,"q3":374.054912,"n":5},"peak_ram_mb_16_mib_input":{"value":1918.242816,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":1917.41952,"q3":1920.2048,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0.065536,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0.065536,"q3":0.065536,"n":5},"ram_mb_per_input_mb":{"value":92.78125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":92.732177734375,"q3":92.898193359375,"n":5}}},{"rank":7,"name":"gemma-4-31B-it","provider":"Google","family":"Published","version":null,"vocab":262144,"models":["gemma-4-31B-it"],"overall":4.71,"english":4.659,"code_avg":3.5049,"languages_avg":5.9661,"bpt":{"code_asm":2.117,"code_cs":4.5274,"code_dart":3.8042,"code_go":3.1827,"code_html":3.1174,"code_kotlin":4.0663,"code_lua":3.1917,"code_php":3.7524,"code_ps":3.6356,"code_r":2.7267,"code_ruby":3.5957,"code_shell":3.0447,"code_sql":2.8651,"code_swift":4.2418,"code_ts":4.0306,"code_c":3.0856,"code_cpp":3.4607,"code_java":4.1809,"code_js":3.675,"code_py":3.5865,"code_rust":3.7147,"en_fineweb":4.659,"wiki_ar":5.2875,"wiki_bn":11.3718,"wiki_de":3.3938,"wiki_es":4.2979,"wiki_fr":3.9926,"wiki_gu":6.1929,"wiki_hi":9.0565,"wiki_id":4.3329,"wiki_ja":4.5893,"wiki_ko":3.7148,"wiki_mr":8.8618,"wiki_ru":5.9768,"wiki_ta":9.4088,"wiki_te":6.7414,"wiki_th":8.0864,"wiki_tr":3.842,"wiki_ur":5.59,"wiki_vi":4.447,"wiki_zh":4.1716},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":6.426287669332646,"provenance":"measured","note":"median of 5 fresh processes","q1":6.3731937836178565,"q3":6.496928971519612,"n":5},"batch_mb_per_s_4_threads":{"value":20.89084307099784,"provenance":"measured","note":"median of 5 fresh processes","q1":20.777198523991068,"q3":21.004698820989567,"n":5},"cpu_cores_busy_batch":{"value":3.551847617761725,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.531887869453192,"q3":3.5882750340083382,"n":5},"cpu_seconds_per_mb":{"value":0.1690639222864304,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.16802319218744302,"q3":0.1726202865834512,"n":5},"latency_p95_us_256_bytes":{"value":53.33300032361876,"provenance":"measured","note":"median of 5 fresh processes","q1":52.24999995334656,"q3":54.58300000782401,"n":5},"decode_mb_per_s":{"value":19.99801044816113,"provenance":"measured","note":"median of 5 fresh processes","q1":19.358661512641568,"q3":20.0630422976614,"n":5},"load_seconds":{"value":0.7190039170000091,"provenance":"measured","note":"median of 5 fresh processes","q1":0.6862907080000014,"q3":0.7275295000000597,"n":5},"load_ram_mb":{"value":309.850112,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":309.850112,"q3":309.850112,"n":5},"peak_ram_mb_1_mib_input":{"value":373.338112,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":373.338112,"q3":373.342208,"n":5},"peak_ram_mb_16_mib_input":{"value":1770.811392,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":1770.807296,"q3":1770.840064,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0.069632,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0.069632,"q3":0.069632,"n":5},"ram_mb_per_input_mb":{"value":83.546630859375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":83.531005859375,"q3":83.546630859375,"n":5}}},{"rank":8,"name":"OpenAI o200k_base (GPT-4o)","provider":"OpenAI","family":"Published","version":null,"vocab":200019,"models":["OpenAI o200k_base (GPT-4o)"],"overall":4.6404,"english":4.7424,"code_avg":3.8524,"languages_avg":5.3264,"bpt":{"code_asm":2.5011,"code_cs":4.8032,"code_dart":4.151,"code_go":3.5193,"code_html":3.5834,"code_kotlin":4.5176,"code_lua":3.6175,"code_php":3.9451,"code_ps":3.7496,"code_r":3.0581,"code_ruby":4.0529,"code_shell":3.358,"code_sql":3.3553,"code_swift":4.4634,"code_ts":4.2001,"code_c":3.3606,"code_cpp":3.8789,"code_java":4.4883,"code_js":3.9776,"code_py":4.1637,"code_rust":4.1552,"en_fineweb":4.7424,"wiki_ar":5.73,"wiki_bn":7.6621,"wiki_de":3.7027,"wiki_es":4.3661,"wiki_fr":4.0961,"wiki_gu":6.4939,"wiki_hi":7.6212,"wiki_id":4.0841,"wiki_ja":3.533,"wiki_ko":3.5902,"wiki_mr":7.0112,"wiki_ru":6.0788,"wiki_ta":7.3747,"wiki_te":6.2667,"wiki_th":6.8635,"wiki_tr":3.7785,"wiki_ur":5.2534,"wiki_vi":4.2522,"wiki_zh":3.4425},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (converted from tiktoken)","encode_mb_per_s_1_thread":{"value":5.101910820251484,"provenance":"measured","note":"median of 5 fresh processes","q1":5.0714635495205,"q3":5.138293990144321,"n":5},"batch_mb_per_s_4_threads":{"value":17.73679441869205,"provenance":"measured","note":"median of 5 fresh processes","q1":17.62775178570308,"q3":17.788999687840956,"n":5},"cpu_cores_busy_batch":{"value":3.614590933457547,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5862186206606927,"q3":3.619274973598692,"n":5},"cpu_seconds_per_mb":{"value":0.20344163363868564,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2031924783228915,"q3":0.20352126796928408,"n":5},"latency_p95_us_256_bytes":{"value":65.54100036737509,"provenance":"measured","note":"median of 5 fresh processes","q1":64.70800008173683,"q3":66.58300026174402,"n":5},"decode_mb_per_s":{"value":39.12072516397016,"provenance":"measured","note":"median of 5 fresh processes","q1":37.16271940307519,"q3":40.102457704134835,"n":5},"load_seconds":{"value":0.5957294170000296,"provenance":"measured","note":"median of 5 fresh processes","q1":0.5924940419999984,"q3":0.6016672920000019,"n":5},"load_ram_mb":{"value":249.864192,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":249.860096,"q3":249.868288,"n":5},"peak_ram_mb_1_mib_input":{"value":321.458176,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":320.503808,"q3":321.8432,"n":5},"peak_ram_mb_16_mib_input":{"value":2471.8336,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2469.187584,"q3":2474.323968,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0,"q3":0,"n":5},"ram_mb_per_input_mb":{"value":129.434814453125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":129.2568359375,"q3":129.58325195312503,"n":5},"native":{"implementation":"tiktoken","encode_mb_per_s_1_thread":{"value":25.11267863118479,"provenance":"measured","note":"median of 5 fresh processes","q1":25.054576960652053,"q3":25.3858707437138,"n":5},"batch_mb_per_s_4_threads":{"value":74.47334099824387,"provenance":"measured","note":"median of 5 fresh processes","q1":71.89185807330907,"q3":74.93683693291769,"n":5},"cpu_seconds_per_mb":{"value":0.05307079754258946,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.052912482585950826,"q3":0.055236899020092406,"n":5},"latency_p95_us_256_bytes":{"value":19.249999922976713,"provenance":"measured","note":"median of 5 fresh processes","q1":18.49999989644857,"q3":20.707999965452473,"n":5},"decode_mb_per_s":{"value":188.6404882960821,"provenance":"measured","note":"median of 5 fresh processes","q1":188.52886841947648,"q3":191.52183450810887,"n":5},"load_seconds":{"value":0.1457424169998376,"provenance":"measured","note":"median of 5 fresh processes","q1":0.14254829100013922,"q3":0.15049458299995422,"n":5},"load_ram_mb":{"value":91.897856,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":91.897856,"q3":91.897856,"n":5},"peak_ram_mb_1_mib_input":{"value":76.345344,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":76.345344,"q3":76.34944,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.973248,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.575936,"q3":18.026496,"n":5}}}},{"rank":9,"name":"North-Micro-Vision-Instruct","provider":"Cohere","family":"Published","version":null,"vocab":255033,"models":["North-Micro-Vision-Instruct"],"overall":4.6396,"english":4.8878,"code_avg":3.9196,"languages_avg":5.1113,"bpt":{"code_asm":2.5615,"code_cs":4.8256,"code_dart":4.2083,"code_go":3.5889,"code_html":3.7046,"code_kotlin":4.5636,"code_lua":3.6623,"code_php":3.9608,"code_ps":3.9278,"code_r":3.1448,"code_ruby":4.1485,"code_shell":3.5131,"code_sql":3.4999,"code_swift":4.4885,"code_ts":4.2099,"code_c":3.387,"code_cpp":3.9162,"code_java":4.5209,"code_js":4.0178,"code_py":4.2227,"code_rust":4.2397,"en_fineweb":4.8878,"wiki_ar":6.0129,"wiki_bn":6.9321,"wiki_de":3.865,"wiki_es":4.4269,"wiki_fr":4.197,"wiki_gu":2.1495,"wiki_hi":7.2446,"wiki_id":4.7278,"wiki_ja":4.8018,"wiki_ko":4.3039,"wiki_mr":5.6715,"wiki_ru":5.9958,"wiki_ta":6.6114,"wiki_te":5.1785,"wiki_th":7.9845,"wiki_tr":3.5908,"wiki_ur":4.458,"wiki_vi":4.7861,"wiki_zh":4.1762},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.009868145514843,"provenance":"measured","note":"median of 5 fresh processes","q1":3.92947061141834,"q3":4.030443939003501,"n":5},"batch_mb_per_s_4_threads":{"value":14.617944898121555,"provenance":"measured","note":"median of 5 fresh processes","q1":14.150318780627517,"q3":14.632415683022046,"n":5},"cpu_cores_busy_batch":{"value":3.66762908148896,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.667441700727189,"q3":3.683267231248008,"n":5},"cpu_seconds_per_mb":{"value":0.25203025818451374,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.25065096296741357,"q3":0.25860414370324913,"n":5},"latency_p95_us_256_bytes":{"value":88.54199995766976,"provenance":"measured","note":"median of 5 fresh processes","q1":87.37500002098386,"q3":91.87499995277904,"n":5},"decode_mb_per_s":{"value":38.502504064913786,"provenance":"measured","note":"median of 5 fresh processes","q1":34.63283446547719,"q3":38.52000742646196,"n":5},"load_seconds":{"value":0.440281541999866,"provenance":"measured","note":"median of 5 fresh processes","q1":0.4303102079998098,"q3":0.4431673340000089,"n":5},"load_ram_mb":{"value":185.212928,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":185.208832,"q3":185.212928,"n":5},"peak_ram_mb_1_mib_input":{"value":254.160896,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":254.1568,"q3":254.160896,"n":5},"peak_ram_mb_16_mib_input":{"value":2461.401088,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2461.392896,"q3":2461.995008,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0.008192,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0.008192,"q3":0.008192,"n":5},"ram_mb_per_input_mb":{"value":132.162841796875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":132.162353515625,"q3":132.1982421875,"n":5}}},{"rank":10,"name":"Akshara 92k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":92000,"models":["Akshara 92k"],"overall":4.5886,"english":4.5649,"code_avg":3.782,"languages_avg":5.4188,"bpt":{"code_asm":2.53,"code_cs":4.682,"code_dart":4.0375,"code_go":3.4765,"code_html":3.6253,"code_kotlin":4.3618,"code_lua":3.5164,"code_php":3.7839,"code_ps":3.8629,"code_r":2.957,"code_ruby":3.9523,"code_shell":3.3084,"code_sql":3.2917,"code_swift":4.4559,"code_ts":4.051,"code_c":3.3447,"code_cpp":3.7482,"code_java":4.3854,"code_js":3.8779,"code_py":4.0547,"code_rust":4.1187,"en_fineweb":4.5649,"wiki_ar":5.4504,"wiki_bn":8.2268,"wiki_de":3.4495,"wiki_es":4.0339,"wiki_fr":3.826,"wiki_gu":6.1873,"wiki_hi":6.9576,"wiki_id":3.8996,"wiki_ja":4.6944,"wiki_ko":3.572,"wiki_mr":7.0766,"wiki_ru":5.6188,"wiki_ta":7.8075,"wiki_te":6.4666,"wiki_th":8.3636,"wiki_tr":3.6951,"wiki_ur":5.0728,"wiki_vi":4.4411,"wiki_zh":4.1172},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":6.100163595476987,"provenance":"measured","note":"median of 5 fresh processes","q1":6.008446439600618,"q3":6.168005805384802,"n":5},"batch_mb_per_s_4_threads":{"value":17.31074049393309,"provenance":"measured","note":"median of 5 fresh processes","q1":17.257512822256132,"q3":17.90969294083765,"n":5},"cpu_cores_busy_batch":{"value":3.5482269309547316,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.495635393329438,"q3":3.582283615394513,"n":5},"cpu_seconds_per_mb":{"value":0.2025573110872209,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.20001926483207288,"q3":0.2029137581657972,"n":5},"latency_p95_us_256_bytes":{"value":63.54199967972818,"provenance":"measured","note":"median of 5 fresh processes","q1":58.37499975314131,"q3":67.50000000010914,"n":5},"decode_mb_per_s":{"value":46.85866863218416,"provenance":"measured","note":"median of 5 fresh processes","q1":46.77488202423642,"q3":47.64206157670607,"n":5},"load_seconds":{"value":0.1078596249999464,"provenance":"measured","note":"median of 5 fresh processes","q1":0.10639008299995112,"q3":0.11712995800007776,"n":5},"load_ram_mb":{"value":45.416448,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":45.416448,"q3":45.416448,"n":5},"peak_ram_mb_1_mib_input":{"value":174.1824,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":174.174208,"q3":174.211072,"n":5},"peak_ram_mb_16_mib_input":{"value":2381.4144,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2381.316096,"q3":2381.471744,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.747968,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.6128,"q3":18.276352,"n":5},"ram_mb_per_input_mb":{"value":139.429443359375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":139.414794921875,"q3":139.46240234375,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":39.1210378725301,"provenance":"measured","note":"median of 5 fresh processes","q1":38.182779392110085,"q3":41.60118715472255,"n":5},"batch_mb_per_s_4_threads":{"value":102.9317037703785,"provenance":"measured","note":"median of 5 fresh processes","q1":98.31125080890791,"q3":109.62298731222458,"n":5},"cpu_seconds_per_mb":{"value":0.028730349811679108,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.027497448244241825,"q3":0.03031946003155331,"n":5},"latency_p95_us_256_bytes":{"value":13.54100004391512,"provenance":"measured","note":"median of 5 fresh processes","q1":12.95900005970907,"q3":13.9579997266992,"n":5},"decode_mb_per_s":{"value":99.18681068335604,"provenance":"measured","note":"median of 5 fresh processes","q1":98.29075211288612,"q3":102.35190719021892,"n":5},"load_seconds":{"value":0.15286491599999863,"provenance":"measured","note":"median of 5 fresh processes","q1":0.1514836250000826,"q3":0.15466254200009644,"n":5},"load_ram_mb":{"value":30.691328,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":30.552064,"q3":30.818304,"n":5},"peak_ram_mb_1_mib_input":{"value":32.530432,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":32.247808,"q3":33.243136,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.424384,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.088512,"q3":17.465344,"n":5}}}},{"rank":11,"name":"Mistral-Large-4-1T-A52B","provider":"Mistral","family":"Published","version":null,"vocab":204800,"models":["Mistral-Large-4-1T-A52B"],"overall":4.5844,"english":4.6092,"code_avg":3.7268,"languages_avg":5.4172,"bpt":{"code_asm":2.3895,"code_cs":4.65,"code_dart":4.0612,"code_go":3.2879,"code_html":3.54,"code_kotlin":4.3828,"code_lua":3.4043,"code_php":3.8332,"code_ps":3.6873,"code_r":3.0086,"code_ruby":3.9526,"code_shell":3.2475,"code_sql":3.2595,"code_swift":4.3209,"code_ts":4.1408,"code_c":3.1843,"code_cpp":3.6843,"code_java":4.3383,"code_js":3.8424,"code_py":4.0356,"code_rust":4.0126,"en_fineweb":4.6092,"wiki_ar":5.9766,"wiki_bn":8.2445,"wiki_de":3.8499,"wiki_es":4.4476,"wiki_fr":4.2279,"wiki_gu":6.3397,"wiki_hi":7.5487,"wiki_id":3.9472,"wiki_ja":3.5443,"wiki_ko":3.6808,"wiki_mr":7.7267,"wiki_ru":6.1905,"wiki_ta":8.0813,"wiki_te":6.5096,"wiki_th":5.4595,"wiki_tr":3.9612,"wiki_ur":5.4653,"wiki_vi":4.3801,"wiki_zh":3.3461},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":5.069619965517145,"provenance":"measured","note":"median of 5 fresh processes","q1":4.944411374013184,"q3":5.130161389281384,"n":5},"batch_mb_per_s_4_threads":{"value":16.036505735404884,"provenance":"measured","note":"median of 5 fresh processes","q1":15.577337956412958,"q3":17.061922351465657,"n":5},"cpu_cores_busy_batch":{"value":3.5718486431320344,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5684167557161284,"q3":3.5754822186916937,"n":5},"cpu_seconds_per_mb":{"value":0.2225183474875011,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.21084858008889,"q3":0.22929775633849928,"n":5},"latency_p95_us_256_bytes":{"value":66.95899992337218,"provenance":"measured","note":"median of 5 fresh processes","q1":66.16700011363719,"q3":67.08299997626455,"n":5},"decode_mb_per_s":{"value":38.47395577739818,"provenance":"measured","note":"median of 5 fresh processes","q1":37.69634192986763,"q3":38.718842024315094,"n":5},"load_seconds":{"value":0.5272523330000354,"provenance":"measured","note":"median of 5 fresh processes","q1":0.5243418330001077,"q3":0.5360050419999425,"n":5},"load_ram_mb":{"value":236.31872,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":236.31872,"q3":236.326912,"n":5},"peak_ram_mb_1_mib_input":{"value":290.353152,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":289.792,"q3":290.783232,"n":5},"peak_ram_mb_16_mib_input":{"value":2466.238464,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2465.366016,"q3":2467.115008,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0.004096,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0.004096,"q3":0.004096,"n":5},"ram_mb_per_input_mb":{"value":130.468017578125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":130.44677734375,"q3":130.48876953125,"n":5}}},{"rank":12,"name":"sarvam-105b","provider":"Sarvam","family":"Published","version":null,"vocab":262144,"models":["sarvam-105b"],"overall":4.4579,"english":4.6528,"code_avg":3.498,"languages_avg":5.2228,"bpt":{"code_asm":2.1172,"code_cs":4.4913,"code_dart":3.804,"code_go":3.1836,"code_html":3.2034,"code_kotlin":4.0479,"code_lua":3.191,"code_php":3.791,"code_ps":3.6356,"code_r":2.6957,"code_ruby":3.4573,"code_shell":3.0447,"code_sql":2.8651,"code_swift":4.192,"code_ts":4.0291,"code_c":3.0788,"code_cpp":3.4607,"code_java":4.1883,"code_js":3.6863,"code_py":3.5798,"code_rust":3.7147,"en_fineweb":4.6528,"wiki_ar":5.2854,"wiki_bn":11.3715,"wiki_de":3.3911,"wiki_es":4.2948,"wiki_fr":3.9921,"wiki_gu":7.4592,"wiki_hi":9.0557,"wiki_id":4.33,"wiki_ja":1.5324,"wiki_ko":1.0258,"wiki_mr":8.8618,"wiki_ru":5.9752,"wiki_ta":9.4067,"wiki_te":7.5686,"wiki_th":1.0133,"wiki_tr":3.1822,"wiki_ur":5.59,"wiki_vi":1.7293,"wiki_zh":4.1683},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"byte_fallback","detail":"test sets where the tokenizer falls back to (nearly) single bytes","sets":[{"set":"wiki_ja","bytes_per_token":{"value":1.5324050486159628,"provenance":"measured"},"split_byte_token_share":{"value":0.7932898682929954,"provenance":"measured"}},{"set":"wiki_ko","bytes_per_token":{"value":1.02577993283464,"provenance":"measured"},"split_byte_token_share":{"value":0.8614514203773997,"provenance":"measured"}},{"set":"wiki_th","bytes_per_token":{"value":1.0132615114516,"provenance":"measured"},"split_byte_token_share":{"value":0.9758997685739368,"provenance":"measured"}},{"set":"wiki_vi","bytes_per_token":{"value":1.7293469891097306,"provenance":"measured"},"split_byte_token_share":{"value":0.5074975538868594,"provenance":"measured"}}]}],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":5.616660252990121,"provenance":"measured","note":"median of 5 fresh processes","q1":5.597913809617128,"q3":5.63931486359084,"n":5},"batch_mb_per_s_4_threads":{"value":17.953576871051627,"provenance":"measured","note":"median of 5 fresh processes","q1":17.901230339315713,"q3":18.587922055381327,"n":5},"cpu_cores_busy_batch":{"value":3.4716281983425694,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.453077630209263,"q3":3.4730279266476978,"n":5},"cpu_seconds_per_mb":{"value":0.1934449024610585,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.1857699650300381,"q3":0.19419499109597585,"n":5},"latency_p95_us_256_bytes":{"value":60.2499999331485,"provenance":"measured","note":"median of 5 fresh processes","q1":60.04200008646876,"q3":67.29200003974256,"n":5},"decode_mb_per_s":{"value":17.318860027422122,"provenance":"measured","note":"median of 5 fresh processes","q1":15.063022196315604,"q3":17.415340016447725,"n":5},"load_seconds":{"value":0.693015374999959,"provenance":"measured","note":"median of 5 fresh processes","q1":0.6841009579998172,"q3":0.7270837920000304,"n":5},"load_ram_mb":{"value":312.578048,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":312.578048,"q3":312.578048,"n":5},"peak_ram_mb_1_mib_input":{"value":372.129792,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":372.125696,"q3":377.229312,"n":5},"peak_ram_mb_16_mib_input":{"value":2000.072704,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":1993.474048,"q3":2003.259392,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0.012288,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0.012288,"q3":0.012288,"n":5},"ram_mb_per_input_mb":{"value":97.684326171875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":97.282958984375,"q3":97.874267578125,"n":5}}},{"rank":13,"name":"Akshara 59k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":59000,"models":["Akshara 59k"],"overall":4.3191,"english":4.3932,"code_avg":3.6578,"languages_avg":4.9061,"bpt":{"code_asm":2.4456,"code_cs":4.5497,"code_dart":3.9258,"code_go":3.3736,"code_html":3.5086,"code_kotlin":4.2174,"code_lua":3.4023,"code_php":3.6761,"code_ps":3.7157,"code_r":2.8623,"code_ruby":3.8347,"code_shell":3.1788,"code_sql":3.1189,"code_swift":4.3075,"code_ts":3.8791,"code_c":3.2484,"code_cpp":3.6347,"code_java":4.26,"code_js":3.7572,"code_py":3.9138,"code_rust":4.0043,"en_fineweb":4.3932,"wiki_ar":4.9483,"wiki_bn":7.1818,"wiki_de":3.2094,"wiki_es":3.7634,"wiki_fr":3.5937,"wiki_gu":5.5904,"wiki_hi":6.3142,"wiki_id":3.5653,"wiki_ja":4.3486,"wiki_ko":3.2803,"wiki_mr":6.3575,"wiki_ru":5.1229,"wiki_ta":6.9774,"wiki_te":5.7668,"wiki_th":7.3413,"wiki_tr":3.325,"wiki_ur":4.5851,"wiki_vi":4.1603,"wiki_zh":3.7847},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":6.198963235580143,"provenance":"measured","note":"median of 5 fresh processes","q1":6.142785722277156,"q3":6.2555780795380285,"n":5},"batch_mb_per_s_4_threads":{"value":17.667729432326173,"provenance":"measured","note":"median of 5 fresh processes","q1":17.166497951509918,"q3":17.810078165382766,"n":5},"cpu_cores_busy_batch":{"value":3.5754968188117333,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5660106350776615,"q3":3.5764043900531157,"n":5},"cpu_seconds_per_mb":{"value":0.202374438238212,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.20022431131205665,"q3":0.20833628385680988,"n":5},"latency_p95_us_256_bytes":{"value":58.165999689663295,"provenance":"measured","note":"median of 5 fresh processes","q1":57.45899989051395,"q3":58.5830000545684,"n":5},"decode_mb_per_s":{"value":48.26985164104428,"provenance":"measured","note":"median of 5 fresh processes","q1":47.21170933430155,"q3":49.15305904263578,"n":5},"load_seconds":{"value":0.06612683299999844,"provenance":"measured","note":"median of 5 fresh processes","q1":0.06587079200016888,"q3":0.06826133300000947,"n":5},"load_ram_mb":{"value":38.756352,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":38.756352,"q3":38.756352,"n":5},"peak_ram_mb_1_mib_input":{"value":172.630016,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":172.593152,"q3":172.6464,"n":5},"peak_ram_mb_16_mib_input":{"value":2407.292928,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2406.936576,"q3":2407.387136,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.341888,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.686528,"q3":18.604032,"n":5},"ram_mb_per_input_mb":{"value":141.488037109375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":141.466796875,"q3":141.49389648437503,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":43.147832703931286,"provenance":"measured","note":"median of 5 fresh processes","q1":41.94876540731698,"q3":43.53332102375039,"n":5},"batch_mb_per_s_4_threads":{"value":111.908222349312,"provenance":"measured","note":"median of 5 fresh processes","q1":111.18645773404668,"q3":113.30136477754456,"n":5},"cpu_seconds_per_mb":{"value":0.025885925967253132,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.025822981466420893,"q3":0.026422384781164147,"n":5},"latency_p95_us_256_bytes":{"value":11.792000009336334,"provenance":"measured","note":"median of 5 fresh processes","q1":11.416999996072263,"q3":12.166999994178695,"n":5},"decode_mb_per_s":{"value":97.06023132026148,"provenance":"measured","note":"median of 5 fresh processes","q1":96.48454810324738,"q3":98.91740955818553,"n":5},"load_seconds":{"value":0.11027433399999607,"provenance":"measured","note":"median of 5 fresh processes","q1":0.10760424999989482,"q3":0.11044145899995784,"n":5},"load_ram_mb":{"value":25.432064,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":25.432064,"q3":25.43616,"n":5},"peak_ram_mb_1_mib_input":{"value":29.229056,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":28.823552,"q3":29.241344,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.260544,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.199104,"q3":17.809408,"n":5}}}},{"rank":14,"name":"Mistral-Nemo-Base-2407","provider":"Mistral","family":"Published","version":null,"vocab":131072,"models":["Mistral-Nemo-Base-2407"],"overall":4.312,"english":4.5515,"code_avg":3.5793,"languages_avg":4.8052,"bpt":{"code_c":3.107,"code_cpp":3.627,"code_java":4.237,"code_js":3.7661,"code_rust":3.8825,"wiki_bn":6.4833,"wiki_gu":4.141,"wiki_id":3.8351,"wiki_mr":5.8503,"wiki_ta":6.6578,"wiki_te":5.3743,"wiki_th":5.6131,"wiki_tr":3.579,"wiki_ur":4.8733,"wiki_vi":4.2809,"code_py":3.874,"en_fineweb":4.5515,"wiki_ar":6.0871,"wiki_de":3.3112,"wiki_es":4.1982,"wiki_fr":4.1348,"wiki_hi":6.6198,"wiki_ja":3.6897,"wiki_ko":3.8007,"wiki_ru":5.6128,"wiki_zh":3.1557,"code_asm":2.121,"code_cs":4.5494,"code_dart":3.9461,"code_go":3.287,"code_html":3.3108,"code_kotlin":4.198,"code_lua":3.3324,"code_php":3.7443,"code_ps":3.5069,"code_r":2.7891,"code_ruby":3.82,"code_shell":3.0491,"code_sql":2.8207,"code_swift":4.2053,"code_ts":3.9918},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":5.466480170437732,"provenance":"measured","note":"median of 5 fresh processes","q1":5.3116203562655855,"q3":5.517293344807472,"n":5},"batch_mb_per_s_4_threads":{"value":17.23201412522955,"provenance":"measured","note":"median of 5 fresh processes","q1":16.968407182053173,"q3":17.504269329655944,"n":5},"cpu_cores_busy_batch":{"value":3.5563506080525533,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.542723693726835,"q3":3.5791808168173396,"n":5},"cpu_seconds_per_mb":{"value":0.20493418127205593,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2048500168448068,"q3":0.20958659053243175,"n":5},"latency_p95_us_256_bytes":{"value":68.7090000610624,"provenance":"measured","note":"median of 5 fresh processes","q1":68.37499995526741,"q3":69.25000002411252,"n":5},"decode_mb_per_s":{"value":42.946988438882826,"provenance":"measured","note":"median of 5 fresh processes","q1":42.55410848886436,"q3":43.65821985876021,"n":5},"load_seconds":{"value":0.20511329200007825,"provenance":"measured","note":"median of 5 fresh processes","q1":0.19820354200010115,"q3":0.20675441699995645,"n":5},"load_ram_mb":{"value":79.0528,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":79.048704,"q3":79.0528,"n":5},"peak_ram_mb_1_mib_input":{"value":194.605056,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":194.51904,"q3":194.658304,"n":5},"peak_ram_mb_16_mib_input":{"value":2571.370496,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2570.027008,"q3":2572.787712,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.547264,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.53088,"q3":17.887232,"n":5},"ram_mb_per_input_mb":{"value":149.272705078125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":149.192626953125,"q3":149.357421875,"n":5}}},{"rank":15,"name":"LFM3-1.7B","provider":"Liquid AI","family":"Published","version":null,"vocab":125033,"models":["LFM3-1.7B"],"overall":4.2133,"english":4.6284,"code_avg":3.5937,"languages_avg":4.4179,"bpt":{"code_asm":2.2993,"code_cs":4.5342,"code_dart":3.8428,"code_go":3.2393,"code_html":3.366,"code_kotlin":4.1536,"code_lua":3.2977,"code_php":3.7026,"code_ps":3.5754,"code_r":2.8605,"code_ruby":3.8119,"code_shell":3.0947,"code_sql":3.2057,"code_swift":4.1655,"code_ts":3.9681,"code_c":3.1437,"code_cpp":3.5369,"code_java":4.2395,"code_js":3.7408,"code_py":3.8632,"code_rust":3.8262,"en_fineweb":4.6284,"wiki_ar":5.7951,"wiki_bn":4.4546,"wiki_de":3.6351,"wiki_es":4.0772,"wiki_fr":3.9882,"wiki_gu":4.3028,"wiki_hi":4.8127,"wiki_id":3.7877,"wiki_ja":4.2088,"wiki_ko":3.7061,"wiki_mr":4.6788,"wiki_ru":5.5291,"wiki_ta":4.7605,"wiki_te":4.222,"wiki_th":6.3697,"wiki_tr":3.4805,"wiki_ur":4.157,"wiki_vi":4.3881,"wiki_zh":3.5853},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.983365371128134,"provenance":"measured","note":"median of 5 fresh processes","q1":4.806476507486226,"q3":5.34534948371401,"n":5},"batch_mb_per_s_4_threads":{"value":16.76986979818239,"provenance":"measured","note":"median of 5 fresh processes","q1":16.38202464429069,"q3":16.853742072768885,"n":5},"cpu_cores_busy_batch":{"value":3.579015847412648,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.55829315949383,"q3":3.5883161772007197,"n":5},"cpu_seconds_per_mb":{"value":0.2140475436027763,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.21235734069974735,"q3":0.2172071668036425,"n":5},"latency_p95_us_256_bytes":{"value":64.41699997594696,"provenance":"measured","note":"median of 5 fresh processes","q1":64.37499996536644,"q3":66.83300011900428,"n":5},"decode_mb_per_s":{"value":37.9316281183514,"provenance":"measured","note":"median of 5 fresh processes","q1":37.66446508870162,"q3":39.84568250166923,"n":5},"load_seconds":{"value":0.3344739170001958,"provenance":"measured","note":"median of 5 fresh processes","q1":0.32873833299981925,"q3":0.3375459169999431,"n":5},"load_ram_mb":{"value":168.439808,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":168.439808,"q3":168.443904,"n":5},"peak_ram_mb_1_mib_input":{"value":257.810432,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":256.344064,"q3":257.859584,"n":5},"peak_ram_mb_16_mib_input":{"value":2572.292096,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2570.715136,"q3":2572.816384,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0.012288,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0.012288,"q3":0.012288,"n":5},"ram_mb_per_input_mb":{"value":140.4375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":140.343505859375,"q3":140.46875,"n":5}}},{"rank":16,"name":"Akshara 49k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":49000,"models":["Akshara 49k"],"overall":4.2021,"english":4.3059,"code_avg":3.5977,"languages_avg":4.7026,"bpt":{"code_asm":2.3997,"code_cs":4.4746,"code_dart":3.8832,"code_go":3.3177,"code_html":3.4267,"code_kotlin":4.156,"code_lua":3.3505,"code_php":3.6275,"code_ps":3.6499,"code_r":2.8203,"code_ruby":3.7715,"code_shell":3.1129,"code_sql":3.0181,"code_swift":4.2315,"code_ts":3.8396,"code_c":3.1982,"code_cpp":3.5843,"code_java":4.1953,"code_js":3.7046,"code_py":3.8455,"code_rust":3.9431,"en_fineweb":4.3059,"wiki_ar":4.756,"wiki_bn":6.7828,"wiki_de":3.1212,"wiki_es":3.6591,"wiki_fr":3.5037,"wiki_gu":5.3299,"wiki_hi":6.0899,"wiki_id":3.4265,"wiki_ja":4.1837,"wiki_ko":3.1667,"wiki_mr":6.085,"wiki_ru":4.9021,"wiki_ta":6.637,"wiki_te":5.5118,"wiki_th":6.9616,"wiki_tr":3.1982,"wiki_ur":4.3576,"wiki_vi":4.0038,"wiki_zh":3.6732},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":6.273736117150162,"provenance":"measured","note":"median of 5 fresh processes","q1":6.253023908066624,"q3":6.294651485427605,"n":5},"batch_mb_per_s_4_threads":{"value":17.597606427027582,"provenance":"measured","note":"median of 5 fresh processes","q1":17.457215779760034,"q3":17.756775096730202,"n":5},"cpu_cores_busy_batch":{"value":3.561898862847338,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5331112483489044,"q3":3.562512854349198,"n":5},"cpu_seconds_per_mb":{"value":0.20279120713008603,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.1989725740795976,"q3":0.2040710786470079,"n":5},"latency_p95_us_256_bytes":{"value":58.45900022904971,"provenance":"measured","note":"median of 5 fresh processes","q1":57.16700025004684,"q3":58.91699993298971,"n":5},"decode_mb_per_s":{"value":49.309272169518415,"provenance":"measured","note":"median of 5 fresh processes","q1":49.11152353280939,"q3":49.31400755671072,"n":5},"load_seconds":{"value":0.05253670900037832,"provenance":"measured","note":"median of 5 fresh processes","q1":0.05247358399992663,"q3":0.054333542000222224,"n":5},"load_ram_mb":{"value":33.431552,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":33.427456,"q3":33.431552,"n":5},"peak_ram_mb_1_mib_input":{"value":178.335744,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":178.065408,"q3":178.495488,"n":5},"peak_ram_mb_16_mib_input":{"value":2466.205696,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2465.374208,"q3":2466.316288,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.555456,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.145856,"q3":17.8176,"n":5},"ram_mb_per_input_mb":{"value":145.688720703125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":145.63916015625,"q3":145.6953125,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":39.730417183112266,"provenance":"measured","note":"median of 5 fresh processes","q1":37.78823476444552,"q3":40.083570953849474,"n":5},"batch_mb_per_s_4_threads":{"value":104.66337423945856,"provenance":"measured","note":"median of 5 fresh processes","q1":103.02778553968233,"q3":104.76936728546434,"n":5},"cpu_seconds_per_mb":{"value":0.02859921543494531,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.027978592193785127,"q3":0.02908727374253457,"n":5},"latency_p95_us_256_bytes":{"value":12.834000244765775,"provenance":"measured","note":"median of 5 fresh processes","q1":12.792000234185252,"q3":13.958999943497474,"n":5},"decode_mb_per_s":{"value":95.15622305737604,"provenance":"measured","note":"median of 5 fresh processes","q1":94.86281507110628,"q3":96.38846820944634,"n":5},"load_seconds":{"value":0.09296616699975856,"provenance":"measured","note":"median of 5 fresh processes","q1":0.09286824999992405,"q3":0.09580345900008068,"n":5},"load_ram_mb":{"value":20.840448,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":20.836352,"q3":20.844544,"n":5},"peak_ram_mb_1_mib_input":{"value":22.052864,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":21.87264,"q3":23.216128,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.854464,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.05984,"q3":18.468864,"n":5}}}},{"rank":17,"name":"DeepSeek-V3","provider":"DeepSeek","family":"Published","version":null,"vocab":128815,"models":["DeepSeek-V3"],"overall":4.2016,"english":4.7317,"code_avg":3.6104,"languages_avg":4.2627,"bpt":{"code_c":3.1622,"code_cpp":3.5698,"code_java":4.2585,"code_js":3.7575,"code_rust":3.8342,"wiki_bn":6.2712,"wiki_gu":3.0665,"wiki_id":3.8642,"wiki_mr":4.313,"wiki_ta":4.9821,"wiki_te":3.4581,"wiki_th":7.595,"wiki_tr":2.6915,"wiki_ur":3.7603,"wiki_vi":3.0334,"code_py":3.8782,"en_fineweb":4.7317,"wiki_ar":4.8951,"wiki_de":3.3487,"wiki_es":3.9035,"wiki_fr":3.7273,"wiki_hi":4.5597,"wiki_ja":4.0931,"wiki_ko":3.2896,"wiki_ru":5.5698,"wiki_zh":4.5686,"code_asm":2.342,"code_cs":4.5803,"code_dart":3.9268,"code_go":3.2677,"code_html":3.4194,"code_kotlin":4.1748,"code_lua":3.3283,"code_php":3.7088,"code_ps":3.5767,"code_r":2.9141,"code_ruby":3.8218,"code_shell":3.1026,"code_sql":3.1634,"code_swift":4.2276,"code_ts":3.8044},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.593484509872189,"provenance":"measured","note":"median of 5 fresh processes","q1":4.561213481913486,"q3":4.6270812985391325,"n":5},"batch_mb_per_s_4_threads":{"value":14.352636211190546,"provenance":"measured","note":"median of 5 fresh processes","q1":14.279768807509049,"q3":14.46610265055963,"n":5},"cpu_cores_busy_batch":{"value":3.6064276215178497,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.596928755710072,"q3":3.609650436190768,"n":5},"cpu_seconds_per_mb":{"value":0.24990969609965827,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2493231677964489,"q3":0.2519885336100984,"n":5},"latency_p95_us_256_bytes":{"value":89.08300003440672,"provenance":"measured","note":"median of 5 fresh processes","q1":87.54200007388135,"q3":93.16700015915558,"n":5},"decode_mb_per_s":{"value":40.39197598096321,"provenance":"measured","note":"median of 5 fresh processes","q1":40.18262663395477,"q3":40.96878937743569,"n":5},"load_seconds":{"value":0.14764133300013782,"provenance":"measured","note":"median of 5 fresh processes","q1":0.14453125,"q3":0.15512591700002076,"n":5},"load_ram_mb":{"value":83.18976,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":83.18976,"q3":83.18976,"n":5},"peak_ram_mb_1_mib_input":{"value":199.45472,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":198.688768,"q3":199.745536,"n":5},"peak_ram_mb_16_mib_input":{"value":2510.675968,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2510.630912,"q3":2511.2576,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.928192,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.748544,"q3":18.108416,"n":5},"ram_mb_per_input_mb":{"value":144.1591796875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":144.15673828125003,"q3":144.19384765625,"n":5}}},{"rank":18,"name":"Falcon-H1-34B-Instruct","provider":"TII (Falcon)","family":"Published","version":null,"vocab":261120,"models":["Falcon-H1-34B-Instruct"],"overall":4.1338,"english":4.5315,"code_avg":3.2498,"languages_avg":4.62,"bpt":{"code_asm":2.1036,"code_cs":4.1522,"code_dart":3.4964,"code_go":3.0099,"code_html":2.9713,"code_kotlin":3.7207,"code_lua":2.9994,"code_php":3.39,"code_ps":3.2959,"code_r":2.5486,"code_ruby":3.3226,"code_shell":2.7724,"code_sql":2.6337,"code_swift":3.8117,"code_ts":3.8973,"code_c":2.8604,"code_cpp":3.2499,"code_java":3.8307,"code_js":3.3071,"code_py":3.37,"code_rust":3.5021,"en_fineweb":4.5315,"wiki_ar":6.7345,"wiki_bn":6.8565,"wiki_de":3.0596,"wiki_es":3.9609,"wiki_fr":3.5863,"wiki_gu":4.6939,"wiki_hi":6.7824,"wiki_id":4.3669,"wiki_ja":4.7022,"wiki_ko":4.1803,"wiki_mr":7.1155,"wiki_ru":6.4765,"wiki_ta":2.1021,"wiki_te":1.9796,"wiki_th":5.1316,"wiki_tr":3.3283,"wiki_ur":5.4215,"wiki_vi":3.5607,"wiki_zh":3.7411},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":2.765068889391423,"provenance":"measured","note":"median of 5 fresh processes","q1":2.7389083970847636,"q3":2.765128679894381,"n":5},"batch_mb_per_s_4_threads":{"value":10.159878525556868,"provenance":"measured","note":"median of 5 fresh processes","q1":10.050722230334545,"q3":10.360540892551512,"n":5},"cpu_cores_busy_batch":{"value":3.6838644674690686,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.6536249873171274,"q3":3.690406076755664,"n":5},"cpu_seconds_per_mb":{"value":0.3633895709068609,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.3555668092693407,"q3":0.3645166112683531,"n":5},"latency_p95_us_256_bytes":{"value":124.2079999883572,"provenance":"measured","note":"median of 5 fresh processes","q1":123.87500009936048,"q3":124.4170000518352,"n":5},"decode_mb_per_s":{"value":32.697688559727155,"provenance":"measured","note":"median of 5 fresh processes","q1":31.341726365224908,"q3":33.74930397972445,"n":5},"load_seconds":{"value":0.4538142919998336,"provenance":"measured","note":"median of 5 fresh processes","q1":0.447092499999826,"q3":0.4607368759999418,"n":5},"load_ram_mb":{"value":172.048384,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":172.048384,"q3":172.048384,"n":5},"peak_ram_mb_1_mib_input":{"value":288.256,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":287.137792,"q3":288.70656,"n":5},"peak_ram_mb_16_mib_input":{"value":2683.24864,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2683.097088,"q3":2683.600896,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0,"q3":0,"n":5},"ram_mb_per_input_mb":{"value":145.377685546875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":145.372314453125,"q3":145.402099609375,"n":5}}},{"rank":19,"name":"Llama-3.2-1B","provider":"Meta","family":"Published","version":null,"vocab":128256,"models":["Llama-3.2-1B"],"overall":4.0801,"english":4.6929,"code_avg":3.8907,"languages_avg":3.6568,"bpt":{"code_c":3.3897,"code_cpp":3.9067,"code_java":4.5829,"code_js":4.0385,"code_rust":4.1821,"wiki_bn":2.2129,"wiki_gu":1.5938,"wiki_id":3.3214,"wiki_mr":4.6552,"wiki_ta":2.0485,"wiki_te":1.574,"wiki_th":6.281,"wiki_tr":3.7997,"wiki_ur":2.9292,"wiki_vi":4.55,"code_py":4.2023,"en_fineweb":4.6929,"wiki_ar":4.7821,"wiki_de":3.151,"wiki_es":3.7222,"wiki_fr":3.5639,"wiki_hi":4.8864,"wiki_ja":3.8583,"wiki_ko":3.5839,"wiki_ru":5.4785,"wiki_zh":3.4868,"code_asm":2.4998,"code_cs":4.9216,"code_dart":4.2085,"code_go":3.5307,"code_html":3.5692,"code_kotlin":4.6348,"code_lua":3.6325,"code_php":3.9733,"code_ps":3.7698,"code_r":3.0753,"code_ruby":4.0797,"code_shell":3.395,"code_sql":3.385,"code_swift":4.6006,"code_ts":4.1277},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.979777107529496,"provenance":"measured","note":"median of 5 fresh processes","q1":4.972462633823275,"q3":4.98280743289368,"n":5},"batch_mb_per_s_4_threads":{"value":16.618196262248443,"provenance":"measured","note":"median of 5 fresh processes","q1":16.467326111026395,"q3":16.677035908702184,"n":5},"cpu_cores_busy_batch":{"value":3.5796053042140357,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.571813812405879,"q3":3.58504672428761,"n":5},"cpu_seconds_per_mb":{"value":0.21540275777978535,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2142971757708496,"q3":0.21770666956592852,"n":5},"latency_p95_us_256_bytes":{"value":67.41700008205953,"provenance":"measured","note":"median of 5 fresh processes","q1":66.62499981757719,"q3":67.95799993142282,"n":5},"decode_mb_per_s":{"value":39.064492844360885,"provenance":"measured","note":"median of 5 fresh processes","q1":38.48001422683172,"q3":39.0731060065189,"n":5},"load_seconds":{"value":0.3169502499999908,"provenance":"measured","note":"median of 5 fresh processes","q1":0.3169401669999843,"q3":0.3193151250000028,"n":5},"load_ram_mb":{"value":179.073024,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":179.068928,"q3":179.073024,"n":5},"peak_ram_mb_1_mib_input":{"value":239.906816,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":239.53408,"q3":240.164864,"n":5},"peak_ram_mb_16_mib_input":{"value":2636.156928,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2636.136448,"q3":2637.656064,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.997824,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.739776,"q3":18.235392,"n":5},"ram_mb_per_input_mb":{"value":144.03564453125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":144.034423828125,"q3":144.125,"n":5}}},{"rank":20,"name":"Kimi-K3","provider":"Moonshot AI (Kimi)","family":"Published","version":null,"vocab":163840,"models":["Kimi-K3"],"overall":4.0412,"english":4.6935,"code_avg":3.8373,"languages_avg":3.5927,"bpt":{"code_asm":2.518,"code_cs":4.7461,"code_dart":4.1605,"code_go":3.5246,"code_html":3.5895,"code_kotlin":4.4853,"code_lua":3.5871,"code_php":3.8735,"code_ps":3.8056,"code_r":3.072,"code_ruby":4.0359,"code_shell":3.4054,"code_sql":3.4106,"code_swift":4.3837,"code_ts":4.0224,"code_c":3.36,"code_cpp":3.8594,"code_java":4.474,"code_js":3.9608,"code_py":4.1318,"code_rust":4.1777,"en_fineweb":4.6935,"wiki_ar":3.8315,"wiki_bn":4.3159,"wiki_de":3.052,"wiki_es":3.4709,"wiki_fr":3.3683,"wiki_gu":3.1929,"wiki_hi":4.6792,"wiki_id":3.3277,"wiki_ja":3.6335,"wiki_ko":2.7817,"wiki_mr":4.5215,"wiki_ru":4.495,"wiki_ta":3.7175,"wiki_te":3.299,"wiki_th":3.6764,"wiki_tr":2.8013,"wiki_ur":4.0208,"wiki_vi":2.3595,"wiki_zh":3.716},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (converted from tiktoken)","encode_mb_per_s_1_thread":{"value":4.938964989355018,"provenance":"measured","note":"median of 5 fresh processes","q1":4.767492476666688,"q3":4.9664928780057656,"n":5},"batch_mb_per_s_4_threads":{"value":16.933337146737262,"provenance":"measured","note":"median of 5 fresh processes","q1":16.351862609405714,"q3":17.09634579773465,"n":5},"cpu_cores_busy_batch":{"value":3.5651845065201857,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5542644567964254,"q3":3.5686436916357844,"n":5},"cpu_seconds_per_mb":{"value":0.21131684902689948,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.20789614920326327,"q3":0.2180292601326937,"n":5},"latency_p95_us_256_bytes":{"value":64.8340001134784,"provenance":"measured","note":"median of 5 fresh processes","q1":63.41700009215856,"q3":71.45800009311642,"n":5},"decode_mb_per_s":{"value":36.614670171051586,"provenance":"measured","note":"median of 5 fresh processes","q1":35.2603091461058,"q3":37.82018196408981,"n":5},"load_seconds":{"value":0.4387297920002311,"provenance":"measured","note":"median of 5 fresh processes","q1":0.4382846260000406,"q3":0.441998125000282,"n":5},"load_ram_mb":{"value":188.178432,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":188.174336,"q3":188.178432,"n":5},"peak_ram_mb_1_mib_input":{"value":260.07552,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":260.07552,"q3":260.07552,"n":5},"peak_ram_mb_16_mib_input":{"value":2572.242944,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2571.79648,"q3":2572.603392,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0,"q3":0,"n":5},"ram_mb_per_input_mb":{"value":138.894287109375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":138.86767578125,"q3":138.91552734375,"n":5},"native":{"implementation":"tiktoken","encode_mb_per_s_1_thread":{"value":24.208677865641175,"provenance":"measured","note":"median of 5 fresh processes","q1":23.10022007742137,"q3":24.27460189874134,"n":5},"batch_mb_per_s_4_threads":{"value":67.95394005965954,"provenance":"measured","note":"median of 5 fresh processes","q1":66.90219298158598,"q3":68.00159234623783,"n":5},"cpu_seconds_per_mb":{"value":0.057708901234594386,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.05748335010661224,"q3":0.05772177624612826,"n":5},"latency_p95_us_256_bytes":{"value":18.459000102666323,"provenance":"measured","note":"median of 5 fresh processes","q1":17.917000150191598,"q3":19.083000097452896,"n":5},"decode_mb_per_s":{"value":188.0990581543668,"provenance":"measured","note":"median of 5 fresh processes","q1":178.80151925666618,"q3":188.40322955806735,"n":5},"load_seconds":{"value":0.0925006669999675,"provenance":"measured","note":"median of 5 fresh processes","q1":0.09022929200000362,"q3":0.09309383400000117,"n":5},"load_ram_mb":{"value":61.571072,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":61.571072,"q3":61.571072,"n":5},"peak_ram_mb_1_mib_input":{"value":55.697408,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":55.697408,"q3":55.709696,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.780736,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.477632,"q3":18.563072,"n":5}}}},{"rank":21,"name":"Hy3","provider":"Tencent","family":"Published","version":null,"vocab":120818,"models":["Hy3"],"overall":3.9654,"english":4.7065,"code_avg":3.6956,"languages_avg":3.4941,"bpt":{"code_asm":2.399,"code_cs":4.6273,"code_dart":4.0179,"code_go":3.3503,"code_html":3.458,"code_kotlin":4.2992,"code_lua":3.3715,"code_php":3.7478,"code_ps":3.6364,"code_r":2.9758,"code_ruby":3.8976,"code_shell":3.2042,"code_sql":3.2688,"code_swift":4.2931,"code_ts":3.911,"code_c":3.243,"code_cpp":3.6957,"code_java":4.3751,"code_js":3.8525,"code_py":4.0026,"code_rust":3.9806,"en_fineweb":4.7065,"wiki_ar":3.9333,"wiki_bn":3.2625,"wiki_de":3.0657,"wiki_es":3.4793,"wiki_fr":3.4432,"wiki_gu":2.3424,"wiki_hi":3.6924,"wiki_id":3.3625,"wiki_ja":3.7223,"wiki_ko":3.1227,"wiki_mr":3.461,"wiki_ru":4.3692,"wiki_ta":4.0135,"wiki_te":2.4124,"wiki_th":5.1233,"wiki_tr":3.0208,"wiki_ur":2.9709,"wiki_vi":3.5783,"wiki_zh":4.0131},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.135631116558927,"provenance":"measured","note":"median of 5 fresh processes","q1":4.12265591811356,"q3":4.220216549664736,"n":5},"batch_mb_per_s_4_threads":{"value":14.15198592956625,"provenance":"measured","note":"median of 5 fresh processes","q1":13.997522586239256,"q3":14.201477918105386,"n":5},"cpu_cores_busy_batch":{"value":3.605940285034379,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.586049779368823,"q3":3.621703444562264,"n":5},"cpu_seconds_per_mb":{"value":0.2556965369318513,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.25480100835182923,"q3":0.2561917480236261,"n":5},"latency_p95_us_256_bytes":{"value":89.50000005825132,"provenance":"measured","note":"median of 5 fresh processes","q1":88.20899984129937,"q3":94.00000004688992,"n":5},"decode_mb_per_s":{"value":35.86769057103556,"provenance":"measured","note":"median of 5 fresh processes","q1":35.409047448975976,"q3":36.776853590940746,"n":5},"load_seconds":{"value":0.1811083749998943,"provenance":"measured","note":"median of 5 fresh processes","q1":0.18062112500001604,"q3":0.18120783299991672,"n":5},"load_ram_mb":{"value":93.503488,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":93.503488,"q3":93.503488,"n":5},"peak_ram_mb_1_mib_input":{"value":199.356416,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":199.35232,"q3":199.417856,"n":5},"peak_ram_mb_16_mib_input":{"value":2540.376064,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2539.077632,"q3":2541.166592,"n":5},"peak_ram_mb_16_mib_chunked":{"value":16.986112,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.900096,"q3":17.637376,"n":5},"ram_mb_per_input_mb":{"value":143.6865234375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":143.66381835937503,"q3":143.7314453125,"n":5}}},{"rank":22,"name":"GLM-5.3-Flash","provider":"Z.ai (GLM)","family":"Published","version":null,"vocab":154856,"models":["GLM-5.3-Flash"],"overall":3.9579,"english":4.6855,"code_avg":3.8796,"languages_avg":3.3085,"bpt":{"code_asm":2.4551,"code_cs":4.9159,"code_dart":4.209,"code_go":3.5254,"code_html":3.5688,"code_kotlin":4.6219,"code_lua":3.6241,"code_php":3.9683,"code_ps":3.7614,"code_r":3.0574,"code_ruby":4.0664,"code_shell":3.3729,"code_sql":3.3195,"code_swift":4.5957,"code_ts":4.1289,"code_c":3.386,"code_cpp":3.9064,"code_java":4.5799,"code_js":4.0374,"code_py":4.195,"code_rust":4.1765,"en_fineweb":4.6855,"wiki_ar":4.4048,"wiki_bn":2.7218,"wiki_de":3.3603,"wiki_es":3.9361,"wiki_fr":3.7276,"wiki_gu":1.5935,"wiki_hi":2.8877,"wiki_id":3.3239,"wiki_ja":3.8218,"wiki_ko":2.8636,"wiki_mr":2.8584,"wiki_ru":5.9906,"wiki_ta":2.0479,"wiki_te":1.5735,"wiki_th":3.1579,"wiki_tr":3.2607,"wiki_ur":3.2102,"wiki_vi":4.292,"wiki_zh":3.829},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.882575809711876,"provenance":"measured","note":"median of 5 fresh processes","q1":4.783285403986078,"q3":5.046999136402947,"n":5},"batch_mb_per_s_4_threads":{"value":16.669548809795145,"provenance":"measured","note":"median of 5 fresh processes","q1":16.343995737264507,"q3":16.85284761351425,"n":5},"cpu_cores_busy_batch":{"value":3.5612438855466726,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5504032077437833,"q3":3.561737918526622,"n":5},"cpu_seconds_per_mb":{"value":0.2136376890689483,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2102932856099572,"q3":0.21722981728689644,"n":5},"latency_p95_us_256_bytes":{"value":65.16599978567683,"provenance":"measured","note":"median of 5 fresh processes","q1":64.16600001557526,"q3":66.0420000144768,"n":5},"decode_mb_per_s":{"value":34.27865346085365,"provenance":"measured","note":"median of 5 fresh processes","q1":34.14832150286263,"q3":36.42605225564783,"n":5},"load_seconds":{"value":0.38684520800006794,"provenance":"measured","note":"median of 5 fresh processes","q1":0.3731782919999205,"q3":0.3926069579999876,"n":5},"load_ram_mb":{"value":175.747072,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":175.747072,"q3":175.775744,"n":5},"peak_ram_mb_1_mib_input":{"value":268.546048,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":268.230656,"q3":268.648448,"n":5},"peak_ram_mb_16_mib_input":{"value":2628.591616,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2628.23936,"q3":2629.791744,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0.02048,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0.02048,"q3":0.02048,"n":5},"ram_mb_per_input_mb":{"value":142.818603515625,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":142.7998046875,"q3":142.95751953125,"n":5}}},{"rank":23,"name":"Akshara 32k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":32000,"models":["Akshara 32k"],"overall":3.9282,"english":4.0927,"code_avg":3.4521,"languages_avg":4.24,"bpt":{"code_asm":2.2915,"code_cs":4.2918,"code_dart":3.7421,"code_go":3.2019,"code_html":3.2486,"code_kotlin":3.9722,"code_lua":3.2236,"code_php":3.5017,"code_ps":3.4958,"code_r":2.7158,"code_ruby":3.6218,"code_shell":2.9773,"code_sql":2.8265,"code_swift":4.0742,"code_ts":3.7134,"code_c":3.0781,"code_cpp":3.4212,"code_java":4.046,"code_js":3.5688,"code_py":3.6943,"code_rust":3.7869,"en_fineweb":4.0927,"wiki_ar":4.3298,"wiki_bn":6.0157,"wiki_de":2.8959,"wiki_es":3.4142,"wiki_fr":3.2698,"wiki_gu":4.729,"wiki_hi":5.4514,"wiki_id":3.1179,"wiki_ja":3.8511,"wiki_ko":2.898,"wiki_mr":5.4347,"wiki_ru":4.4207,"wiki_ta":5.9043,"wiki_te":4.9143,"wiki_th":6.0852,"wiki_tr":2.9202,"wiki_ur":3.9457,"wiki_vi":3.5941,"wiki_zh":3.3674},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":6.303510437376889,"provenance":"measured","note":"median of 5 fresh processes","q1":6.114958198355796,"q3":6.381434276626128,"n":5},"batch_mb_per_s_4_threads":{"value":17.277981622498476,"provenance":"measured","note":"median of 5 fresh processes","q1":16.969863289683705,"q3":17.50034971107797,"n":5},"cpu_cores_busy_batch":{"value":3.531046994243594,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.521034285318306,"q3":3.5422166992960378,"n":5},"cpu_seconds_per_mb":{"value":0.2050133387503752,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2030878092476439,"q3":0.20746221362934417,"n":5},"latency_p95_us_256_bytes":{"value":57.66699996456737,"provenance":"measured","note":"median of 5 fresh processes","q1":57.45799990108935,"q3":58.54200003341248,"n":5},"decode_mb_per_s":{"value":46.35135997715321,"provenance":"measured","note":"median of 5 fresh processes","q1":45.87138545053736,"q3":47.085390975300854,"n":5},"load_seconds":{"value":0.03534070899991093,"provenance":"measured","note":"median of 5 fresh processes","q1":0.03528599999981452,"q3":0.0369946250000055,"n":5},"load_ram_mb":{"value":24.612864,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":24.612864,"q3":24.612864,"n":5},"peak_ram_mb_1_mib_input":{"value":176.607232,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":176.533504,"q3":176.771072,"n":5},"peak_ram_mb_16_mib_input":{"value":2486.562816,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2485.219328,"q3":2487.762944,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.046976,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.772544,"q3":18.419712,"n":5},"ram_mb_per_input_mb":{"value":147.15576171875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":147.07568359375,"q3":147.227294921875,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":42.53190270744731,"provenance":"measured","note":"median of 5 fresh processes","q1":41.40957775792447,"q3":42.55086535378701,"n":5},"batch_mb_per_s_4_threads":{"value":110.03894128296491,"provenance":"measured","note":"median of 5 fresh processes","q1":105.7101972536139,"q3":113.2780285046388,"n":5},"cpu_seconds_per_mb":{"value":0.02632153052414888,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.025935518604272456,"q3":0.027332695781836273,"n":5},"latency_p95_us_256_bytes":{"value":11.12500001454464,"provenance":"measured","note":"median of 5 fresh processes","q1":10.916999940491223,"q3":12.334000075497896,"n":5},"decode_mb_per_s":{"value":92.08316318702454,"provenance":"measured","note":"median of 5 fresh processes","q1":91.72270818743596,"q3":92.39182152658292,"n":5},"load_seconds":{"value":0.07983766699999251,"provenance":"measured","note":"median of 5 fresh processes","q1":0.0760976669998854,"q3":0.08004070800006957,"n":5},"load_ram_mb":{"value":18.190336,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":18.18624,"q3":18.190336,"n":5},"peak_ram_mb_1_mib_input":{"value":23.158784,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":23.150592,"q3":23.187456,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.149376,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.584128,"q3":18.407424,"n":5}}}},{"rank":24,"name":"ERNIE-4.5-21B-A3B-PT","provider":"Baidu","family":"Published","version":null,"vocab":101304,"models":["ERNIE-4.5-21B-A3B-PT"],"overall":3.8853,"english":4.4011,"code_avg":3.3734,"languages_avg":3.8814,"bpt":{"code_asm":2.155,"code_cs":4.2678,"code_dart":3.7009,"code_go":2.9665,"code_html":3.0619,"code_kotlin":3.9024,"code_lua":3.0118,"code_php":3.4634,"code_ps":3.535,"code_r":2.6334,"code_ruby":3.5069,"code_shell":2.9851,"code_sql":2.787,"code_swift":4.0467,"code_ts":3.8571,"code_c":2.8791,"code_cpp":3.4025,"code_java":4.0773,"code_js":3.5089,"code_py":3.4868,"code_rust":3.6067,"en_fineweb":4.4011,"wiki_ar":3.9523,"wiki_bn":4.8256,"wiki_de":2.8707,"wiki_es":3.5594,"wiki_fr":3.3614,"wiki_gu":3.7242,"wiki_hi":5.2307,"wiki_id":3.5464,"wiki_ja":3.33,"wiki_ko":2.4829,"wiki_mr":5.0162,"wiki_ru":4.4464,"wiki_ta":4.8868,"wiki_te":3.6725,"wiki_th":4.4248,"wiki_tr":3.1555,"wiki_ur":3.9206,"wiki_vi":3.4736,"wiki_zh":3.8661},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":6.455317587551054,"provenance":"measured","note":"median of 5 fresh processes","q1":6.135972573213207,"q3":6.509016394963281,"n":5},"batch_mb_per_s_4_threads":{"value":20.487115783013383,"provenance":"measured","note":"median of 5 fresh processes","q1":20.01462707042647,"q3":20.547415605708625,"n":5},"cpu_cores_busy_batch":{"value":3.4673331154399127,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.4499990535951177,"q3":3.5124164209491826,"n":5},"cpu_seconds_per_mb":{"value":0.17094200498739798,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.1705714907665901,"q3":0.17323995612005327,"n":5},"latency_p95_us_256_bytes":{"value":58.832999911828665,"provenance":"measured","note":"median of 5 fresh processes","q1":57.95800007035723,"q3":59.58400015515508,"n":5},"decode_mb_per_s":{"value":19.625329580720596,"provenance":"measured","note":"median of 5 fresh processes","q1":18.770204291667856,"q3":20.175473635126412,"n":5},"load_seconds":{"value":0.22060670800010485,"provenance":"measured","note":"median of 5 fresh processes","q1":0.2186067920001733,"q3":0.22601158299994495,"n":5},"load_ram_mb":{"value":108.675072,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":108.675072,"q3":108.675072,"n":5},"peak_ram_mb_1_mib_input":{"value":165.261312,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":164.163584,"q3":165.322752,"n":5},"peak_ram_mb_16_mib_input":{"value":1883.11552,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":1880.662016,"q3":1904.549888,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.698816,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.93696,"q3":17.92,"n":5},"ram_mb_per_input_mb":{"value":104.678466796875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":104.531982421875,"q3":105.847412109375,"n":5}}},{"rank":25,"name":"cl100k_base (GPT-4, OLMo-2-0425-1B, Olmo-3-1025-7B, phi-4)","provider":"OpenAI","family":"Published","version":null,"vocab":100277,"models":["OpenAI cl100k_base (GPT-4)","OLMo-2-0425-1B","Olmo-3-1025-7B","phi-4"],"overall":3.7359,"english":4.6914,"code_avg":3.8723,"languages_avg":2.644,"bpt":{"code_asm":2.4992,"code_cs":4.8988,"code_dart":4.2041,"code_go":3.5294,"code_html":3.533,"code_kotlin":4.6222,"code_lua":3.5982,"code_php":3.97,"code_ps":3.768,"code_r":3.0618,"code_ruby":4.0089,"code_shell":3.3917,"code_sql":3.3815,"code_swift":4.5891,"code_ts":3.9882,"code_c":3.3836,"code_cpp":3.9063,"code_java":4.5803,"code_js":4.0296,"code_py":4.1926,"code_rust":4.1818,"en_fineweb":4.6914,"wiki_ar":2.615,"wiki_bn":2.197,"wiki_de":3.1417,"wiki_es":3.7145,"wiki_fr":3.5579,"wiki_gu":1.5934,"wiki_hi":2.6125,"wiki_id":3.2902,"wiki_ja":2.6141,"wiki_ko":2.2853,"wiki_mr":2.5863,"wiki_ru":3.6319,"wiki_ta":2.0483,"wiki_te":1.574,"wiki_th":2.9863,"wiki_tr":2.8183,"wiki_ur":2.0264,"wiki_vi":2.6936,"wiki_zh":2.2487},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (converted from tiktoken)","encode_mb_per_s_1_thread":{"value":5.153221493593562,"provenance":"measured","note":"median of 5 fresh processes","q1":5.101702636960398,"q3":5.19896650664927,"n":5},"batch_mb_per_s_4_threads":{"value":16.422928599858064,"provenance":"measured","note":"median of 5 fresh processes","q1":16.343199716368368,"q3":16.71586612937975,"n":5},"cpu_cores_busy_batch":{"value":3.519135287307303,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5109858677711996,"q3":3.5221516695946185,"n":5},"cpu_seconds_per_mb":{"value":0.21258265340158997,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2100391233452332,"q3":0.2144655046253479,"n":5},"latency_p95_us_256_bytes":{"value":65.74999997610576,"provenance":"measured","note":"median of 5 fresh processes","q1":62.791000345896464,"q3":84.12500000076761,"n":5},"decode_mb_per_s":{"value":36.9393414973548,"provenance":"measured","note":"median of 5 fresh processes","q1":35.73547012610242,"q3":38.24051348435033,"n":5},"load_seconds":{"value":0.2929883750002773,"provenance":"measured","note":"median of 5 fresh processes","q1":0.2891960420001851,"q3":0.2957059999998819,"n":5},"load_ram_mb":{"value":160.456704,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":160.452608,"q3":160.456704,"n":5},"peak_ram_mb_1_mib_input":{"value":231.231488,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":231.231488,"q3":231.297024,"n":5},"peak_ram_mb_16_mib_input":{"value":2687.71328,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2687.438848,"q3":2688.016384,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.260544,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.047552,"q3":17.330176,"n":5},"ram_mb_per_input_mb":{"value":148.69970703125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":148.696044921875,"q3":148.7119140625,"n":5},"native":{"implementation":"tiktoken","encode_mb_per_s_1_thread":{"value":19.71889776783207,"provenance":"measured","note":"median of 5 fresh processes","q1":18.84703351082206,"q3":20.008905249603483,"n":5},"batch_mb_per_s_4_threads":{"value":46.55900518851056,"provenance":"measured","note":"median of 5 fresh processes","q1":43.79388111042827,"q3":51.848860675754,"n":5},"cpu_seconds_per_mb":{"value":0.08492953673085811,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.0763106702134939,"q3":0.09054924083925046,"n":5},"latency_p95_us_256_bytes":{"value":20.708999727503397,"provenance":"measured","note":"median of 5 fresh processes","q1":20.667000171670225,"q3":22.290999936558364,"n":5},"decode_mb_per_s":{"value":191.80648101361635,"provenance":"measured","note":"median of 5 fresh processes","q1":188.95918319043187,"q3":193.53821773446285,"n":5},"load_seconds":{"value":0.07270533400014756,"provenance":"measured","note":"median of 5 fresh processes","q1":0.07246958300015649,"q3":0.07311799999979485,"n":5},"load_ram_mb":{"value":48.766976,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":48.766976,"q3":48.771072,"n":5},"peak_ram_mb_1_mib_input":{"value":43.13088,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":43.13088,"q3":43.159552,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.386944,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":18.243584,"q3":18.640896,"n":5}}}},{"rank":26,"name":"OpenAI p50k_base (Codex)","provider":"OpenAI","family":"Published","version":null,"vocab":50281,"models":["OpenAI p50k_base (Codex)"],"overall":3.2165,"english":4.6591,"code_avg":3.2019,"languages_avg":1.7884,"bpt":{"code_asm":2.2921,"code_cs":4.0215,"code_dart":3.4779,"code_go":2.8186,"code_html":2.9378,"code_kotlin":3.6489,"code_lua":2.9298,"code_php":3.2511,"code_ps":3.3222,"code_r":2.6295,"code_ruby":3.2811,"code_shell":2.7908,"code_sql":2.8017,"code_swift":3.785,"code_ts":3.4253,"code_c":2.7859,"code_cpp":3.1851,"code_java":3.8114,"code_js":3.2924,"code_py":3.3188,"code_rust":3.4336,"en_fineweb":4.6591,"wiki_ar":1.8465,"wiki_bn":1.3396,"wiki_de":2.7027,"wiki_es":2.9993,"wiki_fr":3.0024,"wiki_gu":1.0123,"wiki_hi":1.6918,"wiki_id":2.6656,"wiki_ja":1.9655,"wiki_ko":1.1704,"wiki_mr":1.682,"wiki_ru":1.7176,"wiki_ta":1.0207,"wiki_te":1.0141,"wiki_th":1.5085,"wiki_tr":2.2562,"wiki_ur":1.4183,"wiki_vi":1.5279,"wiki_zh":1.4374},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"byte_fallback","detail":"test sets where the tokenizer falls back to (nearly) single bytes","sets":[{"set":"wiki_bn","bytes_per_token":{"value":1.33957277920174,"provenance":"measured"},"split_byte_token_share":{"value":0.5971711511428,"provenance":"measured"}},{"set":"wiki_gu","bytes_per_token":{"value":1.0123481947145003,"provenance":"measured"},"split_byte_token_share":{"value":0.927668602182073,"provenance":"measured"}},{"set":"wiki_ko","bytes_per_token":{"value":1.1704074058388076,"provenance":"measured"},"split_byte_token_share":{"value":0.800793100604299,"provenance":"measured"}},{"set":"wiki_ta","bytes_per_token":{"value":1.0206642334440994,"provenance":"measured"},"split_byte_token_share":{"value":0.947330617647118,"provenance":"measured"}},{"set":"wiki_te","bytes_per_token":{"value":1.014070553316086,"provenance":"measured"},"split_byte_token_share":{"value":0.9391352305316568,"provenance":"measured"}},{"set":"wiki_ur","bytes_per_token":{"value":1.418344131757258,"provenance":"measured"},"split_byte_token_share":{"value":0.5737919778426233,"provenance":"measured"}},{"set":"wiki_zh","bytes_per_token":{"value":1.4374053798988284,"provenance":"measured"},"split_byte_token_share":{"value":0.667344514070483,"provenance":"measured"}}]}],"ranked":true,"resources":{"library":"tokenizers (converted from tiktoken)","encode_mb_per_s_1_thread":{"value":5.033180687959843,"provenance":"measured","note":"median of 5 fresh processes","q1":4.988326943952645,"q3":5.0736678747005595,"n":5},"batch_mb_per_s_4_threads":{"value":14.126312341626818,"provenance":"measured","note":"median of 5 fresh processes","q1":14.121296533748598,"q3":14.12667908683695,"n":5},"cpu_cores_busy_batch":{"value":3.479774456382404,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.4232377921248203,"q3":3.481527017858286,"n":5},"cpu_seconds_per_mb":{"value":0.24275214339138776,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2424166778130887,"q3":0.2463328271546391,"n":5},"latency_p95_us_256_bytes":{"value":64.29200038837735,"provenance":"measured","note":"median of 5 fresh processes","q1":63.499999669147655,"q3":64.45799999710289,"n":5},"decode_mb_per_s":{"value":33.18989830935828,"provenance":"measured","note":"median of 5 fresh processes","q1":32.07674185188733,"q3":33.241454751372615,"n":5},"load_seconds":{"value":0.13893637499995748,"provenance":"measured","note":"median of 5 fresh processes","q1":0.138239583000086,"q3":0.14304120799988596,"n":5},"load_ram_mb":{"value":80.216064,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":80.216064,"q3":80.216064,"n":5},"peak_ram_mb_1_mib_input":{"value":216.608768,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":216.608768,"q3":216.612864,"n":5},"peak_ram_mb_16_mib_input":{"value":3049.263104,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":3049.189376,"q3":3049.435136,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.120704,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.89952,"q3":18.14528,"n":5},"ram_mb_per_input_mb":{"value":176.546875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":176.54248046875003,"q3":176.553955078125,"n":5},"native":{"implementation":"tiktoken","encode_mb_per_s_1_thread":{"value":19.471356296940503,"provenance":"measured","note":"median of 5 fresh processes","q1":19.126967563461037,"q3":19.823081075696624,"n":5},"batch_mb_per_s_4_threads":{"value":57.774697014819,"provenance":"measured","note":"median of 5 fresh processes","q1":56.93341749470764,"q3":60.09178194137764,"n":5},"cpu_seconds_per_mb":{"value":0.06811548694604964,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.06554167676997433,"q3":0.06936555519553209,"n":5},"latency_p95_us_256_bytes":{"value":20.0829999812413,"provenance":"measured","note":"median of 5 fresh processes","q1":19.583999801398026,"q3":20.166000012977747,"n":5},"decode_mb_per_s":{"value":160.8849104287948,"provenance":"measured","note":"median of 5 fresh processes","q1":155.67390986114853,"q3":161.1063421911218,"n":5},"load_seconds":{"value":0.03768454200007909,"provenance":"measured","note":"median of 5 fresh processes","q1":0.03750716600006854,"q3":0.04000120800037621,"n":5},"load_ram_mb":{"value":26.29632,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":26.29632,"q3":26.29632,"n":5},"peak_ram_mb_1_mib_input":{"value":25.731072,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":25.68192,"q3":25.731072,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.157568,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.69472,"q3":18.501632,"n":5}}}},{"rank":27,"name":"SmolLM2-135M","provider":"Hugging Face","family":"Published","version":null,"vocab":49152,"models":["SmolLM2-135M"],"overall":3.2034,"english":4.5429,"code_avg":3.1345,"languages_avg":1.9328,"bpt":{"code_c":2.727,"code_cpp":3.1395,"code_java":3.758,"code_js":3.2918,"code_rust":3.3578,"wiki_bn":1.4724,"wiki_gu":1.0118,"wiki_id":2.6579,"wiki_mr":2.3236,"wiki_ta":1.6092,"wiki_te":1.0109,"wiki_th":1.6121,"wiki_tr":2.2222,"wiki_ur":1.5729,"wiki_vi":1.7162,"code_py":3.4088,"en_fineweb":4.5429,"wiki_ar":1.9445,"wiki_de":2.4786,"wiki_es":2.93,"wiki_fr":2.9511,"wiki_hi":2.3784,"wiki_ja":1.648,"wiki_ko":1.1791,"wiki_ru":2.358,"wiki_zh":1.6462,"code_asm":1.9593,"code_cs":3.9463,"code_dart":3.4225,"code_go":2.867,"code_html":2.7792,"code_kotlin":3.5828,"code_lua":2.9169,"code_php":3.2872,"code_ps":3.2819,"code_r":2.4755,"code_ruby":3.2661,"code_shell":2.6977,"code_sql":2.5175,"code_swift":3.758,"code_ts":3.3839},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"byte_fallback","detail":"test sets where the tokenizer falls back to (nearly) single bytes","sets":[{"set":"wiki_gu","bytes_per_token":{"value":1.0117643700038066,"provenance":"measured"},"split_byte_token_share":{"value":0.9269623144271032,"provenance":"measured"}},{"set":"wiki_ko","bytes_per_token":{"value":1.179067375663562,"provenance":"measured"},"split_byte_token_share":{"value":0.7803621632788711,"provenance":"measured"}},{"set":"wiki_te","bytes_per_token":{"value":1.010956362487654,"provenance":"measured"},"split_byte_token_share":{"value":0.9358313040070696,"provenance":"measured"}},{"set":"wiki_zh","bytes_per_token":{"value":1.6462281297146368,"provenance":"measured"},"split_byte_token_share":{"value":0.5461531780472034,"provenance":"measured"}}]}],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.853248808026626,"provenance":"measured","note":"median of 5 fresh processes","q1":4.824940646454353,"q3":4.899484800901388,"n":5},"batch_mb_per_s_4_threads":{"value":12.374920625155946,"provenance":"measured","note":"median of 5 fresh processes","q1":12.127153555075385,"q3":12.633565211752764,"n":5},"cpu_cores_busy_batch":{"value":3.4985777456728875,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.457418936443337,"q3":3.5161412931353415,"n":5},"cpu_seconds_per_mb":{"value":0.2793891808416967,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.27692719252505443,"q3":0.28669884942697854,"n":5},"latency_p95_us_256_bytes":{"value":80.41699999239427,"provenance":"measured","note":"median of 5 fresh processes","q1":79.62500012581586,"q3":81.95899999918765,"n":5},"decode_mb_per_s":{"value":32.44802029987467,"provenance":"measured","note":"median of 5 fresh processes","q1":32.22535120544784,"q3":33.72718610168552,"n":5},"load_seconds":{"value":0.050100375000056374,"provenance":"measured","note":"median of 5 fresh processes","q1":0.04943820800008325,"q3":0.05065804199966806,"n":5},"load_ram_mb":{"value":34.832384,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":34.832384,"q3":34.832384,"n":5},"peak_ram_mb_1_mib_input":{"value":207.511552,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":207.245312,"q3":208.134144,"n":5},"peak_ram_mb_16_mib_input":{"value":3006.599168,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":3003.424768,"q3":3007.803392,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.543168,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.969728,"q3":17.702912,"n":5},"ram_mb_per_input_mb":{"value":177.715576171875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":177.5263671875,"q3":177.787353515625,"n":5}}},{"rank":28,"name":"Phi-3.5-mini-instruct","provider":"Microsoft","family":"Published","version":null,"vocab":32011,"models":["Phi-3.5-mini-instruct"],"overall":3.1695,"english":4.0442,"code_avg":3.0768,"languages_avg":2.3875,"bpt":{"code_c":2.5499,"code_cpp":3.0396,"code_java":3.6715,"code_js":3.2083,"code_rust":3.3722,"wiki_bn":2.1215,"wiki_gu":1.0817,"wiki_id":2.5877,"wiki_mr":2.4741,"wiki_ta":2.3278,"wiki_te":1.066,"wiki_th":2.6903,"wiki_tr":2.2604,"wiki_ur":1.7332,"wiki_vi":2.0154,"code_py":3.2572,"en_fineweb":4.0442,"wiki_ar":2.0156,"wiki_de":2.8092,"wiki_es":3.4709,"wiki_fr":3.3774,"wiki_hi":2.4207,"wiki_ja":2.4488,"wiki_ko":1.6281,"wiki_ru":4.8331,"wiki_zh":2.0012,"code_asm":1.8866,"code_cs":3.9459,"code_dart":3.4529,"code_go":2.601,"code_html":2.8104,"code_kotlin":3.5861,"code_lua":2.7111,"code_php":3.185,"code_ps":3.2211,"code_r":2.4631,"code_ruby":3.2831,"code_shell":2.6957,"code_sql":2.5216,"code_swift":3.7886,"code_ts":3.3626},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"byte_fallback","detail":"test sets where the tokenizer falls back to (nearly) single bytes","sets":[{"set":"wiki_gu","bytes_per_token":{"value":1.081700177644471,"provenance":"measured"},"split_byte_token_share":{"value":0.8841872002376733,"provenance":"measured"}},{"set":"wiki_te","bytes_per_token":{"value":1.066041773398184,"provenance":"measured"},"split_byte_token_share":{"value":0.902970772957554,"provenance":"measured"}}]}],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":6.788077586874321,"provenance":"measured","note":"median of 5 fresh processes","q1":6.781886443853547,"q3":6.87281856579862,"n":5},"batch_mb_per_s_4_threads":{"value":20.144123357075937,"provenance":"measured","note":"median of 5 fresh processes","q1":20.137997757743214,"q3":20.48385563393946,"n":5},"cpu_cores_busy_batch":{"value":3.429526846427324,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.414632821740462,"q3":3.4297443481240077,"n":5},"cpu_seconds_per_mb":{"value":0.169624223714293,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.1672564137227595,"q3":0.17031208312679666,"n":5},"latency_p95_us_256_bytes":{"value":49.70899999534595,"provenance":"measured","note":"median of 5 fresh processes","q1":49.33300010634411,"q3":50.7919999108708,"n":5},"decode_mb_per_s":{"value":16.656788731075494,"provenance":"measured","note":"median of 5 fresh processes","q1":16.40120282645318,"q3":16.842082066379636,"n":5},"load_seconds":{"value":0.04443212499973015,"provenance":"measured","note":"median of 5 fresh processes","q1":0.043551125000021784,"q3":0.04617033400000992,"n":5},"load_ram_mb":{"value":30.879744,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":30.879744,"q3":30.879744,"n":5},"peak_ram_mb_1_mib_input":{"value":138.862592,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":138.768384,"q3":138.932224,"n":5},"peak_ram_mb_16_mib_input":{"value":2057.474048,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2055.565312,"q3":2057.8304,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.625088,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.965632,"q3":17.768448,"n":5},"ram_mb_per_input_mb":{"value":120.52392578125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":120.410400390625,"q3":120.545166015625,"n":5}}},{"rank":29,"name":"gpt2","provider":"OpenAI","family":"Published","version":null,"vocab":50257,"models":["gpt2"],"overall":2.9504,"english":4.6591,"code_avg":2.4051,"languages_avg":1.7871,"bpt":{"code_c":2.3221,"code_cpp":2.461,"code_java":2.7997,"code_js":2.4135,"code_rust":2.2879,"wiki_bn":1.3305,"wiki_gu":1.0123,"wiki_id":2.6621,"wiki_mr":1.6818,"wiki_ta":1.0205,"wiki_te":1.014,"wiki_th":1.5085,"wiki_tr":2.2561,"wiki_ur":1.4178,"wiki_vi":1.5231,"code_py":2.2877,"en_fineweb":4.6591,"wiki_ar":1.8464,"wiki_de":2.7027,"wiki_es":2.9992,"wiki_fr":3.0023,"wiki_hi":1.6872,"wiki_ja":1.9648,"wiki_ko":1.1704,"wiki_ru":1.7174,"wiki_zh":1.4374,"code_asm":2.0056,"code_cs":2.1996,"code_dart":2.6525,"code_go":2.7016,"code_html":2.0877,"code_kotlin":2.3454,"code_lua":2.5877,"code_php":2.2399,"code_ps":2.39,"code_r":2.3696,"code_ruby":2.4978,"code_shell":2.483,"code_sql":2.4008,"code_swift":2.4827,"code_ts":2.4913},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"byte_fallback","detail":"test sets where the tokenizer falls back to (nearly) single bytes","sets":[{"set":"wiki_bn","bytes_per_token":{"value":1.330538411612696,"provenance":"measured"},"split_byte_token_share":{"value":0.5931437001698029,"provenance":"measured"}},{"set":"wiki_gu","bytes_per_token":{"value":1.0123327715409298,"provenance":"measured"},"split_byte_token_share":{"value":0.9276544691061788,"provenance":"measured"}},{"set":"wiki_ko","bytes_per_token":{"value":1.1703891437411118,"provenance":"measured"},"split_byte_token_share":{"value":0.8007806056715393,"provenance":"measured"}},{"set":"wiki_ta","bytes_per_token":{"value":1.020505692775702,"provenance":"measured"},"split_byte_token_share":{"value":0.9471834679534248,"provenance":"measured"}},{"set":"wiki_te","bytes_per_token":{"value":1.0140420821267278,"provenance":"measured"},"split_byte_token_share":{"value":0.939108863237099,"provenance":"measured"}},{"set":"wiki_ur","bytes_per_token":{"value":1.4178493637195146,"provenance":"measured"},"split_byte_token_share":{"value":0.5735918191331862,"provenance":"measured"}},{"set":"wiki_zh","bytes_per_token":{"value":1.437360970677452,"provenance":"measured"},"split_byte_token_share":{"value":0.6673238961914392,"provenance":"measured"}}]}],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.814825216097814,"provenance":"measured","note":"median of 5 fresh processes","q1":4.813821967040831,"q3":4.819629790355922,"n":5},"batch_mb_per_s_4_threads":{"value":12.407483704422464,"provenance":"measured","note":"median of 5 fresh processes","q1":12.314218491580338,"q3":12.439353137060534,"n":5},"cpu_cores_busy_batch":{"value":3.4413917119159008,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.417085669690939,"q3":3.4452696797853495,"n":5},"cpu_seconds_per_mb":{"value":0.276965340707377,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.275581515393626,"q3":0.2774910703450097,"n":5},"latency_p95_us_256_bytes":{"value":73.50000009864743,"provenance":"measured","note":"median of 5 fresh processes","q1":72.45799997690483,"q3":74.1250000828586,"n":5},"decode_mb_per_s":{"value":29.806463251233527,"provenance":"measured","note":"median of 5 fresh processes","q1":29.531230642237514,"q3":29.983672418984067,"n":5},"load_seconds":{"value":0.04961520899996685,"provenance":"measured","note":"median of 5 fresh processes","q1":0.04851966699993682,"q3":0.0536567499999876,"n":5},"load_ram_mb":{"value":35.856384,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":35.856384,"q3":35.897344,"n":5},"peak_ram_mb_1_mib_input":{"value":240.410624,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":239.464448,"q3":242.450432,"n":5},"peak_ram_mb_16_mib_input":{"value":3192.500224,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":3187.777536,"q3":3193.532416,"n":5},"peak_ram_mb_16_mib_chunked":{"value":24.22784,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":23.94112,"q3":24.879104,"n":5},"ram_mb_per_input_mb":{"value":189.016357421875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":188.75244140625,"q3":189.07861328125,"n":5}}},{"rank":null,"name":"Qwen3.8-27B","provider":"Alibaba (Qwen)","family":"Published","version":null,"vocab":248077,"models":["Qwen3.8-27B"],"overall":4.4561,"english":4.5894,"code_avg":3.8087,"languages_avg":4.9703,"bpt":{"code_asm":2.2372,"code_cs":4.8866,"code_dart":4.1762,"code_go":3.4836,"code_html":3.4516,"code_kotlin":4.5436,"code_lua":3.5617,"code_php":3.9464,"code_ps":3.7039,"code_r":2.9431,"code_ruby":4.0392,"code_shell":3.2869,"code_sql":3.0189,"code_swift":4.5594,"code_ts":4.1354,"code_c":3.3162,"code_cpp":3.8864,"code_java":4.5407,"code_js":4.0012,"code_py":4.1346,"code_rust":4.1299,"en_fineweb":4.5894,"wiki_ar":5.8604,"wiki_bn":5.4412,"wiki_de":3.3573,"wiki_es":4.1697,"wiki_fr":3.9187,"wiki_gu":3.7196,"wiki_hi":5.7384,"wiki_id":4.4462,"wiki_ja":4.6274,"wiki_ko":4.0002,"wiki_mr":5.346,"wiki_ru":6.041,"wiki_ta":5.5818,"wiki_te":4.3099,"wiki_th":10.6791,"wiki_tr":3.6317,"wiki_ur":4.3226,"wiki_vi":4.625,"wiki_zh":4.6197},"lossless":false,"roundtrip":{"value":733,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"not_lossless_nfc","detail":"733 inputs decode to canonically equivalent (NFC) but not identical text","example":"                \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\","}],"ranked":false,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.178264118415175,"provenance":"measured","note":"median of 5 fresh processes","q1":4.007467155386007,"q3":4.17966360199197,"n":5},"batch_mb_per_s_4_threads":{"value":13.768086388465784,"provenance":"measured","note":"median of 5 fresh processes","q1":13.390027005597226,"q3":14.101835965023056,"n":5},"cpu_cores_busy_batch":{"value":3.638189226073153,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.6343470122944024,"q3":3.641795968586866,"n":5},"cpu_seconds_per_mb":{"value":0.2645099591790607,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.25799401121222765,"q3":0.27142193296362976,"n":5},"latency_p95_us_256_bytes":{"value":90.45800015883287,"provenance":"measured","note":"median of 5 fresh processes","q1":88.33399988361634,"q3":100.91700005432358,"n":5},"decode_mb_per_s":{"value":40.20413132029879,"provenance":"measured","note":"median of 5 fresh processes","q1":40.108593477309256,"q3":40.25545147393336,"n":5},"load_seconds":{"value":0.30378187499991327,"provenance":"measured","note":"median of 5 fresh processes","q1":0.29920245799985423,"q3":0.3053127919997678,"n":5},"load_ram_mb":{"value":115.396608,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":115.396608,"q3":115.396608,"n":5},"peak_ram_mb_1_mib_input":{"value":218.599424,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":218.456064,"q3":218.607616,"n":5},"peak_ram_mb_16_mib_input":{"value":2551.861248,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2550.76352,"q3":2551.967744,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.567744,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.518592,"q3":17.911808,"n":5},"ram_mb_per_input_mb":{"value":142.276611328125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":142.21142578125,"q3":142.282958984375,"n":5}}},{"rank":null,"name":"MiniMax-M2.5","provider":"MiniMax","family":"Published","version":null,"vocab":200054,"models":["MiniMax-M2.5"],"overall":4.1609,"english":4.8212,"code_avg":3.7795,"languages_avg":3.8821,"bpt":{"code_asm":2.4624,"code_cs":4.7131,"code_dart":4.1087,"code_go":3.4665,"code_html":3.572,"code_kotlin":4.4332,"code_lua":3.4963,"code_php":3.8494,"code_ps":3.7617,"code_r":3.0321,"code_ruby":4.0457,"code_shell":3.2975,"code_sql":3.2777,"code_swift":4.2971,"code_ts":3.9891,"code_c":3.2764,"code_cpp":3.7721,"code_java":4.4184,"code_js":3.9024,"code_py":4.082,"code_rust":4.1159,"en_fineweb":4.8212,"wiki_ar":5.9019,"wiki_bn":3.4478,"wiki_de":3.4946,"wiki_es":4.6555,"wiki_fr":4.3824,"wiki_gu":1.8605,"wiki_hi":4.4576,"wiki_id":4.6221,"wiki_ja":5.3348,"wiki_ko":3.9978,"wiki_mr":4.0235,"wiki_ru":5.3633,"wiki_ta":3.9104,"wiki_te":3.1816,"wiki_th":2.829,"wiki_tr":2.8323,"wiki_ur":2.4749,"wiki_vi":2.4347,"wiki_zh":4.5556},"lossless":false,"roundtrip":{"value":733,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"not_lossless_nfc","detail":"733 inputs decode to canonically equivalent (NFC) but not identical text","example":"                \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\","}],"ranked":false,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.2863295702025015,"provenance":"measured","note":"median of 5 fresh processes","q1":4.106062903165659,"q3":4.314534054925623,"n":5},"batch_mb_per_s_4_threads":{"value":13.28344203289287,"provenance":"measured","note":"median of 5 fresh processes","q1":13.277353389014594,"q3":13.417999827684188,"n":5},"cpu_cores_busy_batch":{"value":3.612286995831657,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.588588708560855,"q3":3.629868860970989,"n":5},"cpu_seconds_per_mb":{"value":0.2705223511392359,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.27027891805079,"q3":0.2719390792602399,"n":5},"latency_p95_us_256_bytes":{"value":86.95799988345243,"provenance":"measured","note":"median of 5 fresh processes","q1":82.7090000257158,"q3":87.5829998676636,"n":5},"decode_mb_per_s":{"value":38.192575850644786,"provenance":"measured","note":"median of 5 fresh processes","q1":37.39748293691054,"q3":38.33750896901568,"n":5},"load_seconds":{"value":0.23188758399999188,"provenance":"measured","note":"median of 5 fresh processes","q1":0.2309566670001004,"q3":0.23225875100024496,"n":5},"load_ram_mb":{"value":73.449472,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":73.445376,"q3":74.268672,"n":5},"peak_ram_mb_1_mib_input":{"value":206.053376,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":205.836288,"q3":206.176256,"n":5},"peak_ram_mb_16_mib_input":{"value":2552.291328,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2552.180736,"q3":2552.840192,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.670144,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.657856,"q3":17.895424,"n":5},"ram_mb_per_input_mb":{"value":145.1064453125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":145.104736328125,"q3":145.13916015625,"n":5}}},{"rank":null,"name":"Seed-OSS-36B-Instruct","provider":"ByteDance","family":"Published","version":null,"vocab":155121,"models":["Seed-OSS-36B-Instruct"],"overall":4.1092,"english":4.5889,"code_avg":3.6172,"languages_avg":4.1215,"bpt":{"code_asm":2.1789,"code_cs":4.609,"code_dart":3.9724,"code_go":3.2974,"code_html":3.3934,"code_kotlin":4.2345,"code_lua":3.3417,"code_php":3.7532,"code_ps":3.5762,"code_r":2.8244,"code_ruby":3.791,"code_shell":3.102,"code_sql":2.9117,"code_swift":4.2194,"code_ts":3.9911,"code_c":3.1688,"code_cpp":3.6621,"code_java":4.328,"code_js":3.8077,"code_py":3.895,"code_rust":3.9025,"en_fineweb":4.5889,"wiki_ar":4.6329,"wiki_bn":4.5524,"wiki_de":3.2217,"wiki_es":3.8853,"wiki_fr":3.7992,"wiki_gu":3.7959,"wiki_hi":4.5887,"wiki_id":3.8145,"wiki_ja":2.9086,"wiki_ko":3.0803,"wiki_mr":4.5994,"wiki_ru":5.7039,"wiki_ta":5.0595,"wiki_te":4.3389,"wiki_th":4.9891,"wiki_tr":3.4382,"wiki_ur":3.8403,"wiki_vi":4.1454,"wiki_zh":3.9149},"lossless":false,"roundtrip":{"value":733,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"not_lossless_nfc","detail":"733 inputs decode to canonically equivalent (NFC) but not identical text","example":"                \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\","}],"ranked":false,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.236921381376127,"provenance":"measured","note":"median of 5 fresh processes","q1":4.188703953896487,"q3":4.247458323581646,"n":5},"batch_mb_per_s_4_threads":{"value":13.967031663098131,"provenance":"measured","note":"median of 5 fresh processes","q1":13.93647462267473,"q3":14.032971454237485,"n":5},"cpu_cores_busy_batch":{"value":3.604842468830048,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.6003777834383697,"q3":3.606908798605146,"n":5},"cpu_seconds_per_mb":{"value":0.25809653445221953,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.25703100803472245,"q3":0.25914680159678755,"n":5},"latency_p95_us_256_bytes":{"value":81.83299996744609,"provenance":"measured","note":"median of 5 fresh processes","q1":81.33300002555188,"q3":82.5830002213479,"n":5},"decode_mb_per_s":{"value":38.05299281016757,"provenance":"measured","note":"median of 5 fresh processes","q1":37.25448327934852,"q3":38.3355817570048,"n":5},"load_seconds":{"value":0.24087216699990677,"provenance":"measured","note":"median of 5 fresh processes","q1":0.2376469589999033,"q3":0.24242649999996504,"n":5},"load_ram_mb":{"value":136.736768,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":136.732672,"q3":136.736768,"n":5},"peak_ram_mb_1_mib_input":{"value":231.493632,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":229.392384,"q3":233.213952,"n":5},"peak_ram_mb_16_mib_input":{"value":2654.588928,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2654.584832,"q3":2654.691328,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.6128,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.924672,"q3":17.92,"n":5},"ram_mb_per_input_mb":{"value":147.872314453125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":147.799560546875,"q3":147.91015625000003,"n":5}}},{"rank":null,"name":"Ling-3.0-tiny","provider":"inclusionAI (Ant Group)","family":"Published","version":null,"vocab":157153,"models":["Ling-3.0-tiny"],"overall":3.9197,"english":4.6174,"code_avg":3.6687,"languages_avg":3.473,"bpt":{"code_asm":2.1806,"code_cs":4.6931,"code_dart":4.0643,"code_go":3.2699,"code_html":3.3821,"code_kotlin":4.3367,"code_lua":3.3417,"code_php":3.7711,"code_ps":3.6854,"code_r":2.8767,"code_ruby":3.9093,"code_shell":3.1993,"code_sql":2.9287,"code_swift":4.3613,"code_ts":3.8945,"code_c":3.1736,"code_cpp":3.7291,"code_java":4.4028,"code_js":3.8366,"code_py":3.989,"code_rust":4.0171,"en_fineweb":4.6174,"wiki_ar":4.2668,"wiki_bn":4.9868,"wiki_de":3.0369,"wiki_es":3.5046,"wiki_fr":3.5059,"wiki_gu":1.7316,"wiki_hi":3.1944,"wiki_id":3.2945,"wiki_ja":3.8206,"wiki_ko":3.456,"wiki_mr":3.0152,"wiki_ru":4.845,"wiki_ta":2.2108,"wiki_te":1.9148,"wiki_th":5.7768,"wiki_tr":3.2019,"wiki_ur":2.4933,"wiki_vi":3.8298,"wiki_zh":3.9018},"lossless":false,"roundtrip":{"value":733,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"not_lossless_nfc","detail":"733 inputs decode to canonically equivalent (NFC) but not identical text","example":"                \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\","}],"ranked":false,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.229124727384775,"provenance":"measured","note":"median of 5 fresh processes","q1":4.186029504625122,"q3":4.309772640935677,"n":5},"batch_mb_per_s_4_threads":{"value":14.130000644302076,"provenance":"measured","note":"median of 5 fresh processes","q1":13.545959144013002,"q3":14.310461923491644,"n":5},"cpu_cores_busy_batch":{"value":3.592977491594083,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.586836403021493,"q3":3.6005073759317026,"n":5},"cpu_seconds_per_mb":{"value":0.25372713701944916,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.25231183945528207,"q3":0.2657993677415632,"n":5},"latency_p95_us_256_bytes":{"value":82.08300005208002,"provenance":"measured","note":"median of 5 fresh processes","q1":80.7499999382344,"q3":82.66600002571067,"n":5},"decode_mb_per_s":{"value":34.64389579633848,"provenance":"measured","note":"median of 5 fresh processes","q1":34.567945515610404,"q3":34.84015594000904,"n":5},"load_seconds":{"value":0.2463905419999719,"provenance":"measured","note":"median of 5 fresh processes","q1":0.24160091700014164,"q3":0.24775787499993385,"n":5},"load_ram_mb":{"value":103.542784,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":103.542784,"q3":103.54688,"n":5},"peak_ram_mb_1_mib_input":{"value":232.214528,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":231.739392,"q3":232.460288,"n":5},"peak_ram_mb_16_mib_input":{"value":2665.668608,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2665.443328,"q3":2667.6224,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.42848,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.32608,"q3":18.305024,"n":5},"ram_mb_per_input_mb":{"value":148.587646484375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":148.55419921875,"q3":148.701171875,"n":5}}},{"rank":null,"name":"Qwen2.5-0.5B (same tokenizer: Qwen3-0.6B)","provider":"Alibaba (Qwen)","family":"Published","version":null,"vocab":151665,"models":["Qwen2.5-0.5B","Qwen3-0.6B"],"overall":3.9029,"english":4.5731,"code_avg":3.7925,"languages_avg":3.3432,"bpt":{"code_c":3.3064,"code_cpp":3.8844,"code_java":4.537,"code_js":3.9962,"code_rust":4.1257,"wiki_bn":2.5106,"wiki_gu":1.7836,"wiki_id":3.2472,"wiki_mr":2.7808,"wiki_ta":2.5062,"wiki_te":1.8285,"wiki_th":5.056,"wiki_tr":3.2563,"wiki_ur":2.7541,"wiki_vi":4.232,"code_py":4.1254,"en_fineweb":4.5731,"wiki_ar":4.7057,"wiki_de":2.907,"wiki_es":3.6416,"wiki_fr":3.5205,"wiki_hi":2.7882,"wiki_ja":3.9299,"wiki_ko":3.1729,"wiki_ru":4.8193,"wiki_zh":4.0806,"code_asm":2.2352,"code_cs":4.8677,"code_dart":4.1713,"code_go":3.4807,"code_html":3.4303,"code_kotlin":4.5346,"code_lua":3.541,"code_php":3.9323,"code_ps":3.6985,"code_r":2.9306,"code_ruby":4.0086,"code_shell":3.2816,"code_sql":3.0164,"code_swift":4.551,"code_ts":3.9883},"lossless":false,"roundtrip":{"value":733,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"not_lossless_nfc","detail":"733 inputs decode to canonically equivalent (NFC) but not identical text","example":"                \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\","}],"ranked":false,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.307057689547919,"provenance":"measured","note":"median of 5 fresh processes","q1":4.279288412295699,"q3":4.338851432013901,"n":5},"batch_mb_per_s_4_threads":{"value":13.201951882847291,"provenance":"measured","note":"median of 5 fresh processes","q1":12.856043004832054,"q3":13.433439100652608,"n":5},"cpu_cores_busy_batch":{"value":3.626957765081496,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.616644265918417,"q3":3.631264141220847,"n":5},"cpu_seconds_per_mb":{"value":0.27472890351871687,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.26922698192324535,"q3":0.28245581784815166,"n":5},"latency_p95_us_256_bytes":{"value":87.00000000771979,"provenance":"measured","note":"median of 5 fresh processes","q1":84.79200005240273,"q3":88.00000023256871,"n":5},"decode_mb_per_s":{"value":37.897640077907525,"provenance":"measured","note":"median of 5 fresh processes","q1":37.72617964936915,"q3":38.63514698416962,"n":5},"load_seconds":{"value":0.15903862499999377,"provenance":"measured","note":"median of 5 fresh processes","q1":0.15626587500037203,"q3":0.16179104200000438,"n":5},"load_ram_mb":{"value":97.73056,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":97.120256,"q3":98.414592,"n":5},"peak_ram_mb_1_mib_input":{"value":209.358848,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":209.129472,"q3":209.678336,"n":5},"peak_ram_mb_16_mib_input":{"value":2646.319104,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2645.385216,"q3":2646.511616,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.010112,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.649664,"q3":18.116608,"n":5},"ram_mb_per_input_mb":{"value":152.4541015625,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":152.400390625,"q3":152.5302734375,"n":5}}},{"rank":null,"name":"sarvam-1","provider":"Sarvam","family":"Published","version":null,"vocab":68096,"models":["sarvam-1"],"overall":3.4984,"english":4.0199,"code_avg":2.7662,"languages_avg":3.7091,"bpt":{"code_asm":1.8677,"code_cs":3.4329,"code_dart":3.0569,"code_go":2.5225,"code_html":2.3615,"code_kotlin":3.1117,"code_lua":2.561,"code_php":2.8166,"code_ps":2.8724,"code_r":2.2358,"code_ruby":2.7856,"code_shell":2.3912,"code_sql":2.2695,"code_swift":3.3245,"code_ts":2.8128,"code_c":2.5425,"code_cpp":2.902,"code_java":3.3767,"code_js":2.9089,"code_py":2.8979,"code_rust":3.0401,"en_fineweb":4.0199,"wiki_ar":1.0024,"wiki_bn":9.2433,"wiki_de":2.1031,"wiki_es":2.3727,"wiki_fr":2.3764,"wiki_gu":6.7435,"wiki_hi":8.5214,"wiki_id":2.2973,"wiki_ja":1.0059,"wiki_ko":1.0103,"wiki_mr":9.1385,"wiki_ru":1.0067,"wiki_ta":9.7478,"wiki_te":7.7864,"wiki_th":1.0103,"wiki_tr":1.7308,"wiki_ur":1.0026,"wiki_vi":1.3378,"wiki_zh":1.0359},"lossless":false,"roundtrip":{"value":215393,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"not_lossless","detail":"215393 inputs do not decode back exactly (0 of them only by NFC)","example":"  ret ; just return until the proper paging code is installed into this location"},{"kind":"byte_fallback","detail":"test sets where the tokenizer falls back to (nearly) single bytes","sets":[{"set":"wiki_ar","bytes_per_token":{"value":1.0024407344343875,"provenance":"measured"},"split_byte_token_share":{"value":0.8928870778135658,"provenance":"measured"}},{"set":"wiki_ja","bytes_per_token":{"value":1.0058575445173383,"provenance":"measured"},"split_byte_token_share":{"value":0.9793514046091366,"provenance":"measured"}},{"set":"wiki_ko","bytes_per_token":{"value":1.0102571305431989,"provenance":"measured"},"split_byte_token_share":{"value":0.8607793187310481,"provenance":"measured"}},{"set":"wiki_ru","bytes_per_token":{"value":1.0066856519698013,"provenance":"measured"},"split_byte_token_share":{"value":0.900065943179265,"provenance":"measured"}},{"set":"wiki_th","bytes_per_token":{"value":1.010280092743623,"provenance":"measured"},"split_byte_token_share":{"value":0.9732240184978016,"provenance":"measured"}},{"set":"wiki_ur","bytes_per_token":{"value":1.0026474655810051,"provenance":"measured"},"split_byte_token_share":{"value":0.8724224601423377,"provenance":"measured"}},{"set":"wiki_vi","bytes_per_token":{"value":1.3377971658395886,"provenance":"measured"},"split_byte_token_share":{"value":0.5186518031494652,"provenance":"measured"}},{"set":"wiki_zh","bytes_per_token":{"value":1.0359201073655957,"provenance":"measured"},"split_byte_token_share":{"value":0.9666526994228896,"provenance":"measured"}}]}],"ranked":false,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":6.321151415335212,"provenance":"measured","note":"median of 5 fresh processes","q1":6.306436262401096,"q3":6.3230548212430735,"n":5},"batch_mb_per_s_4_threads":{"value":16.82872400173529,"provenance":"measured","note":"median of 5 fresh processes","q1":16.379430910362814,"q3":17.234958655190237,"n":5},"cpu_cores_busy_batch":{"value":3.2861305919883934,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.2289697635499537,"q3":3.2922173007086304,"n":5},"cpu_seconds_per_mb":{"value":0.19074854124927193,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.1873500150566107,"q3":0.20099705043022803,"n":5},"latency_p95_us_256_bytes":{"value":54.99999997482519,"provenance":"measured","note":"median of 5 fresh processes","q1":53.874999935032974,"q3":56.12499990093056,"n":5},"decode_mb_per_s":{"value":15.417834483448631,"provenance":"measured","note":"median of 5 fresh processes","q1":14.422796936108409,"q3":15.65840171959222,"n":5},"load_seconds":{"value":0.13710370900025737,"provenance":"measured","note":"median of 5 fresh processes","q1":0.13492304100009278,"q3":0.14654233300007036,"n":5},"load_ram_mb":{"value":85.950464,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":85.950464,"q3":85.958656,"n":5},"peak_ram_mb_1_mib_input":{"value":205.59872,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":205.500416,"q3":205.668352,"n":5},"peak_ram_mb_16_mib_input":{"value":2489.655296,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2487.9104,"q3":2519.396352,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.637376,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.629184,"q3":17.682432,"n":5},"ram_mb_per_input_mb":{"value":142.76440429687503,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":142.666748046875,"q3":144.54345703125,"n":5}}},{"rank":null,"name":"pythia-160m","provider":"EleutherAI","family":"Published","version":null,"vocab":50277,"models":["pythia-160m"],"overall":3.4933,"english":4.6661,"code_avg":3.3695,"languages_avg":2.4444,"bpt":{"code_c":3.0195,"code_cpp":3.3283,"code_java":3.9737,"code_js":3.497,"code_rust":3.5577,"wiki_bn":1.9431,"wiki_gu":1.7327,"wiki_id":2.7658,"wiki_mr":2.5061,"wiki_ta":1.9794,"wiki_te":1.7713,"wiki_th":2.5502,"wiki_tr":2.4713,"wiki_ur":1.9755,"wiki_vi":2.0947,"code_py":3.4684,"en_fineweb":4.6661,"wiki_ar":2.6299,"wiki_de":2.9481,"wiki_es":3.3734,"wiki_fr":3.2841,"wiki_hi":2.59,"wiki_ja":2.681,"wiki_ko":1.5455,"wiki_ru":3.4578,"wiki_zh":2.1433,"code_asm":2.3978,"code_cs":4.2377,"code_dart":3.608,"code_go":2.9939,"code_html":3.1039,"code_kotlin":3.8132,"code_lua":3.0447,"code_php":3.5156,"code_ps":3.456,"code_r":2.7444,"code_ruby":3.4305,"code_shell":2.9555,"code_sql":2.9881,"code_swift":3.9752,"code_ts":3.6503},"lossless":false,"roundtrip":{"value":733,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"not_lossless_nfc","detail":"733 inputs decode to canonically equivalent (NFC) but not identical text","example":"                \"יום רִאשׁוֹן\", \"יום שֵׁנִי\", \"יום שְׁלִישִׁי\", \"יום רְבִיעִי\","},{"kind":"byte_fallback","detail":"test sets where the tokenizer falls back to (nearly) single bytes","sets":[{"set":"wiki_ko","bytes_per_token":{"value":1.5454556157634018,"provenance":"measured"},"split_byte_token_share":{"value":0.5991952882863526,"provenance":"measured"}}]}],"ranked":false,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.724995212054687,"provenance":"measured","note":"median of 5 fresh processes","q1":4.715288721532525,"q3":4.767939642189444,"n":5},"batch_mb_per_s_4_threads":{"value":13.191500272194986,"provenance":"measured","note":"median of 5 fresh processes","q1":12.986857161371269,"q3":13.25527623421092,"n":5},"cpu_cores_busy_batch":{"value":3.537306295162905,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.53672222724792,"q3":3.5438245302255855,"n":5},"cpu_seconds_per_mb":{"value":0.2681402355793314,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.26810614064138066,"q3":0.2726636562982291,"n":5},"latency_p95_us_256_bytes":{"value":73.3329999320631,"provenance":"measured","note":"median of 5 fresh processes","q1":71.54200000059063,"q3":73.3329999320631,"n":5},"decode_mb_per_s":{"value":36.310007055338886,"provenance":"measured","note":"median of 5 fresh processes","q1":36.04379219659033,"q3":37.0972005070531,"n":5},"load_seconds":{"value":0.04969120799978555,"provenance":"measured","note":"median of 5 fresh processes","q1":0.04936454200003482,"q3":0.05005616699963866,"n":5},"load_ram_mb":{"value":37.154816,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":37.154816,"q3":37.154816,"n":5},"peak_ram_mb_1_mib_input":{"value":197.931008,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":197.246976,"q3":198.172672,"n":5},"peak_ram_mb_16_mib_input":{"value":2817.212416,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2817.040384,"q3":2821.394432,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.65376,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.108992,"q3":17.87904,"n":5},"ram_mb_per_input_mb":{"value":166.404541015625,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":166.400390625,"q3":166.663330078125,"n":5}}}],"sets":[{"id":"en_fineweb","label":"English web","group":"English"},{"id":"code_py","label":"Python","group":"Code"},{"id":"code_js","label":"JavaScript","group":"Code"},{"id":"code_ts","label":"TypeScript","group":"Code"},{"id":"code_java","label":"Java","group":"Code"},{"id":"code_cs","label":"C#","group":"Code"},{"id":"code_cpp","label":"C++","group":"Code"},{"id":"code_c","label":"C","group":"Code"},{"id":"code_go","label":"Go","group":"Code"},{"id":"code_rust","label":"Rust","group":"Code"},{"id":"code_php","label":"PHP","group":"Code"},{"id":"code_ruby","label":"Ruby","group":"Code"},{"id":"code_swift","label":"Swift","group":"Code"},{"id":"code_kotlin","label":"Kotlin","group":"Code"},{"id":"code_dart","label":"Dart","group":"Code"},{"id":"code_lua","label":"Lua","group":"Code"},{"id":"code_r","label":"R","group":"Code"},{"id":"code_sql","label":"SQL","group":"Code"},{"id":"code_shell","label":"Bash/Shell","group":"Code"},{"id":"code_ps","label":"PowerShell","group":"Code"},{"id":"code_html","label":"HTML/CSS","group":"Code"},{"id":"code_asm","label":"Assembly","group":"Code"},{"id":"wiki_de","label":"German","group":"Languages"},{"id":"wiki_fr","label":"French","group":"Languages"},{"id":"wiki_es","label":"Spanish","group":"Languages"},{"id":"wiki_ru","label":"Russian","group":"Languages"},{"id":"wiki_zh","label":"Chinese","group":"Languages"},{"id":"wiki_ja","label":"Japanese","group":"Languages"},{"id":"wiki_ko","label":"Korean","group":"Languages"},{"id":"wiki_ar","label":"Arabic","group":"Languages"},{"id":"wiki_ur","label":"Urdu","group":"Languages"},{"id":"wiki_hi","label":"Hindi","group":"Languages"},{"id":"wiki_mr","label":"Marathi","group":"Languages"},{"id":"wiki_gu","label":"Gujarati","group":"Languages"},{"id":"wiki_bn","label":"Bengali","group":"Languages"},{"id":"wiki_ta","label":"Tamil","group":"Languages"},{"id":"wiki_te","label":"Telugu","group":"Languages"},{"id":"wiki_th","label":"Thai","group":"Languages"},{"id":"wiki_vi","label":"Vietnamese","group":"Languages"},{"id":"wiki_tr","label":"Turkish","group":"Languages"},{"id":"wiki_id","label":"Indonesian","group":"Languages"}],"providers":[{"name":"Akshara","color":"#0f7b6c","dark_color":"#3cc2ad","logo":null,"mark":"monogram","tokenizers":9},{"name":"OpenAI","color":"#1f1f1f","dark_color":"#d9d9d9","logo":"logos/openai.svg","mark":"logo","tokenizers":4},{"name":"Google","color":"#4285f4","dark_color":"#4285f4","logo":null,"mark":"icon","tokenizers":2},{"name":"Meta","color":"#0467df","dark_color":"#3b8cf0","logo":null,"mark":"icon","tokenizers":1},{"name":"Alibaba (Qwen)","color":"#ff6a00","dark_color":"#ff6a00","logo":null,"mark":"icon","tokenizers":3},{"name":"DeepSeek","color":"#4d6bfe","dark_color":"#4d6bfe","logo":null,"mark":"icon","tokenizers":1},{"name":"Mistral","color":"#d9370c","dark_color":"#f0592a","logo":null,"mark":"icon","tokenizers":2},{"name":"Microsoft","color":"#e5a000","dark_color":"#e5a000","logo":"logos/microsoft.svg","mark":"logo","tokenizers":2},{"name":"Sarvam","color":"#7c3aed","dark_color":"#9d6cf5","logo":"logos/sarvam-black.svg","mark":"logo","tokenizers":2},{"name":"Moonshot AI (Kimi)","color":"#111827","dark_color":"#cbd5e1","logo":null,"mark":"icon","tokenizers":1},{"name":"Z.ai (GLM)","color":"#0ea5e9","dark_color":"#0ea5e9","logo":null,"mark":"icon","tokenizers":1},{"name":"MiniMax","color":"#e73562","dark_color":"#e73562","logo":null,"mark":"icon","tokenizers":1},{"name":"Cohere","color":"#39594d","dark_color":"#6f9a88","logo":"logos/cohere.svg","mark":"logo","tokenizers":1},{"name":"Tencent","color":"#1d6fe0","dark_color":"#1d6fe0","logo":null,"mark":"icon","tokenizers":1},{"name":"Baidu","color":"#2932e1","dark_color":"#5a62f0","logo":null,"mark":"icon","tokenizers":1},{"name":"ByteDance","color":"#3c8cff","dark_color":"#3c8cff","logo":null,"mark":"icon","tokenizers":1},{"name":"NVIDIA","color":"#76b900","dark_color":"#76b900","logo":null,"mark":"icon","tokenizers":0},{"name":"IBM","color":"#0f62fe","dark_color":"#4589ff","logo":null,"mark":"monogram","tokenizers":0},{"name":"Hugging Face","color":"#e6b400","dark_color":"#ffd21e","logo":null,"mark":"icon","tokenizers":1},{"name":"Ai2","color":"#f0529c","dark_color":"#f0529c","logo":"logos/ai2.png","mark":"logo","tokenizers":2},{"name":"EleutherAI","color":"#6b7280","dark_color":"#9ca3af","logo":"logos/eleutherai.png","mark":"logo","tokenizers":1},{"name":"TII (Falcon)","color":"#b45309","dark_color":"#d97706","logo":"logos/tii.png","mark":"logo","tokenizers":1},{"name":"Liquid AI","color":"#7e22ce","dark_color":"#a855f7","logo":"logos/liquid-ai.png","mark":"logo","tokenizers":1},{"name":"inclusionAI (Ant Group)","color":"#38a3e8","dark_color":"#38a3e8","logo":null,"mark":"monogram","tokenizers":1},{"name":"Xiaomi","color":"#ff6900","dark_color":"#ff6900","logo":null,"mark":"icon","tokenizers":0},{"name":"01.AI","color":"#15803d","dark_color":"#22c55e","logo":"logos/01ai.svg","mark":"logo","tokenizers":0},{"name":"Shanghai AI Lab (InternLM)","color":"#1e40af","dark_color":"#5b7fe0","logo":"logos/internlm.png","mark":"logo","tokenizers":0},{"name":"StepFun","color":"#0d9488","dark_color":"#2dd4bf","logo":"logos/stepfun.svg","mark":"logo","tokenizers":0}]}