{"generated":"2026-11-08","unit":"bytes_per_token","text_unit":"lines","measure":"UTF-8 bytes per token on held-out text; higher is better","ranking":"rank counts lossless tokenizers only, by overall score; a tokenizer that does not decode every input back exactly has rank null and a flag saying why; tokenizers with identical IDs on every input are one row","resources_measured_on":{"cpu":"Apple M4 Pro (container)","logical_cpus":14,"memory_bytes":18829856768,"gpu":null,"os":"Linux-7.0.14-linuxkit-aarch64-with-glibc2.41","python":"3.13.18","libraries":{"tokenizers":"0.23.3","tiktoken":"0.14.0","transformers":"5.22.2","numpy":"2.5.4"},"git_commit":"ef2fc972f4e24836ebc9fea119b9338548969a1d","git_dirty":false,"config_version":"1.3","library":"every tokenizer on the Hugging Face tokenizers library (tiktoken files converted, Akshara exported); resources.native is each tokenizer's own runtime where it differs, never mixed with the library numbers","note":"each value is the median of 5 fresh processes run in one seeded random order over all tokenizers; q1 and q3 give the spread, and values whose q1-q3 ranges overlap are ties"},"query":{"provider":["openai","akshara"],"family":null,"q":null,"sets":[],"sort":"rank","order":"asc","offset":0,"limit":null},"total":13,"count":13,"tokenizers":[{"rank":1,"name":"Akshara 448k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":448000,"models":["Akshara 448k"],"overall":5.409,"english":4.9363,"code_avg":4.0776,"languages_avg":7.2132,"bpt":{"code_asm":2.7413,"code_c":3.5702,"code_cpp":3.9995,"code_cs":5.0448,"code_dart":4.2901,"code_go":3.7281,"code_html":3.843,"code_java":4.722,"code_js":4.1326,"code_kotlin":4.7655,"code_lua":3.856,"code_php":4.0435,"code_ps":4.2526,"code_py":4.3606,"code_r":3.2167,"code_ruby":4.2121,"code_rust":4.383,"code_shell":3.6305,"code_sql":3.6156,"code_swift":4.8209,"code_ts":4.4003,"en_fineweb":4.9363,"wiki_ar":7.2141,"wiki_bn":11.6908,"wiki_de":4.1961,"wiki_es":4.8297,"wiki_fr":4.5989,"wiki_gu":8.6161,"wiki_hi":9.1979,"wiki_id":4.9855,"wiki_ja":5.9287,"wiki_ko":4.5465,"wiki_mr":9.9304,"wiki_ru":7.3757,"wiki_ta":10.9733,"wiki_te":9.2259,"wiki_th":12.3622,"wiki_tr":4.9241,"wiki_ur":6.3696,"wiki_vi":4.9869,"wiki_zh":5.0979},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":5.620347250340921,"provenance":"measured","note":"median of 5 fresh processes","q1":5.4628430832448185,"q3":5.638177126563517,"n":5},"batch_mb_per_s_4_threads":{"value":18.300612613557615,"provenance":"measured","note":"median of 5 fresh processes","q1":18.190271144082107,"q3":18.779172937414017,"n":5},"cpu_cores_busy_batch":{"value":3.602732836677368,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.586244613345892,"q3":3.605149734177987,"n":5},"cpu_seconds_per_mb":{"value":0.19461342897082637,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.19197595901550035,"q3":0.19861731913171873,"n":5},"latency_p95_us_256_bytes":{"value":63.45899964799173,"provenance":"measured","note":"median of 5 fresh processes","q1":61.58399992273189,"q3":65.66699994436931,"n":5},"decode_mb_per_s":{"value":43.08721156622042,"provenance":"measured","note":"median of 5 fresh processes","q1":38.015456932945334,"q3":44.60498553642126,"n":5},"load_seconds":{"value":0.6785901250000279,"provenance":"measured","note":"median of 5 fresh processes","q1":0.6715609589996347,"q3":0.6974505419998422,"n":5},"load_ram_mb":{"value":204.791808,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":204.791808,"q3":205.451264,"n":5},"peak_ram_mb_1_mib_input":{"value":257.273856,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":256.765952,"q3":258.854912,"n":5},"peak_ram_mb_16_mib_input":{"value":2394.68544,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2393.84576,"q3":2396.40576,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.57184,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.769024,"q3":18.415616,"n":5},"ram_mb_per_input_mb":{"value":130.41162109375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":130.26196289062503,"q3":130.486083984375,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":36.18286900914715,"provenance":"measured","note":"median of 5 fresh processes","q1":34.1841535617166,"q3":36.61716876583317,"n":5},"batch_mb_per_s_4_threads":{"value":96.03981959090198,"provenance":"measured","note":"median of 5 fresh processes","q1":94.16075364712611,"q3":97.95374011367092,"n":5},"cpu_seconds_per_mb":{"value":0.03312668939821483,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.032212086727031373,"q3":0.03355299533566941,"n":5},"latency_p95_us_256_bytes":{"value":19.458000224403804,"provenance":"measured","note":"median of 5 fresh processes","q1":19.250000150350388,"q3":20.166000012977747,"n":5},"decode_mb_per_s":{"value":112.98578979403972,"provenance":"measured","note":"median of 5 fresh processes","q1":112.44912130252726,"q3":113.2155370183665,"n":5},"load_seconds":{"value":0.8073415829999249,"provenance":"measured","note":"median of 5 fresh processes","q1":0.7976622080000197,"q3":0.8185076249999952,"n":5},"load_ram_mb":{"value":93.540352,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":93.536256,"q3":93.540352,"n":5},"peak_ram_mb_1_mib_input":{"value":88.43264,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":87.703552,"q3":88.604672,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.707008,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.686528,"q3":17.887232,"n":5}}}},{"rank":2,"name":"Akshara 248k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":248000,"models":["Akshara 248k"],"overall":5.1408,"english":4.8394,"code_avg":3.9914,"languages_avg":6.5915,"bpt":{"code_asm":2.6835,"code_cs":4.9354,"code_dart":4.2171,"code_go":3.648,"code_html":3.7873,"code_kotlin":4.6319,"code_lua":3.7525,"code_php":3.9549,"code_ps":4.1292,"code_r":3.1425,"code_ruby":4.1493,"code_shell":3.5268,"code_sql":3.5431,"code_swift":4.7112,"code_ts":4.295,"code_c":3.5042,"code_cpp":3.9276,"code_java":4.6261,"code_js":4.0622,"code_py":4.2687,"code_rust":4.3223,"en_fineweb":4.8394,"wiki_ar":6.5818,"wiki_bn":10.5595,"wiki_de":3.9407,"wiki_es":4.5835,"wiki_fr":4.3493,"wiki_gu":7.7535,"wiki_hi":8.4888,"wiki_id":4.6375,"wiki_ja":5.4672,"wiki_ko":4.1718,"wiki_mr":8.9641,"wiki_ru":6.762,"wiki_ta":9.8155,"wiki_te":8.1998,"wiki_th":10.9216,"wiki_tr":4.4721,"wiki_ur":5.9721,"wiki_vi":4.8656,"wiki_zh":4.7322},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":5.836628228431435,"provenance":"measured","note":"median of 5 fresh processes","q1":5.729098732557851,"q3":5.852981716836457,"n":5},"batch_mb_per_s_4_threads":{"value":18.40640894684117,"provenance":"measured","note":"median of 5 fresh processes","q1":18.07314929157135,"q3":18.44548721846948,"n":5},"cpu_cores_busy_batch":{"value":3.599999563918848,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5902364777768137,"q3":3.600140658904089,"n":5},"cpu_seconds_per_mb":{"value":0.19559169142126065,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.19464037112459176,"q3":0.1991904955711153,"n":5},"latency_p95_us_256_bytes":{"value":61.37500008662755,"provenance":"measured","note":"median of 5 fresh processes","q1":60.58300004951889,"q3":65.91700002900325,"n":5},"decode_mb_per_s":{"value":41.01579788879382,"provenance":"measured","note":"median of 5 fresh processes","q1":40.03215785134015,"q3":43.44499781195673,"n":5},"load_seconds":{"value":0.33785904199999095,"provenance":"measured","note":"median of 5 fresh processes","q1":0.33691641699988395,"q3":0.34502274999999827,"n":5},"load_ram_mb":{"value":119.328768,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":119.328768,"q3":119.328768,"n":5},"peak_ram_mb_1_mib_input":{"value":220.925952,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":220.91776,"q3":220.971008,"n":5},"peak_ram_mb_16_mib_input":{"value":2374.10304,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2373.025792,"q3":2375.737344,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.403328,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.616896,"q3":18.522112,"n":5},"ram_mb_per_input_mb":{"value":131.10302734375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":131.0390625,"q3":131.19409179687503,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":41.148355701987875,"provenance":"measured","note":"median of 5 fresh processes","q1":38.179145933126726,"q3":41.25354307372325,"n":5},"batch_mb_per_s_4_threads":{"value":106.473221629358,"provenance":"measured","note":"median of 5 fresh processes","q1":104.02521484246932,"q3":109.93066708625295,"n":5},"cpu_seconds_per_mb":{"value":0.028916322200501585,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.028385108761659932,"q3":0.02962468626100365,"n":5},"latency_p95_us_256_bytes":{"value":15.874999917286914,"provenance":"measured","note":"median of 5 fresh processes","q1":15.624999832652977,"q3":16.2080000336573,"n":5},"decode_mb_per_s":{"value":110.3553940136642,"provenance":"measured","note":"median of 5 fresh processes","q1":109.80790261679124,"q3":110.50125141473536,"n":5},"load_seconds":{"value":0.4160123340000154,"provenance":"measured","note":"median of 5 fresh processes","q1":0.41419583400011106,"q3":0.4196818330001406,"n":5},"load_ram_mb":{"value":72.159232,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":72.155136,"q3":72.159232,"n":5},"peak_ram_mb_1_mib_input":{"value":63.01696,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":63.01696,"q3":63.021056,"n":5},"peak_ram_mb_16_mib_chunked":{"value":19.304448,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":19.214336,"q3":19.39456,"n":5}}}},{"rank":3,"name":"Akshara 200k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":200000,"models":["Akshara 200k"],"overall":5.0315,"english":4.7918,"code_avg":3.9523,"languages_avg":6.3505,"bpt":{"code_asm":2.6446,"code_cs":4.877,"code_dart":4.1926,"code_go":3.6145,"code_html":3.7629,"code_kotlin":4.5733,"code_lua":3.7102,"code_php":3.9198,"code_ps":4.0742,"code_r":3.1143,"code_ruby":4.1136,"code_shell":3.4859,"code_sql":3.5022,"code_swift":4.6727,"code_ts":4.2449,"code_c":3.4704,"code_cpp":3.8865,"code_java":4.5848,"code_js":4.0293,"code_py":4.2306,"code_rust":4.2934,"en_fineweb":4.7918,"wiki_ar":6.3332,"wiki_bn":10.075,"wiki_de":3.842,"wiki_es":4.4673,"wiki_fr":4.2451,"wiki_gu":7.4366,"wiki_hi":8.1685,"wiki_id":4.4894,"wiki_ja":5.3145,"wiki_ko":4.0399,"wiki_mr":8.577,"wiki_ru":6.5089,"wiki_ta":9.3755,"wiki_te":7.8364,"wiki_th":10.4265,"wiki_tr":4.3145,"wiki_ur":5.7932,"wiki_vi":4.8026,"wiki_zh":4.6132},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":5.734110759561312,"provenance":"measured","note":"median of 5 fresh processes","q1":5.429966938050952,"q3":5.844051229124779,"n":5},"batch_mb_per_s_4_threads":{"value":18.650088442681785,"provenance":"measured","note":"median of 5 fresh processes","q1":18.585399546469404,"q3":18.674387758522915,"n":5},"cpu_cores_busy_batch":{"value":3.5789323405534805,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.572936159223736,"q3":3.6132156532798265,"n":5},"cpu_seconds_per_mb":{"value":0.1927506055427878,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.19189894737243665,"q3":0.19449302377037084,"n":5},"latency_p95_us_256_bytes":{"value":61.70799997562426,"provenance":"measured","note":"median of 5 fresh processes","q1":59.04099998588208,"q3":61.70899996504886,"n":5},"decode_mb_per_s":{"value":45.14105379322546,"provenance":"measured","note":"median of 5 fresh processes","q1":44.88950725625062,"q3":45.64243052736785,"n":5},"load_seconds":{"value":0.2555477079999946,"provenance":"measured","note":"median of 5 fresh processes","q1":0.2500330009997924,"q3":0.2636542090003786,"n":5},"load_ram_mb":{"value":84.148224,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":84.148224,"q3":84.148224,"n":5},"peak_ram_mb_1_mib_input":{"value":191.758336,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":191.75424,"q3":191.758336,"n":5},"peak_ram_mb_16_mib_input":{"value":2359.476224,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2359.369728,"q3":2361.46688,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.129472,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.068032,"q3":17.199104,"n":5},"ram_mb_per_input_mb":{"value":134.06909179687503,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":134.06494140625,"q3":134.18994140625,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":37.12108302421878,"provenance":"measured","note":"median of 5 fresh processes","q1":36.41212202025844,"q3":40.28306496977799,"n":5},"batch_mb_per_s_4_threads":{"value":107.02999409867677,"provenance":"measured","note":"median of 5 fresh processes","q1":105.7416243454292,"q3":107.8570836187658,"n":5},"cpu_seconds_per_mb":{"value":0.029026475076958005,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.02871485211261057,"q3":0.029048648707932997,"n":5},"latency_p95_us_256_bytes":{"value":16.041999970184406,"provenance":"measured","note":"median of 5 fresh processes","q1":15.95799994902336,"q3":16.95800006018544,"n":5},"decode_mb_per_s":{"value":108.66943691211009,"provenance":"measured","note":"median of 5 fresh processes","q1":108.54476329906092,"q3":110.9778271687773,"n":5},"load_seconds":{"value":0.3114650829999732,"provenance":"measured","note":"median of 5 fresh processes","q1":0.3081390420002208,"q3":0.3124225839999326,"n":5},"load_ram_mb":{"value":51.34336,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":51.34336,"q3":51.34336,"n":5},"peak_ram_mb_1_mib_input":{"value":57.91744,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":57.901056,"q3":58.642432,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.096704,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.842752,"q3":17.567744,"n":5}}}},{"rank":4,"name":"Akshara 151k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":151000,"models":["Akshara 151k"],"overall":4.8772,"english":4.7185,"code_avg":3.8937,"languages_avg":6.0195,"bpt":{"code_asm":2.6032,"code_cs":4.813,"code_dart":4.143,"code_go":3.5683,"code_html":3.722,"code_kotlin":4.4921,"code_lua":3.6472,"code_php":3.8753,"code_ps":3.9948,"code_r":3.0438,"code_ruby":4.0555,"code_shell":3.4289,"code_sql":3.4237,"code_swift":4.5994,"code_ts":4.1744,"code_c":3.4318,"code_cpp":3.8389,"code_java":4.515,"code_js":3.983,"code_py":4.1745,"code_rust":4.2397,"en_fineweb":4.7185,"wiki_ar":6.03,"wiki_bn":9.3852,"wiki_de":3.7056,"wiki_es":4.3179,"wiki_fr":4.12,"wiki_gu":6.954,"wiki_hi":7.7859,"wiki_id":4.2931,"wiki_ja":5.0919,"wiki_ko":3.8708,"wiki_mr":8.0431,"wiki_ru":6.1917,"wiki_ta":8.8353,"wiki_te":7.3259,"wiki_th":9.6561,"wiki_tr":4.0739,"wiki_ur":5.548,"wiki_vi":4.7028,"wiki_zh":4.439},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":6.116414017986229,"provenance":"measured","note":"median of 5 fresh processes","q1":6.0974718698952675,"q3":6.128655757761346,"n":5},"batch_mb_per_s_4_threads":{"value":18.45893944644569,"provenance":"measured","note":"median of 5 fresh processes","q1":18.424939107275005,"q3":18.56070840705968,"n":5},"cpu_cores_busy_batch":{"value":3.5890226977657993,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5785923410417104,"q3":3.591094386953844,"n":5},"cpu_seconds_per_mb":{"value":0.19385141902893324,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.192804728276458,"q3":0.1945450006687853,"n":5},"latency_p95_us_256_bytes":{"value":59.750000218627974,"provenance":"measured","note":"median of 5 fresh processes","q1":59.29200005994062,"q3":71.33400004022405,"n":5},"decode_mb_per_s":{"value":46.21559142784512,"provenance":"measured","note":"median of 5 fresh processes","q1":46.12766144661114,"q3":46.52376839601841,"n":5},"load_seconds":{"value":0.199583500000017,"provenance":"measured","note":"median of 5 fresh processes","q1":0.1833465839999917,"q3":0.20041354199975103,"n":5},"load_ram_mb":{"value":93.712384,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":93.36832,"q3":93.71648,"n":5},"peak_ram_mb_1_mib_input":{"value":188.841984,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":188.833792,"q3":188.850176,"n":5},"peak_ram_mb_16_mib_input":{"value":2368.565248,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2368.323584,"q3":2368.7168,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.649664,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.010688,"q3":17.743872,"n":5},"ram_mb_per_input_mb":{"value":136.063720703125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":136.041259765625,"q3":136.068115234375,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":41.239480939779185,"provenance":"measured","note":"median of 5 fresh processes","q1":41.192756585857424,"q3":43.16628297529017,"n":5},"batch_mb_per_s_4_threads":{"value":111.58481435588936,"provenance":"measured","note":"median of 5 fresh processes","q1":103.2065184299624,"q3":115.39791728780762,"n":5},"cpu_seconds_per_mb":{"value":0.027358445804903996,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.026594051601615668,"q3":0.028802831358091977,"n":5},"latency_p95_us_256_bytes":{"value":14.416999874811154,"provenance":"measured","note":"median of 5 fresh processes","q1":14.20800026608049,"q3":15.250000160449416,"n":5},"decode_mb_per_s":{"value":110.06677227775393,"provenance":"measured","note":"median of 5 fresh processes","q1":104.97068436158962,"q3":111.11994214786446,"n":5},"load_seconds":{"value":0.23353245799989963,"provenance":"measured","note":"median of 5 fresh processes","q1":0.23245729200016285,"q3":0.23717933299997185,"n":5},"load_ram_mb":{"value":45.367296,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":45.3632,"q3":45.367296,"n":5},"peak_ram_mb_1_mib_input":{"value":46.661632,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":46.641152,"q3":47.104,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.3296,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.473536,"q3":18.55488,"n":5}}}},{"rank":5,"name":"Akshara 120k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":120000,"models":["Akshara 120k"],"overall":4.7469,"english":4.6553,"code_avg":3.8468,"languages_avg":5.7387,"bpt":{"code_asm":2.573,"code_cs":4.7558,"code_dart":4.1107,"code_go":3.5334,"code_html":3.6805,"code_kotlin":4.4315,"code_lua":3.5823,"code_php":3.837,"code_ps":3.9378,"code_r":3.0069,"code_ruby":4.0117,"code_shell":3.374,"code_sql":3.3728,"code_swift":4.5379,"code_ts":4.1158,"code_c":3.3894,"code_cpp":3.8081,"code_java":4.4626,"code_js":3.9454,"code_py":4.1251,"code_rust":4.1919,"en_fineweb":4.6553,"wiki_ar":5.752,"wiki_bn":8.811,"wiki_de":3.598,"wiki_es":4.1929,"wiki_fr":3.9884,"wiki_gu":6.5997,"wiki_hi":7.3936,"wiki_id":4.1076,"wiki_ja":4.8985,"wiki_ko":3.7331,"wiki_mr":7.591,"wiki_ru":5.9348,"wiki_ta":8.331,"wiki_te":6.9625,"wiki_th":9.0311,"wiki_tr":3.8994,"wiki_ur":5.3396,"wiki_vi":4.5816,"wiki_zh":4.2892},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":5.967619867743156,"provenance":"measured","note":"median of 5 fresh processes","q1":5.956208157401425,"q3":6.016314594826091,"n":5},"batch_mb_per_s_4_threads":{"value":18.09350513047292,"provenance":"measured","note":"median of 5 fresh processes","q1":17.71619448610454,"q3":18.27981824551409,"n":5},"cpu_cores_busy_batch":{"value":3.5869782359774,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.581509474669802,"q3":3.5968366058587065,"n":5},"cpu_seconds_per_mb":{"value":0.19794448056600464,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.1962261433785127,"q3":0.203311929818789,"n":5},"latency_p95_us_256_bytes":{"value":62.000000013995304,"provenance":"measured","note":"median of 5 fresh processes","q1":59.959000054732314,"q3":66.79100010842376,"n":5},"decode_mb_per_s":{"value":46.44357459654735,"provenance":"measured","note":"median of 5 fresh processes","q1":45.12073969902209,"q3":47.21170720911533,"n":5},"load_seconds":{"value":0.1461671250000336,"provenance":"measured","note":"median of 5 fresh processes","q1":0.14010229199993773,"q3":0.15664995900033318,"n":5},"load_ram_mb":{"value":81.068032,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":81.068032,"q3":81.068032,"n":5},"peak_ram_mb_1_mib_input":{"value":187.088896,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":186.998784,"q3":187.138048,"n":5},"peak_ram_mb_16_mib_input":{"value":2378.186752,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2377.9328,"q3":2378.264576,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.088512,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.826368,"q3":17.661952,"n":5},"ram_mb_per_input_mb":{"value":137.31005859375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":137.30590820312503,"q3":137.3251953125,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":40.06823361453194,"provenance":"measured","note":"median of 5 fresh processes","q1":36.20469114167294,"q3":40.9232847662627,"n":5},"batch_mb_per_s_4_threads":{"value":109.3360270854944,"provenance":"measured","note":"median of 5 fresh processes","q1":108.50220877359422,"q3":109.78547342923947,"n":5},"cpu_seconds_per_mb":{"value":0.027261644792260505,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.02709283908548316,"q3":0.02751866817065875,"n":5},"latency_p95_us_256_bytes":{"value":14.457999895967076,"provenance":"measured","note":"median of 5 fresh processes","q1":13.292000005549198,"q3":14.87499997665509,"n":5},"decode_mb_per_s":{"value":107.71359840311378,"provenance":"measured","note":"median of 5 fresh processes","q1":100.94939648822276,"q3":107.86364610800392,"n":5},"load_seconds":{"value":0.2049808749998192,"provenance":"measured","note":"median of 5 fresh processes","q1":0.19523945800006004,"q3":0.2066935839998223,"n":5},"load_ram_mb":{"value":41.598976,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":41.598976,"q3":41.603072,"n":5},"peak_ram_mb_1_mib_input":{"value":44.07296,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":44.068864,"q3":44.089344,"n":5},"peak_ram_mb_16_mib_chunked":{"value":16.994304,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":16.994304,"q3":18.2272,"n":5}}}},{"rank":8,"name":"OpenAI o200k_base (GPT-4o)","provider":"OpenAI","family":"Published","version":null,"vocab":200019,"models":["OpenAI o200k_base (GPT-4o)"],"overall":4.6404,"english":4.7424,"code_avg":3.8524,"languages_avg":5.3264,"bpt":{"code_asm":2.5011,"code_cs":4.8032,"code_dart":4.151,"code_go":3.5193,"code_html":3.5834,"code_kotlin":4.5176,"code_lua":3.6175,"code_php":3.9451,"code_ps":3.7496,"code_r":3.0581,"code_ruby":4.0529,"code_shell":3.358,"code_sql":3.3553,"code_swift":4.4634,"code_ts":4.2001,"code_c":3.3606,"code_cpp":3.8789,"code_java":4.4883,"code_js":3.9776,"code_py":4.1637,"code_rust":4.1552,"en_fineweb":4.7424,"wiki_ar":5.73,"wiki_bn":7.6621,"wiki_de":3.7027,"wiki_es":4.3661,"wiki_fr":4.0961,"wiki_gu":6.4939,"wiki_hi":7.6212,"wiki_id":4.0841,"wiki_ja":3.533,"wiki_ko":3.5902,"wiki_mr":7.0112,"wiki_ru":6.0788,"wiki_ta":7.3747,"wiki_te":6.2667,"wiki_th":6.8635,"wiki_tr":3.7785,"wiki_ur":5.2534,"wiki_vi":4.2522,"wiki_zh":3.4425},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (converted from tiktoken)","encode_mb_per_s_1_thread":{"value":5.101910820251484,"provenance":"measured","note":"median of 5 fresh processes","q1":5.0714635495205,"q3":5.138293990144321,"n":5},"batch_mb_per_s_4_threads":{"value":17.73679441869205,"provenance":"measured","note":"median of 5 fresh processes","q1":17.62775178570308,"q3":17.788999687840956,"n":5},"cpu_cores_busy_batch":{"value":3.614590933457547,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5862186206606927,"q3":3.619274973598692,"n":5},"cpu_seconds_per_mb":{"value":0.20344163363868564,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2031924783228915,"q3":0.20352126796928408,"n":5},"latency_p95_us_256_bytes":{"value":65.54100036737509,"provenance":"measured","note":"median of 5 fresh processes","q1":64.70800008173683,"q3":66.58300026174402,"n":5},"decode_mb_per_s":{"value":39.12072516397016,"provenance":"measured","note":"median of 5 fresh processes","q1":37.16271940307519,"q3":40.102457704134835,"n":5},"load_seconds":{"value":0.5957294170000296,"provenance":"measured","note":"median of 5 fresh processes","q1":0.5924940419999984,"q3":0.6016672920000019,"n":5},"load_ram_mb":{"value":249.864192,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":249.860096,"q3":249.868288,"n":5},"peak_ram_mb_1_mib_input":{"value":321.458176,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":320.503808,"q3":321.8432,"n":5},"peak_ram_mb_16_mib_input":{"value":2471.8336,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2469.187584,"q3":2474.323968,"n":5},"peak_ram_mb_16_mib_chunked":{"value":0,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":0,"q3":0,"n":5},"ram_mb_per_input_mb":{"value":129.434814453125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":129.2568359375,"q3":129.58325195312503,"n":5},"native":{"implementation":"tiktoken","encode_mb_per_s_1_thread":{"value":25.11267863118479,"provenance":"measured","note":"median of 5 fresh processes","q1":25.054576960652053,"q3":25.3858707437138,"n":5},"batch_mb_per_s_4_threads":{"value":74.47334099824387,"provenance":"measured","note":"median of 5 fresh processes","q1":71.89185807330907,"q3":74.93683693291769,"n":5},"cpu_seconds_per_mb":{"value":0.05307079754258946,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.052912482585950826,"q3":0.055236899020092406,"n":5},"latency_p95_us_256_bytes":{"value":19.249999922976713,"provenance":"measured","note":"median of 5 fresh processes","q1":18.49999989644857,"q3":20.707999965452473,"n":5},"decode_mb_per_s":{"value":188.6404882960821,"provenance":"measured","note":"median of 5 fresh processes","q1":188.52886841947648,"q3":191.52183450810887,"n":5},"load_seconds":{"value":0.1457424169998376,"provenance":"measured","note":"median of 5 fresh processes","q1":0.14254829100013922,"q3":0.15049458299995422,"n":5},"load_ram_mb":{"value":91.897856,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":91.897856,"q3":91.897856,"n":5},"peak_ram_mb_1_mib_input":{"value":76.345344,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":76.345344,"q3":76.34944,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.973248,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.575936,"q3":18.026496,"n":5}}}},{"rank":10,"name":"Akshara 92k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":92000,"models":["Akshara 92k"],"overall":4.5886,"english":4.5649,"code_avg":3.782,"languages_avg":5.4188,"bpt":{"code_asm":2.53,"code_cs":4.682,"code_dart":4.0375,"code_go":3.4765,"code_html":3.6253,"code_kotlin":4.3618,"code_lua":3.5164,"code_php":3.7839,"code_ps":3.8629,"code_r":2.957,"code_ruby":3.9523,"code_shell":3.3084,"code_sql":3.2917,"code_swift":4.4559,"code_ts":4.051,"code_c":3.3447,"code_cpp":3.7482,"code_java":4.3854,"code_js":3.8779,"code_py":4.0547,"code_rust":4.1187,"en_fineweb":4.5649,"wiki_ar":5.4504,"wiki_bn":8.2268,"wiki_de":3.4495,"wiki_es":4.0339,"wiki_fr":3.826,"wiki_gu":6.1873,"wiki_hi":6.9576,"wiki_id":3.8996,"wiki_ja":4.6944,"wiki_ko":3.572,"wiki_mr":7.0766,"wiki_ru":5.6188,"wiki_ta":7.8075,"wiki_te":6.4666,"wiki_th":8.3636,"wiki_tr":3.6951,"wiki_ur":5.0728,"wiki_vi":4.4411,"wiki_zh":4.1172},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":6.100163595476987,"provenance":"measured","note":"median of 5 fresh processes","q1":6.008446439600618,"q3":6.168005805384802,"n":5},"batch_mb_per_s_4_threads":{"value":17.31074049393309,"provenance":"measured","note":"median of 5 fresh processes","q1":17.257512822256132,"q3":17.90969294083765,"n":5},"cpu_cores_busy_batch":{"value":3.5482269309547316,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.495635393329438,"q3":3.582283615394513,"n":5},"cpu_seconds_per_mb":{"value":0.2025573110872209,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.20001926483207288,"q3":0.2029137581657972,"n":5},"latency_p95_us_256_bytes":{"value":63.54199967972818,"provenance":"measured","note":"median of 5 fresh processes","q1":58.37499975314131,"q3":67.50000000010914,"n":5},"decode_mb_per_s":{"value":46.85866863218416,"provenance":"measured","note":"median of 5 fresh processes","q1":46.77488202423642,"q3":47.64206157670607,"n":5},"load_seconds":{"value":0.1078596249999464,"provenance":"measured","note":"median of 5 fresh processes","q1":0.10639008299995112,"q3":0.11712995800007776,"n":5},"load_ram_mb":{"value":45.416448,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":45.416448,"q3":45.416448,"n":5},"peak_ram_mb_1_mib_input":{"value":174.1824,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":174.174208,"q3":174.211072,"n":5},"peak_ram_mb_16_mib_input":{"value":2381.4144,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2381.316096,"q3":2381.471744,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.747968,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.6128,"q3":18.276352,"n":5},"ram_mb_per_input_mb":{"value":139.429443359375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":139.414794921875,"q3":139.46240234375,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":39.1210378725301,"provenance":"measured","note":"median of 5 fresh processes","q1":38.182779392110085,"q3":41.60118715472255,"n":5},"batch_mb_per_s_4_threads":{"value":102.9317037703785,"provenance":"measured","note":"median of 5 fresh processes","q1":98.31125080890791,"q3":109.62298731222458,"n":5},"cpu_seconds_per_mb":{"value":0.028730349811679108,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.027497448244241825,"q3":0.03031946003155331,"n":5},"latency_p95_us_256_bytes":{"value":13.54100004391512,"provenance":"measured","note":"median of 5 fresh processes","q1":12.95900005970907,"q3":13.9579997266992,"n":5},"decode_mb_per_s":{"value":99.18681068335604,"provenance":"measured","note":"median of 5 fresh processes","q1":98.29075211288612,"q3":102.35190719021892,"n":5},"load_seconds":{"value":0.15286491599999863,"provenance":"measured","note":"median of 5 fresh processes","q1":0.1514836250000826,"q3":0.15466254200009644,"n":5},"load_ram_mb":{"value":30.691328,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":30.552064,"q3":30.818304,"n":5},"peak_ram_mb_1_mib_input":{"value":32.530432,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":32.247808,"q3":33.243136,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.424384,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.088512,"q3":17.465344,"n":5}}}},{"rank":13,"name":"Akshara 59k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":59000,"models":["Akshara 59k"],"overall":4.3191,"english":4.3932,"code_avg":3.6578,"languages_avg":4.9061,"bpt":{"code_asm":2.4456,"code_cs":4.5497,"code_dart":3.9258,"code_go":3.3736,"code_html":3.5086,"code_kotlin":4.2174,"code_lua":3.4023,"code_php":3.6761,"code_ps":3.7157,"code_r":2.8623,"code_ruby":3.8347,"code_shell":3.1788,"code_sql":3.1189,"code_swift":4.3075,"code_ts":3.8791,"code_c":3.2484,"code_cpp":3.6347,"code_java":4.26,"code_js":3.7572,"code_py":3.9138,"code_rust":4.0043,"en_fineweb":4.3932,"wiki_ar":4.9483,"wiki_bn":7.1818,"wiki_de":3.2094,"wiki_es":3.7634,"wiki_fr":3.5937,"wiki_gu":5.5904,"wiki_hi":6.3142,"wiki_id":3.5653,"wiki_ja":4.3486,"wiki_ko":3.2803,"wiki_mr":6.3575,"wiki_ru":5.1229,"wiki_ta":6.9774,"wiki_te":5.7668,"wiki_th":7.3413,"wiki_tr":3.325,"wiki_ur":4.5851,"wiki_vi":4.1603,"wiki_zh":3.7847},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":6.198963235580143,"provenance":"measured","note":"median of 5 fresh processes","q1":6.142785722277156,"q3":6.2555780795380285,"n":5},"batch_mb_per_s_4_threads":{"value":17.667729432326173,"provenance":"measured","note":"median of 5 fresh processes","q1":17.166497951509918,"q3":17.810078165382766,"n":5},"cpu_cores_busy_batch":{"value":3.5754968188117333,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5660106350776615,"q3":3.5764043900531157,"n":5},"cpu_seconds_per_mb":{"value":0.202374438238212,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.20022431131205665,"q3":0.20833628385680988,"n":5},"latency_p95_us_256_bytes":{"value":58.165999689663295,"provenance":"measured","note":"median of 5 fresh processes","q1":57.45899989051395,"q3":58.5830000545684,"n":5},"decode_mb_per_s":{"value":48.26985164104428,"provenance":"measured","note":"median of 5 fresh processes","q1":47.21170933430155,"q3":49.15305904263578,"n":5},"load_seconds":{"value":0.06612683299999844,"provenance":"measured","note":"median of 5 fresh processes","q1":0.06587079200016888,"q3":0.06826133300000947,"n":5},"load_ram_mb":{"value":38.756352,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":38.756352,"q3":38.756352,"n":5},"peak_ram_mb_1_mib_input":{"value":172.630016,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":172.593152,"q3":172.6464,"n":5},"peak_ram_mb_16_mib_input":{"value":2407.292928,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2406.936576,"q3":2407.387136,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.341888,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.686528,"q3":18.604032,"n":5},"ram_mb_per_input_mb":{"value":141.488037109375,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":141.466796875,"q3":141.49389648437503,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":43.147832703931286,"provenance":"measured","note":"median of 5 fresh processes","q1":41.94876540731698,"q3":43.53332102375039,"n":5},"batch_mb_per_s_4_threads":{"value":111.908222349312,"provenance":"measured","note":"median of 5 fresh processes","q1":111.18645773404668,"q3":113.30136477754456,"n":5},"cpu_seconds_per_mb":{"value":0.025885925967253132,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.025822981466420893,"q3":0.026422384781164147,"n":5},"latency_p95_us_256_bytes":{"value":11.792000009336334,"provenance":"measured","note":"median of 5 fresh processes","q1":11.416999996072263,"q3":12.166999994178695,"n":5},"decode_mb_per_s":{"value":97.06023132026148,"provenance":"measured","note":"median of 5 fresh processes","q1":96.48454810324738,"q3":98.91740955818553,"n":5},"load_seconds":{"value":0.11027433399999607,"provenance":"measured","note":"median of 5 fresh processes","q1":0.10760424999989482,"q3":0.11044145899995784,"n":5},"load_ram_mb":{"value":25.432064,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":25.432064,"q3":25.43616,"n":5},"peak_ram_mb_1_mib_input":{"value":29.229056,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":28.823552,"q3":29.241344,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.260544,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.199104,"q3":17.809408,"n":5}}}},{"rank":16,"name":"Akshara 49k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":49000,"models":["Akshara 49k"],"overall":4.2021,"english":4.3059,"code_avg":3.5977,"languages_avg":4.7026,"bpt":{"code_asm":2.3997,"code_cs":4.4746,"code_dart":3.8832,"code_go":3.3177,"code_html":3.4267,"code_kotlin":4.156,"code_lua":3.3505,"code_php":3.6275,"code_ps":3.6499,"code_r":2.8203,"code_ruby":3.7715,"code_shell":3.1129,"code_sql":3.0181,"code_swift":4.2315,"code_ts":3.8396,"code_c":3.1982,"code_cpp":3.5843,"code_java":4.1953,"code_js":3.7046,"code_py":3.8455,"code_rust":3.9431,"en_fineweb":4.3059,"wiki_ar":4.756,"wiki_bn":6.7828,"wiki_de":3.1212,"wiki_es":3.6591,"wiki_fr":3.5037,"wiki_gu":5.3299,"wiki_hi":6.0899,"wiki_id":3.4265,"wiki_ja":4.1837,"wiki_ko":3.1667,"wiki_mr":6.085,"wiki_ru":4.9021,"wiki_ta":6.637,"wiki_te":5.5118,"wiki_th":6.9616,"wiki_tr":3.1982,"wiki_ur":4.3576,"wiki_vi":4.0038,"wiki_zh":3.6732},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":6.273736117150162,"provenance":"measured","note":"median of 5 fresh processes","q1":6.253023908066624,"q3":6.294651485427605,"n":5},"batch_mb_per_s_4_threads":{"value":17.597606427027582,"provenance":"measured","note":"median of 5 fresh processes","q1":17.457215779760034,"q3":17.756775096730202,"n":5},"cpu_cores_busy_batch":{"value":3.561898862847338,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5331112483489044,"q3":3.562512854349198,"n":5},"cpu_seconds_per_mb":{"value":0.20279120713008603,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.1989725740795976,"q3":0.2040710786470079,"n":5},"latency_p95_us_256_bytes":{"value":58.45900022904971,"provenance":"measured","note":"median of 5 fresh processes","q1":57.16700025004684,"q3":58.91699993298971,"n":5},"decode_mb_per_s":{"value":49.309272169518415,"provenance":"measured","note":"median of 5 fresh processes","q1":49.11152353280939,"q3":49.31400755671072,"n":5},"load_seconds":{"value":0.05253670900037832,"provenance":"measured","note":"median of 5 fresh processes","q1":0.05247358399992663,"q3":0.054333542000222224,"n":5},"load_ram_mb":{"value":33.431552,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":33.427456,"q3":33.431552,"n":5},"peak_ram_mb_1_mib_input":{"value":178.335744,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":178.065408,"q3":178.495488,"n":5},"peak_ram_mb_16_mib_input":{"value":2466.205696,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2465.374208,"q3":2466.316288,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.555456,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.145856,"q3":17.8176,"n":5},"ram_mb_per_input_mb":{"value":145.688720703125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":145.63916015625,"q3":145.6953125,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":39.730417183112266,"provenance":"measured","note":"median of 5 fresh processes","q1":37.78823476444552,"q3":40.083570953849474,"n":5},"batch_mb_per_s_4_threads":{"value":104.66337423945856,"provenance":"measured","note":"median of 5 fresh processes","q1":103.02778553968233,"q3":104.76936728546434,"n":5},"cpu_seconds_per_mb":{"value":0.02859921543494531,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.027978592193785127,"q3":0.02908727374253457,"n":5},"latency_p95_us_256_bytes":{"value":12.834000244765775,"provenance":"measured","note":"median of 5 fresh processes","q1":12.792000234185252,"q3":13.958999943497474,"n":5},"decode_mb_per_s":{"value":95.15622305737604,"provenance":"measured","note":"median of 5 fresh processes","q1":94.86281507110628,"q3":96.38846820944634,"n":5},"load_seconds":{"value":0.09296616699975856,"provenance":"measured","note":"median of 5 fresh processes","q1":0.09286824999992405,"q3":0.09580345900008068,"n":5},"load_ram_mb":{"value":20.840448,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":20.836352,"q3":20.844544,"n":5},"peak_ram_mb_1_mib_input":{"value":22.052864,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":21.87264,"q3":23.216128,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.854464,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.05984,"q3":18.468864,"n":5}}}},{"rank":23,"name":"Akshara 32k","provider":"Akshara","family":"Akshara","version":"0.5.0","vocab":32000,"models":["Akshara 32k"],"overall":3.9282,"english":4.0927,"code_avg":3.4521,"languages_avg":4.24,"bpt":{"code_asm":2.2915,"code_cs":4.2918,"code_dart":3.7421,"code_go":3.2019,"code_html":3.2486,"code_kotlin":3.9722,"code_lua":3.2236,"code_php":3.5017,"code_ps":3.4958,"code_r":2.7158,"code_ruby":3.6218,"code_shell":2.9773,"code_sql":2.8265,"code_swift":4.0742,"code_ts":3.7134,"code_c":3.0781,"code_cpp":3.4212,"code_java":4.046,"code_js":3.5688,"code_py":3.6943,"code_rust":3.7869,"en_fineweb":4.0927,"wiki_ar":4.3298,"wiki_bn":6.0157,"wiki_de":2.8959,"wiki_es":3.4142,"wiki_fr":3.2698,"wiki_gu":4.729,"wiki_hi":5.4514,"wiki_id":3.1179,"wiki_ja":3.8511,"wiki_ko":2.898,"wiki_mr":5.4347,"wiki_ru":4.4207,"wiki_ta":5.9043,"wiki_te":4.9143,"wiki_th":6.0852,"wiki_tr":2.9202,"wiki_ur":3.9457,"wiki_vi":3.5941,"wiki_zh":3.3674},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (exported from Akshara)","encode_mb_per_s_1_thread":{"value":6.303510437376889,"provenance":"measured","note":"median of 5 fresh processes","q1":6.114958198355796,"q3":6.381434276626128,"n":5},"batch_mb_per_s_4_threads":{"value":17.277981622498476,"provenance":"measured","note":"median of 5 fresh processes","q1":16.969863289683705,"q3":17.50034971107797,"n":5},"cpu_cores_busy_batch":{"value":3.531046994243594,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.521034285318306,"q3":3.5422166992960378,"n":5},"cpu_seconds_per_mb":{"value":0.2050133387503752,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2030878092476439,"q3":0.20746221362934417,"n":5},"latency_p95_us_256_bytes":{"value":57.66699996456737,"provenance":"measured","note":"median of 5 fresh processes","q1":57.45799990108935,"q3":58.54200003341248,"n":5},"decode_mb_per_s":{"value":46.35135997715321,"provenance":"measured","note":"median of 5 fresh processes","q1":45.87138545053736,"q3":47.085390975300854,"n":5},"load_seconds":{"value":0.03534070899991093,"provenance":"measured","note":"median of 5 fresh processes","q1":0.03528599999981452,"q3":0.0369946250000055,"n":5},"load_ram_mb":{"value":24.612864,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":24.612864,"q3":24.612864,"n":5},"peak_ram_mb_1_mib_input":{"value":176.607232,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":176.533504,"q3":176.771072,"n":5},"peak_ram_mb_16_mib_input":{"value":2486.562816,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2485.219328,"q3":2487.762944,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.046976,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.772544,"q3":18.419712,"n":5},"ram_mb_per_input_mb":{"value":147.15576171875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":147.07568359375,"q3":147.227294921875,"n":5},"native":{"implementation":"akshara-native (Rust)","encode_mb_per_s_1_thread":{"value":42.53190270744731,"provenance":"measured","note":"median of 5 fresh processes","q1":41.40957775792447,"q3":42.55086535378701,"n":5},"batch_mb_per_s_4_threads":{"value":110.03894128296491,"provenance":"measured","note":"median of 5 fresh processes","q1":105.7101972536139,"q3":113.2780285046388,"n":5},"cpu_seconds_per_mb":{"value":0.02632153052414888,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.025935518604272456,"q3":0.027332695781836273,"n":5},"latency_p95_us_256_bytes":{"value":11.12500001454464,"provenance":"measured","note":"median of 5 fresh processes","q1":10.916999940491223,"q3":12.334000075497896,"n":5},"decode_mb_per_s":{"value":92.08316318702454,"provenance":"measured","note":"median of 5 fresh processes","q1":91.72270818743596,"q3":92.39182152658292,"n":5},"load_seconds":{"value":0.07983766699999251,"provenance":"measured","note":"median of 5 fresh processes","q1":0.0760976669998854,"q3":0.08004070800006957,"n":5},"load_ram_mb":{"value":18.190336,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":18.18624,"q3":18.190336,"n":5},"peak_ram_mb_1_mib_input":{"value":23.158784,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":23.150592,"q3":23.187456,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.149376,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.584128,"q3":18.407424,"n":5}}}},{"rank":25,"name":"cl100k_base (GPT-4, OLMo-2-0425-1B, Olmo-3-1025-7B, phi-4)","provider":"OpenAI","family":"Published","version":null,"vocab":100277,"models":["OpenAI cl100k_base (GPT-4)","OLMo-2-0425-1B","Olmo-3-1025-7B","phi-4"],"overall":3.7359,"english":4.6914,"code_avg":3.8723,"languages_avg":2.644,"bpt":{"code_asm":2.4992,"code_cs":4.8988,"code_dart":4.2041,"code_go":3.5294,"code_html":3.533,"code_kotlin":4.6222,"code_lua":3.5982,"code_php":3.97,"code_ps":3.768,"code_r":3.0618,"code_ruby":4.0089,"code_shell":3.3917,"code_sql":3.3815,"code_swift":4.5891,"code_ts":3.9882,"code_c":3.3836,"code_cpp":3.9063,"code_java":4.5803,"code_js":4.0296,"code_py":4.1926,"code_rust":4.1818,"en_fineweb":4.6914,"wiki_ar":2.615,"wiki_bn":2.197,"wiki_de":3.1417,"wiki_es":3.7145,"wiki_fr":3.5579,"wiki_gu":1.5934,"wiki_hi":2.6125,"wiki_id":3.2902,"wiki_ja":2.6141,"wiki_ko":2.2853,"wiki_mr":2.5863,"wiki_ru":3.6319,"wiki_ta":2.0483,"wiki_te":1.574,"wiki_th":2.9863,"wiki_tr":2.8183,"wiki_ur":2.0264,"wiki_vi":2.6936,"wiki_zh":2.2487},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[],"ranked":true,"resources":{"library":"tokenizers (converted from tiktoken)","encode_mb_per_s_1_thread":{"value":5.153221493593562,"provenance":"measured","note":"median of 5 fresh processes","q1":5.101702636960398,"q3":5.19896650664927,"n":5},"batch_mb_per_s_4_threads":{"value":16.422928599858064,"provenance":"measured","note":"median of 5 fresh processes","q1":16.343199716368368,"q3":16.71586612937975,"n":5},"cpu_cores_busy_batch":{"value":3.519135287307303,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.5109858677711996,"q3":3.5221516695946185,"n":5},"cpu_seconds_per_mb":{"value":0.21258265340158997,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2100391233452332,"q3":0.2144655046253479,"n":5},"latency_p95_us_256_bytes":{"value":65.74999997610576,"provenance":"measured","note":"median of 5 fresh processes","q1":62.791000345896464,"q3":84.12500000076761,"n":5},"decode_mb_per_s":{"value":36.9393414973548,"provenance":"measured","note":"median of 5 fresh processes","q1":35.73547012610242,"q3":38.24051348435033,"n":5},"load_seconds":{"value":0.2929883750002773,"provenance":"measured","note":"median of 5 fresh processes","q1":0.2891960420001851,"q3":0.2957059999998819,"n":5},"load_ram_mb":{"value":160.456704,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":160.452608,"q3":160.456704,"n":5},"peak_ram_mb_1_mib_input":{"value":231.231488,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":231.231488,"q3":231.297024,"n":5},"peak_ram_mb_16_mib_input":{"value":2687.71328,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":2687.438848,"q3":2688.016384,"n":5},"peak_ram_mb_16_mib_chunked":{"value":17.260544,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.047552,"q3":17.330176,"n":5},"ram_mb_per_input_mb":{"value":148.69970703125,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":148.696044921875,"q3":148.7119140625,"n":5},"native":{"implementation":"tiktoken","encode_mb_per_s_1_thread":{"value":19.71889776783207,"provenance":"measured","note":"median of 5 fresh processes","q1":18.84703351082206,"q3":20.008905249603483,"n":5},"batch_mb_per_s_4_threads":{"value":46.55900518851056,"provenance":"measured","note":"median of 5 fresh processes","q1":43.79388111042827,"q3":51.848860675754,"n":5},"cpu_seconds_per_mb":{"value":0.08492953673085811,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.0763106702134939,"q3":0.09054924083925046,"n":5},"latency_p95_us_256_bytes":{"value":20.708999727503397,"provenance":"measured","note":"median of 5 fresh processes","q1":20.667000171670225,"q3":22.290999936558364,"n":5},"decode_mb_per_s":{"value":191.80648101361635,"provenance":"measured","note":"median of 5 fresh processes","q1":188.95918319043187,"q3":193.53821773446285,"n":5},"load_seconds":{"value":0.07270533400014756,"provenance":"measured","note":"median of 5 fresh processes","q1":0.07246958300015649,"q3":0.07311799999979485,"n":5},"load_ram_mb":{"value":48.766976,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":48.766976,"q3":48.771072,"n":5},"peak_ram_mb_1_mib_input":{"value":43.13088,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":43.13088,"q3":43.159552,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.386944,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":18.243584,"q3":18.640896,"n":5}}}},{"rank":26,"name":"OpenAI p50k_base (Codex)","provider":"OpenAI","family":"Published","version":null,"vocab":50281,"models":["OpenAI p50k_base (Codex)"],"overall":3.2165,"english":4.6591,"code_avg":3.2019,"languages_avg":1.7884,"bpt":{"code_asm":2.2921,"code_cs":4.0215,"code_dart":3.4779,"code_go":2.8186,"code_html":2.9378,"code_kotlin":3.6489,"code_lua":2.9298,"code_php":3.2511,"code_ps":3.3222,"code_r":2.6295,"code_ruby":3.2811,"code_shell":2.7908,"code_sql":2.8017,"code_swift":3.785,"code_ts":3.4253,"code_c":2.7859,"code_cpp":3.1851,"code_java":3.8114,"code_js":3.2924,"code_py":3.3188,"code_rust":3.4336,"en_fineweb":4.6591,"wiki_ar":1.8465,"wiki_bn":1.3396,"wiki_de":2.7027,"wiki_es":2.9993,"wiki_fr":3.0024,"wiki_gu":1.0123,"wiki_hi":1.6918,"wiki_id":2.6656,"wiki_ja":1.9655,"wiki_ko":1.1704,"wiki_mr":1.682,"wiki_ru":1.7176,"wiki_ta":1.0207,"wiki_te":1.0141,"wiki_th":1.5085,"wiki_tr":2.2562,"wiki_ur":1.4183,"wiki_vi":1.5279,"wiki_zh":1.4374},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"byte_fallback","detail":"test sets where the tokenizer falls back to (nearly) single bytes","sets":[{"set":"wiki_bn","bytes_per_token":{"value":1.33957277920174,"provenance":"measured"},"split_byte_token_share":{"value":0.5971711511428,"provenance":"measured"}},{"set":"wiki_gu","bytes_per_token":{"value":1.0123481947145003,"provenance":"measured"},"split_byte_token_share":{"value":0.927668602182073,"provenance":"measured"}},{"set":"wiki_ko","bytes_per_token":{"value":1.1704074058388076,"provenance":"measured"},"split_byte_token_share":{"value":0.800793100604299,"provenance":"measured"}},{"set":"wiki_ta","bytes_per_token":{"value":1.0206642334440994,"provenance":"measured"},"split_byte_token_share":{"value":0.947330617647118,"provenance":"measured"}},{"set":"wiki_te","bytes_per_token":{"value":1.014070553316086,"provenance":"measured"},"split_byte_token_share":{"value":0.9391352305316568,"provenance":"measured"}},{"set":"wiki_ur","bytes_per_token":{"value":1.418344131757258,"provenance":"measured"},"split_byte_token_share":{"value":0.5737919778426233,"provenance":"measured"}},{"set":"wiki_zh","bytes_per_token":{"value":1.4374053798988284,"provenance":"measured"},"split_byte_token_share":{"value":0.667344514070483,"provenance":"measured"}}]}],"ranked":true,"resources":{"library":"tokenizers (converted from tiktoken)","encode_mb_per_s_1_thread":{"value":5.033180687959843,"provenance":"measured","note":"median of 5 fresh processes","q1":4.988326943952645,"q3":5.0736678747005595,"n":5},"batch_mb_per_s_4_threads":{"value":14.126312341626818,"provenance":"measured","note":"median of 5 fresh processes","q1":14.121296533748598,"q3":14.12667908683695,"n":5},"cpu_cores_busy_batch":{"value":3.479774456382404,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.4232377921248203,"q3":3.481527017858286,"n":5},"cpu_seconds_per_mb":{"value":0.24275214339138776,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.2424166778130887,"q3":0.2463328271546391,"n":5},"latency_p95_us_256_bytes":{"value":64.29200038837735,"provenance":"measured","note":"median of 5 fresh processes","q1":63.499999669147655,"q3":64.45799999710289,"n":5},"decode_mb_per_s":{"value":33.18989830935828,"provenance":"measured","note":"median of 5 fresh processes","q1":32.07674185188733,"q3":33.241454751372615,"n":5},"load_seconds":{"value":0.13893637499995748,"provenance":"measured","note":"median of 5 fresh processes","q1":0.138239583000086,"q3":0.14304120799988596,"n":5},"load_ram_mb":{"value":80.216064,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":80.216064,"q3":80.216064,"n":5},"peak_ram_mb_1_mib_input":{"value":216.608768,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":216.608768,"q3":216.612864,"n":5},"peak_ram_mb_16_mib_input":{"value":3049.263104,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":3049.189376,"q3":3049.435136,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.120704,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.89952,"q3":18.14528,"n":5},"ram_mb_per_input_mb":{"value":176.546875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":176.54248046875003,"q3":176.553955078125,"n":5},"native":{"implementation":"tiktoken","encode_mb_per_s_1_thread":{"value":19.471356296940503,"provenance":"measured","note":"median of 5 fresh processes","q1":19.126967563461037,"q3":19.823081075696624,"n":5},"batch_mb_per_s_4_threads":{"value":57.774697014819,"provenance":"measured","note":"median of 5 fresh processes","q1":56.93341749470764,"q3":60.09178194137764,"n":5},"cpu_seconds_per_mb":{"value":0.06811548694604964,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.06554167676997433,"q3":0.06936555519553209,"n":5},"latency_p95_us_256_bytes":{"value":20.0829999812413,"provenance":"measured","note":"median of 5 fresh processes","q1":19.583999801398026,"q3":20.166000012977747,"n":5},"decode_mb_per_s":{"value":160.8849104287948,"provenance":"measured","note":"median of 5 fresh processes","q1":155.67390986114853,"q3":161.1063421911218,"n":5},"load_seconds":{"value":0.03768454200007909,"provenance":"measured","note":"median of 5 fresh processes","q1":0.03750716600006854,"q3":0.04000120800037621,"n":5},"load_ram_mb":{"value":26.29632,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":26.29632,"q3":26.29632,"n":5},"peak_ram_mb_1_mib_input":{"value":25.731072,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":25.68192,"q3":25.731072,"n":5},"peak_ram_mb_16_mib_chunked":{"value":18.157568,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":17.69472,"q3":18.501632,"n":5}}}},{"rank":29,"name":"gpt2","provider":"OpenAI","family":"Published","version":null,"vocab":50257,"models":["gpt2"],"overall":2.9504,"english":4.6591,"code_avg":2.4051,"languages_avg":1.7871,"bpt":{"code_c":2.3221,"code_cpp":2.461,"code_java":2.7997,"code_js":2.4135,"code_rust":2.2879,"wiki_bn":1.3305,"wiki_gu":1.0123,"wiki_id":2.6621,"wiki_mr":1.6818,"wiki_ta":1.0205,"wiki_te":1.014,"wiki_th":1.5085,"wiki_tr":2.2561,"wiki_ur":1.4178,"wiki_vi":1.5231,"code_py":2.2877,"en_fineweb":4.6591,"wiki_ar":1.8464,"wiki_de":2.7027,"wiki_es":2.9992,"wiki_fr":3.0023,"wiki_hi":1.6872,"wiki_ja":1.9648,"wiki_ko":1.1704,"wiki_ru":1.7174,"wiki_zh":1.4374,"code_asm":2.0056,"code_cs":2.1996,"code_dart":2.6525,"code_go":2.7016,"code_html":2.0877,"code_kotlin":2.3454,"code_lua":2.5877,"code_php":2.2399,"code_ps":2.39,"code_r":2.3696,"code_ruby":2.4978,"code_shell":2.483,"code_sql":2.4008,"code_swift":2.4827,"code_ts":2.4913},"lossless":true,"roundtrip":{"value":0,"provenance":"measured","note":"lines, sentences and requests that do not decode back exactly"},"flags":[{"kind":"byte_fallback","detail":"test sets where the tokenizer falls back to (nearly) single bytes","sets":[{"set":"wiki_bn","bytes_per_token":{"value":1.330538411612696,"provenance":"measured"},"split_byte_token_share":{"value":0.5931437001698029,"provenance":"measured"}},{"set":"wiki_gu","bytes_per_token":{"value":1.0123327715409298,"provenance":"measured"},"split_byte_token_share":{"value":0.9276544691061788,"provenance":"measured"}},{"set":"wiki_ko","bytes_per_token":{"value":1.1703891437411118,"provenance":"measured"},"split_byte_token_share":{"value":0.8007806056715393,"provenance":"measured"}},{"set":"wiki_ta","bytes_per_token":{"value":1.020505692775702,"provenance":"measured"},"split_byte_token_share":{"value":0.9471834679534248,"provenance":"measured"}},{"set":"wiki_te","bytes_per_token":{"value":1.0140420821267278,"provenance":"measured"},"split_byte_token_share":{"value":0.939108863237099,"provenance":"measured"}},{"set":"wiki_ur","bytes_per_token":{"value":1.4178493637195146,"provenance":"measured"},"split_byte_token_share":{"value":0.5735918191331862,"provenance":"measured"}},{"set":"wiki_zh","bytes_per_token":{"value":1.437360970677452,"provenance":"measured"},"split_byte_token_share":{"value":0.6673238961914392,"provenance":"measured"}}]}],"ranked":true,"resources":{"library":"tokenizers","encode_mb_per_s_1_thread":{"value":4.814825216097814,"provenance":"measured","note":"median of 5 fresh processes","q1":4.813821967040831,"q3":4.819629790355922,"n":5},"batch_mb_per_s_4_threads":{"value":12.407483704422464,"provenance":"measured","note":"median of 5 fresh processes","q1":12.314218491580338,"q3":12.439353137060534,"n":5},"cpu_cores_busy_batch":{"value":3.4413917119159008,"provenance":"measured","note":"CPU time / wall time during the 4-thread batch (4.0 = all four cores busy); median of 5 fresh processes","q1":3.417085669690939,"q3":3.4452696797853495,"n":5},"cpu_seconds_per_mb":{"value":0.276965340707377,"provenance":"measured","note":"CPU seconds (user + system, all threads) per MB encoded in the 4-thread batch; median of 5 fresh processes","q1":0.275581515393626,"q3":0.2774910703450097,"n":5},"latency_p95_us_256_bytes":{"value":73.50000009864743,"provenance":"measured","note":"median of 5 fresh processes","q1":72.45799997690483,"q3":74.1250000828586,"n":5},"decode_mb_per_s":{"value":29.806463251233527,"provenance":"measured","note":"median of 5 fresh processes","q1":29.531230642237514,"q3":29.983672418984067,"n":5},"load_seconds":{"value":0.04961520899996685,"provenance":"measured","note":"median of 5 fresh processes","q1":0.04851966699993682,"q3":0.0536567499999876,"n":5},"load_ram_mb":{"value":35.856384,"provenance":"measured","note":"resident-memory growth from loading, read 1 s after load and a real-text warm-up (configs 1.2 and 1.3; earlier configs read it at return); median of 5 fresh processes","q1":35.856384,"q3":35.897344,"n":5},"peak_ram_mb_1_mib_input":{"value":240.410624,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 1 MiB; median of 5 fresh processes","q1":239.464448,"q3":242.450432,"n":5},"peak_ram_mb_16_mib_input":{"value":3192.500224,"provenance":"measured","note":"peak resident memory over the process before loading, while encoding 16 MiB in one call; on the tokenizers library mostly that call's output buffers; reported, not scored; median of 5 fresh processes","q1":3187.777536,"q3":3193.532416,"n":5},"peak_ram_mb_16_mib_chunked":{"value":24.22784,"provenance":"measured","note":"peak resident memory above the loaded and warmed-up tokenizer (config 1.3), encoding the 16 MiB in 64 Ki-character pieces whose outputs are dropped; median of 5 fresh processes","q1":23.94112,"q3":24.879104,"n":5},"ram_mb_per_input_mb":{"value":189.016357421875,"provenance":"measured","note":"peak resident memory above the loaded tokenizer, per MB of input, encoding 16 MiB in one call; median of 5 fresh processes","q1":188.75244140625,"q3":189.07861328125,"n":5}}}],"sets":[{"id":"en_fineweb","label":"English web","group":"English"},{"id":"code_py","label":"Python","group":"Code"},{"id":"code_js","label":"JavaScript","group":"Code"},{"id":"code_ts","label":"TypeScript","group":"Code"},{"id":"code_java","label":"Java","group":"Code"},{"id":"code_cs","label":"C#","group":"Code"},{"id":"code_cpp","label":"C++","group":"Code"},{"id":"code_c","label":"C","group":"Code"},{"id":"code_go","label":"Go","group":"Code"},{"id":"code_rust","label":"Rust","group":"Code"},{"id":"code_php","label":"PHP","group":"Code"},{"id":"code_ruby","label":"Ruby","group":"Code"},{"id":"code_swift","label":"Swift","group":"Code"},{"id":"code_kotlin","label":"Kotlin","group":"Code"},{"id":"code_dart","label":"Dart","group":"Code"},{"id":"code_lua","label":"Lua","group":"Code"},{"id":"code_r","label":"R","group":"Code"},{"id":"code_sql","label":"SQL","group":"Code"},{"id":"code_shell","label":"Bash/Shell","group":"Code"},{"id":"code_ps","label":"PowerShell","group":"Code"},{"id":"code_html","label":"HTML/CSS","group":"Code"},{"id":"code_asm","label":"Assembly","group":"Code"},{"id":"wiki_de","label":"German","group":"Languages"},{"id":"wiki_fr","label":"French","group":"Languages"},{"id":"wiki_es","label":"Spanish","group":"Languages"},{"id":"wiki_ru","label":"Russian","group":"Languages"},{"id":"wiki_zh","label":"Chinese","group":"Languages"},{"id":"wiki_ja","label":"Japanese","group":"Languages"},{"id":"wiki_ko","label":"Korean","group":"Languages"},{"id":"wiki_ar","label":"Arabic","group":"Languages"},{"id":"wiki_ur","label":"Urdu","group":"Languages"},{"id":"wiki_hi","label":"Hindi","group":"Languages"},{"id":"wiki_mr","label":"Marathi","group":"Languages"},{"id":"wiki_gu","label":"Gujarati","group":"Languages"},{"id":"wiki_bn","label":"Bengali","group":"Languages"},{"id":"wiki_ta","label":"Tamil","group":"Languages"},{"id":"wiki_te","label":"Telugu","group":"Languages"},{"id":"wiki_th","label":"Thai","group":"Languages"},{"id":"wiki_vi","label":"Vietnamese","group":"Languages"},{"id":"wiki_tr","label":"Turkish","group":"Languages"},{"id":"wiki_id","label":"Indonesian","group":"Languages"}],"providers":[{"name":"Akshara","color":"#0f7b6c","dark_color":"#3cc2ad","logo":null,"mark":"monogram","tokenizers":9},{"name":"OpenAI","color":"#1f1f1f","dark_color":"#d9d9d9","logo":"logos/openai.svg","mark":"logo","tokenizers":4},{"name":"Google","color":"#4285f4","dark_color":"#4285f4","logo":null,"mark":"icon","tokenizers":2},{"name":"Meta","color":"#0467df","dark_color":"#3b8cf0","logo":null,"mark":"icon","tokenizers":1},{"name":"Alibaba (Qwen)","color":"#ff6a00","dark_color":"#ff6a00","logo":null,"mark":"icon","tokenizers":3},{"name":"DeepSeek","color":"#4d6bfe","dark_color":"#4d6bfe","logo":null,"mark":"icon","tokenizers":1},{"name":"Mistral","color":"#d9370c","dark_color":"#f0592a","logo":null,"mark":"icon","tokenizers":2},{"name":"Microsoft","color":"#e5a000","dark_color":"#e5a000","logo":"logos/microsoft.svg","mark":"logo","tokenizers":2},{"name":"Sarvam","color":"#7c3aed","dark_color":"#9d6cf5","logo":"logos/sarvam-black.svg","mark":"logo","tokenizers":2},{"name":"Moonshot AI (Kimi)","color":"#111827","dark_color":"#cbd5e1","logo":null,"mark":"icon","tokenizers":1},{"name":"Z.ai (GLM)","color":"#0ea5e9","dark_color":"#0ea5e9","logo":null,"mark":"icon","tokenizers":1},{"name":"MiniMax","color":"#e73562","dark_color":"#e73562","logo":null,"mark":"icon","tokenizers":1},{"name":"Cohere","color":"#39594d","dark_color":"#6f9a88","logo":"logos/cohere.svg","mark":"logo","tokenizers":1},{"name":"Tencent","color":"#1d6fe0","dark_color":"#1d6fe0","logo":null,"mark":"icon","tokenizers":1},{"name":"Baidu","color":"#2932e1","dark_color":"#5a62f0","logo":null,"mark":"icon","tokenizers":1},{"name":"ByteDance","color":"#3c8cff","dark_color":"#3c8cff","logo":null,"mark":"icon","tokenizers":1},{"name":"NVIDIA","color":"#76b900","dark_color":"#76b900","logo":null,"mark":"icon","tokenizers":0},{"name":"IBM","color":"#0f62fe","dark_color":"#4589ff","logo":null,"mark":"monogram","tokenizers":0},{"name":"Hugging Face","color":"#e6b400","dark_color":"#ffd21e","logo":null,"mark":"icon","tokenizers":1},{"name":"Ai2","color":"#f0529c","dark_color":"#f0529c","logo":"logos/ai2.png","mark":"logo","tokenizers":2},{"name":"EleutherAI","color":"#6b7280","dark_color":"#9ca3af","logo":"logos/eleutherai.png","mark":"logo","tokenizers":1},{"name":"TII (Falcon)","color":"#b45309","dark_color":"#d97706","logo":"logos/tii.png","mark":"logo","tokenizers":1},{"name":"Liquid AI","color":"#7e22ce","dark_color":"#a855f7","logo":"logos/liquid-ai.png","mark":"logo","tokenizers":1},{"name":"inclusionAI (Ant Group)","color":"#38a3e8","dark_color":"#38a3e8","logo":null,"mark":"monogram","tokenizers":1},{"name":"Xiaomi","color":"#ff6900","dark_color":"#ff6900","logo":null,"mark":"icon","tokenizers":0},{"name":"01.AI","color":"#15803d","dark_color":"#22c55e","logo":"logos/01ai.svg","mark":"logo","tokenizers":0},{"name":"Shanghai AI Lab (InternLM)","color":"#1e40af","dark_color":"#5b7fe0","logo":"logos/internlm.png","mark":"logo","tokenizers":0},{"name":"StepFun","color":"#0d9488","dark_color":"#2dd4bf","logo":"logos/stepfun.svg","mark":"logo","tokenizers":0}]}