1 | 2 | 2 | 2 | 2 | 2 | 2 | 2 | 2 | 2 | 2 | 2 | 2 | 2 | 2 | 2 | 1 | 1 | 1 | 2 | 2 | 4 | 4 | 4 | 4 | 4 | 4 | 4 | 5 | 5 | 5 | 5 | 5 | 6 | 6 | 7 | 7 |
10
|
11 | 12 | 13 | 13 |
14
|
15
|
31
|
32
|
31
|
38
|
44
|
43
|
42
|
52
|
49
|
53
|
51
|
48
|
51
|
54
|
56
|
2020 | 2021 | 2022 | 2023 | 2024 | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
Tokenizers is used at run time in 116 crates (of which 14 optionally). It is a direct run-time dependency in 82 crates. It's used only as a dev dependency in 5 crates (of which 4 directly).
Number of dependers | Tokenizers version | Downloads/month |
---|---|---|
33 | 0.21.0 | 60K |
12 | 0.20.4 | 17K |
1 | 0.20.3 | 11K |
33 | 0.19.1 | 31K |
22 | 0.15.2 | 29K |
6 | 0.14.1 | 4.4K |
9 | 0.13.4 | 5.8K |
4 | 0.11.3 | 0 |
1 | 0.10.1 | 92 |
Depender (with downloads and own rev deps) | Tokenizers version | |||
---|---|---|---|---|
26K | 14 | text-splitter | optional | ^0.21 |
4.3K | 10 | fastembed | ^0.19 | |
1.8K | valentinus | >=0.13.4 | ||
1.7K | rsllm | ^0.19.1 | ||
1.5K | aleph-alpha-client | ^0.21.0 | ||
4.3K | 6 | lancedb | optional | ^0.19.1 |
950 | gpt_sovits_rs | ^0.20.0 | ||
800 | sbv2_core | ^0.20.0 | ||
800 | toktrie_hf_tokenizers | >=0.20.0, <1.0.0 | ||
700 | outlines-core | =0.20.3 | ||
2.4K | 17 | rust-bert | optional | ^0.20 |
440 | rai-models | ^0.15.0 | ||
420 | musicgpt | ^0.19.1 | ||
400 | 1 | rbert | ^0.21.0 | |
400 | embed_anything | ^0.20.2 | ||
380 | 2 | claude-tokenizer | ^0.20.0 | |
330 | 1 | kalosm-llama | ^0.21.0 | |
290 | 4 | candle-datasets | ^0.19.1 | |
280 | mpnet-rs | ^0.15.1 | ||
280 | 1 | rwhisper | ^0.21.0 | |
270 | 1 | rwuerstchen | ^0.21.0 | |
270 | 4 | candle-examples | ^0.19.1 | |
260 | 1 | kalosm-ocr | ^0.21.0 | |
230 | gline-rs | ^0.21.0 | ||
220 | rphi | ^0.19.1 | ||
220 | candle_embed | ^0.19.1 | ||
190 | 3 | llm_models | ^0.21.0 | |
190 | lsp-ai | ^0.14.1 | ||
180 | simple-whisper | ^0.19.1 | ||
140 | llms-from-scratch-rs | ^0.21.0 | ||
140 | bundle_repo | ^0.21 | ||
130 | gte-rs | ^0.21.0 | ||
110 | moshi-backend | ^0.15.2 | ||
110 | prefrontal | ^0.15 | ||
2 | beet_ml | ^0.19.1 | ||
smartgpt | ^0.13.3 | |||
sbert | ^0.11 | |||
reco-forge | ^0.15.0 | |||
edge-transformers | ^0.13 | |||
promptbox | ^0.15.0 | |||
fashion-clip-rs | ^0.15 | |||
modeling | ^0.11.0 | |||
1 | wonnx-preprocessing | ^0.13.3 | ||
ct2rs | ^0.21.0 | |||
2 | diffusion_rs_common | ^0.21.0 | ||
enfer_core | ^0.15.0 | |||
glimpse | ^0.21.0 | |||
usls | ^0.15.2 | |||
1 | diffuse_rs_core | ^0.21.0 | ||
1 | diffusion_rs_core | ^0.21.0 | ||
4 | sapiens | ^0.14.1 | ||
dataflow_nlp | ^0.11 | |||
2 | diffuse_rs_common | ^0.21.0 | ||
aiochatgpt | ^0.13.2 | |||
coze | ^0.15.2 | |||
callm | ^0.19 | |||
moz_genai | ^0.15.0 | |||
cracken | ~0.11.0 | |||
1 | clip_txt | ^0.13.3 | ||
spoilers | ^0 | |||
wimbd | ^0.15.1 | |||
glowrs | ^0.19.1 | |||
aleph-alpha-api | ^0.15.0 | |||
ragtime | ^0.19 | |||
oxidized-transformers | ^0.15 | |||
rusty-dawg | ^0.20.0 | |||
yolors | ^0.15.0 | |||
bge | ^0.15.2 | |||
bm42 | ^0.19 | |||
codegeex4-candle | ^0.19.1 | |||
heapswap | ^0.15.1 | |||
1 | stable-diffusion | ^0.15.0 | ||
anonymize_rs | ^0.13.3 | |||
bridge-common | ^0.19.1 | |||
candle-chat-types | ^0.13.4 | |||
inference_core | ^0.15.0 | |||
rmistral | ^0.13.4 | |||
token-counter | ^0.19.1 | |||
180 | norma | optional | ^0.20.0 | |
170 | sentinellm_ai | optional | ^0.20 | |
aleph-alpha-tokenizer | optional | ^0.10.1 | ||
code-splitter | optional | ^0.19.1 | ||
181K | 48 | ort | dev | >=0.13.4 |
280 | nano-vectordb-rs | dev | ^0.21.0 | |
190 | mtc-token-healing | dev | ^0.20.1 | |
170 | 3 | general-sam | dev | ^0.15.2 |