{"slug":"openlair-huggingface-tokenizers","source_name":"openlair/huggingface-tokenizers","name":"Openlair/Huggingface Tokenizers","description":"Fast tokenizers optimized for research and production. Rust-based implementation tokenizes 1GB in <20 seconds. Supports BPE, WordPiece, and Unigram algorithms. Train custom vocabularies, track alignments, handle padding/truncation. Integrates seamlessly with transformers. Use when you need high-performance tokenization or custom tokenizer training.","version":1,"lift":{"pass_rate_delta_pts":0,"pass_rate_pct":90.9,"total_cases":22,"passed_cases":20,"tokens_delta_pct":223.6,"turns_delta_pct":0,"verdict":"mixed","benchmark_model":"gemini-3.6-flash","grading_method":"judged","completed_at":"2026-08-07T20:01:38.920279+00:00"},"skill_score":null,"benchmark_models":[{"model":"gemini-3.6-flash","headline":true,"delta_pts":0,"with_pass_pct":90.9,"without_pass_pct":90.9,"tokens_delta_pct":223.6,"turns_delta_pct":0,"total_cases":22,"cases_aggregated":22,"verdict":"mixed","never_hurt":false,"completed_at":"2026-08-07T20:01:38.920279+00:00","run_id":"3fa679a4-f8c1-4034-8535-c20295ba4d71","version_number":1,"is_latest_version":true,"gate":null}],"trust":{"skill_safety":"passed","safety_status":"clean","intent_verdict":"safe","content_status":"clean","indexable":true},"license":"MIT","install_count":0,"manifest_hash":"245a82b2d0d99a09046379f4119c64b57529faf3f3e5477a5aefd87c04a11465","raw_url":"https://app.decimal.ai/s/openlair-huggingface-tokenizers/SKILL.md","scorecard_url":"https://app.decimal.ai/skills/openlair-huggingface-tokenizers"}