BDEPEND=test? ( sci-ml/datasets[python_single_target_python3_11(-)?,python_single_target_python3_12(-)?,python_single_target_python3_13(-)?] ) python_single_target_python3_11? ( dev-python/setuptools-rust[python_targets_python3_11(-)] ) python_single_target_python3_12? ( dev-python/setuptools-rust[python_targets_python3_12(-)] ) python_single_target_python3_13? ( dev-python/setuptools-rust[python_targets_python3_13(-)] ) test? ( dev-libs/oniguruma python_single_target_python3_11? ( >=dev-python/pytest-7.4.4[python_targets_python3_11(-)] ) python_single_target_python3_12? ( >=dev-python/pytest-7.4.4[python_targets_python3_12(-)] ) python_single_target_python3_13? ( >=dev-python/pytest-7.4.4[python_targets_python3_13(-)] ) ) || ( >=dev-lang/rust-bin-1.82.0:* >=dev-lang/rust-1.82.0:* ) python_single_target_python3_11? ( dev-lang/python:3.11 ) python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 ) python_single_target_python3_11? ( >=dev-python/gpep517-16[python_targets_python3_11(-)] >=dev-util/maturin-1.8.2[python_targets_python3_11(-)] ) python_single_target_python3_12? ( >=dev-python/gpep517-16[python_targets_python3_12(-)] >=dev-util/maturin-1.8.2[python_targets_python3_12(-)] ) python_single_target_python3_13? ( >=dev-python/gpep517-16[python_targets_python3_13(-)] >=dev-util/maturin-1.8.2[python_targets_python3_13(-)] )
DEFINED_PHASES=compile configure install prepare setup test unpack
DEPEND=python_single_target_python3_11? ( dev-lang/python:3.11 ) python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 )
DESCRIPTION=Implementation of today's most used tokenizers
EAPI=8
HOMEPAGE=https://github.com/huggingface/tokenizers
INHERIT=cargo distutils-r1
IUSE=test debug python_single_target_python3_11 python_single_target_python3_12 python_single_target_python3_13 debug
KEYWORDS=~amd64
LICENSE=Apache-2.0 Apache-2.0 Apache-2.0-with-LLVM-exceptions BSD-2 BSD ISC MIT MPL-2.0 Unicode-DFS-2016
RDEPEND=dev-libs/oniguruma python_single_target_python3_11? ( dev-lang/python:3.11 ) python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 )
REQUIRED_USE=^^ ( python_single_target_python3_11 python_single_target_python3_12 python_single_target_python3_13 )
RESTRICT=!test? ( test )
SLOT=0
SRC_URI=https://github.com/huggingface/tokenizers/archive/refs/tags/v0.21.4.tar.gz -> tokenizers-0.21.4.gh.tar.gz https://dev.gentoo.org/~tupone/distfiles/tokenizers-0.21.4-crates.tar.xz https://dev.gentoo.org/~tupone/distfiles/tokenizers-python-0.21.4-crates.tar.xz
_eclasses_=cargo	a1af74aa9e34cd73aeac642cd0dd54d8	distutils-r1	37143302bc375f4878f7b5f7497da8cc	flag-o-matic	a7afe42e95fb46ce9691605acfb24672	multibuild	4650a65187015567b4e041bb9bfdb364	multilib	a199af442898963433d6f038a88cce9c	multiprocessing	1e32df7deee68372153dca65f4a7c21f	ninja-utils	2df4e452cea39a9ec8fb543ce059f8d6	out-of-source-utils	dbf9e34ee8964084651e25907fa8f52c	python-single-r1	47d8ac9be330c9366112b78fd4f8ce57	python-utils-r1	b7eb9c3a6dc6662368634584aab44a2c	rust	2c958ff95a0fd21c1cb19e961cb2946d	rust-toolchain	76468983281b0a7fc167ca224f84ecfd	toolchain-funcs	a0b29008c671a362b505f96fa80ce9c0
_md5_=aea58fbb9f880f8ff07e76880e992cc9