huggingface/transformers · #48656
Update tokenizer gguf support
src/transformers/integrations/__init__.py0 + / 6 −
@@ -61,10 +61,7 @@ "ggml": [ "GGUF_CONFIG_DEFAULTS_MAPPING", "GGUF_CONFIG_MAPPING",- "GGUF_TOKENIZER_MAPPING", "_gguf_parse_value",- "load_dequant_gguf_tensor",- "load_gguf", ], "higgs": [ "HiggsLinear",@@ -221,10 +218,7 @@ from .ggml import ( GGUF_CONFIG_DEFAULTS_MAPPING, GGUF_CONFIG_MAPPING,- GGUF_TOKENIZER_MAPPING, _gguf_parse_value,- load_dequant_gguf_tensor,- load_gguf, ) from .higgs import HiggsLinear, dequantize_higgs, quantize_with_higgs, replace_with_higgs_linear from .hqq import prepare_for_hqq_linearsrc/transformers/integrations/ggml.py0 + / 477 −
@@ -19,14 +19,7 @@ from array import array -import numpy as np-from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors-from tokenizers.models import BPE, Unigram--from .. import AddedToken-from ..convert_slow_tokenizer import GemmaConverter, GPT2Converter, LlamaConverter, Qwen2Converter, T5Converter from ..utils import logging-from ..utils.logging import tqdm logger = logging.get_logger(__name__)@@ -339,30 +332,6 @@ }, } -GGUF_TOKENIZER_MAPPING = {- "tokenizer": {- "ggml.model": "tokenizer_type",- "ggml.tokens": "tokens",- "ggml.scores": "scores",- "ggml.token_type": "token_type",- "ggml.merges": "merges",- "ggml.bos_token_id": "bos_token_id",- "ggml.eos_token_id": "eos_token_id",- "ggml.unknown_token_id": "unk_token_id",- "ggml.padding_token_id": "pad_token_id",- "ggml.add_space_prefix": "add_prefix_space",- },- "tokenizer_config": {- "chat_template": "chat_template",- "ggml.model": "model_type",- "ggml.bos_token_id": "bos_token_id",- "ggml.eos_token_id": "eos_token_id",- "ggml.unknown_token_id": "unk_token_id",- "ggml.padding_token_id": "pad_token_id",- },-}--# We only need to set here the parameters that default to different values between transformers and llamacpp. GGUF_CONFIG_DEFAULTS_MAPPING = { "qwen3_moe": { # NOTE: Qwen3MoeConfig defaults to false but llama.cpp needs this to be true.@@ -401,449 +370,3 @@ def _gguf_parse_value(_value, data_type): elif data_type == 9: _value = _gguf_parse_value(_value, array_data_type) return _value---class GGUFTokenizerSkeleton:- def __init__(self, dict_):- for k, v in dict_.items():- setattr(self, k, v)-- if not hasattr(self, "merges"):- if not hasattr(self, "tokens") or not hasattr(self, "scores"):- raise ValueError(- "tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated."- )- tokens = self.tokens- scores = self.scores- vocab = {t: scores[i] for i, t in enumerate(tokens)}-- logger.warning("Merges were not in checkpoint, building merges on the fly.")- merges = []- for merge, piece_score in tqdm(vocab.items()):- local = []- for index in range(1, len(merge)):- piece_l, piece_r = merge[:index], merge[index:]- if piece_l in tokens and piece_r in tokens:- local.append((piece_l, piece_r, piece_score))- local = sorted(local, key=lambda x: (vocab[x[0]], vocab[x[1]]), reverse=True)- merges.extend(local)- merges = sorted(merges, key=lambda val: val[2], reverse=True)- merges = [(val[0], val[1]) for val in merges]- self.merges = merges- else:- self.merges = [tuple(merge.split(" ")) for merge in self.merges]- if not hasattr(self, "scores"):- self.scores = [None for _ in range(len(self.tokens))]-- if not hasattr(self, "added_tokens"):- self.added_tokens = []-- if not hasattr(self, "unk_token_id"):- self.unk_token_id = None-- # Llama2 uses the field `unknown_token_id`- if hasattr(self, "unknown_token_id") and self.unk_token_id is None:- self.unk_token_id = self.unknown_token_id---class GGUFLlamaConverter(LlamaConverter):- def __init__(self, tokenizer_dict):- self.proto = GGUFTokenizerSkeleton(tokenizer_dict)- self.original_tokenizer = self.proto- self.additional_kwargs = {}- self.is_llama_3_tokenizer = getattr(self.proto, "tokenizer_type", "llama") != "llama"-- def vocab(self, proto):- return list(zip(proto.tokens, proto.scores))-- def merges(self, proto):- return proto.merges-- def tokenizer(self, proto):- vocab_scores = self.vocab(self.proto)- merges = self.merges(self.proto)- bpe_vocab = {word: i for i, (word, _score) in enumerate(vocab_scores)}-- unk_token = proto.tokens[proto.unk_token_id] if proto.unk_token_id is not None else None- bos_token = proto.tokens[proto.bos_token_id] if getattr(proto, "bos_token_id", None) is not None else None- eos_token = proto.tokens[proto.bos_token_id] if getattr(proto, "eos_token_id", None) is not None else None-- tokenizer = Tokenizer(- BPE(- bpe_vocab,- merges,- unk_token=unk_token,- fuse_unk=True,- byte_fallback=True,- )- )-- special_tokens = []-- if not hasattr(self.proto, "token_type"):- if unk_token is not None:- special_tokens.append(AddedToken(unk_token, normalized=False, special=True))-- if bos_token is not None:- special_tokens.append(AddedToken(bos_token, normalized=False, special=True))-- if eos_token is not None:- special_tokens.append(AddedToken(eos_token, normalized=False, special=True))- else:- # 3 stands for special tokens- special_tokens_idx = np.where(np.array(self.proto.token_type) == 3)[0]-- for idx in special_tokens_idx:- special_tokens.append(AddedToken(self.proto.tokens[idx], normalized=False, special=True))-- if len(special_tokens) != 0:- tokenizer.add_special_tokens(special_tokens)-- if len(self.proto.added_tokens) != 0:- tokenizer.add_tokens(- [AddedToken(added_token, normalized=False, special=False) for added_token in self.proto.added_tokens]- )-- self.additional_kwargs["unk_token"] = unk_token- self.additional_kwargs["eos_token"] = bos_token- self.additional_kwargs["bos_token"] = eos_token-- if self.is_llama_3_tokenizer:- self.additional_kwargs["add_prefix_space"] = None- self.additional_kwargs["clean_up_tokenization_spaces"] = True-- self.additional_kwargs["legacy"] = False- self.original_tokenizer.legacy = False-- return tokenizer-- def decoder(self, replacement, add_prefix_space):- sequence = [- decoders.ByteFallback(),- decoders.Fuse(),- decoders.Replace("▁", " "),- ]-- if self.is_llama_3_tokenizer:- sequence += [decoders.ByteLevel(add_prefix_space=False, trim_offsets=False, use_regex=True)]-- if add_prefix_space:- sequence += [decoders.Strip(content=" ", left=1)]- return decoders.Sequence(sequence)-- def converted(self):- # Copied partly from converted method in SpmConverter class- tokenizer = self.tokenizer(self.proto)-- # Tokenizer assemble- normalizer = self.normalizer(self.proto)- if normalizer is not None:- tokenizer.normalizer = normalizer-- replacement = "▁"- add_prefix_space = True- if hasattr(self.original_tokenizer, "add_prefix_space"):- add_prefix_space = self.original_tokenizer.add_prefix_space-- pre_tokenizer = self.pre_tokenizer(replacement, add_prefix_space)- if pre_tokenizer is not None:- tokenizer.pre_tokenizer = pre_tokenizer-- tokenizer.decoder = self.decoder(replacement, add_prefix_space)- post_processor = self.post_processor()- if post_processor:- tokenizer.post_processor = post_processor-- # HACK: patch the llama-3 tokenizer to use the corresponding pre-tokenizer- # and normalizer- if self.is_llama_3_tokenizer:- tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(- add_prefix_space=False, trim_offsets=False, use_regex=True- )- # This is tricky as the additional kwargs are passed after legacy is force-set in LlamaTokenizer's- # init.- tokenizer.normalizer = normalizers.Sequence([])-- return tokenizer---class GGUFQwen2Converter(Qwen2Converter):- def __init__(self, tokenizer_dict):- self.original_tokenizer = GGUFTokenizerSkeleton(tokenizer_dict)- self.additional_kwargs = {}-- def converted(self) -> Tokenizer:- vocab = {word: i for i, word in enumerate(self.original_tokenizer.tokens)}- merges = self.original_tokenizer.merges- tokenizer = super().converted(vocab, merges)-- tokenizer.add_special_tokens(- [- AddedToken("<|endoftext|>", normalized=False, special=True),- AddedToken("<|im_start|>", normalized=False, special=True),- AddedToken("<|im_end|>", normalized=False, special=True),- ]- )- return tokenizer---class GGUFPhi3Converter(LlamaConverter):- def __init__(self, tokenizer_dict):- self.proto = GGUFTokenizerSkeleton(tokenizer_dict)- self.original_tokenizer = self.proto- self.additional_kwargs = {}-- def vocab(self, proto):- return list(zip(proto.tokens, proto.scores))-- def merges(self, proto):- return proto.merges-- def tokenizer(self, proto):- vocab_scores = self.vocab(self.proto)- merges = self.merges(self.proto)- bpe_vocab = {word: i for i, (word, _score) in enumerate(vocab_scores)}-- tokenizer = Tokenizer(BPE(bpe_vocab, merges))- # add the special tokens from phi3 tokenizer config- tokenizer.add_special_tokens(- [- AddedToken("</s>", rstrip=True, lstrip=False, normalized=False, special=True),- AddedToken("<|endoftext|>", normalized=False, special=True),- AddedToken("<|assistant|>", rstrip=True, normalized=False, special=True),- AddedToken("<|placeholder1|>", rstrip=True, normalized=False, special=True),- AddedToken("<|placeholder2|>", rstrip=True, normalized=False, special=True),- AddedToken("<|placeholder3|>", rstrip=True, normalized=False, special=True),- AddedToken("<|placeholder4|>", rstrip=True, normalized=False, special=True),- AddedToken("<|system|>", rstrip=True, normalized=False, special=True),- AddedToken("<|end|>", rstrip=True, normalized=False, special=True),- AddedToken("<|placeholder5|>", rstrip=True, normalized=False, special=True),- AddedToken("<|placeholder6|>", rstrip=True, normalized=False, special=True),- AddedToken("<|user|>", rstrip=True, normalized=False, special=True),- ]- )-- self.additional_kwargs["unk_token"] = (- proto.tokens[proto.unk_token_id] if proto.unk_token_id is not None else None- )- self.additional_kwargs["eos_token"] = (- proto.tokens[proto.eos_token_id] if proto.eos_token_id is not None else None- )- self.additional_kwargs["bos_token"] = (- proto.tokens[proto.bos_token_id] if proto.bos_token_id is not None else None- )- self.additional_kwargs["pad_token"] = (- proto.tokens[proto.pad_token_id] if proto.pad_token_id is not None else None- )-- return tokenizer-- def decoder(self, replacement, add_prefix_space):- sequence = [- decoders.ByteFallback(),- decoders.Fuse(),- decoders.Replace(replacement, " "),- ]-- if add_prefix_space:- sequence += [decoders.Strip(content=" ", left=1)]- return decoders.Sequence(sequence)-- def converted(self) -> Tokenizer:- tokenizer = self.tokenizer(self.proto)-- replacement = "▁"- add_prefix_space = True- if hasattr(self.original_tokenizer, "add_prefix_space"):- add_prefix_space = self.original_tokenizer.add_prefix_space-- tokenizer.decoder = self.decoder(replacement, add_prefix_space)-- return tokenizer---class GGUFGPTConverter(GPT2Converter):- def __init__(self, tokenizer_dict):- self.original_tokenizer = GGUFTokenizerSkeleton(tokenizer_dict)- self.additional_kwargs = {}-- def converted(self) -> Tokenizer:- vocab = {word: i for i, word in enumerate(self.original_tokenizer.tokens)}- merges = self.original_tokenizer.merges- tokenizer = super().converted(vocab, merges)- return tokenizer---class GGUFT5Converter(T5Converter):- def __init__(self, tokenizer_dict):- # set dummy data to avoid unnecessary merges calculation- tokenizer_dict["merges"] = ["dummy text"]-- self.proto = GGUFTokenizerSkeleton(tokenizer_dict)- self.token2id = {k: v for v, k in enumerate(self.proto.tokens)}- self.original_tokenizer = self.proto- self.additional_kwargs = {}-- def vocab(self, proto):- return list(zip(proto.tokens, proto.scores))-- def normalizer(self, proto):- if getattr(self.original_tokenizer, "legacy", True):- sequence = []- if getattr(self.original_tokenizer, "add_prefix_space", True):- sequence += [normalizers.Prepend(prepend="▁")]- sequence += [normalizers.Replace(pattern=" ", content="▁")]- return normalizers.Sequence(sequence)- return None # non-legacy, no normalizer-- def post_processor(self):- return processors.TemplateProcessing(- single=["$A", "</s>"],- pair=["$A", "</s>", "$B", "</s>"],- special_tokens=[- ("</s>", self.token2id["</s>"]),- ],- )-- def converted(self) -> Tokenizer:- vocab_scores = self.vocab(self.proto)- tokenizer = Tokenizer(- Unigram(- vocab_scores,- unk_id=self.proto.unk_token_id,- byte_fallback=False,- )- )-- # Tokenizer assemble- normalizer = self.normalizer(self.proto)- if normalizer is not None:- tokenizer.normalizer = normalizer-- replacement = "▁"- add_prefix_space = True- if hasattr(self.original_tokenizer, "add_prefix_space"):- add_prefix_space = self.original_tokenizer.add_prefix_space-- pre_tokenizer = self.pre_tokenizer(replacement, add_prefix_space)- if pre_tokenizer is not None:- tokenizer.pre_tokenizer = pre_tokenizer-- tokenizer.decoder = self.decoder(replacement, add_prefix_space)- post_processor = self.post_processor()- if post_processor:- tokenizer.post_processor = post_processor-- return tokenizer---class GGUFGemmaConverter(GemmaConverter):- def __init__(self, tokenizer_dict):- # set dummy data to avoid unnecessary merges calculation- tokenizer_dict["merges"] = ["dummy text"]-- self.proto = GGUFTokenizerSkeleton(tokenizer_dict)- self.original_tokenizer = self.proto- self.additional_kwargs = {}-- def vocab(self, proto):- original_vocab = list(zip(proto.tokens, proto.scores))- updated_vocab = []-- for token, score in original_vocab:- if token == "<0x09>":- updated_vocab.append(("\t", score))- elif " " in token and len(token.strip()) == 0:- underscores = "▁" * len(token)- updated_vocab.append((underscores, score))- else:- updated_vocab.append((token, score))-- return updated_vocab-- def normalizer(self, proto):- return normalizers.Replace(" ", "▁")-- def decoder(self, replacement, add_prefix_space):- sequence = [- decoders.Replace("▁", " "),- decoders.ByteFallback(),- decoders.Fuse(),- ]-- if add_prefix_space:- sequence += [decoders.Strip(content=" ", left=1)]- return decoders.Sequence(sequence)-- def converted(self) -> Tokenizer:- vocab_scores = self.vocab(self.proto)- tokenizer = Tokenizer(- Unigram(- vocab_scores,- unk_id=self.proto.unk_token_id,- byte_fallback=self.handle_byte_fallback,- )- )-- normalizer = self.normalizer(self.proto)- if normalizer is not None:- tokenizer.normalizer = normalizer-- replacement = "▁"- add_prefix_space = True- if hasattr(self.original_tokenizer, "add_prefix_space"):- add_prefix_space = self.original_tokenizer.add_prefix_space-- tokenizer.decoder = self.decoder(replacement, add_prefix_space)- pre_tokenizer = self.pre_tokenizer(replacement, add_prefix_space)- if pre_tokenizer is not None:- tokenizer.pre_tokenizer = pre_tokenizer-- return tokenizer---GGUF_TO_FAST_CONVERTERS = {- "llama": GGUFLlamaConverter,- "qwen2": GGUFQwen2Converter,- "qwen2_moe": GGUFQwen2Converter,- "qwen3": GGUFQwen2Converter,- "qwen3_moe": GGUFQwen2Converter,- "qwen3_5_text": GGUFQwen2Converter,- "qwen3_5_moe_text": GGUFQwen2Converter,- "phi3": GGUFPhi3Converter,- "bloom": GGUFGPTConverter,- "falcon": GGUFGPTConverter,- "stablelm": GGUFGPTConverter,- "gpt2": GGUFGPTConverter,- "starcoder2": GGUFGPTConverter,- "t5": GGUFT5Converter,- "mamba": GGUFGPTConverter,- "nemotron": GGUFGPTConverter,- "gemma2": GGUFGemmaConverter,- "gemma3_text": GGUFGemmaConverter,- "gemma4_text": GGUFGemmaConverter,- "umt5": GGUFT5Converter,- "deci": GGUFLlamaConverter,- "decilm": GGUFLlamaConverter,- "minimax_m2": GGUFQwen2Converter,-}---def convert_gguf_tokenizer(architecture: str, tokenizer_dict) -> tuple[Tokenizer, dict]:- """- Utilities to convert a slow tokenizer instance in a fast tokenizer instance.-- Args:- architecture (`str`): The model architecture derived from gguf file.- transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):- Instance of a slow tokenizer to convert in the backend tokenizer for- [`~tokenization_utils_base.PreTrainedTokenizerFast`].-- Return:- A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a- [`~tokenization_utils_base.PreTrainedTokenizerFast`]- """- tokenizer_class_name = architecture- converter = GGUF_TO_FAST_CONVERTERS[tokenizer_class_name](tokenizer_dict)- fast_tokenizer = converter.converted()- return fast_tokenizer, converter.additional_kwargssrc/transformers/integrations/gguf/__init__.py31 + / 28 −
@@ -11,33 +11,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License.-from .gguf_config_mapping import GGUF_CONFIG_ARCHS, get_gguf_config-from .gguf_conversion_mapping import GGUF_ARCHS-from .gguf_tokenizer_mapping import GGUF_TOKENIZER_ARCHS, get_gguf_tokenizer-from .reader import (- GgufHeader,- load_gguf_state_dict,- read_gguf_metadata,-)-from .utils import (- get_gguf_conversion_mapping,- get_gguf_plan,- is_gguf_arch_supported,- replace_with_gguf_modules,-)+from typing import TYPE_CHECKING +from ...utils import _LazyModule -__all__ = [- "GGUF_ARCHS",- "GGUF_CONFIG_ARCHS",- "GGUF_TOKENIZER_ARCHS",- "get_gguf_tokenizer",- "get_gguf_config",- "get_gguf_conversion_mapping",- "get_gguf_plan",- "is_gguf_arch_supported",- "load_gguf_state_dict",- "read_gguf_metadata",- "GgufHeader",- "replace_with_gguf_modules",-]++_import_structure = {+ "gguf_config_mapping": ["GGUF_CONFIG_ARCHS", "get_gguf_config"],+ "gguf_conversion_mapping": ["GGUF_ARCHS"],+ "gguf_tokenizer_mapping": ["GGUF_TOKENIZER_MAPPING", "convert_gguf_tokenizer", "get_gguf_tokenizer"],+ "reader": ["GgufHeader", "load_gguf_state_dict", "read_gguf_metadata"],+ "utils": [+ "get_gguf_conversion_mapping",+ "get_gguf_plan",+ "is_gguf_arch_supported",+ "replace_with_gguf_modules",+ ],+}++if TYPE_CHECKING:+ from .gguf_config_mapping import GGUF_CONFIG_ARCHS, get_gguf_config+ from .gguf_conversion_mapping import GGUF_ARCHS+ from .gguf_tokenizer_mapping import GGUF_TOKENIZER_MAPPING, convert_gguf_tokenizer, get_gguf_tokenizer+ from .reader import GgufHeader, load_gguf_state_dict, read_gguf_metadata+ from .utils import (+ get_gguf_conversion_mapping,+ get_gguf_plan,+ is_gguf_arch_supported,+ replace_with_gguf_modules,+ )+else:+ import sys++ sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)src/transformers/integrations/gguf/gguf_tokenizer_mapping.py257 + / 34 −
@@ -1,4 +1,5 @@-# Copyright 2026 The HuggingFace Team. All rights reserved.+# Copyright 2024 The ggml.ai team and The HuggingFace Inc. team. and pygguf author (github.com/99991)+# https://github.com/99991/pygguf # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.@@ -13,49 +14,271 @@ # limitations under the License. """Building a tokenizer from a GGUF file's metadata. -A GGUF repo ships no `tokenizer.json`: the vocabulary, merges and special token ids are metadata keys.-The renaming table is llama.cpp's convention, shared with `integrations.ggml.GGUF_TOKENIZER_MAPPING`;-the only per-architecture fact is which converter reads the result.+A GGUF has no `tokenizer.json`. It stores the vocabulary, the merges and the special token ids as+metadata, under llama.cpp's key names. The tokenizer it describes is almost always one transformers+already has, so we read those out and hand them to that class, which brings its own normalizer,+pre-tokenizer and decoder:++ architecture, tokenizer_dict, config = get_gguf_tokenizer("model.gguf")+ backend, _ = convert_gguf_tokenizer(architecture, tokenizer_dict)++To support a new file, add an entry to `GGUF_TOKENIZER_TYPES`, or a builder if it needs more. """ -from .reader import read_gguf_metadata+import re+from functools import partial++from tokenizers import AddedToken, Regex, Tokenizer, normalizers, pre_tokenizers++from ...convert_slow_tokenizer import bytes_to_unicode+from ...models.gemma.tokenization_gemma import GemmaTokenizer+from ...models.gpt2.tokenization_gpt2 import GPT2Tokenizer+from ...models.llama.tokenization_llama import LlamaTokenizer+from ...models.qwen2.tokenization_qwen2 import PRETOKENIZE_REGEX as _QWEN2_SPLIT+from ...models.qwen3_5.tokenization_qwen3_5 import PRETOKENIZE_REGEX as _QWEN35_SPLIT+from ...models.t5.tokenization_t5 import T5Tokenizer+from ...tokenization_utils_base import generate_merges+from ...utils import logging -# `general.architecture` -> the `model_type` whose `GGUF_TO_FAST_CONVERTERS` entry reads this vocabulary.-GGUF_TOKENIZER_ARCHS = {- "qwen35": "qwen3_5_text",- "qwen35moe": "qwen3_5_moe_text",+logger = logging.get_logger(__name__)+++GGUF_TOKENIZER_MAPPING = {+ "tokenizer": {+ "ggml.model": "tokenizer_type",+ "ggml.pre": "pre_tokenizer_type",+ "ggml.tokens": "tokens",+ "ggml.scores": "scores",+ "ggml.token_type": "token_type",+ "ggml.merges": "merges",+ "ggml.precompiled_charsmap": "precompiled_charsmap",+ "ggml.bos_token_id": "bos_token_id",+ "ggml.eos_token_id": "eos_token_id",+ "ggml.unknown_token_id": "unk_token_id",+ "ggml.padding_token_id": "pad_token_id",+ "ggml.add_space_prefix": "add_prefix_space",+ },+ "tokenizer_config": {+ "chat_template": "chat_template",+ }, } -# The metadata arrays a tokenizer needs in full; everything else the reader can leave as a count.-_VOCABULARY_KEYS = ("tokenizer.ggml.tokens", "tokenizer.ggml.merges") +_SPECIAL_TOKENS = {+ "bos_token": "bos_token_id",+ "eos_token": "eos_token_id",+ "unk_token": "unknown_token_id",+ "pad_token": "padding_token_id",+} -def get_gguf_tokenizer(gguf_path: str) -> tuple[str, dict, dict]:- """`(model_type, tokenizer_dict, tokenizer_config)` for the tokenizer this file describes. - Raises for an architecture with no entry above; callers with a fallback check- `GGUF_TOKENIZER_ARCHS` first.- """- from ..ggml import GGUF_TOKENIZER_MAPPING+def get_gguf_tokenizer(gguf_path: str) -> tuple[str, dict, dict]:+ """`(architecture, tokenizer_dict, tokenizer_config)` for the tokenizer this file describes."""+ from .reader import read_gguf_metadata - metadata, _ = read_gguf_metadata(gguf_path, _VOCABULARY_KEYS)+ # Only these two are needed in full; the reader leaves every other array as a count.+ metadata, _ = read_gguf_metadata(gguf_path, ("tokenizer.ggml.tokens", "tokenizer.ggml.merges")) architecture = metadata["general.architecture"]- if architecture not in GGUF_TOKENIZER_ARCHS:- raise ValueError(- f"Cannot build a tokenizer from a GGUF file of architecture {architecture!r}. "- f"Supported: {sorted(GGUF_TOKENIZER_ARCHS)}."- )- sections = {- section: {- name: metadata[f"tokenizer.{key}"] for key, name in renames.items() if f"tokenizer.{key}" in metadata- }- for section, renames in GGUF_TOKENIZER_MAPPING.items()+ grouped = {+ group: {name: metadata[f"tokenizer.{key}"] for key, name in renames.items() if f"tokenizer.{key}" in metadata}+ for group, renames in GGUF_TOKENIZER_MAPPING.items() }- tokenizer, tokenizer_config = sections["tokenizer"], sections["tokenizer_config"]- # A GGUF names its special tokens by id, the tokenizer wants the strings. Undeclared ones are stated- # as `None` rather than left out, or the sentencepiece `<s>`/`</s>` fallback invents two tokens.- for name in ("bos_token", "eos_token", "pad_token", "unk_token"):- token_id = tokenizer_config.get(f"{name}_id")+ tokenizer, tokenizer_config = grouped["tokenizer"], grouped["tokenizer_config"]++ for name, key in _SPECIAL_TOKENS.items():+ token_id = metadata.get(f"tokenizer.ggml.{key}") tokenizer_config[name] = tokenizer["tokens"][token_id] if token_id is not None else None- return GGUF_TOKENIZER_ARCHS[architecture], tokenizer, tokenizer_config+ return architecture, tokenizer, tokenizer_config+++def convert_gguf_tokenizer(architecture: str, tokenizer_dict: dict) -> tuple[Tokenizer, dict]:+ tokenizer_type = tokenizer_dict.get("tokenizer_type")+ tokenizer = select_tokenizer_builder(architecture, tokenizer_type)(tokenizer_dict)+ tokenizer = add_gguf_special_tokens(tokenizer, tokenizer_dict)+ tokenizer = set_split_regex(tokenizer, tokenizer_dict)+ return tokenizer.backend_tokenizer, {}+++def sentencepiece_tokenizer(tokenizer_dict, tokenizer_class=LlamaTokenizer, vocab_scores=None):+ """The vocabulary as the file spells it, and merges from the file or recovered from it."""+ vocab = {token: index for index, token in enumerate(tokenizer_dict["tokens"])}+ return tokenizer_class(vocab=vocab, merges=get_merges(tokenizer_dict, vocab_scores))+++def phi3_tokenizer(tokenizer_dict):+ """Sentencepiece, but the prefix space comes from a normalizer, not from `Metaspace`.++ There is no `Phi3Tokenizer` to hand this to. Phi-3's shape lives in the `tokenizer.json` of its+ repo, which a GGUF does not carry, so we rebuild it.++ It only shows on text starting with a space: `" hi"` becomes `\u2581\u2581\u2581hi` here and `\u2581\u2581hi`+ with `Metaspace`. Nothing in a GGUF says which a model wants, so this is picked by architecture.+ """+ tokenizer = sentencepiece_tokenizer(tokenizer_dict)+ tokenizer.backend_tokenizer.normalizer = normalizers.Sequence(+ [normalizers.Prepend(prepend="\u2581"), normalizers.Replace(pattern=" ", content="\u2581")]+ )+ tokenizer.backend_tokenizer.pre_tokenizer = None+ return tokenizer+++def gemma_tokenizer(tokenizer_dict):+ """Sentencepiece, with two fixes Gemma files need.++ 1. A token of two or more spaces is stored as real spaces (" "), while `GemmaTokenizer` looks+ it up as "\u2581\u2581". Rename those tokens first, so the merges built from them match.+ 2. The file lists no merges, so they get rebuilt from the vocabulary, cut into pairs and sorted+ best-first. Elsewhere the score each token carries is the right sort key, but Gemma's scores+ do not follow its merge order -- the order the tokens appear in the file does.+ """+ respelled = dict(+ tokenizer_dict,+ tokens=["\u2581" * len(t) if " " in t and not t.strip() else t for t in tokenizer_dict["tokens"]],+ )+ positions = {token: -index for index, token in enumerate(respelled["tokens"])}+ return sentencepiece_tokenizer(respelled, GemmaTokenizer, vocab_scores=positions)+++def byte_level_tokenizer(tokenizer_dict):+ """Byte-level BPE"""+ spelling = bytes_to_unicode()+ alphabet = set(spelling.values())+ vocab = {+ token if set(token) <= alphabet else "".join(spelling[byte] for byte in token.encode("utf-8")): index+ for index, token in enumerate(tokenizer_dict["tokens"])+ }+ return GPT2Tokenizer(vocab=vocab, merges=get_merges(tokenizer_dict))+++def unigram_tokenizer(tokenizer_dict):+ """Unigram: a vocabulary of `(token, score)` pairs, and no merges.++ T5 writes a blank in a prompt as `<extra_id_0>`..`<extra_id_99>`, but the file calls those+ `[PAD32000]`... We rename the first 100 back, which is how many T5 has by default.+ """+ count = 100+ placeholder = re.compile(r"^\[PAD\d+\]$")+ tokens = list(tokenizer_dict["tokens"])+ blanks = [index for index, token in enumerate(tokens) if placeholder.match(token)]+ for offset, index in enumerate(blanks[:count]):+ tokens[index] = f"<extra_id_{count - 1 - offset}>"+ return T5Tokenizer(vocab=list(zip(tokens, tokenizer_dict["scores"])))+++def get_merges(tokenizer_dict, vocab_scores=None):+ """The merges the file gives, or the ones its vocabulary implies."""+ if "merges" in tokenizer_dict:+ return [tuple(merge.split(" ")) for merge in tokenizer_dict["merges"]]++ tokens = tokenizer_dict["tokens"]+ logger.info("No merges in the file; deriving them from the vocabulary.")+ if vocab_scores is None:+ vocab_scores = dict(zip(tokens, tokenizer_dict["scores"]))+ return generate_merges({token: index for index, token in enumerate(tokens)}, vocab_scores)+++def add_gguf_special_tokens(tokenizer, tokenizer_dict):+ """Add the tokens the file marks as control, like `<|endoftext|>`, as special tokens.++ Without this they get cut into pieces instead of staying whole. In a byte-level vocabulary every+ real token is written in the byte alphabet, so one marked control that uses any other character+ is a llama.cpp mislabel -- adding it would push the vocabulary past the size the model expects,+ so it is skipped.+ """+ byte_level = tokenizer_dict.get("tokenizer_type") == "gpt2"+ spelled = set(bytes_to_unicode().values()) if byte_level else None+ control = [+ AddedToken(token, normalized=False, special=True)+ for token, token_type in zip(tokenizer_dict["tokens"], tokenizer_dict.get("token_type") or ())+ if token_type == 3 and (spelled is None or set(token) <= spelled) # 3 is CONTROL+ ]+ if control:+ tokenizer.add_special_tokens({"additional_special_tokens": control}, replace_extra_special_tokens=False)+ return tokenizer+++def select_tokenizer_builder(architecture: str, tokenizer_type: str | None):+ """The builder for the tokenizer a file describes, from `tokenizer.ggml.model`."""+ if tokenizer_type == "llama" and architecture in GGUF_SENTENCEPIECE_BUILDERS:+ return GGUF_SENTENCEPIECE_BUILDERS[architecture]+ if tokenizer_type in GGUF_TOKENIZER_TYPES:+ return GGUF_TOKENIZER_TYPES[tokenizer_type]+ raise ValueError(+ f"Cannot build a tokenizer from a GGUF file stating tokenizer {tokenizer_type!r}. "+ f"Supported: {sorted(GGUF_TOKENIZER_TYPES)}."+ )+++_LLAMA3_SPLIT = (+ r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}"+ r"| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"+)+++_TEKKEN_SPLIT = (+ r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+"+ r"|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*"+ r"|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+"+)+++_GPT4O_SPLIT = (+ r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?"+ r"|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?"+ r"|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+"+)++GGUF_PRE_TOKENIZER_SPLITS = {+ "llama3": _LLAMA3_SPLIT,+ "llama-v3": _LLAMA3_SPLIT,+ "llama-bpe": _LLAMA3_SPLIT,+ "falcon3": _LLAMA3_SPLIT,+ "falcon-h1": _LLAMA3_SPLIT,+ "pixtral": _LLAMA3_SPLIT,+ "midm-2.0": _LLAMA3_SPLIT,+ "lfm2": _LLAMA3_SPLIT,+ "jina-v5-nano": _LLAMA3_SPLIT,+ "dbrx": _LLAMA3_SPLIT,+ "smaug-bpe": _LLAMA3_SPLIT,+ "tekken": _TEKKEN_SPLIT,+ "qwen2": _QWEN2_SPLIT,+ "deepseek-r1-qwen": _QWEN2_SPLIT,+ "kormo": _QWEN2_SPLIT,+ "f2llmv2": _QWEN2_SPLIT,+ "megrez": _QWEN2_SPLIT,+ "qwen35": _QWEN35_SPLIT,+ "gpt-4o": _GPT4O_SPLIT,+ "llama4": _GPT4O_SPLIT,+ "kanana2": _GPT4O_SPLIT,+ "talkie": _GPT4O_SPLIT,+ "minimax-m2": _GPT4O_SPLIT,+}+++def set_split_regex(tokenizer, tokenizer_dict):+ """Split text the way this vocabulary's merges were learned."""+ split = GGUF_PRE_TOKENIZER_SPLITS.get(tokenizer_dict.get("pre_tokenizer_type"))+ if split is not None:+ tokenizer.backend_tokenizer.pre_tokenizer = pre_tokenizers.Sequence(+ [+ pre_tokenizers.Split(Regex(split), behavior="isolated"),+ pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),+ ]+ )+ return tokenizer+++GGUF_SENTENCEPIECE_BUILDERS = {+ "gemma": gemma_tokenizer,+ "gemma2": gemma_tokenizer,+ "gemma3": gemma_tokenizer,+ "phi3": phi3_tokenizer,+}++GGUF_TOKENIZER_TYPES = {+ "gpt2": byte_level_tokenizer,+ "llama": sentencepiece_tokenizer,+ "t5": unigram_tokenizer,+ "gemma4": partial(sentencepiece_tokenizer, tokenizer_class=GemmaTokenizer),+}src/transformers/modeling_gguf_pytorch_utils.py1 + / 1 −
@@ -22,9 +22,9 @@ from .integrations import ( GGUF_CONFIG_DEFAULTS_MAPPING, GGUF_CONFIG_MAPPING,- GGUF_TOKENIZER_MAPPING, _gguf_parse_value, )+from .integrations.gguf import GGUF_TOKENIZER_MAPPING from .utils import is_torch_available from .utils.import_utils import is_gguf_available from .utils.logging import get_loggersrc/transformers/tokenization_utils_tokenizers.py2 + / 13 −
@@ -36,9 +36,7 @@ from transformers.utils.hub import cached_file from .convert_slow_tokenizer import SpmConverter-from .integrations.ggml import convert_gguf_tokenizer from .integrations.mistral.constants import is_tekken_vocab_filename-from .modeling_gguf_pytorch_utils import load_gguf_checkpoint from .tokenization_utils_base import ( INIT_TOKENIZER_DOCSTRING, BatchEncoding,@@ -393,19 +391,10 @@ def __init__(self, *args, **kwargs): # We have a serialization from tokenizers which let us directly build the backend fast_tokenizer = TokenizerFast.from_file(fast_tokenizer_file) elif gguf_file is not None:- # We need to convert a slow tokenizer to build the backend- from .integrations.gguf import GGUF_TOKENIZER_ARCHS, get_gguf_tokenizer, read_gguf_metadata+ from .integrations.gguf import convert_gguf_tokenizer, get_gguf_tokenizer gguf_path = cached_file(kwargs.get("name_or_path", ""), gguf_file, **kwargs)- # The fast reader materializes the vocabulary and nothing else.- metadata, _ = read_gguf_metadata(gguf_path)- if metadata["general.architecture"] in GGUF_TOKENIZER_ARCHS:- architecture, tokenizer_dict, tokenizer_config = get_gguf_tokenizer(gguf_path)- else:- gguf_param = load_gguf_checkpoint(gguf_path)- architecture = gguf_param["config"]["model_type"]- tokenizer_dict = gguf_param["tokenizer"]- tokenizer_config = gguf_param["tokenizer_config"]+ architecture, tokenizer_dict, tokenizer_config = get_gguf_tokenizer(gguf_path) fast_tokenizer, additional_kwargs = convert_gguf_tokenizer(architecture, tokenizer_dict) kwargs.update(tokenizer_config) if len(additional_kwargs) > 0:tests/quantization/ggml/test_ggml.py7 + / 7 −
@@ -1060,14 +1060,14 @@ def test_deci_config_mapping(self): self.assertIsNone(deci_mapping["rope.dimension_count"]) def test_deci_architecture_mapping(self):- """Test that Deci architectures are mapped to GGUFLlamaConverter."""- from transformers.integrations.ggml import GGUF_TO_FAST_CONVERTERS, GGUFLlamaConverter-- self.assertIn("deci", GGUF_TO_FAST_CONVERTERS)- self.assertIn("decilm", GGUF_TO_FAST_CONVERTERS)+ """Test that Deci architectures use the plain sentencepiece tokenizer."""+ from transformers.integrations.gguf.gguf_tokenizer_mapping import (+ select_tokenizer_builder,+ sentencepiece_tokenizer,+ ) - self.assertEqual(GGUF_TO_FAST_CONVERTERS["deci"], GGUFLlamaConverter)- self.assertEqual(GGUF_TO_FAST_CONVERTERS["decilm"], GGUFLlamaConverter)+ self.assertEqual(select_tokenizer_builder("deci", "llama"), sentencepiece_tokenizer)+ self.assertEqual(select_tokenizer_builder("decilm", "llama"), sentencepiece_tokenizer) @unittest.skipUnless(is_gguf_available("0.16.0"), "test requires gguf version >= 0.16.0") def test_qwen3_q8_0(self):tests/quantization/ggml/test_gguf_integration.py163 + / 37 −
@@ -22,6 +22,7 @@ AutoConfig, AutoModelForCausalLM, AutoTokenizer,+ GenerationConfig, GgufConfig, Qwen3_5ForCausalLM, Qwen3_5MoeForCausalLM,@@ -64,22 +65,24 @@ def test_every_type_matches_the_reference(self): self.assertTrue(np.array_equal(ours, theirs.reshape(-1)[: ours.size], equal_nan=True)) -class GgufModelIntegrationTesterMixin:- """Tests every integrated architecture must pass."""+class GgufTokenizerTesterMixin:+ """A tokenizer built from a GGUF file's metadata is the one the reference repo ships.++ Needs no model, so it also covers architectures the loading path does not support yet.+ """ tokenizer_texts = ( "The capital of France is Paris.", "def f(x):\n return x ** 2\n", " leading and trailing ",+ "a\tb\n\nc d",+ "I don't think it's 3.14159 or -42,000", "\u65e5\u672c\u8a9e \U0001f680 \u00fcn\u00efc\u00f4de",+ "\u0391\u0392\u0393 \u0411\u0413\u0414 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \u05e2\u05d1\u05e8\u05d9\u05ea",+ "https://example.com/a_b?c=1&d=2#e", "<|im_start|>user\nhi<|im_end|>", ) - # Per-parameter relative tolerances, for an architecture whose conversion cannot be exact. Empty- # is the expectation: the transforms run in the float type the file stores and `Cast` rounds once,- # at the end, so a converted weight is bit-for-bit what a safetensors checkpoint holds.- inexact_params: dict[str, float] = {}- chat = ( {"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"},@@ -88,15 +91,80 @@ class GgufModelIntegrationTesterMixin: @classmethod def setUpClass(cls):- # One load for the whole class: these checkpoints are several GB. No dtype is passed, so this- # covers `auto` resolving to the one the file was written in.- # The loading report comes back with it, for `test_load_accounts_for_every_key`: free here,- # another multi-GB load if that test asked for its own.- cls.model, cls.loading_info = cls.load_gguf_model(cls.gguf_file, output_loading_info=True)- # Built once too: a vocabulary of a few hundred thousand tokens is not free to assemble. cls.tokenizer = AutoTokenizer.from_pretrained(cls.gguf_repo, gguf_file=cls.quantized_gguf_file) cls.reference_tokenizer = AutoTokenizer.from_pretrained(cls.reference_repo) + def test_tokenizer_matches_transformers(self):+ from_gguf, reference = self.tokenizer, self.reference_tokenizer++ for text in self.tokenizer_texts:+ with self.subTest(text=text):+ self.assertEqual(+ from_gguf(text, add_special_tokens=False).input_ids,+ reference(text, add_special_tokens=False).input_ids,+ )++ def test_decodes_like_transformers(self):+ """The decoder comes from the tokenizer class, and nothing else here exercises it."""+ from_gguf, reference = self.tokenizer, self.reference_tokenizer+ for text in self.tokenizer_texts:+ with self.subTest(text=text):+ ids = reference(text, add_special_tokens=False).input_ids+ self.assertEqual(from_gguf.decode(ids), reference.decode(ids))++ def test_the_models_own_special_tokens_survive(self):+ """A model's own markers must come back as single ids rather than be split into characters."""+ from_gguf, reference = self.tokenizer, self.reference_tokenizer+ markers = [token.content for token in reference.added_tokens_decoder.values() if token.special]+ if not markers:+ self.skipTest("the reference declares no special tokens")+ for marker in markers[:6]:+ with self.subTest(marker=marker):+ text = f"{marker}hello{marker}"+ self.assertEqual(+ from_gguf(text, add_special_tokens=False).input_ids,+ reference(text, add_special_tokens=False).input_ids,+ )++ def test_special_tokens_match_transformers(self):+ """Stated by id in the file, so only the vocabulary turns them back into strings."""+ from_gguf, reference = self.tokenizer, self.reference_tokenizer++ accepted = {reference.eos_token}+ try:+ eos_ids = GenerationConfig.from_pretrained(self.reference_repo).eos_token_id+ except OSError:+ eos_ids = None+ if eos_ids is not None:+ eos_ids = [eos_ids] if isinstance(eos_ids, int) else eos_ids+ accepted.update(reference.convert_ids_to_tokens(token_id) for token_id in eos_ids)+ self.assertIn(from_gguf.eos_token, accepted)+ # `bos` only where the reference has one: llama.cpp writes an id even for a tokenizer+ # declaring none, and the encodings above prove it is never actually emitted.+ if reference.bos_token is not None:+ self.assertEqual(from_gguf.bos_token, reference.bos_token)++ def test_chat_template_matches_transformers(self):+ """The template rides along in the metadata, and formats a conversation the same way."""+ if self.reference_tokenizer.chat_template is None:+ self.skipTest("the reference repo ships no chat template")+ self.assertEqual(+ self.tokenizer.apply_chat_template(self.chat, tokenize=False),+ self.reference_tokenizer.apply_chat_template(self.chat, tokenize=False),+ )+++class GgufModelIntegrationTesterMixin(GgufTokenizerTesterMixin):+ """Tests every integrated architecture must pass."""++ # Per-parameter relative tolerances, for an architecture whose conversion cannot be exact.+ inexact_params: dict[str, float] = {}++ @classmethod+ def setUpClass(cls):+ super().setUpClass()+ cls.model, cls.loading_info = cls.load_gguf_model(cls.gguf_file, output_loading_info=True)+ @classmethod def tearDownClass(cls): del cls.model@@ -131,30 +199,10 @@ def completion(self, model): output = model.generate(**inputs, max_new_tokens=8, do_sample=False) return self.reference_tokenizer.decode(output[0, inputs.input_ids.shape[1] :]) - def test_tokenizer_matches_transformers(self):- """A GGUF repo ships no tokenizer files either, so the one built from the metadata is the same."""- from_gguf, reference = self.tokenizer, AutoTokenizer.from_pretrained(self.reference_repo)-- # Not the vocabulary sizes: a GGUF states one flat token list, where the reference keeps its- # special tokens as additions on top of a smaller base. What has to agree is what they encode to.- for text in self.tokenizer_texts:- with self.subTest(text=text):- self.assertEqual(from_gguf(text).input_ids, reference(text).input_ids)-- # Stated by id in the file, so only the vocabulary turns them back into strings. Encodings- # cannot catch this: a tokenizer handed no special tokens invents a sentencepiece pair and- # appends it, which agrees with the reference on every text and puts two ids past the end of- # the embedding -- hence the bound below rather than another comparison.- self.assertEqual(from_gguf.eos_token, reference.eos_token)- self.assertEqual(from_gguf.bos_token, reference.bos_token)- self.assertLessEqual(len(from_gguf), self.model.get_input_embeddings().weight.shape[0])-- # The template rides along in the metadata, and formats a conversation the same way.- self.assertIsNotNone(from_gguf.chat_template)- self.assertEqual(- from_gguf.apply_chat_template(self.chat, tokenize=False),- reference.apply_chat_template(self.chat, tokenize=False),- )+ def test_vocabulary_fits_the_embedding(self):+ """A tokenizer handed no special tokens invents a sentencepiece pair and appends it, which+ agrees with the reference on every text but puts two ids past the end of the embedding."""+ self.assertLessEqual(len(self.tokenizer), self.model.get_input_embeddings().weight.shape[0]) def test_state_dict_matches_transformers(self): """The headline test: same values as the safetensors checkpoint, tensor by tensor."""@@ -431,3 +479,81 @@ class Qwen35MoeLargeGgufModelTest(GgufModelIntegrationTesterMixin, unittest.Test @unittest.skip("the bf16 checkpoint is ~70GB and split across shards the reader cannot open") def test_state_dict_matches_transformers(self): pass+++@slow+class Qwen3GgufTokenizerTest(GgufTokenizerTesterMixin, unittest.TestCase):+ gguf_repo = "unsloth/Qwen3-0.6B-GGUF"+ quantized_gguf_file = "Qwen3-0.6B-Q8_0.gguf"+ reference_repo = "Qwen/Qwen3-0.6B"+++@slow+class LlamaGgufTokenizerTest(GgufTokenizerTesterMixin, unittest.TestCase):+ gguf_repo = "unsloth/Llama-3.1-8B-Instruct-GGUF"+ quantized_gguf_file = "Llama-3.1-8B-Instruct-Q4_K_M.gguf"+ reference_repo = "meta-llama/Llama-3.1-8B-Instruct"+++@slow+class Qwen25GgufTokenizerTest(GgufTokenizerTesterMixin, unittest.TestCase):+ gguf_repo = "Qwen/Qwen2.5-7B-Instruct-GGUF"+ quantized_gguf_file = "qwen2.5-7b-instruct-q4_k_m-00001-of-00002.gguf"+ reference_repo = "Qwen/Qwen2.5-7B-Instruct"+++@slow+class MistralGgufTokenizerTest(GgufTokenizerTesterMixin, unittest.TestCase):+ """Tekken: byte-level, and written as `llama` like the sentencepiece Mistrals before it."""++ gguf_repo = "bartowski/Ministral-8B-Instruct-2410-GGUF"+ quantized_gguf_file = "Ministral-8B-Instruct-2410-Q4_K_M.gguf"+ reference_repo = "mistralai/Ministral-8B-Instruct-2410"+++@slow+class TinyLlamaGgufTokenizerTest(GgufTokenizerTesterMixin, unittest.TestCase):+ """The sentencepiece side of `llama`, which the Mistral case covered until it went byte-level."""++ gguf_repo = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"+ quantized_gguf_file = "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"+ reference_repo = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"+++@slow+class Phi3GgufTokenizerTest(GgufTokenizerTesterMixin, unittest.TestCase):+ gguf_repo = "bartowski/Phi-3.5-mini-instruct-GGUF"+ quantized_gguf_file = "Phi-3.5-mini-instruct-Q4_K_M.gguf"+ reference_repo = "microsoft/Phi-3.5-mini-instruct"+++@slow+class T5GgufTokenizerTest(GgufTokenizerTesterMixin, unittest.TestCase):+ gguf_repo = "Felladrin/gguf-LaMini-Flan-T5-248M"+ quantized_gguf_file = "LaMini-Flan-T5-248M.Q8_0.gguf"+ reference_repo = "MBZUAI/LaMini-Flan-T5-248M"+++@slow+class Gemma4GgufTokenizerTest(GgufTokenizerTesterMixin, unittest.TestCase):+ """`tokenizer.ggml.model = "gemma4"`: a tokenizer type of its own, and a BPE carrying its own merges."""++ gguf_repo = "unsloth/gemma-4-E4B-it-GGUF"+ quantized_gguf_file = "gemma-4-E4B-it-Q4_K_M.gguf"+ reference_repo = "google/gemma-4-E4B-it"+++@slow+class FlanT5GgufTokenizerTest(GgufTokenizerTesterMixin, unittest.TestCase):+ """A second T5, to keep the `[PAD...]` -> `<extra_id_...>` renaming honest."""++ gguf_repo = "Felladrin/gguf-flan-t5-small"+ quantized_gguf_file = "flan-t5-small.Q8_0.gguf"+ reference_repo = "google/flan-t5-small"+++@slow+class GemmaGgufTokenizerTest(GgufTokenizerTesterMixin, unittest.TestCase):+ gguf_repo = "unsloth/gemma-3-1b-it-GGUF"+ quantized_gguf_file = "gemma-3-1b-it-Q4_K_M.gguf"+ reference_repo = "google/gemma-3-1b-it"