Skip to content

vllm.tokenizers.mistral

Functions:

mistral_common_tekkenizer(tokenizer)

Return the underlying Tekkenizer for a MistralCommonBackend.

Source code in vllm/tokenizers/mistral.py 
181
182
183
184
185
186
def mistral_common_tekkenizer(tokenizer: object) -> "Tekkenizer | None":
    """Return the underlying `Tekkenizer` for a `MistralCommonBackend`."""
    mistral = getattr(tokenizer, "tokenizer", None)
    instruct = getattr(mistral, "instruct_tokenizer", None)
    tekken = getattr(instruct, "tokenizer", None)
    return tekken if isinstance(tekken, Tekkenizer) else None

tekken_convert_ids_to_tokens(tokenizer, ids)

Convert ids to pieces, using raw bytes for byte-fallback tokens.

Source code in vllm/tokenizers/mistral.py 
189
190
191
192
193
194
195
196
197
198
199
200
201
def tekken_convert_ids_to_tokens(
    tokenizer: "Tekkenizer", ids: Sequence[int]
) -> list[str | bytes]:
    """Convert ids to pieces, using raw `bytes` for byte-fallback tokens."""
    tokens: list[str | bytes] = [tokenizer.id_to_piece(i) for i in ids]
    if any("�" in t for t in tokens):
        tokens = [
            tokenizer.id_to_byte_piece(i, SpecialTokenPolicy.KEEP)
            if i >= tokenizer.num_special_tokens
            else tokenizer.decode([i], SpecialTokenPolicy.KEEP)
            for i in ids
        ]
    return tokens

tekken_convert_tokens_to_string(tokenizer, tokens)

Reassemble pieces from tekken_convert_ids_to_tokens into text.

Source code in vllm/tokenizers/mistral.py 
204
205
206
207
208
209
210
211
def tekken_convert_tokens_to_string(
    tokenizer: "Tekkenizer", tokens: Sequence[str | bytes]
) -> str:
    """Reassemble pieces from `tekken_convert_ids_to_tokens` into text."""
    if any(isinstance(t, bytes) for t in tokens):
        ids = [_tekken_token_to_id(tokenizer, t) for t in tokens]
        return tokenizer.decode(ids, SpecialTokenPolicy.KEEP)
    return "".join(cast(Sequence[str], tokens))

truncate_tool_call_ids(request)

Truncates tool call IDs for Mistral's ID requirements.

Source code in vllm/tokenizers/mistral.py 
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
def truncate_tool_call_ids(request: "MistralChatCompletionRequest"):
    """Truncates tool call IDs for Mistral's ID requirements."""
    for i, message in enumerate(request.messages):
        if message.get("role") == "assistant":
            tool_calls = message.get("tool_calls", [])
            for tool_call in tool_calls:
                if len(tool_call["id"]) > 9:
                    logger.warning(
                        "Truncating tool call ID: %s to %s",
                        tool_call["id"],
                        tool_call["id"][-9:],
                    )
                    tool_call["id"] = tool_call["id"][-9:]

            request.messages[i]["tool_calls"] = tool_calls

        elif message.get("role") in {"tool_results", "tool"}:
            if "tool_call_id" in message:
                tool_call_id = message["tool_call_id"]

                if len(tool_call_id) > 9:
                    logger.warning(
                        "Truncating tool_call_id: %s to %s",
                        tool_call_id,
                        tool_call_id[-9:],
                    )
                    tool_call_id = tool_call_id[-9:]
                request.messages[i]["tool_call_id"] = tool_call_id