vllm.entrypoints.generate.base.protocol

Classes:

• SpeculativeDecodingMetrics –

  Per-request speculative-decoding acceptance metrics.

Functions:

• structured_outputs_from_response_format –

  Apply response_format overrides to structured_outputs.

• validate_cache_salt –

  Validate cache salts before they reach downstream cache backends.

• validate_request_mm_kwargs –

  Reject untrusted per-request multimodal kwarg overrides.

• validate_structural_tag_response_format –

  Validate structural tags before they are sent to the engine.

SpeculativeDecodingMetrics

Bases: OpenAIBaseModel

Per-request speculative-decoding acceptance metrics.

Experimental, subject to change. Only populated for single-sequence requests (n == 1); null for n > 1, mirroring the timing metrics.

Source code in vllm/entrypoints/generate/base/protocol.py

class SpeculativeDecodingMetrics(OpenAIBaseModel):
    """Per-request speculative-decoding acceptance metrics.

    Experimental, subject to change. Only populated for single-sequence requests
    (`n == 1`); `null` for `n > 1`, mirroring the timing metrics.
    """

    mean_acceptance_length: float
    draft_acceptance_rate: float
    # Dense histogram: index j holds the number of verify steps that accepted
    # exactly j draft tokens (length num_spec_tokens + 1). Excludes the
    # always-accepted bonus token.
    acceptance_histogram: list[int]
    num_spec_steps: int
    num_accepted_draft_tokens: int
    num_draft_tokens: int
    num_spec_tokens: int
    # Ordered per-verify-step arrays; populated only at the `detailed` level.
    per_step_accepted: list[int] | None = None
    per_step_drafted: list[int] | None = None

structured_outputs_from_response_format(structured_outputs, response_format)

Apply response_format overrides to structured_outputs.

Source code in vllm/entrypoints/generate/base/protocol.py

def structured_outputs_from_response_format(
    structured_outputs: StructuredOutputsParams | None,
    response_format: AnyResponseFormat | None,
) -> StructuredOutputsParams | None:
    """Apply ``response_format`` overrides to ``structured_outputs``."""
    if response_format is None or response_format.type == "text":
        return structured_outputs

    overrides: dict[str, Any]
    if response_format.type == "json_object":
        overrides = {"json_object": True}
    elif response_format.type == "json_schema":
        json_schema = response_format.json_schema
        assert json_schema is not None
        overrides = {"json": json_schema.json_schema}
    else:
        assert isinstance(
            response_format,
            (
                LegacyStructuralTagResponseFormat,
                StructuralTagResponseFormat,
            ),
        )
        overrides = {
            "structural_tag": json.dumps(response_format.model_dump(by_alias=True))
        }

    if structured_outputs is None:
        return StructuredOutputsParams(**overrides)

    return replace(structured_outputs, **overrides)

validate_cache_salt(cache_salt)

Validate cache salts before they reach downstream cache backends.

Source code in vllm/entrypoints/generate/base/protocol.py

def validate_cache_salt(cache_salt: object) -> None:
    """Validate cache salts before they reach downstream cache backends."""
    if cache_salt is None:
        return
    if not isinstance(cache_salt, str) or not cache_salt:
        raise VLLMValidationError(
            "Parameter 'cache_salt' must be a non-empty string if provided.",
            parameter="cache_salt",
        )
    if len(cache_salt) > _MAX_CACHE_SALT_LENGTH or any(
        char in _CACHE_SALT_FORBIDDEN_CHARS for char in cache_salt
    ):
        raise VLLMValidationError(
            "Parameter 'cache_salt' must be at most 128 characters and must "
            "not contain '@', '/', '\\\\', or NUL.",
            parameter="cache_salt",
        )

validate_request_mm_kwargs(*, mm_processor_kwargs, media_io_kwargs, trust_request_mm_kwargs)

Reject untrusted per-request multimodal kwarg overrides.

Source code in vllm/entrypoints/generate/base/protocol.py

def validate_request_mm_kwargs(
    *,
    mm_processor_kwargs: dict[str, Any] | None,
    media_io_kwargs: dict[str, dict[str, Any]] | None,
    trust_request_mm_kwargs: bool,
) -> None:
    """Reject untrusted per-request multimodal kwarg overrides."""
    if trust_request_mm_kwargs:
        return

    if mm_processor_kwargs:
        raise VLLMValidationError(
            "Per-request mm_processor_kwargs are disabled by default because "
            "they can change multimodal preprocessing resource usage. Start "
            "the server with --trust-request-mm-kwargs only when clients "
            "are trusted.",
            parameter="mm_processor_kwargs",
        )
    if media_io_kwargs:
        raise VLLMValidationError(
            "Per-request media_io_kwargs are disabled by default because "
            "they can change multimodal media loading resource usage. Start "
            "the server with --trust-request-mm-kwargs only when clients "
            "are trusted.",
            parameter="media_io_kwargs",
        )

validate_structural_tag_response_format(response_format)

Validate structural tags before they are sent to the engine.

Engine-side validation reports malformed structural tags as generation failures. OpenAI request parsing should classify them as bad requests.

Source code in vllm/entrypoints/generate/base/protocol.py

def validate_structural_tag_response_format(
    response_format: AnyStructuralTagResponseFormat | dict[str, Any],
) -> None:
    """Validate structural tags before they are sent to the engine.

    Engine-side validation reports malformed structural tags as generation
    failures. OpenAI request parsing should classify them as bad requests.
    """
    from pydantic import TypeAdapter, ValidationError

    if isinstance(response_format, dict):
        try:
            response_format = TypeAdapter(
                AnyStructuralTagResponseFormat
            ).validate_python(response_format)
        except ValidationError as exc:
            raise VLLMValidationError(
                "Invalid response_format structural_tag specification.",
                parameter="response_format",
            ) from exc

    try:
        payload = json.dumps(response_format.model_dump(by_alias=True))
        validate_structural_tag_payload(payload, parameter="response_format")
    except (TypeError, ValueError) as exc:
        raise VLLMValidationError(
            "Invalid response_format structural_tag specification.",
            parameter="response_format",
        ) from exc