vllm.model_executor.models.transformers.multimodal

Transformers modeling backend mixin for multi-modal models.

Classes:

• MultiModalMixin –
• MultiModalProcessor –

  Locates placeholders from the text_replacement_offsets the HF processor

MultiModalMixin

Bases: SupportsMultiModal, SupportsMRoPE, Base

Methods:

• get_language_model –

  Transformers modeling backend multimodal classes do not contain a separate

• get_mm_mapping –

  Get the module prefix in multimodal models

Source code in vllm/model_executor/models/transformers/multimodal.py

class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
    def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
        # Skip SupportsMRoPE.__init__ and call the next class in MRO
        super(SupportsMRoPE, self).__init__(vllm_config=vllm_config, prefix=prefix)

    def _find_encoder_classes(
        self, model: "PreTrainedModel"
    ) -> dict[str, type["PreTrainedModel"]]:
        """Modalities whose encoder cannot be told apart from the model itself are
        omitted, as are those `get_encoder` rejects."""
        encoder_classes: dict[str, type[PreTrainedModel]] = {}
        for modality in _MODALITY_TO_TOKEN_TYPE_ID:
            try:
                encoder_cls = type(model.get_encoder(modality=modality))
            except (TypeError, ValueError):
                continue
            if encoder_cls is not type(model):
                encoder_classes[modality] = encoder_cls
        return encoder_classes

    @contextmanager
    def _mark_model_components(self, vllm_config: "VllmConfig"):
        model_config = vllm_config.model_config
        encoder_classes = self._pre_trained_model_classes.encoders
        if not encoder_classes:
            logger.debug("No encoders identified, so no components will be marked")
            yield
            return

        if model_config.skip_tokenizer_init:
            # Determining the supported modalities needs the HF processor, which in
            # turn needs a tokenizer
            mm_config = model_config.get_multimodal_config()
            if mm_config.mm_encoder_only or any(
                mm_config.get_limit_per_prompt(modality) == 0
                for modality in encoder_classes
            ):
                logger.warning_once(
                    "Unable to determine the supported modalities without a "
                    "tokenizer, so no model components will be skipped."
                )
            yield
            return

        # Modalities we don't serve report a limit of 999, which would stop their
        # encoder ever being skipped
        supported_modalities = MULTIMODAL_REGISTRY.get_processing_info(
            model_config
        ).supported_mm_limits

        # One encoder often serves several modalities, and may only be skipped when
        # all of them are disabled, so mark it once for the whole set
        modalities_by_encoder = defaultdict(set)
        for modality, encoder_cls in encoder_classes.items():
            if modality in supported_modalities:
                modalities_by_encoder[encoder_cls].add(modality)

        with ExitStack() as stack:
            stack.enter_context(
                self._mark_language_model(
                    vllm_config, targets=self._pre_trained_model_classes.decoder
                )
            )
            for encoder_cls, modalities in modalities_by_encoder.items():
                stack.enter_context(
                    self._mark_tower_model(vllm_config, modalities, targets=encoder_cls)
                )
            yield

    def _decorate_for_torch_compile(self):
        """Decorate the model's decoder and encoder classes to indicate to vLLM
        that they support torch compile if `can_enable_torch_compile` and
        `should_torch_compile_mm_encoder` are True respectively.
        """
        super()._decorate_for_torch_compile()
        # Decorate the encoder model classes to support torch compile if needed
        if self.compilation_config.compile_mm_encoder:
            encoder_classes = self._pre_trained_model_classes.encoders
            if not encoder_classes:
                raise ValueError(
                    "Unable to infer any encoder classes from the model. "
                    "You must either: update the model so that "
                    "https://huggingface.co/docs/transformers/en/main_classes/model#transformers.PreTrainedModel.get_encoder"
                    " can detect the encoders correctly, or remove "
                    "'compile_mm_encoder'."
                )
            logger.warning_once(
                "Multimodal encoder compilation with the Transformers modeling backend "
                "is an experimental feature. It relies on:\n"
                "- The encoder being torch compilable.\n"
                "- All encoder tensor inputs must be type hinted as either "
                "`torch.Tensor` or `torch.FloatTensor`.\n"
                "- The 0-th dimension of all tensor inputs to the encoder being the "
                "dynamic dimension (e.g. sequence length, number of patches).\n"
                "Please report any issues you encounter to help us improve it."
            )
            # One encoder can serve several modalities, and must only be decorated once
            for encoder_cls in dict.fromkeys(encoder_classes.values()):
                self._decorate_cls_for_torch_compile(
                    cls=encoder_cls,
                    # TODO: properly infer dynamic_arg_dims based on the encoder's
                    # forward method signature. We assume dim 0 for all tensor inputs.
                    dynamic_arg_dims=None,
                    enable_if=should_torch_compile_mm_encoder,
                    is_encoder=True,
                )

    def forward(
        self,
        input_ids: torch.Tensor | None,
        positions: torch.Tensor,
        intermediate_tensors: IntermediateTensors | None = None,
        inputs_embeds: torch.Tensor | None = None,
        **kwargs: object,
    ) -> torch.Tensor | IntermediateTensors:
        # Positions shape handling for MRoPE models
        if self.model_config.uses_mrope:
            # [3, seq_len] -> [3, 1, seq_len]
            positions = positions[:, None].contiguous()
        model_output = super().forward(
            input_ids, positions, intermediate_tensors, inputs_embeds
        )
        return model_output

    def get_language_model(self) -> torch.nn.Module:
        """Transformers modeling backend multimodal classes do not contain a separate
        vLLM language model class. Therefore, in order to return a language model vLLM
        class, we use a wrapper to give `self` the same interface as a text model."""
        # Exclude self and object
        bases = self.__class__.mro()[1:-1]
        # Keep only classes defined in `vllm.model_executor.models.transformers`
        bases = [b for b in bases if ".transformers." in b.__module__]
        # Exclude MultiModalMixin itself
        bases = [b for b in bases if b is not MultiModalMixin]

        class LanguageModel(*bases):  # type: ignore[misc]
            def __init__(self, multimodal_model):
                # Don't call super().__init__() to avoid re-initialization
                self.__dict__.update(multimodal_model.__dict__)

            model = getattr_iter(self.model, ("language_model", "text_model"), None)

        return LanguageModel(self)

    def get_mm_mapping(self) -> MultiModelKeys:
        """Get the module prefix in multimodal models"""
        for name in ("language_model", "text_model"):
            if getattr(self.model, name, None) is not None:
                return MultiModelKeys.from_string_field(language_model=f"model.{name}")
        raise ValueError(
            "Could not locate the language model submodule for LoRA support"
        )

    def _split_embeddings(
        self, embeddings: torch.Tensor, split_sizes: list[int]
    ) -> list[torch.Tensor]:
        total_expected = sum(split_sizes)

        # Flatten to 2D: [total_tokens, hidden_dim]
        if embeddings.ndim > 2:
            embeddings = embeddings.reshape(-1, embeddings.shape[-1])

        total_tokens = embeddings.shape[0]
        if total_tokens == total_expected:
            # Direct match: split_sizes are actual token counts
            token_split_sizes = split_sizes
        elif total_expected > 0 and total_tokens % total_expected == 0:
            # Uniform expansion: each item expands to N tokens
            tokens_per_item = total_tokens // total_expected
            token_split_sizes = [s * tokens_per_item for s in split_sizes]
        elif total_expected > 0:
            # TODO: make this an error once we know profiling never relies on it
            if total_tokens == 0:
                raise ValueError(
                    "Encoder returned empty embeddings. "
                    f"Expected {total_expected} tokens from "
                    f"split_sizes={split_sizes}"
                )
            # Keep the counts out of the message: `warning_once` keys its cache on
            # the args, so varying them would log on every new pair
            logger.warning_once(
                "Encoder returned a different number of tokens than expected; "
                "padding or truncating to fit. The embeddings are not trustworthy "
                "outside of memory profiling."
            )
            logger.debug(
                "Encoder returned %s tokens but %s were expected",
                total_tokens,
                total_expected,
            )
            if total_tokens < total_expected:
                repeat_factor = (total_expected + total_tokens - 1) // total_tokens
                embeddings = embeddings.repeat(repeat_factor, 1)
            embeddings = embeddings[:total_expected]
            token_split_sizes = split_sizes
        else:
            return []

        return list(torch.split(embeddings, token_split_sizes, dim=0))

    def _process_audio_input(self, **kwargs) -> list[torch.Tensor] | None:
        input_features: torch.Tensor | None = kwargs.pop("input_features", None)
        if input_features is None:
            input_features = kwargs.pop("input_values", None)
        if input_features is None:
            return None

        num_audio_tokens = kwargs.pop("num_audio_tokens")
        kwargs.pop("token_type_ids", None)
        kwargs.pop("mm_token_type_ids", None)

        # Per-audio token counts are needed as Python ints to split.
        with gpu_sync_allowed():
            split_sizes = num_audio_tokens.flatten().tolist()
        if isinstance(input_features, torch.Tensor):
            # HuggingFace's `get_audio_features` implementations branch on
            # per-sample feature lengths internally.
            with gpu_sync_allowed():
                audio_output = self.model.get_audio_features(
                    input_features, return_dict=True, **kwargs
                )
            return self._split_embeddings(audio_output.pooler_output, split_sizes)

        # Audios the processor left un-padded arrive as a list once their
        # lengths differ. Encode them one at a time so that none of them is
        # padded to match another.
        embeddings: list[torch.Tensor] = []
        for index, features in enumerate(input_features):
            audio_output = self.model.get_audio_features(
                features.unsqueeze(0),
                return_dict=True,
                **self._select_item_kwargs(kwargs, index, len(input_features)),
            )
            embeddings.extend(
                self._split_embeddings(audio_output.pooler_output, [split_sizes[index]])
            )
        return embeddings

    def _process_image_input(self, **kwargs) -> list[torch.Tensor] | None:
        pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
        image_embeds: torch.Tensor | None = kwargs.pop("image_embeds", None)
        # Model might use `image_patches` instead of `pixel_values`
        if pixel_values is None:
            pixel_values = kwargs.pop("image_patches", None)

        if image_embeds is not None:
            return [image_embeds]

        if pixel_values is None:
            return None

        num_image_patches = kwargs.pop("num_image_patches")

        split_sizes = num_image_patches.flatten().tolist()
        if isinstance(pixel_values, torch.Tensor):
            vision_embeddings = self._get_image_features(pixel_values, **kwargs)
            if isinstance(vision_embeddings, torch.Tensor):
                return self._split_embeddings(vision_embeddings, split_sizes)
            return list(vision_embeddings)

        # Images the processor left un-padded arrive as a list once their
        # shapes differ. Encode them one at a time so that none of them is
        # padded to match another.
        embeddings: list[torch.Tensor] = []
        for index, image in enumerate(pixel_values):
            features = self._get_image_features(
                image.unsqueeze(0),
                **self._select_item_kwargs(kwargs, index, len(pixel_values)),
            )
            # Encoders which return one entry per image return a single entry
            if not isinstance(features, torch.Tensor):
                features = torch.cat(list(features))
            embeddings.extend(self._split_embeddings(features, [split_sizes[index]]))
        return embeddings

    def _select_item_kwargs(
        self, kwargs: dict[str, Any], index: int, num_items: int
    ) -> dict[str, Any]:
        """Narrow the entries of `kwargs` that hold one row per item down to the item
        at `index`. Length is all there is to match on, so an unrelated entry of the
        same length is narrowed too."""
        return {
            key: value[index : index + 1]
            if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
            else value
            for key, value in kwargs.items()
        }

    def _get_image_features(self, pixel_values: torch.Tensor, **kwargs) -> Any:
        # grid_thw fields are registered keep_on_cpu; restore the on-device
        # placement that HF get_image_features implementations expect.
        for key, value in kwargs.items():
            if isinstance(value, torch.Tensor) and value.is_cpu:
                kwargs[key] = async_tensor_h2d(value, pixel_values.device)

        # The underlying HuggingFace `get_image_features` implementations
        # contain model-internal syncs (e.g. Idefics3 filters all-zero
        # padding images via boolean-mask indexing, LlavaOnevision
        # branches on per-sample batch counts).
        with gpu_sync_allowed():
            features = self.model.get_image_features(pixel_values, **kwargs)

        # Transformers `v5`, `self.get_image_features` returns a tuple
        # containing the features and optionally attentions/hidden_states
        # After v5 is settled, we can enable qwen3-vl with several outputs
        # from `self.get_image_features`
        if isinstance(features, tuple):
            return features[0]
        if isinstance(features, ModelOutput):
            return features.pooler_output
        return features

    def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings:
        # Each helper detects its own inputs. We are called once per modality, so the
        # leftovers a helper forwards to the HF model can't belong to the other one.
        embeddings: list[torch.Tensor] = []
        for process_input in (self._process_audio_input, self._process_image_input):
            embeddings.extend(process_input(**kwargs) or [])
        return embeddings

    def get_mrope_input_positions(
        self,
        input_tokens: list[int],
        mm_features: list[MultiModalFeatureSpec],
    ) -> tuple[torch.Tensor, int]:
        kwargs = MultiModalFeatureSpec.gather_kwargs(
            mm_features,
            {
                "image_grid_thw",
                "video_grid_thw",
                "second_per_grid_ts",
                "audio_feature_lengths",
                "use_audio_in_video",
            },
        )
        if any(v for k, v in kwargs.items() if k not in {"image_grid_thw"}):
            raise NotImplementedError(
                "Transformers modeling backend only supports images."
            )

        image_grid_thw = kwargs.get("image_grid_thw", [])
        video_grid_thw = kwargs.get("video_grid_thw", [])

        image_grid_thw = torch.stack(image_grid_thw) if image_grid_thw else None
        video_grid_thw = torch.stack(video_grid_thw) if video_grid_thw else None

        # `get_rope_index` doesn't always accept arbitrary `kwargs`
        if not hasattr(self, "_get_rope_index_kwarg_names"):
            import inspect

            params = inspect.signature(self.model.get_rope_index).parameters
            self._get_rope_index_kwarg_names = (
                None
                if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
                else frozenset(params)
            )

        kwarg_names = self._get_rope_index_kwarg_names

        def accepts_kwarg(name: str) -> bool:
            return kwarg_names is None or name in kwarg_names

        # Drop a grid the model can't accept only when there is nothing to pass.
        kwargs = {
            name: value
            for name, value in (
                ("image_grid_thw", image_grid_thw),
                ("video_grid_thw", video_grid_thw),
            )
            if value is not None or accepts_kwarg(name)
        }
        if accepts_kwarg("mm_token_type_ids"):
            mm_token_type_ids = torch.zeros(len(input_tokens), dtype=torch.int)
            for feature in mm_features:
                position = feature.mm_position
                offset, length = position.offset, position.length
                mm_token_type_id = _MODALITY_TO_TOKEN_TYPE_ID[feature.modality]
                mm_token_type_ids[offset : offset + length] = mm_token_type_id
            kwargs["mm_token_type_ids"] = mm_token_type_ids.unsqueeze(0)

        mrope_positions, mrope_position_delta = self.model.get_rope_index(
            input_ids=torch.tensor(input_tokens).unsqueeze(0),
            **kwargs,
        )

        mrope_positions = mrope_positions[:, 0]
        mrope_position_delta = mrope_position_delta[0].item()

        return mrope_positions, mrope_position_delta

_decorate_for_torch_compile()

Decorate the model's decoder and encoder classes to indicate to vLLM that they support torch compile if can_enable_torch_compile and should_torch_compile_mm_encoder are True respectively.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _decorate_for_torch_compile(self):
    """Decorate the model's decoder and encoder classes to indicate to vLLM
    that they support torch compile if `can_enable_torch_compile` and
    `should_torch_compile_mm_encoder` are True respectively.
    """
    super()._decorate_for_torch_compile()
    # Decorate the encoder model classes to support torch compile if needed
    if self.compilation_config.compile_mm_encoder:
        encoder_classes = self._pre_trained_model_classes.encoders
        if not encoder_classes:
            raise ValueError(
                "Unable to infer any encoder classes from the model. "
                "You must either: update the model so that "
                "https://huggingface.co/docs/transformers/en/main_classes/model#transformers.PreTrainedModel.get_encoder"
                " can detect the encoders correctly, or remove "
                "'compile_mm_encoder'."
            )
        logger.warning_once(
            "Multimodal encoder compilation with the Transformers modeling backend "
            "is an experimental feature. It relies on:\n"
            "- The encoder being torch compilable.\n"
            "- All encoder tensor inputs must be type hinted as either "
            "`torch.Tensor` or `torch.FloatTensor`.\n"
            "- The 0-th dimension of all tensor inputs to the encoder being the "
            "dynamic dimension (e.g. sequence length, number of patches).\n"
            "Please report any issues you encounter to help us improve it."
        )
        # One encoder can serve several modalities, and must only be decorated once
        for encoder_cls in dict.fromkeys(encoder_classes.values()):
            self._decorate_cls_for_torch_compile(
                cls=encoder_cls,
                # TODO: properly infer dynamic_arg_dims based on the encoder's
                # forward method signature. We assume dim 0 for all tensor inputs.
                dynamic_arg_dims=None,
                enable_if=should_torch_compile_mm_encoder,
                is_encoder=True,
            )

_find_encoder_classes(model)

Modalities whose encoder cannot be told apart from the model itself are omitted, as are those get_encoder rejects.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _find_encoder_classes(
    self, model: "PreTrainedModel"
) -> dict[str, type["PreTrainedModel"]]:
    """Modalities whose encoder cannot be told apart from the model itself are
    omitted, as are those `get_encoder` rejects."""
    encoder_classes: dict[str, type[PreTrainedModel]] = {}
    for modality in _MODALITY_TO_TOKEN_TYPE_ID:
        try:
            encoder_cls = type(model.get_encoder(modality=modality))
        except (TypeError, ValueError):
            continue
        if encoder_cls is not type(model):
            encoder_classes[modality] = encoder_cls
    return encoder_classes

_select_item_kwargs(kwargs, index, num_items)

Narrow the entries of kwargs that hold one row per item down to the item at index. Length is all there is to match on, so an unrelated entry of the same length is narrowed too.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _select_item_kwargs(
    self, kwargs: dict[str, Any], index: int, num_items: int
) -> dict[str, Any]:
    """Narrow the entries of `kwargs` that hold one row per item down to the item
    at `index`. Length is all there is to match on, so an unrelated entry of the
    same length is narrowed too."""
    return {
        key: value[index : index + 1]
        if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
        else value
        for key, value in kwargs.items()
    }

get_language_model()

Transformers modeling backend multimodal classes do not contain a separate vLLM language model class. Therefore, in order to return a language model vLLM class, we use a wrapper to give self the same interface as a text model.

Source code in vllm/model_executor/models/transformers/multimodal.py

def get_language_model(self) -> torch.nn.Module:
    """Transformers modeling backend multimodal classes do not contain a separate
    vLLM language model class. Therefore, in order to return a language model vLLM
    class, we use a wrapper to give `self` the same interface as a text model."""
    # Exclude self and object
    bases = self.__class__.mro()[1:-1]
    # Keep only classes defined in `vllm.model_executor.models.transformers`
    bases = [b for b in bases if ".transformers." in b.__module__]
    # Exclude MultiModalMixin itself
    bases = [b for b in bases if b is not MultiModalMixin]

    class LanguageModel(*bases):  # type: ignore[misc]
        def __init__(self, multimodal_model):
            # Don't call super().__init__() to avoid re-initialization
            self.__dict__.update(multimodal_model.__dict__)

        model = getattr_iter(self.model, ("language_model", "text_model"), None)

    return LanguageModel(self)

get_mm_mapping()

Get the module prefix in multimodal models

Source code in vllm/model_executor/models/transformers/multimodal.py

def get_mm_mapping(self) -> MultiModelKeys:
    """Get the module prefix in multimodal models"""
    for name in ("language_model", "text_model"):
        if getattr(self.model, name, None) is not None:
            return MultiModelKeys.from_string_field(language_model=f"model.{name}")
    raise ValueError(
        "Could not locate the language model submodule for LoRA support"
    )

MultiModalProcessor

Bases: BaseMultiModalProcessor[MultiModalProcessingInfo]

Locates placeholders from the text_replacement_offsets the HF processor reports, expressing each one as a PromptUpdate.

Stating the expansion as an update is what lets it be rebuilt from an unexpanded prompt, so this processor takes the base class's processing path and with it the multi-modal processor cache.

Source code in vllm/model_executor/models/transformers/multimodal.py

class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
    """Locates placeholders from the `text_replacement_offsets` the HF processor
    reports, expressing each one as a `PromptUpdate`.

    Stating the expansion as an update is what lets it be rebuilt from an
    unexpanded prompt, so this processor takes the base class's processing path
    and with it the multi-modal processor cache.
    """

    def _get_modality_field_names(self, modality: str) -> set[str]:
        """Names of the fields the sub-processor for `modality` produces."""
        # TODO: use else branch only once huggingface/transformers#44394 lands.
        if modality == "audio":
            sub_processor = self.info._get_audio_processor()
        else:
            processor = self.info.get_hf_processor()
            sub_processor = getattr(processor, f"{modality}_processor", None)

        # Pre-computed embeddings bypass the sub-processor entirely
        names = {f"{modality}_embeds"}
        for name in getattr(sub_processor, "model_input_names", None) or ():
            # Companion masks are emitted but not always declared
            names.update((name, f"{name}_mask"))
        return names

    def _partition_keys_by_modality(
        self,
        keys: list[str],
        modalities: list[str],
    ) -> dict[str, list[str]]:
        """Attribute each HF processor output key to the modality that produced it."""
        if len(modalities) == 1:
            return {modalities[0]: keys}

        claimed = {m: self._get_modality_field_names(m) for m in modalities}

        owned: dict[str, list[str]] = {modality: [] for modality in modalities}
        unclaimed = []
        for key in keys:
            for modality in modalities:
                if key in claimed[modality]:
                    owned[modality].append(key)
                    break
            else:
                unclaimed.append(key)

        if unclaimed:
            logger.warning_once(
                "Unable to attribute %s to any of the modalities %s, so they "
                "will not be passed to the model. Add them to the relevant "
                "sub-processor's `model_input_names` to fix this.",
                tuple(unclaimed),
                tuple(modalities),
            )

        return owned

    def _get_slice_dim(self, data: Any, total_rows: int) -> int:
        """Which dimension of a field holds the rows belonging to each item.

        Some processors (e.g., Idefics3) return image fields with a leading batch
        dimension, putting the rows one dimension further in.
        """
        if not isinstance(data, torch.Tensor) or data.ndim < 2:
            return 0
        if data.shape[0] != total_rows and data.shape[1] == total_rows:
            return 1
        return 0

    def _get_mm_fields_config(
        self,
        hf_inputs: "BatchFeature",
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        # HF Processors always return a mask but vLLM doesn't need it
        hf_inputs.pop("attention_mask", None)

        # Absent if the modality had no items
        sizes = {
            modality: hf_inputs.get(key)
            for modality, key in _MODALITY_SIZE_KEYS.items()
        }
        modalities = [m for m, size in sizes.items() if size is not None]

        # Keys we wrote ourselves, rather than ones a sub-processor produced
        own_keys = set(_MODALITY_SIZE_KEYS.values()) | {
            f"{modality}_replacement_{suffix}"
            for modality in modalities
            for suffix in ("ids", "sizes")
        }
        keys = [key for key in hf_inputs if key not in own_keys]
        owned = self._partition_keys_by_modality(keys, modalities)

        # Un-padded fields are already one entry per item, so index rather than slice
        mm_fields: dict[str, MultiModalFieldConfig] = {
            key: MultiModalFieldConfig.batched(modality)
            if modality == "audio" or isinstance(hf_inputs[key], list)
            else MultiModalFieldConfig.flat_from_sizes(
                modality,
                sizes[modality],
                dim=self._get_slice_dim(hf_inputs[key], int(sizes[modality].sum())),
            )
            for modality in modalities
            for key in owned[modality]
        }

        for modality in modalities:
            # One row per item, and only ever read on the CPU
            mm_fields[_MODALITY_SIZE_KEYS[modality]] = MultiModalFieldConfig.batched(
                modality, keep_on_cpu=True
            )
            replacement_sizes = hf_inputs.get(f"{modality}_replacement_sizes")
            if replacement_sizes is not None:
                mm_fields[f"{modality}_replacement_ids"] = (
                    MultiModalFieldConfig.flat_from_sizes(
                        modality, replacement_sizes, keep_on_cpu=True
                    )
                )

        if "image" in modalities:
            # Always one row per item, whatever they describe
            mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched(
                "image", keep_on_cpu=True
            )
            # TODO: route to "video" once the video modality is supported
            mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched(
                "image", keep_on_cpu=True
            )

        return mm_fields

    def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
        return self.dummy_inputs.get_dummy_text(mm_counts)

    def _unpad_audios(
        self,
        hf_inputs: "BatchFeature",
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
    ) -> None:
        """Replace the audio fields with each audio processed on its own.

        Processors pad every audio up to the longest in the call, which would leave
        an item's data dependent on what it was processed with. Unlike images,
        nothing in the output states how long each one really is, and processors
        pad a lone audio too, so the only way to know what an audio produces by
        itself is to process it by itself.
        """
        audios = mm_data.get("audio")
        if not isinstance(audios, Iterable) or not audios:
            return
        if len({len(audio) for audio in audios}) == 1:
            return

        alone = [
            self.info.ctx.call_hf_processor(
                self.info.get_hf_processor(**mm_kwargs),
                dict(
                    text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
                ),
                mm_kwargs,
            )
            for audio in audios
        ]

        for key in self._get_modality_field_names("audio"):
            if isinstance(hf_inputs.get(key), torch.Tensor):
                hf_inputs[key] = [output[key][0] for output in alone]

    def _unpad_images(self, hf_inputs: "BatchFeature") -> None:
        """Trim each image back to its own size when the processor padded them all
        to the largest in the batch.

        An image's data has to depend on nothing but that image, or the multi-modal
        processor cache would store it under that image's hash and later reuse it
        beside a different neighbour. Padding is re-applied when the encoder runs.
        """
        pixel_values = hf_inputs.get("pixel_values")
        image_sizes = hf_inputs.get("image_sizes")
        if not isinstance(pixel_values, torch.Tensor):
            return
        if not isinstance(image_sizes, torch.Tensor):
            return
        if pixel_values.ndim != 4 or len(pixel_values) != len(image_sizes):
            return

        # The sizes describe the trailing dimensions only if the largest of them is
        # what the batch was padded up to. Otherwise they mean something else, as
        # in llava-onevision, where they are the sizes before any processing.
        maxima = image_sizes.max(dim=0).values
        if maxima.tolist() != list(pixel_values.shape[-2:]):
            return

        hf_inputs["pixel_values"] = [
            image[..., :height, :width]
            for image, (height, width) in zip(pixel_values, image_sizes.tolist())
        ]

    def _get_prompt_updates(
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        out_mm_kwargs: MultiModalKwargsItems,
    ) -> Sequence[PromptUpdate]:
        """Replace each modality's placeholder token with the token ids that item's
        replacement text encodes to, marking which of them hold embeddings."""
        hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
        tokenizer = self.info.get_tokenizer()

        def get_target_token_ids(token: str | int | Sequence[int]) -> list[int]:
            if isinstance(token, str):
                return cached_encode(tokenizer, token, add_special_tokens=False)
            if isinstance(token, int):
                return [token]
            return list(token)

        updates = []
        for modality, items in out_mm_kwargs.items():
            # Popped so they are neither cached nor sent to the model; the updates
            # they produce are cached alongside the item instead
            replacements = []
            for item in items:
                ids = item.pop(f"{modality}_replacement_ids").data
                assert isinstance(ids, torch.Tensor)
                replacements.append(
                    PromptUpdateDetails.select_token_id(
                        ids.tolist(), _get_embed_token_id(ids)
                    )
                )
            token = getattr(hf_processor, f"{modality}_token")
            updates.append(
                PromptReplacement(
                    modality=modality,
                    target=get_target_token_ids(token),
                    replacement=replacements.__getitem__,
                )
            )
        return updates

    def _get_num_image_patches(
        self,
        hf_inputs: "BatchFeature",
        mm_data: Mapping[str, object],
        num_images: int,
    ) -> torch.Tensor:
        """How many rows of the image fields belong to each image.

        Taken from whichever per-image count the processor reports,
        and checked against the data it has to slice.
        """
        if (grid := hf_inputs.get("image_grid_thw")) is not None:
            num_patches = grid.prod(-1)
        elif (counts := self._get_num_patches_per_image(mm_data)) is not None:
            num_patches = torch.tensor(counts)
        else:
            num_patches = torch.ones(num_images, dtype=torch.long)

        image_data = hf_inputs.get("pixel_values", hf_inputs.get("image_patches"))
        if isinstance(image_data, torch.Tensor):
            total = int(num_patches.sum())
            rows = image_data.shape[self._get_slice_dim(image_data, total)]
            if rows != total:
                raise ValueError(
                    f"{type(self.info.get_hf_processor()).__name__} returned "
                    f"{rows} row(s) of image data for {num_images} image(s), which "
                    f"cannot be split into the {num_patches.tolist()} row(s) per "
                    "image derived from its outputs, so the rows cannot be "
                    "attributed to an image. Gemma3 does this when "
                    "`do_pan_and_scan` crops an image."
                )
        return num_patches

    def _get_num_patches_per_image(
        self, mm_data: Mapping[str, object]
    ) -> list[int] | None:
        """Ask the HF processor how many rows of image data each image produces."""
        images = mm_data.get("images")
        if not isinstance(images, Iterable) or not images:
            return None
        try:
            sizes = [(image.height, image.width) for image in images]
            mm_tokens = self.info.get_hf_processor()._get_num_multimodal_tokens(
                image_sizes=sizes, **self.info.ctx.get_merged_mm_kwargs({})
            )
            return list(mm_tokens["num_image_patches"])
        except (AttributeError, KeyError, TypeError):
            return None

    def _apply_hf_processor_main(
        self,
        mm_items: MultiModalDataItems,
        hf_kwargs: Mapping[str, object],
    ) -> "BatchFeature":
        hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
            mm_items, hf_kwargs
        )

        if not hf_data:
            return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)

        prompt_text = hf_data.pop("text")
        assert isinstance(prompt_text, str)

        # Ask for the replacement each placeholder expands to, and record it as
        # per-item fields: its token ids, and the tokens or patches behind them
        if has_mm_data := any(hf_data.values()):
            hf_data = {**hf_data, "return_text_replacement_offsets": True}

        try:
            hf_inputs = self.info.ctx.call_hf_processor(
                self.info.get_hf_processor(**hf_kwargs),
                dict(text=prompt_text, **hf_data),
                hf_kwargs,
            )
        except ValueError:
            if any(hf_data.values()):
                raise
            # Some processors reject a prompt holding placeholders with
            # no data to go with them, so tokenize it without them
            tokenizer = self.info.get_tokenizer()
            hf_inputs = transformers.BatchFeature(
                dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
            )
        self._unpad_images(hf_inputs)
        self._unpad_audios(hf_inputs, hf_data, hf_kwargs)

        # Drop the inputs the model would reject
        hf_inputs.pop("mm_token_type_ids", None)
        hf_inputs.pop("token_type_ids", None)

        offsets = hf_inputs.pop("text_replacement_offsets", None)
        # Some processors return an empty batch as a tensor rather than a list
        if offsets is None or len(offsets) == 0 or len(offsets[0]) == 0:
            if has_mm_data:
                raise ValueError(
                    f"{type(self.info.get_hf_processor()).__name__} returned no "
                    "text replacement offsets, so the Transformers modeling backend "
                    "cannot locate the placeholder of each item. Its `__call__` has "
                    "to reach `ProcessorMixin.get_text_with_replacements` with one "
                    "replacement per item, which usually means implementing "
                    "`replace_<modality>_token`. Please report this to transformers "
                    "so it can be fixed."
                )
            hf_inputs.pop("input_ids", None)
            return self._finalize_hf_mm_data(
                hf_data, hf_kwargs, passthrough_data, hf_inputs
            )

        tokenizer = self.info.get_tokenizer()
        replacements = defaultdict[str, list[list[int]]](list)
        for entry in offsets[0]:
            replacements[entry["type"]].append(
                cached_encode(tokenizer, entry["replacement"], add_special_tokens=False)
            )

        for modality, seqs in replacements.items():
            hf_inputs[f"{modality}_replacement_ids"] = torch.tensor(
                [token_id for seq in seqs for token_id in seq]
            )
            hf_inputs[f"{modality}_replacement_sizes"] = torch.tensor(
                [len(seq) for seq in seqs]
            )
            if modality == "image":
                hf_inputs["num_image_patches"] = self._get_num_image_patches(
                    hf_inputs, hf_data, len(seqs)
                )
            elif modality == "audio":
                counts = []
                for seq in seqs:
                    ids = torch.tensor(seq)
                    counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
                hf_inputs["num_audio_tokens"] = torch.tensor(counts)

        hf_inputs.pop("input_ids")

        return self._finalize_hf_mm_data(
            hf_data, hf_kwargs, passthrough_data, hf_inputs
        )

_get_modality_field_names(modality)

Names of the fields the sub-processor for modality produces.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _get_modality_field_names(self, modality: str) -> set[str]:
    """Names of the fields the sub-processor for `modality` produces."""
    # TODO: use else branch only once huggingface/transformers#44394 lands.
    if modality == "audio":
        sub_processor = self.info._get_audio_processor()
    else:
        processor = self.info.get_hf_processor()
        sub_processor = getattr(processor, f"{modality}_processor", None)

    # Pre-computed embeddings bypass the sub-processor entirely
    names = {f"{modality}_embeds"}
    for name in getattr(sub_processor, "model_input_names", None) or ():
        # Companion masks are emitted but not always declared
        names.update((name, f"{name}_mask"))
    return names

_get_num_image_patches(hf_inputs, mm_data, num_images)

How many rows of the image fields belong to each image.

Taken from whichever per-image count the processor reports, and checked against the data it has to slice.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _get_num_image_patches(
    self,
    hf_inputs: "BatchFeature",
    mm_data: Mapping[str, object],
    num_images: int,
) -> torch.Tensor:
    """How many rows of the image fields belong to each image.

    Taken from whichever per-image count the processor reports,
    and checked against the data it has to slice.
    """
    if (grid := hf_inputs.get("image_grid_thw")) is not None:
        num_patches = grid.prod(-1)
    elif (counts := self._get_num_patches_per_image(mm_data)) is not None:
        num_patches = torch.tensor(counts)
    else:
        num_patches = torch.ones(num_images, dtype=torch.long)

    image_data = hf_inputs.get("pixel_values", hf_inputs.get("image_patches"))
    if isinstance(image_data, torch.Tensor):
        total = int(num_patches.sum())
        rows = image_data.shape[self._get_slice_dim(image_data, total)]
        if rows != total:
            raise ValueError(
                f"{type(self.info.get_hf_processor()).__name__} returned "
                f"{rows} row(s) of image data for {num_images} image(s), which "
                f"cannot be split into the {num_patches.tolist()} row(s) per "
                "image derived from its outputs, so the rows cannot be "
                "attributed to an image. Gemma3 does this when "
                "`do_pan_and_scan` crops an image."
            )
    return num_patches

_get_num_patches_per_image(mm_data)

Ask the HF processor how many rows of image data each image produces.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _get_num_patches_per_image(
    self, mm_data: Mapping[str, object]
) -> list[int] | None:
    """Ask the HF processor how many rows of image data each image produces."""
    images = mm_data.get("images")
    if not isinstance(images, Iterable) or not images:
        return None
    try:
        sizes = [(image.height, image.width) for image in images]
        mm_tokens = self.info.get_hf_processor()._get_num_multimodal_tokens(
            image_sizes=sizes, **self.info.ctx.get_merged_mm_kwargs({})
        )
        return list(mm_tokens["num_image_patches"])
    except (AttributeError, KeyError, TypeError):
        return None

_get_prompt_updates(mm_items, hf_processor_mm_kwargs, out_mm_kwargs)

Replace each modality's placeholder token with the token ids that item's replacement text encodes to, marking which of them hold embeddings.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _get_prompt_updates(
    self,
    mm_items: MultiModalDataItems,
    hf_processor_mm_kwargs: Mapping[str, object],
    out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
    """Replace each modality's placeholder token with the token ids that item's
    replacement text encodes to, marking which of them hold embeddings."""
    hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
    tokenizer = self.info.get_tokenizer()

    def get_target_token_ids(token: str | int | Sequence[int]) -> list[int]:
        if isinstance(token, str):
            return cached_encode(tokenizer, token, add_special_tokens=False)
        if isinstance(token, int):
            return [token]
        return list(token)

    updates = []
    for modality, items in out_mm_kwargs.items():
        # Popped so they are neither cached nor sent to the model; the updates
        # they produce are cached alongside the item instead
        replacements = []
        for item in items:
            ids = item.pop(f"{modality}_replacement_ids").data
            assert isinstance(ids, torch.Tensor)
            replacements.append(
                PromptUpdateDetails.select_token_id(
                    ids.tolist(), _get_embed_token_id(ids)
                )
            )
        token = getattr(hf_processor, f"{modality}_token")
        updates.append(
            PromptReplacement(
                modality=modality,
                target=get_target_token_ids(token),
                replacement=replacements.__getitem__,
            )
        )
    return updates

_get_slice_dim(data, total_rows)

Which dimension of a field holds the rows belonging to each item.

Some processors (e.g., Idefics3) return image fields with a leading batch dimension, putting the rows one dimension further in.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _get_slice_dim(self, data: Any, total_rows: int) -> int:
    """Which dimension of a field holds the rows belonging to each item.

    Some processors (e.g., Idefics3) return image fields with a leading batch
    dimension, putting the rows one dimension further in.
    """
    if not isinstance(data, torch.Tensor) or data.ndim < 2:
        return 0
    if data.shape[0] != total_rows and data.shape[1] == total_rows:
        return 1
    return 0

_partition_keys_by_modality(keys, modalities)

Attribute each HF processor output key to the modality that produced it.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _partition_keys_by_modality(
    self,
    keys: list[str],
    modalities: list[str],
) -> dict[str, list[str]]:
    """Attribute each HF processor output key to the modality that produced it."""
    if len(modalities) == 1:
        return {modalities[0]: keys}

    claimed = {m: self._get_modality_field_names(m) for m in modalities}

    owned: dict[str, list[str]] = {modality: [] for modality in modalities}
    unclaimed = []
    for key in keys:
        for modality in modalities:
            if key in claimed[modality]:
                owned[modality].append(key)
                break
        else:
            unclaimed.append(key)

    if unclaimed:
        logger.warning_once(
            "Unable to attribute %s to any of the modalities %s, so they "
            "will not be passed to the model. Add them to the relevant "
            "sub-processor's `model_input_names` to fix this.",
            tuple(unclaimed),
            tuple(modalities),
        )

    return owned

_unpad_audios(hf_inputs, mm_data, mm_kwargs)

Replace the audio fields with each audio processed on its own.

Processors pad every audio up to the longest in the call, which would leave an item's data dependent on what it was processed with. Unlike images, nothing in the output states how long each one really is, and processors pad a lone audio too, so the only way to know what an audio produces by itself is to process it by itself.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _unpad_audios(
    self,
    hf_inputs: "BatchFeature",
    mm_data: Mapping[str, object],
    mm_kwargs: Mapping[str, object],
) -> None:
    """Replace the audio fields with each audio processed on its own.

    Processors pad every audio up to the longest in the call, which would leave
    an item's data dependent on what it was processed with. Unlike images,
    nothing in the output states how long each one really is, and processors
    pad a lone audio too, so the only way to know what an audio produces by
    itself is to process it by itself.
    """
    audios = mm_data.get("audio")
    if not isinstance(audios, Iterable) or not audios:
        return
    if len({len(audio) for audio in audios}) == 1:
        return

    alone = [
        self.info.ctx.call_hf_processor(
            self.info.get_hf_processor(**mm_kwargs),
            dict(
                text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
            ),
            mm_kwargs,
        )
        for audio in audios
    ]

    for key in self._get_modality_field_names("audio"):
        if isinstance(hf_inputs.get(key), torch.Tensor):
            hf_inputs[key] = [output[key][0] for output in alone]

_unpad_images(hf_inputs)

Trim each image back to its own size when the processor padded them all to the largest in the batch.

An image's data has to depend on nothing but that image, or the multi-modal processor cache would store it under that image's hash and later reuse it beside a different neighbour. Padding is re-applied when the encoder runs.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _unpad_images(self, hf_inputs: "BatchFeature") -> None:
    """Trim each image back to its own size when the processor padded them all
    to the largest in the batch.

    An image's data has to depend on nothing but that image, or the multi-modal
    processor cache would store it under that image's hash and later reuse it
    beside a different neighbour. Padding is re-applied when the encoder runs.
    """
    pixel_values = hf_inputs.get("pixel_values")
    image_sizes = hf_inputs.get("image_sizes")
    if not isinstance(pixel_values, torch.Tensor):
        return
    if not isinstance(image_sizes, torch.Tensor):
        return
    if pixel_values.ndim != 4 or len(pixel_values) != len(image_sizes):
        return

    # The sizes describe the trailing dimensions only if the largest of them is
    # what the batch was padded up to. Otherwise they mean something else, as
    # in llava-onevision, where they are the sizes before any processing.
    maxima = image_sizes.max(dim=0).values
    if maxima.tolist() != list(pixel_values.shape[-2:]):
        return

    hf_inputs["pixel_values"] = [
        image[..., :height, :width]
        for image, (height, width) in zip(pixel_values, image_sizes.tolist())
    ]

_get_embed_token_id(replacement_ids)

The token an expansion repeats is the one holding the embeddings.

Source code in vllm/model_executor/models/transformers/multimodal.py

def _get_embed_token_id(replacement_ids: torch.Tensor) -> int:
    """The token an expansion repeats is the one holding the embeddings."""
    return int(replacement_ids.mode().values)