Skip to content

vllm.multimodal.registry

Classes:

DummyInputsBuilderFactory

Bases: Protocol[_I]

Constructs a BaseDummyInputsBuilder instance from the context.

Source code in vllm/multimodal/registry.py 
42
43
44
45
46
47
48
class DummyInputsBuilderFactory(Protocol[_I]):  # type: ignore[misc]
    """Constructs a
    [`BaseDummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
    instance from the context.
    """

    def __call__(self, info: _I) -> BaseDummyInputsBuilder[_I]: ...

MultiModalProcessorFactory

Bases: Protocol[_I]

Constructs a BaseMultiModalProcessor instance from the context.

Source code in vllm/multimodal/registry.py 
51
52
53
54
55
56
57
58
59
60
61
class MultiModalProcessorFactory(Protocol[_I]):  # type: ignore[misc]
    """Constructs a
    [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor]
    instance from the context.
    """

    def __call__(
        self,
        info: _I,
        dummy_inputs: BaseDummyInputsBuilder[_I],
    ) -> BaseMultiModalProcessor[_I]: ...

MultiModalRegistry

A registry that dispatches data processing according to the model.

Methods:

  • create_processor –

    Create a multi-modal processor for a specific model and tokenizer.

  • register_processor –

    Register a multi-modal processor to a model class. The processor

Source code in vllm/multimodal/registry.py 
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
class MultiModalRegistry:
    """A registry that dispatches data processing according to the model."""

    def register_processor(
        self,
        processor: MultiModalProcessorFactory[_I],
        *,
        info: ProcessingInfoFactory[_I],
        dummy_inputs: DummyInputsBuilderFactory[_I],
    ):
        """Register a multi-modal processor to a model class. The processor
        is constructed lazily, hence a factory method should be passed.

        When the model receives multi-modal data, the provided function is
        invoked to transform the data into a dictionary of model inputs.
        """

        def wrapper(model_cls: N) -> N:
            if "_processor_factory" in model_cls.__dict__:
                logger.warning(
                    "Model class %s already has a multi-modal processor "
                    "registered to %s. It is overwritten by the new one.",
                    model_cls,
                    self,
                )

            model_cls._processor_factory = _ProcessorFactories(
                info=info,
                dummy_inputs=dummy_inputs,
                processor=processor,
            )

            return model_cls

        return wrapper

    def _get_model_cls(self, model_config: "ModelConfig") -> "SupportsMultiModal":
        # Avoid circular import
        from vllm.model_executor.model_loader import get_model_architecture

        model_cls, _ = get_model_architecture(model_config)
        if not hasattr(model_cls, "_processor_factory"):
            raise ValueError(
                f"Model class {model_cls.__name__} has no registered "
                "multimodal processor"
            )
        return cast("SupportsMultiModal", model_cls)

    def _create_processing_ctx(
        self,
        model_config: "ModelConfig",
        tokenizer: TokenizerLike | None = None,
    ) -> InputProcessingContext:
        if tokenizer is None:
            tokenizer = cached_tokenizer_from_config(model_config)

        return InputProcessingContext(model_config, tokenizer)

    def _create_processing_info(
        self,
        model_config: "ModelConfig",
        tokenizer: TokenizerLike | None = None,
    ) -> BaseProcessingInfo:
        model_cls = self._get_model_cls(model_config)
        factories = model_cls._processor_factory
        ctx = self._create_processing_ctx(model_config, tokenizer)
        return factories.info(ctx)

    def get_processing_info(self, model_config: "ModelConfig") -> BaseProcessingInfo:
        return self._create_processing_info(model_config, tokenizer=None)

    def create_processor(
        self,
        model_config: "ModelConfig",
        *,
        tokenizer: TokenizerLike | None = None,
    ) -> BaseMultiModalProcessor[BaseProcessingInfo]:
        """Create a multi-modal processor for a specific model and tokenizer."""
        if not model_config.is_multimodal_model:
            model_name = model_config.served_model_name or model_config.model
            raise ValueError(f"{model_name} is not a multimodal model")

        model_cls = self._get_model_cls(model_config)
        factories = model_cls._processor_factory

        ctx = self._create_processing_ctx(model_config, tokenizer)

        return factories.build_processor(ctx)

create_processor(model_config, *, tokenizer=None)

Create a multi-modal processor for a specific model and tokenizer.

Source code in vllm/multimodal/registry.py 
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
def create_processor(
    self,
    model_config: "ModelConfig",
    *,
    tokenizer: TokenizerLike | None = None,
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
    """Create a multi-modal processor for a specific model and tokenizer."""
    if not model_config.is_multimodal_model:
        model_name = model_config.served_model_name or model_config.model
        raise ValueError(f"{model_name} is not a multimodal model")

    model_cls = self._get_model_cls(model_config)
    factories = model_cls._processor_factory

    ctx = self._create_processing_ctx(model_config, tokenizer)

    return factories.build_processor(ctx)

register_processor(processor, *, info, dummy_inputs)

Register a multi-modal processor to a model class. The processor is constructed lazily, hence a factory method should be passed.

When the model receives multi-modal data, the provided function is invoked to transform the data into a dictionary of model inputs.

Source code in vllm/multimodal/registry.py 
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
def register_processor(
    self,
    processor: MultiModalProcessorFactory[_I],
    *,
    info: ProcessingInfoFactory[_I],
    dummy_inputs: DummyInputsBuilderFactory[_I],
):
    """Register a multi-modal processor to a model class. The processor
    is constructed lazily, hence a factory method should be passed.

    When the model receives multi-modal data, the provided function is
    invoked to transform the data into a dictionary of model inputs.
    """

    def wrapper(model_cls: N) -> N:
        if "_processor_factory" in model_cls.__dict__:
            logger.warning(
                "Model class %s already has a multi-modal processor "
                "registered to %s. It is overwritten by the new one.",
                model_cls,
                self,
            )

        model_cls._processor_factory = _ProcessorFactories(
            info=info,
            dummy_inputs=dummy_inputs,
            processor=processor,
        )

        return model_cls

    return wrapper

ProcessingInfoFactory

Bases: Protocol[_I_co]

Constructs a BaseMultiModalProcessor instance from the context.

Source code in vllm/multimodal/registry.py 
30
31
32
33
34
35
36
37
38
39
class ProcessingInfoFactory(Protocol[_I_co]):
    """Constructs a
    [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor]
    instance from the context.
    """

    def __call__(
        self,
        ctx: InputProcessingContext,
    ) -> _I_co: ...