Skip to content

vllm.model_executor.model_loader.dummy_loader

Classes:

  • DummyModelLoader –

    Model loader that will set model weights to random values.

DummyModelLoader

Bases: BaseModelLoader

Model loader that will set model weights to random values.

Source code in vllm/model_executor/model_loader/dummy_loader.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
class DummyModelLoader(BaseModelLoader):
    """Model loader that will set model weights to random values."""

    def __init__(self, load_config: LoadConfig):
        super().__init__(load_config)
        if load_config.model_loader_extra_config:
            raise ValueError(
                f"Model loader extra config is not supported for "
                f"load format {load_config.load_format}"
            )

    def download_model(self, model_config: ModelConfig) -> None:
        pass  # Nothing to download

    def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
        for layer in model.modules():
            info = get_layerwise_info(layer)
            if info.can_load():
                self._process_online_quant_layer(layer, info)
            else:
                # NOTE(woosuk): For accurate performance evaluation, we assign
                # random values to the weights.
                # Initialize only local tensors: state_dict() would recursively
                # initialize descendants that model.modules() also visits.
                # Exclude nonpersistent buffers, matching state_dict().
                for name, tensor in get_layer_tensors(layer).items():
                    if name not in layer._non_persistent_buffers_set:
                        initialize_single_dummy_weight(tensor)

    def _process_online_quant_layer(
        self,
        layer: nn.Module,
        info: LayerReloadingInfo,
    ) -> None:
        """Materialize, apply dummy weights, and run quantization processing."""
        materialize_layer(layer, info)

        for tensor in get_layer_tensors(layer).values():
            initialize_single_dummy_weight(tensor)

        for param in get_layer_tensors(layer).values():
            param.weight_loader = _get_original_loader(param)

        quant_method = getattr(layer, "quant_method", None)
        if isinstance(quant_method, QuantizeMethodBase):
            quant_method.process_weights_after_loading(layer)

        info.reset()

_process_online_quant_layer(layer, info)

Materialize, apply dummy weights, and run quantization processing.

Source code in vllm/model_executor/model_loader/dummy_loader.py 
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
def _process_online_quant_layer(
    self,
    layer: nn.Module,
    info: LayerReloadingInfo,
) -> None:
    """Materialize, apply dummy weights, and run quantization processing."""
    materialize_layer(layer, info)

    for tensor in get_layer_tensors(layer).values():
        initialize_single_dummy_weight(tensor)

    for param in get_layer_tensors(layer).values():
        param.weight_loader = _get_original_loader(param)

    quant_method = getattr(layer, "quant_method", None)
    if isinstance(quant_method, QuantizeMethodBase):
        quant_method.process_weights_after_loading(layer)

    info.reset()