Skip to content

vllm.model_executor.models.llama_eagle

Classes:

LlamaDecoderLayer

Bases: LlamaDecoderLayer

Methods:

Source code in vllm/model_executor/models/llama_eagle.py 
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
class LlamaDecoderLayer(BaseLlamaDecoderLayer):
    def __init__(
        self,
        vllm_config: VllmConfig,
        disable_input_layernorm: bool,
        prefix: str = "",
        config: LlamaConfig | None = None,
    ) -> None:
        super().__init__(vllm_config, prefix=prefix, config=config)

        # Skip the input_layernorm
        # https://github.com/SafeAILab/EAGLE/blob/35c78f6cdc19a73e05cf5c330b4c358dad970c6a/eagle/model/cnets.py#L427
        if disable_input_layernorm:
            del self.input_layernorm
            self.input_layernorm = nn.Identity()

    def get_quant_config(self, vllm_config: VllmConfig) -> QuantizationConfig | None:
        """Use drafter's quantization config instead of verifier's."""
        return get_draft_quant_config(vllm_config)

get_quant_config(vllm_config)

Use drafter's quantization config instead of verifier's.

Source code in vllm/model_executor/models/llama_eagle.py 
59
60
61
def get_quant_config(self, vllm_config: VllmConfig) -> QuantizationConfig | None:
    """Use drafter's quantization config instead of verifier's."""
    return get_draft_quant_config(vllm_config)