Skip to content

vllm.model_executor.layers.quantization.quark.utils

Functions:

canonicalize_quark_packed_int4(packed_weight, *, pack_reorder, is_symmetric, pack_factor=8)

Convert Quark export nibble layout to AWQ checkpoint layout.

Source code in vllm/model_executor/layers/quantization/quark/utils.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
def canonicalize_quark_packed_int4(
    packed_weight: torch.Tensor,
    *,
    pack_reorder: bool,
    is_symmetric: bool,
    pack_factor: int = 8,
) -> torch.Tensor:
    """Convert Quark export nibble layout to AWQ checkpoint layout."""
    from vllm.model_executor.layers.quantization.auto_awq import (
        _REVERSE_AWQ_PACK_ORDER,
    )

    if pack_reorder:
        source_order = torch.tensor(
            _REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
        )
    else:
        source_order = torch.arange(
            pack_factor, device=packed_weight.device, dtype=torch.int32
        )
    target_order = torch.tensor(
        _REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
    )
    source_shifts = source_order * 4
    target_shifts = target_order * 4

    values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF
    if is_symmetric:
        values = values ^ 0x8
    packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1)
    return packed.to(torch.int32)

parse_w4a16_int4_weight_config(weight_config)

Parse required W4A16 INT4/UINT4 weight fields from Quark config.

Source code in vllm/model_executor/layers/quantization/quark/utils.py 
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
def parse_w4a16_int4_weight_config(
    weight_config: Mapping[str, Any],
) -> tuple[int, bool]:
    """Parse required W4A16 INT4/UINT4 weight fields from Quark config."""
    if "group_size" not in weight_config:
        raise ValueError(
            "Quark W4A16 INT4/UINT4 configs must specify weight.group_size"
        )
    if "symmetric" not in weight_config:
        raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric")

    group_size = weight_config["group_size"]
    is_symmetric = weight_config["symmetric"]
    if not isinstance(group_size, int) or group_size <= 0:
        raise ValueError(
            f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}"
        )
    if not isinstance(is_symmetric, bool):
        raise ValueError(
            f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}"
        )
    return group_size, is_symmetric