Skip to content

vllm.model_executor.model_loader.weight_cache.protocol

WeightCacheKey fingerprinting and socket protocol for the weight cache daemon.

The protocol uses pickle over a Unix domain socket and is only intended for communication between trusted local processes owned by the same user. The sockets live in a per-user private directory (mode 0700) and the daemon restricts the socket file permissions to the owner (0600). Both the daemon and the engine verify that the directory and socket are owned by the current user and are not group/world accessible before trusting them, so a different local user cannot pre-plant a malicious socket at a predictable path.

Classes:

Functions:

CacheConfigMismatchError

Bases: Exception

Raised when the daemon's cached weights don't match the engine.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
59
60
class CacheConfigMismatchError(Exception):
    """Raised when the daemon's cached weights don't match the engine."""

TensorEntry dataclass

A single cached tensor.

CUDA tensors are exported as torch.multiprocessing reduction args (CUDA IPC handles); non-CUDA tensors are shipped by value.

Attributes:

  • kind (str) –

    Either "param" or "buffer".

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
@dataclass
class TensorEntry:
    """A single cached tensor.

    CUDA tensors are exported as `torch.multiprocessing` reduction args
    (CUDA IPC handles); non-CUDA tensors are shipped by value.
    """

    kind: str
    """Either "param" or "buffer"."""
    ipc_args: tuple | None = None
    cpu_tensor: torch.Tensor | None = None

    @classmethod
    def from_tensor(cls, tensor: torch.Tensor, kind: str) -> "TensorEntry":
        tensor = tensor.detach()
        if tensor.is_cuda:
            _, ipc_args = reduce_tensor(tensor)
            return cls(kind=kind, ipc_args=ipc_args)
        return cls(kind=kind, cpu_tensor=tensor.cpu())

    def rebuild(self, device_index: int) -> torch.Tensor:
        if self.ipc_args is None:
            assert self.cpu_tensor is not None
            return self.cpu_tensor
        args = list(self.ipc_args)
        # Index 6 of the args from reduce_tensor is the device index. It must
        # be retargeted to the local index since the daemon and the engine may
        # have different CUDA_VISIBLE_DEVICES mappings.
        args[6] = device_index
        return rebuild_cuda_tensor(*args)

kind instance-attribute

Either "param" or "buffer".

UnsupportedPlatformForIPCError

Bases: Exception

Raised when the current platform cannot share CUDA IPC handles.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
67
68
class UnsupportedPlatformForIPCError(Exception):
    """Raised when the current platform cannot share CUDA IPC handles."""

UnsupportedQuantForIPCError

Bases: Exception

Raised when a quantization method is not verified for IPC weight sharing.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
63
64
class UnsupportedQuantForIPCError(Exception):
    """Raised when a quantization method is not verified for IPC weight sharing."""

WeightCacheKey dataclass

Fingerprint of the cached weights.

Any mismatch between the daemon's and the engine's fingerprint means the cached weights cannot be reused and the engine must load from disk.

Methods:

Attributes:

  • is_draft (bool) –

    Daemon group the weights come from; False is the target model.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
@dataclass(frozen=True)
class WeightCacheKey:
    """Fingerprint of the cached weights.

    Any mismatch between the daemon's and the engine's fingerprint means the
    cached weights cannot be reused and the engine must load from disk.
    """

    checkpoint: str
    model_arch: str
    tp_size: int
    tp_rank: int
    dtype: str
    quantization: str | None
    quant_config_hash: str
    revision: str | None
    vllm_version: str
    pp_size: int = 1
    pp_rank: int = 0
    is_draft: bool = False
    """Daemon group the weights come from; False is the target model."""
    dp_size: int = 1
    dp_rank: int = 0

    @classmethod
    def from_model_config(
        cls,
        model_config: ModelConfig,
        tp_size: int,
        tp_rank: int,
        *,
        pp_size: int = 1,
        pp_rank: int = 0,
        is_draft: bool = False,
        dp_size: int = 1,
        dp_rank: int = 0,
    ) -> "WeightCacheKey":
        """Build the fingerprint for a model configuration.

        Must be called before weight loading: process_weights_after_loading
        may mutate hf_config.quantization_config, which would change the hash
        between the daemon and the engine.

        The checkpoint is identified by a hash of its safetensors metadata when
        the weights are available locally, so a daemon and engine referencing
        identical weights in different directories still match; otherwise it
        falls back to the model path.
        """
        hf_config = model_config.hf_config
        arch = ",".join(getattr(hf_config, "architectures", None) or [])
        quant_config = getattr(hf_config, "quantization_config", None)
        checkpoint = hash_checkpoint(model_config.model) or model_config.model
        return cls(
            checkpoint=checkpoint,
            model_arch=arch,
            tp_size=tp_size,
            tp_rank=tp_rank,
            pp_size=pp_size,
            pp_rank=pp_rank,
            dtype=str(model_config.dtype),
            quantization=model_config.quantization,
            quant_config_hash=_hash_quant_config(quant_config),
            revision=model_config.revision,
            vllm_version=vllm.version.__version__,
            is_draft=is_draft,
            dp_size=dp_size,
            dp_rank=dp_rank,
        )

    def mismatched_fields(self, other: "WeightCacheKey") -> list[str]:
        return [
            f.name
            for f in fields(self)
            if getattr(self, f.name) != getattr(other, f.name)
        ]

is_draft = False class-attribute instance-attribute

Daemon group the weights come from; False is the target model.

from_model_config(model_config, tp_size, tp_rank, *, pp_size=1, pp_rank=0, is_draft=False, dp_size=1, dp_rank=0) classmethod

Build the fingerprint for a model configuration.

Must be called before weight loading: process_weights_after_loading may mutate hf_config.quantization_config, which would change the hash between the daemon and the engine.

The checkpoint is identified by a hash of its safetensors metadata when the weights are available locally, so a daemon and engine referencing identical weights in different directories still match; otherwise it falls back to the model path.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
@classmethod
def from_model_config(
    cls,
    model_config: ModelConfig,
    tp_size: int,
    tp_rank: int,
    *,
    pp_size: int = 1,
    pp_rank: int = 0,
    is_draft: bool = False,
    dp_size: int = 1,
    dp_rank: int = 0,
) -> "WeightCacheKey":
    """Build the fingerprint for a model configuration.

    Must be called before weight loading: process_weights_after_loading
    may mutate hf_config.quantization_config, which would change the hash
    between the daemon and the engine.

    The checkpoint is identified by a hash of its safetensors metadata when
    the weights are available locally, so a daemon and engine referencing
    identical weights in different directories still match; otherwise it
    falls back to the model path.
    """
    hf_config = model_config.hf_config
    arch = ",".join(getattr(hf_config, "architectures", None) or [])
    quant_config = getattr(hf_config, "quantization_config", None)
    checkpoint = hash_checkpoint(model_config.model) or model_config.model
    return cls(
        checkpoint=checkpoint,
        model_arch=arch,
        tp_size=tp_size,
        tp_rank=tp_rank,
        pp_size=pp_size,
        pp_rank=pp_rank,
        dtype=str(model_config.dtype),
        quantization=model_config.quantization,
        quant_config_hash=_hash_quant_config(quant_config),
        revision=model_config.revision,
        vllm_version=vllm.version.__version__,
        is_draft=is_draft,
        dp_size=dp_size,
        dp_rank=dp_rank,
    )

WeightCacheState

Bases: NamedTuple

Client-side decode of a daemon's get_state response payload.

Attributes:

  • aliases (dict[str, str]) –

    Duplicate (tied) weight names aliased to their canonical entry.

  • attrs (dict[str, bool]) –

    Python-side flags set by load_weights, e.g. EAGLE ownership flags.

  • entries (dict[str, TensorEntry]) –

    Model tensors, exported as CUDA IPC handles or shipped by value.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
385
386
387
388
389
390
391
392
393
class WeightCacheState(NamedTuple):
    """Client-side decode of a daemon's get_state response payload."""

    entries: dict[str, TensorEntry]
    """Model tensors, exported as CUDA IPC handles or shipped by value."""
    aliases: dict[str, str]
    """Duplicate (tied) weight names aliased to their canonical entry."""
    attrs: dict[str, bool]
    """Python-side flags set by load_weights, e.g. EAGLE ownership flags."""

aliases instance-attribute

Duplicate (tied) weight names aliased to their canonical entry.

attrs instance-attribute

Python-side flags set by load_weights, e.g. EAGLE ownership flags.

entries instance-attribute

Model tensors, exported as CUDA IPC handles or shipped by value.

WeightCacheUnavailableError

Bases: Exception

Raised when no weight cache daemon is reachable or usable.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
55
56
class WeightCacheUnavailableError(Exception):
    """Raised when no weight cache daemon is reachable or usable."""

_safetensors_header(path)

Return the raw safetensors header (length prefix + JSON) of a file.

The header carries tensor names, dtypes, shapes and byte offsets, so it is a content fingerprint of the shard without reading any weight bytes.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
239
240
241
242
243
244
245
246
247
248
def _safetensors_header(path: str) -> bytes:
    """Return the raw safetensors header (length prefix + JSON) of a file.

    The header carries tensor names, dtypes, shapes and byte offsets, so it is
    a content fingerprint of the shard without reading any weight bytes.
    """
    with open(path, "rb") as f:
        size_bytes = f.read(8)
        (header_len,) = struct.unpack("<Q", size_bytes)
        return size_bytes + f.read(header_len)

check_ipc_platform_support()

Hard-error unless the current platform can share CUDA IPC handles.

Only CUDA/ROCm tensors get a real IPC handle from TensorEntry; other platforms (e.g. XPU) would silently ship every tensor by value instead.

Raises:

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def check_ipc_platform_support() -> None:
    """Hard-error unless the current platform can share CUDA IPC handles.

    Only CUDA/ROCm tensors get a real IPC handle from ``TensorEntry``; other
    platforms (e.g. XPU) would silently ship every tensor by value instead.

    Raises:
        UnsupportedPlatformForIPCError: If the current platform is not
            CUDA/ROCm.

    """
    if current_platform.is_cuda_alike():
        return
    raise UnsupportedPlatformForIPCError(
        f"platform {current_platform.device_name!r} does not support CUDA IPC "
        "weight sharing; only CUDA and ROCm are supported. Use the default "
        "--load-format for this platform."
    )

check_ipc_quant_support(model)

Hard-error unless every quant method supports pre-processed weights.

Parameters:

  • model

    (Module) –

    The model to inspect (weights need not be loaded).

Raises:

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
def check_ipc_quant_support(model: torch.nn.Module) -> None:
    """Hard-error unless every quant method supports pre-processed weights.

    Args:
        model: The model to inspect (weights need not be loaded).

    Raises:
        UnsupportedQuantForIPCError: If any quant method does not declare
            ``supports_pre_processed_weights``.

    """
    for name, module in model.named_modules():
        quant_method = getattr(module, "quant_method", None)
        if (
            isinstance(quant_method, QuantizeMethodBase)
            and not quant_method.supports_pre_processed_weights
        ):
            raise UnsupportedQuantForIPCError(
                f"layer {name or '<root>'}: {type(quant_method).__name__} "
                "does not support loading from pre-processed weights."
            )

ensure_private_socket_dir(directory, strict_perms=True)

Create the socket directory (if needed) locked down to the owner.

Called by the daemon before binding. Existing directories are re-checked and, for the auto-derived path, tightened so a pre-existing world-writable directory is rejected.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
153
154
155
156
157
158
159
160
161
162
163
def ensure_private_socket_dir(directory: str, strict_perms: bool = True) -> None:
    """Create the socket directory (if needed) locked down to the owner.

    Called by the daemon before binding. Existing directories are re-checked
    and, for the auto-derived path, tightened so a pre-existing world-writable
    directory is rejected.
    """
    os.makedirs(directory, mode=0o700, exist_ok=True)
    if strict_perms:
        os.chmod(directory, 0o700)
    verify_private_dir(directory, strict_perms=strict_perms)

get_current_device_uuid()

UUID of the physical GPU backing the current accelerator device.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
115
116
117
def get_current_device_uuid() -> str:
    """UUID of the physical GPU backing the current accelerator device."""
    return current_platform.get_device_uuid(torch.accelerator.current_device_index())

get_socket_dir(socket_dir=None)

Return the directory that holds the daemon sockets.

When no explicit directory is given, use a per-user private directory under the system temp dir so its path is unpredictable to other users and can be locked down to mode 0700.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
120
121
122
123
124
125
126
127
128
129
130
131
def get_socket_dir(socket_dir: str | None = None) -> str:
    """Return the directory that holds the daemon sockets.

    When no explicit directory is given, use a per-user private directory
    under the system temp dir so its path is unpredictable to other users and
    can be locked down to mode 0700.
    """
    if socket_dir is not None:
        return socket_dir
    return os.path.join(
        tempfile.gettempdir(), SOCKET_DIR_TEMPLATE.format(uid=_current_uid())
    )

get_socket_path(gpu_uuid, socket_dir=None, *, is_draft=False)

Socket path of a daemon group; is_draft=False is the target.

The GPU uuid is hashed to keep the name well under the AF_UNIX path limit (~108 bytes) even with the draft role suffix.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
def get_socket_path(
    gpu_uuid: str,
    socket_dir: str | None = None,
    *,
    is_draft: bool = False,
) -> str:
    """Socket path of a daemon group; ``is_draft=False`` is the target.

    The GPU uuid is hashed to keep the name well under the AF_UNIX path
    limit (~108 bytes) even with the draft role suffix.
    """
    gpu_id = safe_hash(gpu_uuid.encode()).hexdigest()
    name = SOCKET_NAME_TEMPLATE.format(
        gpu_uuid=gpu_id,
        role=format_socket_role_suffix(is_draft),
    )
    return os.path.join(get_socket_dir(socket_dir), name)

hash_checkpoint(model)

Fingerprint checkpoint content from local safetensors metadata.

Hashes each shard's safetensors header so a daemon and an engine pointing at identical weights in different directories produce the same key. Returns None when local safetensors files can't be located (e.g. an undownloaded Hugging Face repo id), leaving the caller to fall back to the model path.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
def hash_checkpoint(model: str) -> str | None:
    """Fingerprint checkpoint content from local safetensors metadata.

    Hashes each shard's safetensors header so a daemon and an engine pointing
    at identical weights in different directories produce the same key. Returns
    None when local safetensors files can't be located (e.g. an undownloaded
    Hugging Face repo id), leaving the caller to fall back to the model path.
    """
    if not os.path.isdir(model):
        return None
    files = glob.glob(os.path.join(model, "*.safetensors"))
    if os.path.isfile(os.path.join(model, SAFE_WEIGHTS_INDEX_NAME)):
        files = filter_duplicate_safetensors_files(
            files, model, SAFE_WEIGHTS_INDEX_NAME
        )
    if not files:
        return None
    hasher = safe_hash(b"", usedforsecurity=False)
    for path in sorted(files, key=os.path.basename):
        hasher.update(os.path.basename(path).encode())
        hasher.update(_safetensors_header(path))
    return hasher.hexdigest()

verify_peer_is_owner(conn)

Best-effort check that the connecting peer runs as the current user.

Uses SO_PEERCRED where available (Linux). Silently returns on platforms that do not expose peer credentials.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
def verify_peer_is_owner(conn: socket.socket) -> None:
    """Best-effort check that the connecting peer runs as the current user.

    Uses SO_PEERCRED where available (Linux). Silently returns on platforms
    that do not expose peer credentials.
    """
    so_peercred = getattr(socket, "SO_PEERCRED", None)
    if so_peercred is None:
        return
    try:
        creds = conn.getsockopt(socket.SOL_SOCKET, so_peercred, struct.calcsize("3i"))
        _, peer_uid, _ = struct.unpack("3i", creds)
    except OSError:
        return
    uid = _current_uid()
    if uid != -1 and peer_uid != uid:
        raise PermissionError(f"Rejecting weight cache connection from uid {peer_uid}")

verify_private_dir(directory, strict_perms=True)

Verify a directory is a real dir owned by us and not world/group readable.

When strict_perms is False the group/world permission bits are not checked; this is used for directories the operator explicitly configured (they own the trust decision), while the auto-derived per-user directory is always checked strictly.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
def verify_private_dir(directory: str, strict_perms: bool = True) -> None:
    """Verify a directory is a real dir owned by us and not world/group readable.

    When ``strict_perms`` is False the group/world permission bits are not
    checked; this is used for directories the operator explicitly configured
    (they own the trust decision), while the auto-derived per-user directory is
    always checked strictly.
    """
    info = os.lstat(directory)
    if stat.S_ISLNK(info.st_mode):
        raise WeightCacheUnavailableError(
            f"Refusing to use symlinked socket directory {directory}"
        )
    if not stat.S_ISDIR(info.st_mode):
        raise WeightCacheUnavailableError(f"{directory} is not a directory")
    uid = _current_uid()
    if uid != -1 and info.st_uid != uid:
        raise WeightCacheUnavailableError(
            f"Socket directory {directory} is not owned by the current user"
        )
    if strict_perms and info.st_mode & 0o077:
        raise WeightCacheUnavailableError(
            f"Socket directory {directory} is group/world accessible"
        )

verify_socket_owner(socket_path, strict_perms=True)

Verify the socket lives in a private dir and is owned by the current user.

Called by the engine before connecting so it never talks to a socket a different user could have planted.

Source code in vllm/model_executor/model_loader/weight_cache/protocol.py 
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def verify_socket_owner(socket_path: str, strict_perms: bool = True) -> None:
    """Verify the socket lives in a private dir and is owned by the current user.

    Called by the engine before connecting so it never talks to a socket a
    different user could have planted.
    """
    verify_private_dir(os.path.dirname(socket_path), strict_perms=strict_perms)
    info = os.lstat(socket_path)
    if stat.S_ISLNK(info.st_mode):
        raise WeightCacheUnavailableError(
            f"Refusing to connect to symlinked socket {socket_path}"
        )
    uid = _current_uid()
    if uid != -1 and info.st_uid != uid:
        raise WeightCacheUnavailableError(
            f"Socket {socket_path} is not owned by the current user"
        )