Skip to content

vllm.utils.system_utils

Functions:

_maybe_force_spawn()

Check if we need to force the use of the spawn multiprocessing start method.

Source code in vllm/utils/system_utils.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def _maybe_force_spawn():
    """Check if we need to force the use of the `spawn` multiprocessing start
    method.
    """
    if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") == "spawn":
        return

    reasons = []
    if is_in_ray_actor():
        # even if we choose to spawn, we need to pass the ray address
        # to the subprocess so that it knows how to connect to the ray cluster.
        # env vars are inherited by subprocesses, even if we use spawn.
        import ray

        os.environ["RAY_ADDRESS"] = ray.get_runtime_context().gcs_address
        reasons.append("In a Ray actor and can only be spawned")

    # Force spawn if NUMA binding is enabled via --numa-bind.
    # NUMA binding uses executable hijacking which requires spawn
    if "--numa-bind" in sys.argv:
        reasons.append("NUMA binding requires spawn method")

    if cuda_is_initialized():
        reasons.append("CUDA is initialized")
    elif xpu_is_initialized():
        reasons.append("XPU is initialized")

    if in_wsl():
        reasons.append("WSL is detected and NVML is not compatible with fork")

    if reasons:
        logger.warning(
            "We must use the `spawn` multiprocessing start method. "
            "Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. "
            "See https://docs.vllm.ai/en/latest/usage/"
            "troubleshooting.html#python-multiprocessing "
            "for more information. Reasons: %s",
            "; ".join(reasons),
        )
        os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"

_sync_visible_devices_env_vars()

Sync HIP/CUDA visibility env vars before spawning (ROCm only).

Source code in vllm/utils/system_utils.py 
111
112
113
114
115
116
117
118
def _sync_visible_devices_env_vars():
    """Sync HIP/CUDA visibility env vars before spawning (ROCm only)."""
    if not current_platform.is_rocm():
        return

    from vllm.platforms.rocm import _sync_hip_cuda_env_vars

    _sync_hip_cuda_env_vars()

decorate_logs(process_name=None, *, skip_if_decorated=False)

Add vLLM process metadata to subsequent log records.

Source code in vllm/utils/system_utils.py 
196
197
198
199
200
201
202
203
def decorate_logs(
    process_name: str | None = None, *, skip_if_decorated: bool = False
) -> None:
    """Add vLLM process metadata to subsequent log records."""
    if process_name is None:
        process_name = get_mp_context().current_process().name

    set_vllm_process_name(process_name, skip_if_set=skip_if_decorated)

find_loaded_library(lib_name)

According to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html, the file /proc/self/maps contains the memory maps of the process, which includes the shared libraries loaded by the process. We can use this file to find the path of the loaded library.

Source code in vllm/utils/system_utils.py 
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
def find_loaded_library(lib_name: str) -> str | None:
    """
    According to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html,
    the file `/proc/self/maps` contains the memory maps of the process, which includes the
    shared libraries loaded by the process. We can use this file to find the path of the
    loaded library.
    """  # noqa
    # Match the mapped file's name, not the whole line: an unrelated library
    # whose name merely contains lib_name (e.g. TileLang's libcudart_stub.so
    # when looking for libcudart) or a directory component containing it must
    # not win. Legitimate filenames are {lib_name}.so[.*] or a name-mangled
    # {lib_name}-<hash>.so[.*], and /proc/self/maps is ordered by mapping
    # address rather than load order, so a substring hit is a
    # nondeterministic hijack.
    with open("/proc/self/maps") as f:
        for line in f:
            start = line.find("/")
            if start == -1:
                continue
            path = line[start:].strip()
            filename = path.rsplit("/", maxsplit=1)[-1]
            if filename.startswith((f"{lib_name}.", f"{lib_name}-")):
                return path
    # the library is not loaded in the current process
    return None

get_mp_context()

Get a multiprocessing context with a particular method (spawn or fork). By default we follow the value of the VLLM_WORKER_MULTIPROC_METHOD to determine the multiprocessing method (default is fork). However, under certain conditions, we may enforce spawn and override the value of VLLM_WORKER_MULTIPROC_METHOD.

Source code in vllm/utils/system_utils.py 
163
164
165
166
167
168
169
170
171
172
173
174
175
176
def get_mp_context():
    """Get a multiprocessing context with a particular method (spawn or fork).
    By default we follow the value of the VLLM_WORKER_MULTIPROC_METHOD to
    determine the multiprocessing method (default is fork). However, under
    certain conditions, we may enforce spawn and override the value of
    VLLM_WORKER_MULTIPROC_METHOD.
    """
    _maybe_force_spawn()
    # (ROCm): Sync GPU visibility env vars so spawned children inherit
    # consistent values. Must run after _maybe_force_spawn and regardless
    # of whether spawn was already set.
    _sync_visible_devices_env_vars()
    mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
    return multiprocessing.get_context(mp_method)

kill_process_tree(pid)

Kills all descendant processes of the given pid by sending SIGKILL.

Parameters:

  • pid

    (int) –

    Process ID of the parent process

Source code in vllm/utils/system_utils.py 
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
def kill_process_tree(pid: int):
    """Kills all descendant processes of the given pid by sending SIGKILL.

    Args:
        pid (int): Process ID of the parent process

    """
    try:
        parent = psutil.Process(pid)
    except psutil.NoSuchProcess:
        return

    # Get all children recursively
    children = parent.children(recursive=True)

    # Send SIGKILL to all children first
    for child in children:
        with contextlib.suppress(ProcessLookupError):
            logger.warning(
                "[shutdown] force killing sub process %s pid %d",
                child.name(),
                child.pid,
            )
            os.kill(child.pid, signal.SIGKILL)

    # Finally kill the parent
    with contextlib.suppress(ProcessLookupError):
        os.kill(pid, signal.SIGKILL)

set_env_var(key, value)

Temporarily set an environment variable.

Source code in vllm/utils/system_utils.py 
43
44
45
46
47
48
49
50
51
52
53
54
@contextlib.contextmanager
def set_env_var(key: str, value: str) -> Iterator[None]:
    """Temporarily set an environment variable."""
    old = os.environ.get(key)
    os.environ[key] = value
    try:
        yield
    finally:
        if old is None:
            os.environ.pop(key, None)
        else:
            os.environ[key] = old

set_process_title(name, suffix='', prefix=envs.VLLM_PROCESS_NAME_PREFIX)

Set the current process title with optional suffix.

Source code in vllm/utils/system_utils.py 
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
def set_process_title(
    name: str,
    suffix: str = "",
    prefix: str = envs.VLLM_PROCESS_NAME_PREFIX,
) -> None:
    """Set the current process title with optional suffix."""
    try:
        import setproctitle
    except ImportError:
        return

    if suffix:
        name = f"{name}_{suffix}"

    setproctitle.setproctitle(f"{prefix}::{name}")

suppress_stdout()

Suppress stdout from C libraries at the file descriptor level.

Only suppresses stdout, not stderr, to preserve error messages. Suppression is disabled when vLLM debug logging is enabled.

Example

with suppress_stdout(): # C library calls that would normally print to stdout torch.distributed.new_group(ranks, backend="gloo")

Source code in vllm/utils/system_utils.py 
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
@contextlib.contextmanager
def suppress_stdout():
    """Suppress stdout from C libraries at the file descriptor level.

    Only suppresses stdout, not stderr, to preserve error messages.
    Suppression is disabled when vLLM debug logging is enabled.

    Example:
        with suppress_stdout():
            # C library calls that would normally print to stdout
            torch.distributed.new_group(ranks, backend="gloo")

    """
    # Don't suppress if logging level is DEBUG
    if logger.isEnabledFor(logging.DEBUG):
        yield
        return

    stdout_fd = sys.stdout.fileno()
    stdout_dup = os.dup(stdout_fd)
    devnull_fd = os.open(os.devnull, os.O_WRONLY)

    try:
        sys.stdout.flush()
        os.dup2(devnull_fd, stdout_fd)
        yield
    finally:
        sys.stdout.flush()
        os.dup2(stdout_dup, stdout_fd)
        os.close(stdout_dup)
        os.close(devnull_fd)

unique_filepath(fn)

Generate a unique file path by trying incrementing integers.

Note: This function has a TOCTOU race condition. Caller should use atomic operations (e.g., open with 'x' mode) when creating the file to ensure thread safety.

Source code in vllm/utils/system_utils.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
def unique_filepath(fn: Callable[[int], Path]) -> Path:
    """Generate a unique file path by trying incrementing integers.

    Note: This function has a TOCTOU race condition.
    Caller should use atomic operations (e.g., open with 'x' mode)
    when creating the file to ensure thread safety.
    """
    i = 0
    while True:
        p = fn(i)
        if not p.exists():
            return p
        i += 1

update_environment_variables(envs_dict)

Update multiple environment variables with logging.

Source code in vllm/utils/system_utils.py 
30
31
32
33
34
35
36
37
38
39
40
def update_environment_variables(envs_dict: dict[str, str]):
    """Update multiple environment variables with logging."""
    for k, v in envs_dict.items():
        if k in os.environ and os.environ[k] != v:
            logger.warning(
                "Overwriting environment variable %s from '%s' to '%s'",
                k,
                os.environ[k],
                v,
            )
        os.environ[k] = v