vllm.benchmarks.throughput

Benchmark offline inference throughput.

Functions:

• assign_loras –

  Attach a LoRA request to each sample (throughput-only post-processing).

• run_vllm_chat –

  Run vLLM chat benchmark. This function is recommended ONLY for benchmarking

• validate_args –

  Validate command-line arguments.

_to_serve_args(args)

Translate throughput args into a namespace the shared get_samples reads.

get_samples (used by bench serve) expects ~45 attributes; the throughput CLI exposes most of them under the same names, and this adapter fills the rest while preserving throughput's existing flag names so no script breaks.

Parameters:

• args

  (Namespace) –

  parsed throughput CLI namespace.

Returns:

• Namespace –

  A namespace satisfying get_samples's attribute reads.

Source code in vllm/benchmarks/throughput.py

def _to_serve_args(args: argparse.Namespace) -> argparse.Namespace:
    """Translate throughput args into a namespace the shared get_samples reads.

    ``get_samples`` (used by ``bench serve``) expects ~45 attributes; the
    throughput CLI exposes most of them under the same names, and this adapter
    fills the rest while preserving throughput's existing flag names so no
    script breaks.

    Args:
        args: parsed throughput CLI namespace.

    Returns:
        A namespace satisfying get_samples's attribute reads.

    """
    d = vars(args).copy()
    # random_*: prefer --random-* over legacy --input/output/prefix-len.
    # Callers without the legacy flags (e.g. bench mm-processor) omit them, so
    # read them optionally. Identity checks, not truthiness: an explicit 0
    # (the default of --random-prefix-len) must not fall through to them.
    input_len = getattr(args, "input_len", None)
    output_len = getattr(args, "output_len", None)
    prefix_len = getattr(args, "prefix_len", None)
    d["random_input_len"] = getattr(args, "random_input_len", input_len)
    d["random_output_len"] = getattr(args, "random_output_len", output_len)
    d["random_prefix_len"] = getattr(args, "random_prefix_len", prefix_len)
    # --output-len maps to the per-dataset output-len entry points get_samples
    # reads. None passes through (each dataset applies its own default),
    # matching prior throughput behaviour of omitting output_len when unset.
    d["hf_output_len"] = output_len
    d["sharegpt_output_len"] = output_len
    # sonnet reads dedicated attrs; fall back to SonnetDataset's own defaults.
    d["sonnet_input_len"] = input_len if input_len is not None else 550
    d["sonnet_output_len"] = output_len if output_len is not None else 150
    d["sonnet_prefix_len"] = prefix_len
    # Explicit --enable-multimodal-chat wins; otherwise auto-enable for the
    # multimodal chat backend (preserves today's vllm-chat handling). Callers
    # without a --backend flag (e.g. bench mm-processor) drive every request
    # through llm.chat, so default to the vllm-chat behaviour.
    backend = getattr(args, "backend", "vllm-chat")
    d["backend"] = backend
    d["enable_multimodal_chat"] = bool(
        getattr(args, "enable_multimodal_chat", False) or backend == "vllm-chat"
    )
    # serve-only attrs throughput never exposed; keep serve's defaults.
    d.setdefault("disable_shuffle", False)
    d.setdefault("skip_chat_template", False)
    d.setdefault("no_stream", False)
    d.setdefault("no_oversample", False)
    d.setdefault("request_id_prefix", "")
    d.setdefault("chat_template_kwargs", None)
    return argparse.Namespace(**d)

assign_loras(requests, args)

Attach a LoRA request to each sample (throughput-only post-processing).

The shared datasets.get_samples path carries no LoRA information, so LoRA assignment is applied here, uniformly across every dataset type. No-op when --lora-path is unset.

Source code in vllm/benchmarks/throughput.py

def assign_loras(requests, args):
    """Attach a LoRA request to each sample (throughput-only post-processing).

    The shared ``datasets.get_samples`` path carries no LoRA information, so
    LoRA assignment is applied here, uniformly across every dataset type. No-op
    when ``--lora-path`` is unset.
    """
    lora_path = getattr(args, "lora_path", None)
    if not lora_path:
        return requests
    max_loras = args.max_loras
    lora_assignment = getattr(args, "lora_assignment", "random")
    for i, req in enumerate(requests):
        req.lora_request = BenchmarkDataset.get_lora_request(
            index=i,
            max_loras=max_loras,
            lora_path=lora_path,
            lora_assignment=lora_assignment,
        )
    return requests

run_vllm_chat(requests, n, engine_args, do_profile, disable_detokenize=False, warmup_requests=None, prequeue_requests=False)

Run vLLM chat benchmark. This function is recommended ONLY for benchmarking multimodal models as it properly handles multimodal inputs and chat formatting. For non-multimodal models, use run_vllm() instead.

Source code in vllm/benchmarks/throughput.py

def run_vllm_chat(
    requests: list[SampleRequest],
    n: int,
    engine_args: EngineArgs,
    do_profile: bool,
    disable_detokenize: bool = False,
    warmup_requests: list[SampleRequest] | None = None,
    prequeue_requests: bool = False,
) -> tuple[float, list[RequestOutput]]:
    """Run vLLM chat benchmark. This function is recommended ONLY for benchmarking
    multimodal models as it properly handles multimodal inputs and chat
    formatting. For non-multimodal models, use run_vllm() instead.
    """
    from vllm import LLM

    llm = LLM.from_engine_args(engine_args)

    all_requests = list(warmup_requests or []) + requests
    assert all(
        llm.llm_engine.model_config.max_model_len
        >= (request.prompt_len + request.expected_output_len)
        for request in all_requests
    ), (
        "Please ensure that max_model_len is greater than the sum of "
        "prompt_len and expected_output_len for all requests."
    )

    if warmup_requests:
        print(f"Warming up with {len(warmup_requests)} requests...")
        _run_vllm_chat_requests(
            llm,
            warmup_requests,
            n,
            disable_detokenize,
            do_profile=False,
            prequeue_requests=prequeue_requests,
        )

    return _run_vllm_chat_requests(
        llm,
        requests,
        n,
        disable_detokenize,
        do_profile=do_profile,
        prequeue_requests=prequeue_requests,
    )

validate_args(args)

Validate command-line arguments.

Source code in vllm/benchmarks/throughput.py

def validate_args(args):
    """Validate command-line arguments."""
    # === Deprecation and Defaulting ===
    if args.dataset is not None:
        warnings.warn(
            "The '--dataset' argument will be deprecated in the next release. "
            "Please use '--dataset-name' and '--dataset-path' instead.",
            stacklevel=2,
        )
        args.dataset_path = args.dataset

    if not getattr(args, "tokenizer", None):
        args.tokenizer = args.model

    # === Backend Validation ===
    valid_backends = {"vllm", "hf", "mii", "vllm-chat"}
    if args.backend not in valid_backends:
        raise ValueError(f"Unsupported backend: {args.backend}")
    if args.prequeue_requests and args.backend not in {"vllm", "vllm-chat"}:
        raise ValueError("--prequeue-requests requires --backend vllm or vllm-chat")
    if args.prequeue_requests and args.async_engine:
        raise ValueError("--prequeue-requests is not supported with --async-engine")

    # === Dataset Configuration ===
    if (
        not args.dataset
        and not args.dataset_path
        and args.dataset_name not in {"prefix_repetition"}
    ):
        print("When dataset path is not set, it will default to random dataset")
        args.dataset_name = "random"
        random_input_len = getattr(args, "random_input_len", None)
        if args.input_len is None and random_input_len is None:
            raise ValueError(
                "Either --input-len or --random-input-len must be provided "
                "for a random dataset"
            )

    # === Dataset Name Specific Checks ===
    # --hf-subset and --hf-split: only used
    # when dataset_name is 'hf'
    if args.dataset_name != "hf" and (
        getattr(args, "hf_subset", None) is not None
        or getattr(args, "hf_split", None) is not None
    ):
        warnings.warn(
            "--hf-subset and --hf-split will be ignored \
                since --dataset-name is not 'hf'.",
            stacklevel=2,
        )

    # --random-range-ratio: only used when dataset_name is 'random',
    # 'random-mm', or 'random-rerank'
    if (
        args.dataset_name not in {"random", "random-mm", "random-rerank"}
        and args.random_range_ratio is not None
    ):
        warnings.warn(
            "--random-range-ratio will be ignored since \
                --dataset-name is not 'random', 'random-mm', or 'random-rerank'.",
            stacklevel=2,
        )

    # --random-batch-size: only used when dataset_name is 'random-rerank'
    if (
        args.dataset_name != "random-rerank"
        and getattr(args, "random_batch_size", None) is not None
    ) and args.random_batch_size != 1:
        warnings.warn(
            "--random-batch-size will be ignored since \
                    --dataset-name is not 'random-rerank'.",
            stacklevel=2,
        )

    # --no-reranker: only used when dataset_name is 'random-rerank'
    if args.dataset_name != "random-rerank" and getattr(args, "no_reranker", False):
        warnings.warn(
            "--no-reranker will be ignored since \
                --dataset-name is not 'random-rerank'.",
            stacklevel=2,
        )

    # --prefix-len: only used when dataset_name is 'random', 'random-mm',
    # 'sonnet', or not set.
    if (
        args.dataset_name not in {"random", "random-mm", "sonnet", None}
        and args.prefix_len is not None
    ):
        warnings.warn(
            "--prefix-len will be ignored since --dataset-name\
                 is not 'random', 'random-mm', 'sonnet', or not set.",
            stacklevel=2,
        )

    # === Random Dataset Argument Conflict Detection ===
    # Check for conflicts between regular and random arguments when using
    # random datasets
    if args.dataset_name in {"random", "random-mm", "random-rerank"}:
        random_input_len = getattr(args, "random_input_len", None)
        random_output_len = getattr(args, "random_output_len", None)
        random_prefix_len = getattr(args, "random_prefix_len", None)

        if args.input_len is not None and random_input_len is not None:
            warnings.warn(
                "Both --input-len and --random-input-len are specified. "
                "The random version (--random-input-len) will be preferred "
                "in this run.",
                stacklevel=2,
            )
        if args.output_len is not None and random_output_len is not None:
            warnings.warn(
                "Both --output-len and --random-output-len are specified. "
                "The random version (--random-output-len) will be preferred "
                "in this run.",
                stacklevel=2,
            )
        if args.prefix_len is not None and random_prefix_len is not None:
            warnings.warn(
                "Both --prefix-len and --random-prefix-len are specified. "
                "The random version (--random-prefix-len) will be preferred "
                "in this run.",
                stacklevel=2,
            )

    # === LoRA Settings ===
    if getattr(args, "enable_lora", False) and args.backend != "vllm":
        raise ValueError("LoRA benchmarking is only supported for vLLM backend")
    if getattr(args, "enable_lora", False) and args.lora_path is None:
        raise ValueError("LoRA path must be provided when enable_lora is True")

    # === Backend-specific Validations ===
    if args.backend == "hf" and args.hf_max_batch_size is None:
        raise ValueError("HF max batch size is required for HF backend")
    if args.backend != "hf" and args.hf_max_batch_size is not None:
        raise ValueError("HF max batch size is only for HF backend.")

    if (
        args.backend in {"hf", "mii"}
        and getattr(args, "quantization", None) is not None
    ):
        raise ValueError("Quantization is only for vLLM backend.")

    if args.backend == "mii" and args.dtype != "auto":
        raise ValueError("dtype must be auto for MII backend.")
    if args.backend == "mii" and args.n != 1:
        raise ValueError("n must be 1 for MII backend.")
    if args.backend == "mii" and args.tokenizer != args.model:
        raise ValueError("Tokenizer must be the same as the model for MII backend.")

    if args.data_parallel_size > 1 and (
        args.distributed_executor_backend != "external_launcher" or args.async_engine
    ):
        # --data-parallel is not supported fully.
        # Old issue: https://github.com/vllm-project/vllm/issues/16222
        # Currently we only support data parallel with external launcher
        # mode (i.e., launch with toruchrun).
        raise ValueError(
            "Data parallel is only supported with external launcher mode "
            "with synchronous engine in offline benchmark, "
            "please use benchmark serving instead"
        )