vllm.models.glm5next.common.multimodal

GLM-5.3-Flash vision tower and multimodal processor.

Classes:

• Glm5NextMultiModalProcessor –

  Glm5NextProcessor extracts image/video features

• Glm5NextProcessingInfo –

  Token-budget geometry for the multimodal checkpoint.

• Glm5NextVisionTransformer –

Glm5NextMultiModalProcessor

Bases: Glm4vMultiModalProcessor

Glm5NextProcessor extracts image/video features only and passes the prompt text through unchanged, so prompt expansion (image token repeat, video frame/timestamp structure) is owned by vLLM's prompt-update machinery — the inherited _get_prompt_updates builds the replacement content and the placeholder scan validates against exactly that.

Source code in vllm/models/glm5next/common/multimodal.py

class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
    """``Glm5NextProcessor`` extracts image/video features
    only and passes the prompt text through unchanged, so prompt expansion
    (image token repeat, video frame/timestamp structure) is owned by vLLM's
    prompt-update machinery — the inherited ``_get_prompt_updates`` builds
    the replacement content and the placeholder scan validates against
    exactly that."""

    def _hf_processor_applies_updates(
        self,
        prompt_text: str,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
        tokenization_kwargs: Mapping[str, object],
    ) -> bool:
        return False

Glm5NextProcessingInfo

Bases: Glm4vProcessingInfo

Token-budget geometry for the multimodal checkpoint.

This checkpoint's processor_config.json ships the token-budget style (min_image_tokens / max_image_tokens) with no size key, so the inherited Glm4v size.longest_edge path does not apply.

Source code in vllm/models/glm5next/common/multimodal.py

class Glm5NextProcessingInfo(Glm4vProcessingInfo):
    """Token-budget geometry for the multimodal checkpoint.

    This checkpoint's ``processor_config.json`` ships the token-budget style
    (``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, so the
    inherited Glm4v ``size.longest_edge`` path does not apply.
    """

    def get_hf_processor(self, **kwargs: object):
        processor = super().get_hf_processor(**kwargs)
        # Cap video inputs to keep encoder profiling from starving the KV cache.
        video_processor = processor.video_processor
        video_processor.max_image_tokens = min(
            video_processor.max_image_tokens, _MAX_VIDEO_TOKENS
        )
        return processor

    @staticmethod
    def _alignment_factor(proc) -> int:
        """Spatial factor canvases align to, as ``smart_resize`` takes it."""
        return proc.patch_size * proc.merge_size * proc.patch_expand_factor

    @classmethod
    def _pixels_per_token(cls, proc) -> int:
        """Pixels one vision token covers, matching ``smart_resize``'s own
        ``temporal_factor * factor ** 2``."""
        return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2

    def _processor_pixel_budget(self, proc) -> tuple[int, int]:
        """(min_pixels, max_pixels) from the processor's token bounds."""
        if proc.min_image_tokens is None or proc.max_image_tokens is None:
            raise ValueError(
                "min_image_tokens and max_image_tokens must be provided by "
                "processor_config.json (or per-call kwargs)."
            )
        pixels_per_token = self._pixels_per_token(proc)
        return (
            proc.min_image_tokens * pixels_per_token,
            proc.max_image_tokens * pixels_per_token,
        )

    def _get_image_max_pixels(self) -> int:
        mm_kwargs = self.ctx.get_merged_mm_kwargs({})
        if (override := mm_kwargs.get("max_pixels")) is not None:
            return int(override)
        return self._processor_pixel_budget(self.get_hf_processor().image_processor)[1]

    def _get_video_max_pixels(self) -> int:
        mm_kwargs = self.ctx.get_merged_mm_kwargs({})
        if (override := mm_kwargs.get("max_pixels")) is not None:
            return int(override)
        return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]

    def get_image_size_with_most_features(self) -> ImageSize:
        # The inherited square probe strands budget whenever the token
        # ceiling is not a perfect square: with max_image_tokens=8000 the
        # square refits to 2492x2492 (89x89 = 7921 tokens) while a
        # 2240x2800 canvas reaches 80x100 = 8000, so the encoder cache came
        # up short of the processor's own maximum and refused valid images
        # (#59539). Factor the ceiling exactly instead of probing a square.
        vision_config = self.get_hf_config().vision_config
        factor = (
            vision_config.patch_size
            * vision_config.spatial_merge_size
            * self.get_hf_processor().image_processor.patch_expand_factor
        )
        pixels_per_token = vision_config.temporal_patch_size * factor * factor
        max_tokens = max(1, self._get_image_max_pixels() // pixels_per_token)
        short_side = math.isqrt(max_tokens)
        while max_tokens % short_side:
            short_side -= 1
        return ImageSize(
            width=(max_tokens // short_side) * factor,
            height=short_side * factor,
        )

    def _get_video_second_idx_glm46v(
        self, metadata: dict[str, Any], total_frames: int
    ) -> list[int]:
        """Timestamps for the video placeholder, from the pixel path's sampler.

        The inherited GLM-4.6V timestamp sampling uses different constants
        than this checkpoint's ``Glm5NextVideoProcessor`` pixel sampling, so
        placeholder count and ``video_grid_thw`` disagree outside 30-300s
        durations. Sampling with the pixel path's own policy makes one
        timestamp per temporal patch == ``grid_t``.
        """
        video_processor = self.get_video_processor()
        temporal_patch_size = int(
            getattr(video_processor, "temporal_patch_size", 1) or 1
        )
        video_fps = float(metadata["fps"])

        if not metadata.get("do_sample_frames", True):
            # The loader already picked the frames and the processor keeps them
            # as they are, so the frame count is what was handed in and
            # ``frames_indices`` only supplies the timestamp values.
            frame_indices = [int(idx) for idx in metadata["frames_indices"]]
            num_frames = int(total_frames)
        else:
            frame_indices = [
                int(idx)
                for idx in video_processor.sample_frames(
                    VideoMetadata(
                        total_num_frames=int(
                            metadata.get("total_num_frames", total_frames)
                        ),
                        fps=video_fps,
                        duration=metadata.get("duration"),
                    )
                )
            ]
            num_frames = len(frame_indices)

        if not frame_indices or num_frames <= 0:
            # ``duration * fps < 1`` selects no frames at all; the
            # pixel path raises an opaque IndexError on the same input a moment
            # later, so say what happened while there is still a request.
            raise ValueError(
                "The GLM-5-Next frame sampler selected no frames for this "
                f"video (fps={video_fps}, duration={metadata.get('duration')!r},"
                f" total_num_frames={metadata.get('total_num_frames')!r}). "
                "Clips shorter than one frame sampling interval cannot be "
                "processed."
            )

        num_padded_frames = num_frames + (-num_frames % temporal_patch_size)
        grid_t = max(num_padded_frames // temporal_patch_size, 1)
        last = len(frame_indices) - 1
        return [
            int(frame_indices[min(i * temporal_patch_size, last)] / video_fps)
            for i in range(grid_t)
        ]

    def _get_vision_info(
        self,
        *,
        image_width: int,
        image_height: int,
        num_frames: int = 16,
        do_resize: bool = True,
        max_image_pixels: int = 28 * 28 * 2 * 30000,
    ) -> tuple[ImageSize, int]:
        """GLM-5.3-Flash canvas geometry for token budgeting and dummy inputs.

        The inherited Glm4v path resolves the pixel budget from
        ``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This
        checkpoint ships token bounds instead, and the alignment factor carries
        ``patch_expand_factor`` — resolve both from the processor so profiling
        matches runtime geometry.
        """
        from transformers.models.glm5_next.image_processing_glm5_next import (
            smart_resize,
        )

        vision_config = self.get_hf_config().vision_config
        patch_size = vision_config.patch_size
        merge_size = vision_config.spatial_merge_size
        temporal_patch_size = vision_config.temporal_patch_size

        image_processor = self.get_hf_processor().image_processor
        factor = self._alignment_factor(image_processor)
        temporal_factor = image_processor.temporal_patch_size
        # `smart_resize` denominates its bounds in vision tokens. Round down,
        # but keep the profiling search viable when the caller's budget is
        # below one aligned canvas of the requested duration.
        max_image_tokens = max(
            max_image_pixels // self._pixels_per_token(image_processor), 1
        )

        if do_resize:
            t = num_frames if num_frames > temporal_factor else temporal_factor
            resized_height, resized_width = smart_resize(
                num_frames=t,
                height=image_height,
                width=image_width,
                temporal_factor=temporal_factor,
                factor=factor,
                min_pixels=1,
                max_pixels=max_image_tokens,
            )
            preprocessed_size = ImageSize(width=resized_width, height=resized_height)
        else:
            preprocessed_size = ImageSize(width=image_width, height=image_height)

        padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
        grid_t = max(padded_num_frames // temporal_patch_size, 1)
        grid_h = preprocessed_size.height // patch_size
        grid_w = preprocessed_size.width // patch_size

        num_patches = grid_t * grid_h * grid_w
        num_vision_tokens = num_patches // (merge_size**2)

        return preprocessed_size, num_vision_tokens

_alignment_factor(proc) staticmethod

Spatial factor canvases align to, as smart_resize takes it.

Source code in vllm/models/glm5next/common/multimodal.py

@staticmethod
def _alignment_factor(proc) -> int:
    """Spatial factor canvases align to, as ``smart_resize`` takes it."""
    return proc.patch_size * proc.merge_size * proc.patch_expand_factor

_get_video_second_idx_glm46v(metadata, total_frames)

Timestamps for the video placeholder, from the pixel path's sampler.

The inherited GLM-4.6V timestamp sampling uses different constants than this checkpoint's Glm5NextVideoProcessor pixel sampling, so placeholder count and video_grid_thw disagree outside 30-300s durations. Sampling with the pixel path's own policy makes one timestamp per temporal patch == grid_t.

Source code in vllm/models/glm5next/common/multimodal.py

def _get_video_second_idx_glm46v(
    self, metadata: dict[str, Any], total_frames: int
) -> list[int]:
    """Timestamps for the video placeholder, from the pixel path's sampler.

    The inherited GLM-4.6V timestamp sampling uses different constants
    than this checkpoint's ``Glm5NextVideoProcessor`` pixel sampling, so
    placeholder count and ``video_grid_thw`` disagree outside 30-300s
    durations. Sampling with the pixel path's own policy makes one
    timestamp per temporal patch == ``grid_t``.
    """
    video_processor = self.get_video_processor()
    temporal_patch_size = int(
        getattr(video_processor, "temporal_patch_size", 1) or 1
    )
    video_fps = float(metadata["fps"])

    if not metadata.get("do_sample_frames", True):
        # The loader already picked the frames and the processor keeps them
        # as they are, so the frame count is what was handed in and
        # ``frames_indices`` only supplies the timestamp values.
        frame_indices = [int(idx) for idx in metadata["frames_indices"]]
        num_frames = int(total_frames)
    else:
        frame_indices = [
            int(idx)
            for idx in video_processor.sample_frames(
                VideoMetadata(
                    total_num_frames=int(
                        metadata.get("total_num_frames", total_frames)
                    ),
                    fps=video_fps,
                    duration=metadata.get("duration"),
                )
            )
        ]
        num_frames = len(frame_indices)

    if not frame_indices or num_frames <= 0:
        # ``duration * fps < 1`` selects no frames at all; the
        # pixel path raises an opaque IndexError on the same input a moment
        # later, so say what happened while there is still a request.
        raise ValueError(
            "The GLM-5-Next frame sampler selected no frames for this "
            f"video (fps={video_fps}, duration={metadata.get('duration')!r},"
            f" total_num_frames={metadata.get('total_num_frames')!r}). "
            "Clips shorter than one frame sampling interval cannot be "
            "processed."
        )

    num_padded_frames = num_frames + (-num_frames % temporal_patch_size)
    grid_t = max(num_padded_frames // temporal_patch_size, 1)
    last = len(frame_indices) - 1
    return [
        int(frame_indices[min(i * temporal_patch_size, last)] / video_fps)
        for i in range(grid_t)
    ]

_get_vision_info(*, image_width, image_height, num_frames=16, do_resize=True, max_image_pixels=28 * 28 * 2 * 30000)

GLM-5.3-Flash canvas geometry for token budgeting and dummy inputs.

The inherited Glm4v path resolves the pixel budget from size.longest_edge and resizes with GLM-4V's smart_resize. This checkpoint ships token bounds instead, and the alignment factor carries patch_expand_factor — resolve both from the processor so profiling matches runtime geometry.

Source code in vllm/models/glm5next/common/multimodal.py

def _get_vision_info(
    self,
    *,
    image_width: int,
    image_height: int,
    num_frames: int = 16,
    do_resize: bool = True,
    max_image_pixels: int = 28 * 28 * 2 * 30000,
) -> tuple[ImageSize, int]:
    """GLM-5.3-Flash canvas geometry for token budgeting and dummy inputs.

    The inherited Glm4v path resolves the pixel budget from
    ``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This
    checkpoint ships token bounds instead, and the alignment factor carries
    ``patch_expand_factor`` — resolve both from the processor so profiling
    matches runtime geometry.
    """
    from transformers.models.glm5_next.image_processing_glm5_next import (
        smart_resize,
    )

    vision_config = self.get_hf_config().vision_config
    patch_size = vision_config.patch_size
    merge_size = vision_config.spatial_merge_size
    temporal_patch_size = vision_config.temporal_patch_size

    image_processor = self.get_hf_processor().image_processor
    factor = self._alignment_factor(image_processor)
    temporal_factor = image_processor.temporal_patch_size
    # `smart_resize` denominates its bounds in vision tokens. Round down,
    # but keep the profiling search viable when the caller's budget is
    # below one aligned canvas of the requested duration.
    max_image_tokens = max(
        max_image_pixels // self._pixels_per_token(image_processor), 1
    )

    if do_resize:
        t = num_frames if num_frames > temporal_factor else temporal_factor
        resized_height, resized_width = smart_resize(
            num_frames=t,
            height=image_height,
            width=image_width,
            temporal_factor=temporal_factor,
            factor=factor,
            min_pixels=1,
            max_pixels=max_image_tokens,
        )
        preprocessed_size = ImageSize(width=resized_width, height=resized_height)
    else:
        preprocessed_size = ImageSize(width=image_width, height=image_height)

    padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
    grid_t = max(padded_num_frames // temporal_patch_size, 1)
    grid_h = preprocessed_size.height // patch_size
    grid_w = preprocessed_size.width // patch_size

    num_patches = grid_t * grid_h * grid_w
    num_vision_tokens = num_patches // (merge_size**2)

    return preprocessed_size, num_vision_tokens

_pixels_per_token(proc) classmethod

Pixels one vision token covers, matching smart_resize's own temporal_factor * factor ** 2.

Source code in vllm/models/glm5next/common/multimodal.py

@classmethod
def _pixels_per_token(cls, proc) -> int:
    """Pixels one vision token covers, matching ``smart_resize``'s own
    ``temporal_factor * factor ** 2``."""
    return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2

_processor_pixel_budget(proc)

(min_pixels, max_pixels) from the processor's token bounds.

Source code in vllm/models/glm5next/common/multimodal.py

def _processor_pixel_budget(self, proc) -> tuple[int, int]:
    """(min_pixels, max_pixels) from the processor's token bounds."""
    if proc.min_image_tokens is None or proc.max_image_tokens is None:
        raise ValueError(
            "min_image_tokens and max_image_tokens must be provided by "
            "processor_config.json (or per-call kwargs)."
        )
    pixels_per_token = self._pixels_per_token(proc)
    return (
        proc.min_image_tokens * pixels_per_token,
        proc.max_image_tokens * pixels_per_token,
    )

Glm5NextVisionTransformer

Bases: Module

Methods:

• prepare_encoder_metadata –

  Compute encoder metadata for eager and CUDA graph execution.

Source code in vllm/models/glm5next/common/multimodal.py

class Glm5NextVisionTransformer(nn.Module):
    # Stacked-weight remap for the GLM-OCR/GLM-4V vision checkpoint layout.
    hf_to_vllm_mapper = WeightsMapper(
        orig_to_new_stacked={
            ".attn.q.": (".attn.qkv.", "q"),
            ".attn.k.": (".attn.qkv.", "k"),
            ".attn.v.": (".attn.qkv.", "v"),
            ".gate_proj": (".gate_up_proj", 0),
            ".up_proj": (".gate_up_proj", 1),
        }
    )

    def __init__(
        self,
        text_config,  # noqa: ANN001
        vision_config,
        norm_eps: float = 1e-6,
        quant_config: QuantizationConfig | None = None,
        input_norm: nn.Module | None = None,
        prefix: str = "",
    ) -> None:
        super().__init__()
        use_data_parallel = is_vit_use_data_parallel()
        self.tp_size = (
            1 if use_data_parallel else get_tensor_model_parallel_world_size()
        )

        patch_size = vision_config.patch_size
        temporal_patch_size = vision_config.temporal_patch_size
        in_channels = vision_config.in_channels
        depth = vision_config.depth
        self.hidden_size = vision_config.hidden_size
        self.num_heads = vision_config.num_heads

        self.patch_size = vision_config.patch_size
        self.spatial_merge_size = vision_config.spatial_merge_size
        self.out_hidden_size = vision_config.out_hidden_size

        swiglu_limit = vision_config.swiglu_limit
        if swiglu_limit is None:
            swiglu_limit = text_config.swiglu_limit
        assert swiglu_limit is not None, (
            "GLM-5.3-Flash vision requires swiglu_limit (vision_config or text_config)"
        )

        # Single construction pass — no abs-pos embeddings / post-conv norm (OCR delta).
        self.patch_embed = Glm5NextVisionPatchEmbed(
            patch_size=patch_size,
            temporal_patch_size=temporal_patch_size,
            in_channels=in_channels,
            hidden_size=self.hidden_size,
        )
        self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()

        norm_layer = partial(RMSNorm, eps=norm_eps)
        head_dim = self.hidden_size // self.num_heads
        self.rotary_pos_emb = get_rope(
            head_size=head_dim,
            max_position=text_config.max_position_embeddings,
            is_neox_style=True,
            rope_parameters={"partial_rotary_factor": 0.5},
        )
        self.blocks = nn.ModuleList(
            [
                Glm5NextVisionBlock(
                    dim=self.hidden_size,
                    num_heads=self.num_heads,
                    mlp_hidden_dim=vision_config.intermediate_size,
                    swiglu_limit=swiglu_limit,
                    norm_layer=norm_layer,
                    quant_config=quant_config,
                    prefix=f"{prefix}.blocks.{layer_idx}",
                )
                for layer_idx in range(depth)
            ]
        )
        # GLM-5.3-Flash merger bottleneck width.
        self.merger = Glm5NextPatchMerger(
            d_model=vision_config.out_hidden_size,
            context_dim=vision_config.projection_intermediate_size,
            swiglu_limit=swiglu_limit,
            quant_config=quant_config,
            bias=False,
            prefix=f"{prefix}.merger",
        )

        self.downsample = Conv2dLayer(
            in_channels=vision_config.hidden_size,
            out_channels=vision_config.out_hidden_size,
            kernel_size=vision_config.spatial_merge_size,
            stride=vision_config.spatial_merge_size,
        )
        self.post_layernorm = RMSNorm(
            vision_config.hidden_size, eps=vision_config.rms_norm_eps
        )

        self.attn_backend = get_vit_attn_backend(
            head_size=head_dim,
            dtype=torch.get_default_dtype(),
        )

    @property
    def dtype(self) -> torch.dtype:
        return self.patch_embed.proj.weight.dtype

    @property
    def device(self) -> torch.device:
        return self.patch_embed.proj.weight.device

    def rot_pos_emb(
        self, grid_thw: list[list[int]]
    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        pos_ids = []
        for t, h, w in grid_thw:
            hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
            wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
            hpos_ids = (
                hpos_ids.reshape(
                    h // self.spatial_merge_size,
                    self.spatial_merge_size,
                    w // self.spatial_merge_size,
                    self.spatial_merge_size,
                )
                .permute(0, 2, 1, 3)
                .flatten()
            )
            wpos_ids = (
                wpos_ids.reshape(
                    h // self.spatial_merge_size,
                    self.spatial_merge_size,
                    w // self.spatial_merge_size,
                    self.spatial_merge_size,
                )
                .permute(0, 2, 1, 3)
                .flatten()
            )
            pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
        pos_ids = torch.cat(pos_ids, dim=0)
        max_grid_size = max(max(h, w) for _, h, w in grid_thw)

        cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)

        pos_ids = async_tensor_h2d(pos_ids, device=cos.device)
        cos_combined = cos[pos_ids].flatten(1)
        sin_combined = sin[pos_ids].flatten(1)
        return cos_combined, sin_combined, pos_ids

    def compute_attn_mask_seqlen(
        self,
        cu_seqlens: torch.Tensor,
    ) -> torch.Tensor | None:
        max_seqlen = None
        if self.attn_backend in {
            AttentionBackendEnum.FLASH_ATTN,
            AttentionBackendEnum.ROCM_AITER_FA,
            AttentionBackendEnum.TRITON_ATTN,
        }:
            max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
        return max_seqlen

    def prepare_encoder_metadata(
        self,
        grid_thw_list: list[list[int]],
        *,
        max_batch_size: int | None = None,
        max_frames_per_batch: int | None = None,
        max_seqlen_override: int | None = None,
        device: torch.device | None = None,
    ) -> dict[str, torch.Tensor | None]:
        """Compute encoder metadata for eager and CUDA graph execution."""
        if device is None:
            device = self.device

        metadata: dict[str, torch.Tensor | None] = {}

        rotary_cos, rotary_sin, _ = self.rot_pos_emb(grid_thw_list)
        metadata["rotary_pos_emb_cos"] = rotary_cos
        metadata["rotary_pos_emb_sin"] = rotary_sin

        grid_thw_np = np.array(grid_thw_list, dtype=np.int32)
        patches_per_frame = grid_thw_np[:, 1] * grid_thw_np[:, 2]
        cu_seqlens = np.repeat(patches_per_frame, grid_thw_np[:, 0]).cumsum(
            dtype=np.int32
        )
        cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])

        pad_to = (
            max_frames_per_batch if max_frames_per_batch is not None else max_batch_size
        )
        if pad_to is not None:
            num_seqs = len(cu_seqlens) - 1
            if num_seqs < pad_to:
                cu_seqlens = np.concatenate(
                    [
                        cu_seqlens,
                        np.full(
                            pad_to - num_seqs,
                            cu_seqlens[-1],
                            dtype=np.int32,
                        ),
                    ]
                )

        metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
            self.attn_backend, cu_seqlens, device
        )

        if max_seqlen_override is not None:
            max_seqlen_val = max_seqlen_override
        else:
            max_seqlen_val = MMEncoderAttention.compute_max_seqlen(
                self.attn_backend, cu_seqlens
            )
        metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)

        metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
            self.attn_backend,
            cu_seqlens,
            self.hidden_size,
            self.tp_size,
            device,
        )

        return metadata

    def forward(
        self,
        x: torch.Tensor,
        grid_thw: torch.Tensor | list[list[int]],
        *,
        encoder_metadata: dict[str, torch.Tensor] | None = None,
    ) -> torch.Tensor:
        # patchify
        x = self.input_norm(x.to(device=self.device), self.dtype)
        x = self.patch_embed(x)

        if encoder_metadata is not None:
            # Encoder CUDA-graph path (PR #49852): rotary/cu_seqlens/max_seqlen are
            # precomputed by prepare_encoder_metadata (which uses rot_pos_emb exactly
            # as the eager rebuild does), so reuse them and skip the per-call CPU
            # rebuild (the low-GPU-util culprit on multimodal workloads).
            rotary_pos_emb_cos = encoder_metadata["rotary_pos_emb_cos"]
            rotary_pos_emb_sin = encoder_metadata["rotary_pos_emb_sin"]
            cu_seqlens = encoder_metadata["cu_seqlens"]
            max_seqlen = encoder_metadata["max_seqlen"]
        else:
            if isinstance(grid_thw, list):
                grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
            rotary_pos_emb_cos, rotary_pos_emb_sin, _ = self.rot_pos_emb(grid_thw)
            cu_seqlens = torch.repeat_interleave(
                grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
            ).cumsum(dim=0, dtype=torch.int32)
            cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
            cu_seqlens = async_tensor_h2d(cu_seqlens, device=self.device)
            max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)

        # transformers
        x = x.unsqueeze(1)
        for blk in self.blocks:
            x = blk(
                x,
                cu_seqlens=cu_seqlens,
                rotary_pos_emb_cos=rotary_pos_emb_cos,
                rotary_pos_emb_sin=rotary_pos_emb_sin,
                max_seqlen=max_seqlen,
            )

        # adapter
        x = self.post_layernorm(x)
        x = x.view(-1, self.spatial_merge_size, self.spatial_merge_size, x.shape[-1])
        x = x.permute(0, 3, 1, 2)
        x = self.downsample(x).view(-1, self.out_hidden_size)
        x = self.merger(x)
        return x

    def load_weights(self, weights) -> set[str]:
        loader = AutoWeightsLoader(self)
        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

prepare_encoder_metadata(grid_thw_list, *, max_batch_size=None, max_frames_per_batch=None, max_seqlen_override=None, device=None)

Compute encoder metadata for eager and CUDA graph execution.

Source code in vllm/models/glm5next/common/multimodal.py

def prepare_encoder_metadata(
    self,
    grid_thw_list: list[list[int]],
    *,
    max_batch_size: int | None = None,
    max_frames_per_batch: int | None = None,
    max_seqlen_override: int | None = None,
    device: torch.device | None = None,
) -> dict[str, torch.Tensor | None]:
    """Compute encoder metadata for eager and CUDA graph execution."""
    if device is None:
        device = self.device

    metadata: dict[str, torch.Tensor | None] = {}

    rotary_cos, rotary_sin, _ = self.rot_pos_emb(grid_thw_list)
    metadata["rotary_pos_emb_cos"] = rotary_cos
    metadata["rotary_pos_emb_sin"] = rotary_sin

    grid_thw_np = np.array(grid_thw_list, dtype=np.int32)
    patches_per_frame = grid_thw_np[:, 1] * grid_thw_np[:, 2]
    cu_seqlens = np.repeat(patches_per_frame, grid_thw_np[:, 0]).cumsum(
        dtype=np.int32
    )
    cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])

    pad_to = (
        max_frames_per_batch if max_frames_per_batch is not None else max_batch_size
    )
    if pad_to is not None:
        num_seqs = len(cu_seqlens) - 1
        if num_seqs < pad_to:
            cu_seqlens = np.concatenate(
                [
                    cu_seqlens,
                    np.full(
                        pad_to - num_seqs,
                        cu_seqlens[-1],
                        dtype=np.int32,
                    ),
                ]
            )

    metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
        self.attn_backend, cu_seqlens, device
    )

    if max_seqlen_override is not None:
        max_seqlen_val = max_seqlen_override
    else:
        max_seqlen_val = MMEncoderAttention.compute_max_seqlen(
            self.attn_backend, cu_seqlens
        )
    metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)

    metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
        self.attn_backend,
        cu_seqlens,
        self.hidden_size,
        self.tp_size,
        device,
    )

    return metadata