Skip to content

vllm.v1.metrics.ray_wrappers

Classes:

RayCounterWrapper

Bases: RayPrometheusMetric

Wraps around ray.util.metrics.Counter to provide same API as prometheus_client.Counter

Source code in vllm/v1/metrics/ray_wrappers.py 
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
class RayCounterWrapper(RayPrometheusMetric):
    """Wraps around ray.util.metrics.Counter to provide same API as
    prometheus_client.Counter"""

    def __init__(
        self,
        name: str,
        documentation: str | None = "",
        labelnames: list[str] | None = None,
    ):
        super().__init__()
        tag_keys = self._get_tag_keys(labelnames)
        name = self._get_sanitized_opentelemetry_name(name)
        self.metric = ray_metrics.Counter(
            name=name,
            description=documentation,
            tag_keys=tag_keys,
        )

    def inc(self, value: int | float = 1.0):
        if value == 0:
            return
        return self.metric.inc(value, tags=self._tags)

RayGaugeWrapper

Bases: RayPrometheusMetric

Wraps around ray.util.metrics.Gauge to provide same API as prometheus_client.Gauge

Source code in vllm/v1/metrics/ray_wrappers.py 
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
class RayGaugeWrapper(RayPrometheusMetric):
    """Wraps around ray.util.metrics.Gauge to provide same API as
    prometheus_client.Gauge"""

    def __init__(
        self,
        name: str,
        documentation: str | None = "",
        labelnames: list[str] | None = None,
        multiprocess_mode: str | None = "",
    ):
        # All Ray metrics are keyed by WorkerId, so multiprocess modes like
        # "mostrecent", "all", "sum" do not apply. This logic can be manually
        # implemented at the observability layer (Prometheus/Grafana).
        del multiprocess_mode

        super().__init__()
        tag_keys = self._get_tag_keys(labelnames)
        name = self._get_sanitized_opentelemetry_name(name)

        self.metric = ray_metrics.Gauge(
            name=name,
            description=documentation,
            tag_keys=tag_keys,
        )

    def set(self, value: int | float):
        return self.metric.set(value, tags=self._tags)

    def set_to_current_time(self):
        # ray metrics doesn't have set_to_current time, https://docs.ray.io/en/latest/_modules/ray/util/metrics.html
        return self.set(time.time())

RayHistogramWrapper

Bases: RayPrometheusMetric

Wraps around ray.util.metrics.Histogram to provide same API as prometheus_client.Histogram

Source code in vllm/v1/metrics/ray_wrappers.py 
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
class RayHistogramWrapper(RayPrometheusMetric):
    """Wraps around ray.util.metrics.Histogram to provide same API as
    prometheus_client.Histogram"""

    def __init__(
        self,
        name: str,
        documentation: str | None = "",
        labelnames: list[str] | None = None,
        buckets: list[float] | None = None,
    ):
        super().__init__()
        tag_keys = self._get_tag_keys(labelnames)
        name = self._get_sanitized_opentelemetry_name(name)

        boundaries = buckets if buckets else []
        self.metric = ray_metrics.Histogram(
            name=name,
            description=documentation,
            tag_keys=tag_keys,
            boundaries=boundaries,
        )

    def observe(self, value: int | float):
        return self.metric.observe(value, tags=self._tags)

RayKVConnectorProm

Bases: KVConnectorProm

RayKVConnectorProm is used by RayMetrics to log Ray metrics. Provides the same metrics as KV connectors but uses Ray's util.metrics library.

Source code in vllm/v1/metrics/ray_wrappers.py 
178
179
180
181
182
183
184
185
186
class RayKVConnectorProm(KVConnectorProm):
    """RayKVConnectorProm is used by RayMetrics to log Ray
    metrics. Provides the same metrics as KV connectors but
    uses Ray's util.metrics library.
    """

    _gauge_cls = RayGaugeWrapper
    _counter_cls = RayCounterWrapper
    _histogram_cls = RayHistogramWrapper

RayPerfMetricsProm

Bases: PerfMetricsProm

RayPerfMetricsProm is used by RayMetrics to log Ray metrics. Provides the same MFU metrics as PerfMetricsProm uses Ray's util.metrics library.

Source code in vllm/v1/metrics/ray_wrappers.py 
189
190
191
192
193
194
195
class RayPerfMetricsProm(PerfMetricsProm):
    """RayPerfMetricsProm is used by RayMetrics to log Ray
    metrics. Provides the same MFU metrics as PerfMetricsProm
    uses Ray's util.metrics library.
    """

    _counter_cls = RayCounterWrapper

RayPrometheusMetric

Source code in vllm/v1/metrics/ray_wrappers.py 
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
class RayPrometheusMetric:
    _is_labeled: bool = False

    def __init__(self):
        if ray_metrics is None:
            raise ImportError("RayPrometheusMetric requires Ray to be installed.")
        self.metric: Metric = None
        self._tags: dict[str, str] = {"ReplicaId": _get_replica_id() or ""}

    @staticmethod
    def _get_tag_keys(labelnames: list[str] | None) -> tuple[str, ...]:
        labels = list(labelnames) if labelnames else []
        labels.append("ReplicaId")
        return tuple(labels)

    def _build_tags(self, *labels, **labelskwargs) -> dict[str, str]:
        if labels:
            # -1 because ReplicaId was added automatically
            expected = len(self.metric._tag_keys) - 1
            if len(labels) != expected:
                raise ValueError(
                    "Number of labels must match the number of tag keys. "
                    f"Expected {expected}, got {len(labels)}"
                )
            labelskwargs.update(zip(self.metric._tag_keys, labels))

        labelskwargs["ReplicaId"] = _get_replica_id() or ""

        return {k: v if isinstance(v, str) else str(v) for k, v in labelskwargs.items()}

    def labels(self, *labels, **labelskwargs) -> "RayPrometheusMetric":
        if self._is_labeled:
            raise ValueError("labels() cannot be called on an already-labeled metric.")
        clone = copy.copy(self)
        clone._tags = self._build_tags(*labels, **labelskwargs)
        clone._is_labeled = True
        return clone

    @staticmethod
    def _get_sanitized_opentelemetry_name(name: str) -> str:
        """For compatibility with Ray + OpenTelemetry, the metric name must be
        sanitized. In particular, this replaces disallowed character (e.g., ':')
        with '_' in the metric name.
        Allowed characters: a-z, A-Z, 0-9, _

        # ruff: noqa: E501
        Ref: https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/src/metrics/instrument_metadata_validator.cc#L22-L23
        Ref: https://github.com/ray-project/ray/blob/master/src/ray/stats/metric.cc#L107
        """
        return re.sub(r"[^a-zA-Z0-9_]", "_", name)

_get_sanitized_opentelemetry_name(name) staticmethod

For compatibility with Ray + OpenTelemetry, the metric name must be sanitized. In particular, this replaces disallowed character (e.g., ':') with '_' in the metric name. Allowed characters: a-z, A-Z, 0-9, _

ruff: noqa: E501

Ref: https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/src/metrics/instrument_metadata_validator.cc#L22-L23 Ref: https://github.com/ray-project/ray/blob/master/src/ray/stats/metric.cc#L107

Source code in vllm/v1/metrics/ray_wrappers.py 
69
70
71
72
73
74
75
76
77
78
79
80
@staticmethod
def _get_sanitized_opentelemetry_name(name: str) -> str:
    """For compatibility with Ray + OpenTelemetry, the metric name must be
    sanitized. In particular, this replaces disallowed character (e.g., ':')
    with '_' in the metric name.
    Allowed characters: a-z, A-Z, 0-9, _

    # ruff: noqa: E501
    Ref: https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/src/metrics/instrument_metadata_validator.cc#L22-L23
    Ref: https://github.com/ray-project/ray/blob/master/src/ray/stats/metric.cc#L107
    """
    return re.sub(r"[^a-zA-Z0-9_]", "_", name)

RayPrometheusStatLogger

Bases: PrometheusStatLogger

RayPrometheusStatLogger uses Ray metrics instead.

Source code in vllm/v1/metrics/ray_wrappers.py 
198
199
200
201
202
203
204
205
206
207
208
209
210
211
class RayPrometheusStatLogger(PrometheusStatLogger):
    """RayPrometheusStatLogger uses Ray metrics instead."""

    _gauge_cls = RayGaugeWrapper
    _counter_cls = RayCounterWrapper
    _histogram_cls = RayHistogramWrapper
    _spec_decoding_cls = RaySpecDecodingProm
    _kv_connector_cls = RayKVConnectorProm
    _perf_metrics_cls = RayPerfMetricsProm

    @staticmethod
    def _unregister_vllm_metrics():
        # No-op on purpose
        pass

RaySpecDecodingProm

Bases: SpecDecodingProm

RaySpecDecodingProm is used by RayMetrics to log to Ray metrics. Provides the same metrics as SpecDecodingProm but uses Ray's util.metrics library.

Source code in vllm/v1/metrics/ray_wrappers.py 
169
170
171
172
173
174
175
class RaySpecDecodingProm(SpecDecodingProm):
    """RaySpecDecodingProm is used by RayMetrics to log to Ray metrics.
    Provides the same metrics as SpecDecodingProm but uses Ray's
    util.metrics library.
    """

    _counter_cls = RayCounterWrapper

_get_replica_id()

Get the current Ray Serve replica ID, or None if not in a Serve context.

Source code in vllm/v1/metrics/ray_wrappers.py 
21
22
23
24
25
26
27
28
def _get_replica_id() -> str | None:
    """Get the current Ray Serve replica ID, or None if not in a Serve context."""
    if ray_serve is None:
        return None
    try:
        return ray_serve.get_replica_context().replica_id.unique_id
    except ray_serve.exceptions.RayServeException:
        return None