Skip to content

vllm.v1.core.sched.request_queue

Classes:

Functions:

FCFSRequestQueue

Bases: deque[Request], RequestQueue

A first-come-first-served queue that supports deque operations.

Methods:

  • __bool__ –

    Check if queue has any requests.

  • __iter__ –

    Iterate over the queue according to FCFS policy.

  • __len__ –

    Get number of requests in queue.

  • add_request –

    Add a request to the queue according to FCFS policy.

  • peek_request –

    Peek at the next request in the queue without removing it.

  • pop_request –

    Pop a request from the queue according to FCFS policy.

  • prepend_request –

    Prepend a request to the front of the queue.

  • prepend_requests –

    Prepend all requests from another queue to the front of this

  • remove_request –

    Remove a specific request from the queue.

  • remove_requests –

    Remove multiple specific requests from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
class FCFSRequestQueue(deque[Request], RequestQueue):
    """A first-come-first-served queue that supports deque operations."""

    def add_request(self, request: Request) -> None:
        """Add a request to the queue according to FCFS policy."""
        self.append(request)

    def pop_request(self) -> Request:
        """Pop a request from the queue according to FCFS policy."""
        return self.popleft()

    def peek_request(self) -> Request:
        """Peek at the next request in the queue without removing it."""
        if not self:
            raise IndexError("peek from an empty queue")
        return self[0]

    def prepend_request(self, request: Request) -> None:
        """Prepend a request to the front of the queue."""
        self.appendleft(request)

    def prepend_requests(self, requests: Iterable[Request]) -> None:
        """Prepend all requests from another queue to the front of this
        queue.

        Note: The requests will be prepended in reverse order of their
        appearance in the `requests` queue.
        """
        self.extendleft(requests)

    def remove_request(self, request: Request) -> None:
        """Remove a specific request from the queue."""
        self.remove(request)

    def remove_requests(self, requests: Iterable[Request]) -> None:
        """Remove multiple specific requests from the queue."""
        requests_to_remove = set(requests)
        filtered_requests = [req for req in self if req not in requests_to_remove]
        # deque does not support in-place filtering, so we need to clear
        # and extend
        self.clear()
        self.extend(filtered_requests)

    def __bool__(self) -> bool:
        """Check if queue has any requests."""
        return len(self) > 0

    def __len__(self) -> int:
        """Get number of requests in queue."""
        return super().__len__()

    def __iter__(self) -> Iterator[Request]:
        """Iterate over the queue according to FCFS policy."""
        return super().__iter__()

__bool__()

Check if queue has any requests.

Source code in vllm/v1/core/sched/request_queue.py 
118
119
120
def __bool__(self) -> bool:
    """Check if queue has any requests."""
    return len(self) > 0

__iter__()

Iterate over the queue according to FCFS policy.

Source code in vllm/v1/core/sched/request_queue.py 
126
127
128
def __iter__(self) -> Iterator[Request]:
    """Iterate over the queue according to FCFS policy."""
    return super().__iter__()

__len__()

Get number of requests in queue.

Source code in vllm/v1/core/sched/request_queue.py 
122
123
124
def __len__(self) -> int:
    """Get number of requests in queue."""
    return super().__len__()

add_request(request)

Add a request to the queue according to FCFS policy.

Source code in vllm/v1/core/sched/request_queue.py 
78
79
80
def add_request(self, request: Request) -> None:
    """Add a request to the queue according to FCFS policy."""
    self.append(request)

peek_request()

Peek at the next request in the queue without removing it.

Source code in vllm/v1/core/sched/request_queue.py 
86
87
88
89
90
def peek_request(self) -> Request:
    """Peek at the next request in the queue without removing it."""
    if not self:
        raise IndexError("peek from an empty queue")
    return self[0]

pop_request()

Pop a request from the queue according to FCFS policy.

Source code in vllm/v1/core/sched/request_queue.py 
82
83
84
def pop_request(self) -> Request:
    """Pop a request from the queue according to FCFS policy."""
    return self.popleft()

prepend_request(request)

Prepend a request to the front of the queue.

Source code in vllm/v1/core/sched/request_queue.py 
92
93
94
def prepend_request(self, request: Request) -> None:
    """Prepend a request to the front of the queue."""
    self.appendleft(request)

prepend_requests(requests)

Prepend all requests from another queue to the front of this queue.

Note: The requests will be prepended in reverse order of their appearance in the requests queue.

Source code in vllm/v1/core/sched/request_queue.py 
 96
 97
 98
 99
100
101
102
103
def prepend_requests(self, requests: Iterable[Request]) -> None:
    """Prepend all requests from another queue to the front of this
    queue.

    Note: The requests will be prepended in reverse order of their
    appearance in the `requests` queue.
    """
    self.extendleft(requests)

remove_request(request)

Remove a specific request from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
105
106
107
def remove_request(self, request: Request) -> None:
    """Remove a specific request from the queue."""
    self.remove(request)

remove_requests(requests)

Remove multiple specific requests from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
109
110
111
112
113
114
115
116
def remove_requests(self, requests: Iterable[Request]) -> None:
    """Remove multiple specific requests from the queue."""
    requests_to_remove = set(requests)
    filtered_requests = [req for req in self if req not in requests_to_remove]
    # deque does not support in-place filtering, so we need to clear
    # and extend
    self.clear()
    self.extend(filtered_requests)

PriorityRequestQueue

Bases: RequestQueue

A priority queue that supports heap operations.

Respects the ordering defined in the Request class, where requests with a smaller value of priority are processed first. If multiple requests have the same priority, the one with the earlier arrival_time is processed first.

Methods:

  • __bool__ –

    Check if queue has any requests.

  • __iter__ –

    Iterate over the queue according to priority policy.

  • __len__ –

    Get number of requests in queue.

  • add_request –

    Add a request to the queue according to priority policy.

  • peek_request –

    Peek at the next request in the queue without removing it.

  • pop_request –

    Pop a request from the queue according to priority policy.

  • prepend_request –

    Add a request to the queue according to priority policy.

  • prepend_requests –

    Add all requests from another queue according to priority policy.

  • remove_request –

    Remove a specific request from the queue.

  • remove_requests –

    Remove multiple specific requests from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
class PriorityRequestQueue(RequestQueue):
    """A priority queue that supports heap operations.

    Respects the ordering defined in the Request class, where
    requests with a smaller value of `priority` are processed first.
    If multiple requests have the same priority, the one with the earlier
    `arrival_time` is processed first.
    """

    def __init__(self) -> None:
        self._heap: list[Request] = []

    def add_request(self, request: Request) -> None:
        """Add a request to the queue according to priority policy."""
        heapq.heappush(self._heap, request)

    def pop_request(self) -> Request:
        """Pop a request from the queue according to priority policy."""
        if not self._heap:
            raise IndexError("pop from empty heap")
        return heapq.heappop(self._heap)

    def peek_request(self) -> Request:
        """Peek at the next request in the queue without removing it."""
        if not self._heap:
            raise IndexError("peek from empty heap")
        return self._heap[0]

    def prepend_request(self, request: Request) -> None:
        """Add a request to the queue according to priority policy.

        Note: In a priority queue, there is no concept of prepending to the
        front. Requests are ordered by (priority, arrival_time)."""
        self.add_request(request)

    def prepend_requests(self, requests: Iterable[Request]) -> None:
        """Add all requests from another queue according to priority policy.

        Note: In a priority queue, there is no concept of prepending to the
        front. Requests are ordered by (priority, arrival_time)."""
        for request in requests:
            self.add_request(request)

    def remove_request(self, request: Request) -> None:
        """Remove a specific request from the queue."""
        self._heap.remove(request)
        heapq.heapify(self._heap)

    def remove_requests(self, requests: Iterable[Request]) -> None:
        """Remove multiple specific requests from the queue."""
        requests_to_remove = requests if isinstance(requests, set) else set(requests)
        self._heap = [r for r in self._heap if r not in requests_to_remove]
        heapq.heapify(self._heap)

    def __bool__(self) -> bool:
        """Check if queue has any requests."""
        return bool(self._heap)

    def __len__(self) -> int:
        """Get number of requests in queue."""
        return len(self._heap)

    def __iter__(self) -> Iterator[Request]:
        """Iterate over the queue according to priority policy."""
        heap_copy = self._heap[:]
        while heap_copy:
            yield heapq.heappop(heap_copy)

__bool__()

Check if queue has any requests.

Source code in vllm/v1/core/sched/request_queue.py 
185
186
187
def __bool__(self) -> bool:
    """Check if queue has any requests."""
    return bool(self._heap)

__iter__()

Iterate over the queue according to priority policy.

Source code in vllm/v1/core/sched/request_queue.py 
193
194
195
196
197
def __iter__(self) -> Iterator[Request]:
    """Iterate over the queue according to priority policy."""
    heap_copy = self._heap[:]
    while heap_copy:
        yield heapq.heappop(heap_copy)

__len__()

Get number of requests in queue.

Source code in vllm/v1/core/sched/request_queue.py 
189
190
191
def __len__(self) -> int:
    """Get number of requests in queue."""
    return len(self._heap)

add_request(request)

Add a request to the queue according to priority policy.

Source code in vllm/v1/core/sched/request_queue.py 
143
144
145
def add_request(self, request: Request) -> None:
    """Add a request to the queue according to priority policy."""
    heapq.heappush(self._heap, request)

peek_request()

Peek at the next request in the queue without removing it.

Source code in vllm/v1/core/sched/request_queue.py 
153
154
155
156
157
def peek_request(self) -> Request:
    """Peek at the next request in the queue without removing it."""
    if not self._heap:
        raise IndexError("peek from empty heap")
    return self._heap[0]

pop_request()

Pop a request from the queue according to priority policy.

Source code in vllm/v1/core/sched/request_queue.py 
147
148
149
150
151
def pop_request(self) -> Request:
    """Pop a request from the queue according to priority policy."""
    if not self._heap:
        raise IndexError("pop from empty heap")
    return heapq.heappop(self._heap)

prepend_request(request)

Add a request to the queue according to priority policy.

Note: In a priority queue, there is no concept of prepending to the front. Requests are ordered by (priority, arrival_time).

Source code in vllm/v1/core/sched/request_queue.py 
159
160
161
162
163
164
def prepend_request(self, request: Request) -> None:
    """Add a request to the queue according to priority policy.

    Note: In a priority queue, there is no concept of prepending to the
    front. Requests are ordered by (priority, arrival_time)."""
    self.add_request(request)

prepend_requests(requests)

Add all requests from another queue according to priority policy.

Note: In a priority queue, there is no concept of prepending to the front. Requests are ordered by (priority, arrival_time).

Source code in vllm/v1/core/sched/request_queue.py 
166
167
168
169
170
171
172
def prepend_requests(self, requests: Iterable[Request]) -> None:
    """Add all requests from another queue according to priority policy.

    Note: In a priority queue, there is no concept of prepending to the
    front. Requests are ordered by (priority, arrival_time)."""
    for request in requests:
        self.add_request(request)

remove_request(request)

Remove a specific request from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
174
175
176
177
def remove_request(self, request: Request) -> None:
    """Remove a specific request from the queue."""
    self._heap.remove(request)
    heapq.heapify(self._heap)

remove_requests(requests)

Remove multiple specific requests from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
179
180
181
182
183
def remove_requests(self, requests: Iterable[Request]) -> None:
    """Remove multiple specific requests from the queue."""
    requests_to_remove = requests if isinstance(requests, set) else set(requests)
    self._heap = [r for r in self._heap if r not in requests_to_remove]
    heapq.heapify(self._heap)

RequestQueue

Bases: ABC

Abstract base class for request queues.

Methods:

  • __bool__ –

    Check if queue has any requests.

  • __iter__ –

    Iterate over the queue according to the policy.

  • __len__ –

    Get number of requests in queue.

  • add_request –

    Add a request to the queue according to the policy.

  • peek_request –

    Peek at the request at the front of the queue without removing it.

  • pop_request –

    Pop a request from the queue according to the policy.

  • prepend_request –

    Prepend a request to the front of the queue.

  • prepend_requests –

    Prepend all requests from another queue to the front of this

  • remove_request –

    Remove a specific request from the queue.

  • remove_requests –

    Remove multiple specific requests from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
class RequestQueue(ABC):
    """Abstract base class for request queues."""

    @abstractmethod
    def add_request(self, request: Request) -> None:
        """Add a request to the queue according to the policy."""
        pass

    @abstractmethod
    def pop_request(self) -> Request:
        """Pop a request from the queue according to the policy."""
        pass

    @abstractmethod
    def peek_request(self) -> Request:
        """Peek at the request at the front of the queue without removing it."""
        pass

    @abstractmethod
    def prepend_request(self, request: Request) -> None:
        """Prepend a request to the front of the queue."""
        pass

    @abstractmethod
    def prepend_requests(self, requests: Iterable[Request]) -> None:
        """Prepend all requests from another queue to the front of this
        queue."""
        pass

    @abstractmethod
    def remove_request(self, request: Request) -> None:
        """Remove a specific request from the queue."""
        pass

    @abstractmethod
    def remove_requests(self, requests: Iterable[Request]) -> None:
        """Remove multiple specific requests from the queue."""
        pass

    @abstractmethod
    def __bool__(self) -> bool:
        """Check if queue has any requests."""
        pass

    @abstractmethod
    def __len__(self) -> int:
        """Get number of requests in queue."""
        pass

    @abstractmethod
    def __iter__(self) -> Iterator[Request]:
        """Iterate over the queue according to the policy."""
        pass

__bool__() abstractmethod

Check if queue has any requests.

Source code in vllm/v1/core/sched/request_queue.py 
59
60
61
62
@abstractmethod
def __bool__(self) -> bool:
    """Check if queue has any requests."""
    pass

__iter__() abstractmethod

Iterate over the queue according to the policy.

Source code in vllm/v1/core/sched/request_queue.py 
69
70
71
72
@abstractmethod
def __iter__(self) -> Iterator[Request]:
    """Iterate over the queue according to the policy."""
    pass

__len__() abstractmethod

Get number of requests in queue.

Source code in vllm/v1/core/sched/request_queue.py 
64
65
66
67
@abstractmethod
def __len__(self) -> int:
    """Get number of requests in queue."""
    pass

add_request(request) abstractmethod

Add a request to the queue according to the policy.

Source code in vllm/v1/core/sched/request_queue.py 
23
24
25
26
@abstractmethod
def add_request(self, request: Request) -> None:
    """Add a request to the queue according to the policy."""
    pass

peek_request() abstractmethod

Peek at the request at the front of the queue without removing it.

Source code in vllm/v1/core/sched/request_queue.py 
33
34
35
36
@abstractmethod
def peek_request(self) -> Request:
    """Peek at the request at the front of the queue without removing it."""
    pass

pop_request() abstractmethod

Pop a request from the queue according to the policy.

Source code in vllm/v1/core/sched/request_queue.py 
28
29
30
31
@abstractmethod
def pop_request(self) -> Request:
    """Pop a request from the queue according to the policy."""
    pass

prepend_request(request) abstractmethod

Prepend a request to the front of the queue.

Source code in vllm/v1/core/sched/request_queue.py 
38
39
40
41
@abstractmethod
def prepend_request(self, request: Request) -> None:
    """Prepend a request to the front of the queue."""
    pass

prepend_requests(requests) abstractmethod

Prepend all requests from another queue to the front of this queue.

Source code in vllm/v1/core/sched/request_queue.py 
43
44
45
46
47
@abstractmethod
def prepend_requests(self, requests: Iterable[Request]) -> None:
    """Prepend all requests from another queue to the front of this
    queue."""
    pass

remove_request(request) abstractmethod

Remove a specific request from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
49
50
51
52
@abstractmethod
def remove_request(self, request: Request) -> None:
    """Remove a specific request from the queue."""
    pass

remove_requests(requests) abstractmethod

Remove multiple specific requests from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
54
55
56
57
@abstractmethod
def remove_requests(self, requests: Iterable[Request]) -> None:
    """Remove multiple specific requests from the queue."""
    pass

SchedulingPolicy

Bases: Enum

Enum for scheduling policies.

Source code in vllm/v1/core/sched/request_queue.py 
13
14
15
16
17
class SchedulingPolicy(Enum):
    """Enum for scheduling policies."""

    FCFS = "fcfs"
    PRIORITY = "priority"

create_request_queue(policy)

Create request queue based on scheduling policy.

Source code in vllm/v1/core/sched/request_queue.py 
200
201
202
203
204
205
206
207
def create_request_queue(policy: SchedulingPolicy) -> RequestQueue:
    """Create request queue based on scheduling policy."""
    if policy == SchedulingPolicy.PRIORITY:
        return PriorityRequestQueue()
    elif policy == SchedulingPolicy.FCFS:
        return FCFSRequestQueue()
    else:
        raise ValueError(f"Unknown scheduling policy: {policy}")