|
@@ -49,8 +49,8 @@ class SchedulingBudget:
|
|
|
"""
|
|
|
token_budget: int
|
|
|
max_num_seqs: int
|
|
|
- _requeset_ids_num_batched_tokens: Set[str] = field(default_factory=set)
|
|
|
- _requeset_ids_num_curr_seqs: Set[str] = field(default_factory=set)
|
|
|
+ _request_ids_num_batched_tokens: Set[str] = field(default_factory=set)
|
|
|
+ _request_ids_num_curr_seqs: Set[str] = field(default_factory=set)
|
|
|
_num_batched_tokens: int = 0
|
|
|
_num_curr_seqs: int = 0
|
|
|
|
|
@@ -64,28 +64,28 @@ class SchedulingBudget:
|
|
|
return self.token_budget - self.num_batched_tokens
|
|
|
|
|
|
def add_num_batched_tokens(self, req_id: str, num_batched_tokens: int):
|
|
|
- if req_id in self._requeset_ids_num_batched_tokens:
|
|
|
+ if req_id in self._request_ids_num_batched_tokens:
|
|
|
return
|
|
|
|
|
|
- self._requeset_ids_num_batched_tokens.add(req_id)
|
|
|
+ self._request_ids_num_batched_tokens.add(req_id)
|
|
|
self._num_batched_tokens += num_batched_tokens
|
|
|
|
|
|
def subtract_num_batched_tokens(self, req_id: str,
|
|
|
num_batched_tokens: int):
|
|
|
- if req_id in self._requeset_ids_num_batched_tokens:
|
|
|
- self._requeset_ids_num_batched_tokens.remove(req_id)
|
|
|
+ if req_id in self._request_ids_num_batched_tokens:
|
|
|
+ self._request_ids_num_batched_tokens.remove(req_id)
|
|
|
self._num_batched_tokens -= num_batched_tokens
|
|
|
|
|
|
def add_num_seqs(self, req_id: str, num_curr_seqs: int):
|
|
|
- if req_id in self._requeset_ids_num_curr_seqs:
|
|
|
+ if req_id in self._request_ids_num_curr_seqs:
|
|
|
return
|
|
|
|
|
|
- self._requeset_ids_num_curr_seqs.add(req_id)
|
|
|
+ self._request_ids_num_curr_seqs.add(req_id)
|
|
|
self._num_curr_seqs += num_curr_seqs
|
|
|
|
|
|
def subtract_num_seqs(self, req_id: str, num_curr_seqs: int):
|
|
|
- if req_id in self._requeset_ids_num_curr_seqs:
|
|
|
- self._requeset_ids_num_curr_seqs.remove(req_id)
|
|
|
+ if req_id in self._request_ids_num_curr_seqs:
|
|
|
+ self._request_ids_num_curr_seqs.remove(req_id)
|
|
|
self._num_curr_seqs -= num_curr_seqs
|
|
|
|
|
|
@property
|