Skip to content

Commit

Permalink
[Core] Optimize evictor-v2 performance (vllm-project#7193)
Browse files Browse the repository at this point in the history
  • Loading branch information
xiaobochen123 authored Aug 6, 2024
1 parent 8d59dbb commit 660470e
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions vllm/core/evictor_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,9 @@ def evict(self) -> Tuple[int, int]:
# at the start of OrderedDict. Loop through all these blocks to
# find the one with maximum number of hashed tokens.
for _id, block in self.free_table.items():
if evicted_block.last_accessed > block.last_accessed or (
evicted_block.last_accessed == block.last_accessed and
if evicted_block.last_accessed < block.last_accessed:
break
if (evicted_block.last_accessed == block.last_accessed and
evicted_block.num_hashed_tokens < block.num_hashed_tokens):
evicted_block = block
evicted_block_id = _id
Expand All @@ -109,6 +110,7 @@ def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,

def update(self, block_id: int, last_accessed: float):
self.free_table[block_id].last_accessed = last_accessed
self.free_table.move_to_end(block_id)

def remove(self, block_id: int):
if block_id not in self.free_table:
Expand Down

0 comments on commit 660470e

Please sign in to comment.