Skip to content

Commit

Permalink
fix assertion in scheduler
Browse files Browse the repository at this point in the history
  • Loading branch information
AlpinDale committed Nov 25, 2024
1 parent 5092183 commit 2242a56
Showing 1 changed file with 12 additions and 2 deletions.
14 changes: 12 additions & 2 deletions aphrodite/processing/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1341,9 +1341,19 @@ def _get_num_new_tokens(self, seq_group: SequenceGroup,
"""
num_new_tokens = 0
seqs = seq_group.get_seqs(status=status)

if not seqs:
return 0

for seq in seqs:
num_new_tokens += seq.get_num_new_tokens()
assert num_new_tokens > 0
new_tokens = seq.get_num_new_tokens()
if new_tokens == 0 and seq.status == SequenceStatus.WAITING:
continue
num_new_tokens += new_tokens

if num_new_tokens == 0:
return 0

# Chunk if a running request cannot fit in.
# If number of seq > 1, it means it is doing beam search in a
# decode phase. Do not chunk in that case.
Expand Down

0 comments on commit 2242a56

Please sign in to comment.