Fix gibberish token problem for prefill also (#723)

missed a line in #665
nod-ai · Dec 20, 2024 · 2776186 · 2776186
1 parent fc9576b
commit 2776186
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/shortfin/python/shortfin_apps/llm/components/service.py b/shortfin/python/shortfin_apps/llm/components/service.py
@@ -383,7 +383,7 @@ async def run(self):
             if self.phase == InferencePhase.PREFILL:
                 seq_lens_host = seq_lens.for_transfer()
                 with seq_lens_host.map(discard=True) as m:
-                    m.fill(0)
+                    m.fill(1)
                     m.items = [len(req.input_token_ids) for req in self.exec_requests]
                 seq_lens_host.copy_to(seq_lens)