From c4f975be781f39a20f1e07de54ce774246f37da1 Mon Sep 17 00:00:00 2001
From: neuroevolutus <19356702+neuroevolutus@users.noreply.github.com>
Date: Mon, 27 Jan 2025 14:02:43 -0600
Subject: [PATCH] Make the parser drive the lexer

---
 scrapscript.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/scrapscript.py b/scrapscript.py
index b639f020..698b5c81 100755
--- a/scrapscript.py
+++ b/scrapscript.py
@@ -18,7 +18,7 @@
 from dataclasses import dataclass
 from enum import auto
 from types import ModuleType
-from typing import Any, Callable, Dict, Iterator, Mapping, Optional, Set, Tuple, Union
+from typing import Any, Callable, Dict, Generator, Iterator, Mapping, Optional, Set, Tuple, Union
 
 readline: Optional[ModuleType]
 try:
@@ -218,6 +218,10 @@ def make_token(self, cls: type, *args: Any) -> Token:
 
         return result
 
+    def read_tokens(self) -> Generator[Token, None, None]:
+        while (token := self.read_token()) and not isinstance(token, EOF):
+            yield token
+
     def read_token(self) -> Token:
         # Consume all whitespace
         while self.has_input():
@@ -407,10 +411,7 @@ def test_next_on_empty_peekable_raises_stop_iteration(self) -> None:
 
 def tokenize(x: str) -> Peekable:
     lexer = Lexer(x)
-    tokens = []
-    while (token := lexer.read_token()) and not isinstance(token, EOF):
-        tokens.append(token)
-    return Peekable(iter(tokens))
+    return Peekable(lexer.read_tokens())
 
 
 @dataclass(frozen=True)