diff --git a/crates/pgt_lexer/src/lib.rs b/crates/pgt_lexer/src/lib.rs index 32bbdd42..44fbca94 100644 --- a/crates/pgt_lexer/src/lib.rs +++ b/crates/pgt_lexer/src/lib.rs @@ -66,7 +66,8 @@ static PATTERN_LEXER: LazyLock = LazyLock::new(|| { #[cfg(windows)] { // On Windows, treat \r\n as a single newline token - Regex::new(r"(?P +)|(?P(\r\n|\n)+)|(?P\t+)").unwrap() + // and treat \r as a whitespace token + Regex::new(r"(?P +)|(?P(\r\n|\n|\r)+)|(?P\t+)").unwrap() } #[cfg(not(windows))] { @@ -206,6 +207,15 @@ mod tests { assert_eq!(tokens[1].kind, SyntaxKind::Tab); } + #[test] + #[cfg(windows)] + fn test_carriage_return() { + let input = "select\r\n\r1"; + let tokens = lex(input).unwrap(); + assert_eq!(tokens[1].kind, SyntaxKind::Newline); + assert_eq!(tokens[2].kind, SyntaxKind::Whitespace); + } + #[test] fn test_newline_tokens() { let input = "select\n1"; @@ -217,7 +227,7 @@ mod tests { fn test_consecutive_newlines() { // Test with multiple consecutive newlines #[cfg(windows)] - let input = "select\r\n\r\n1"; + let input = "select\r\n\r\n\r1"; #[cfg(not(windows))] let input = "select\n\n1";