From 27284a37efa3fb1a385e3cca0b80204e076a4274 Mon Sep 17 00:00:00 2001 From: shayyz-code Date: Wed, 29 Jul 2026 10:42:04 +0630 Subject: [PATCH] fix: harden lexer and parser error handling --- TODO.md | 5 ++- src/lexer.rs | 86 ++++++++++++++++++++++++++++--------- src/parser.rs | 54 ++++++++++++++++++++--- tests/language_specs.rs | 94 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 209 insertions(+), 30 deletions(-) diff --git a/TODO.md b/TODO.md index 0c7010e..a0bd712 100644 --- a/TODO.md +++ b/TODO.md @@ -10,12 +10,13 @@ Snapshot: 2026-07-29. - `cargo fmt --all -- --check` passes and is enforced for pull requests. - `cargo check --all-targets` passes. -- `cargo test` passes and is enforced for pull requests, including all 48 integration specifications and the library doctest. +- `cargo test` passes and is enforced for pull requests, including all 58 integration specifications and the library doctest. - `cargo clippy --all-targets --all-features -- -D warnings` passes and is enforced for pull requests. - `mdbook build docs` reproducibly generates ignored documentation output from `docs/src/`. - English and Burmese READMEs distinguish the available v0.1.11 interpreter from the planned compiled backend platform. - Both shipped examples execute against the v0.1.11 interpreter; the donut runs until interrupted. - The crate exposes checked file/source execution APIs and typed I/O, parse, and runtime error categories. +- Checked parsing reports malformed literals, comments, non-ASCII identifiers, and unexpected EOF without panicking or hanging. - Lexer, parser, interpreter, type-inference, examples, mdBook documentation, and cargo-dist release assets exist. ### Quality gaps @@ -76,7 +77,7 @@ Snapshot: 2026-07-29. - [x] Require `cargo clippy --all-targets --all-features -- -D warnings` in pull-request CI. - [x] Separate generated mdBook output from sources and define one reproducible documentation build command. - [x] Reconcile README commands, branch names, CI claims, supported features, and examples with executable behavior. -- [ ] Add focused lexer/parser error tests for malformed strings, comments, UTF-8 input, and unexpected EOF. +- [x] Add focused lexer/parser error tests for malformed strings, comments, UTF-8 input, and unexpected EOF. - [ ] Audit release workflow actions, permissions, secrets, installers, and generated cargo-dist configuration. - [ ] Add dependency, license, and supply-chain checks appropriate to Rust and future native runtime dependencies. diff --git a/src/lexer.rs b/src/lexer.rs index 38ff670..56e6c4e 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -96,13 +96,20 @@ impl Lexer { } pub fn advance(&mut self) { - self.pos += 1; + if self.pos < self.input.len() { + self.pos += self.at().len_utf8(); + } } pub fn peek_next_token(&mut self) -> Token { + self.peek_next_token_checked() + .unwrap_or_else(|message| panic!("{message}")) + } + + pub(crate) fn peek_next_token_checked(&mut self) -> Result { let current_pos = self.pos; - let next_token = self.next_token(); + let next_token = self.next_token_checked(); self.pos = current_pos; next_token @@ -119,24 +126,33 @@ impl Lexer { } pub fn next_token(&mut self) -> Token { + self.next_token_checked() + .unwrap_or_else(|message| panic!("{message}")) + } + + pub(crate) fn next_token_checked(&mut self) -> Result { if self.pos >= self.input.len() { - return Token::EOF; + return Ok(Token::EOF); } let current_char = self.at(); - match current_char { + let token = match current_char { '/' => { self.advance(); if self.at() == '/' { self.advance(); - while self.at() != '/' && self.peek_next_char() != '/' { + + while self.at() != '\0' { + if self.at() == '/' && self.peek_next_char() == '/' { + self.advance(); + self.advance(); + return self.next_token_checked(); + } self.advance(); } - self.advance(); // above while miss 1 char - self.advance(); // first / - self.advance(); // second / - self.next_token() + + return Err("Unterminated comment".to_string()); } else { Token::Divide } @@ -172,44 +188,70 @@ impl Lexer { '\"' => { self.advance(); let mut str_val = String::new(); - while self.at() != '\"' && self.at() != '\0' { + + loop { + if self.at() == '\0' { + return Err("Unterminated string literal".to_string()); + } + + if self.at() == '\"' { + self.advance(); + break; + } + if self.at() == '\\' { self.advance(); - match self.at() { + let escaped = self.at(); + if escaped == '\0' { + return Err("Unterminated string literal".to_string()); + } + match escaped { 'n' => str_val.push('\n'), 't' => str_val.push('\t'), '\\' => str_val.push('\\'), '\"' => str_val.push('\"'), - _ => panic!("Unknown escape sequence"), + _ => return Err(format!("Unknown escape sequence: \\{escaped}")), } } else { str_val.push(self.at()); } self.advance(); } - self.advance(); // Consume closing quote Token::String(str_val) } '\'' => { self.advance(); let mut char_val: char = ' '; - while self.at() != '\'' && self.at() != '\0' { + + loop { + if self.at() == '\0' { + return Err("Unterminated character literal".to_string()); + } + + if self.at() == '\'' { + self.advance(); + break; + } + if self.at() == '\\' { self.advance(); - match self.at() { + let escaped = self.at(); + if escaped == '\0' { + return Err("Unterminated character literal".to_string()); + } + match escaped { 'n' => char_val = '\n', 't' => char_val = '\t', '\\' => char_val = '\\', '\"' => char_val = '\"', '\'' => char_val = '\'', - _ => panic!("Unknown escape sequence"), + _ => return Err(format!("Unknown escape sequence: \\{escaped}")), } } else { char_val = self.at(); } self.advance(); } - self.advance(); // Consume closing quote Token::Char(char_val) } @@ -315,7 +357,7 @@ impl Lexer { 'a'..='z' | 'A'..='Z' | '_' => { let id_str: String = self.input[self.pos..] .chars() - .take_while(|c| c.is_alphanumeric() || *c == '_') + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_') .collect(); self.pos += id_str.len(); @@ -363,9 +405,11 @@ impl Lexer { _ if current_char.is_whitespace() => { self.advance(); - self.next_token() // Skip whitespace and get the next token + return self.next_token_checked(); // Skip whitespace and get the next token } - _ => panic!("Unexpected character: {}", current_char), - } + _ => return Err(format!("Unexpected character: {current_char}")), + }; + + Ok(token) } } diff --git a/src/parser.rs b/src/parser.rs index c1321f3..a4e9b52 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -10,6 +10,7 @@ use crate::visitor::ScopedSymbolTable; pub struct Parser { lexer: Lexer, current_token: Token, + lexer_error: Option, defined_types: HashMap, current_struct: Option, } @@ -19,6 +20,7 @@ impl Parser { let mut parser = Parser { lexer, current_token: Token::EOF, + lexer_error: None, defined_types: HashMap::new(), current_struct: None, }; @@ -28,11 +30,28 @@ impl Parser { // Advance the lexer and update the current token fn advance(&mut self) { - self.current_token = self.lexer.next_token(); + if self.lexer_error.is_some() { + self.current_token = Token::EOF; + return; + } + + match self.lexer.next_token_checked() { + Ok(token) => self.current_token = token, + Err(message) => { + self.current_token = Token::EOF; + self.lexer_error = Some(LangError::parse(message)); + } + } } fn peek_token(&mut self) -> Token { - self.lexer.peek_next_token() + match self.lexer.peek_next_token_checked() { + Ok(token) => token, + Err(message) => { + self.lexer_error = Some(LangError::parse(message)); + Token::EOF + } + } } // Check the current token and advance if it matches the expected one @@ -52,6 +71,11 @@ impl Parser { if self.current_token == expected { self.advance(); Ok(()) + } else if self.current_token == Token::EOF { + Err(LangError::parse(format!( + "Unexpected end of input, expected: {:?}", + expected + ))) } else { Err(LangError::parse(format!( "Unexpected token: {:?}, expected: {:?}", @@ -187,6 +211,9 @@ impl Parser { Ok(Expr::UnaryOp(Token::Minus, Box::new(expr))) } + Token::EOF => Err(LangError::parse( + "Unexpected end of input while parsing expression".to_string(), + )), _ => Err(LangError::parse(format!( "Unexpected token in primary expression: {:?}", self.current_token @@ -949,15 +976,28 @@ impl Parser { while self.current_token != Token::EOF { statements.push(self.parse_statement()); } + + if let Some(error) = self.lexer_error.take() { + panic!("{}", error.message); + } + statements } pub fn parse_checked(&mut self) -> Result, LangError> { - let mut statements = Vec::new(); - while self.current_token != Token::EOF { - let statement = self.parse_statement_checked()?; - statements.push(statement); + let parse_result = (|| { + let mut statements = Vec::new(); + while self.current_token != Token::EOF { + let statement = self.parse_statement_checked()?; + statements.push(statement); + } + Ok(statements) + })(); + + if let Some(error) = self.lexer_error.take() { + Err(error) + } else { + parse_result } - Ok(statements) } } diff --git a/tests/language_specs.rs b/tests/language_specs.rs index e07862c..89daece 100644 --- a/tests/language_specs.rs +++ b/tests/language_specs.rs @@ -34,6 +34,17 @@ fn run_checked_with_temp_file(label: &str, source: &str) -> Result result } +fn assert_parse_error(source: &str, expected_message: &str) { + let error = run_source_checked(source.to_string()).expect_err("expected parse error"); + assert_eq!(error.kind, LangErrorKind::Parse); + assert!( + error.message.contains(expected_message), + "expected error message {:?} to contain {:?}", + error.message, + expected_message + ); +} + #[test] fn spec_value_display_preserves_legacy_output() { assert_eq!(Value::Int(10).to_string(), "10"); @@ -105,6 +116,89 @@ fn spec_lexer_skips_inline_comment_block() { ); } +#[test] +fn spec_lexer_reads_utf8_string_literal() { + let mut lexer = Lexer::new(r#""မင်္ဂလာပါ""#.to_string()); + + assert_eq!(lexer.next_token(), Token::String("မင်္ဂလာပါ".to_string())); + assert_eq!(lexer.next_token(), Token::EOF); +} + +#[test] +fn spec_lexer_reads_utf8_character_literal() { + let mut lexer = Lexer::new("'ပ'".to_string()); + + assert_eq!(lexer.next_token(), Token::Char('ပ')); + assert_eq!(lexer.next_token(), Token::EOF); +} + +#[test] +fn spec_lexer_skips_utf8_comment_containing_single_slash() { + let mut lexer = Lexer::new("poo x <: 1; // မှတ်ချက် / slash ပါသည် // return x;".to_string()); + + let mut tokens = Vec::new(); + loop { + let token = lexer.next_token(); + tokens.push(token.clone()); + if token == Token::EOF { + break; + } + } + + assert_eq!( + tokens, + vec![ + Token::Poo, + Token::Identifier("x".to_string()), + Token::ShortAssignment, + Token::Int(1), + Token::SemiColon, + Token::Return, + Token::Identifier("x".to_string()), + Token::SemiColon, + Token::EOF, + ] + ); +} + +#[test] +fn spec_checked_api_reports_parse_error_for_unterminated_string() { + assert_parse_error(r#"return "unfinished;"#, "Unterminated string literal"); +} + +#[test] +fn spec_checked_api_reports_parse_error_for_unknown_string_escape() { + assert_parse_error(r#"return "\q";"#, "Unknown escape sequence: \\q"); +} + +#[test] +fn spec_checked_api_reports_parse_error_for_unterminated_character_literal() { + assert_parse_error("return 'ပ;", "Unterminated character literal"); +} + +#[test] +fn spec_checked_api_reports_parse_error_for_unterminated_comment() { + assert_parse_error( + "poo x <: 1; // comment never closes", + "Unterminated comment", + ); +} + +#[test] +fn spec_checked_api_rejects_non_ascii_identifier() { + assert_parse_error("poo နာမည် <: 1;", "Unexpected character: န"); +} + +#[test] +fn spec_checked_api_reports_unexpected_eof_in_parenthesized_expression() { + assert_parse_error("return (1 + 2", "Unexpected end of input"); +} + +#[test] +fn spec_checked_api_reports_unexpected_eof_before_closing_block() { + assert_parse_error("if true { return 1;", "Unexpected end of input"); +} + #[test] fn spec_parser_respects_multiplication_precedence() { let lexer = Lexer::new("poo result <: 1 + 2 * 3;".to_string());