fix: harden lexer and parser error handling

TODO.md

@@ -10,12 +10,13 @@ Snapshot: 2026-07-29.
 
 - `cargo fmt --all -- --check` passes and is enforced for pull requests.
 - `cargo check --all-targets` passes.
-- `cargo test` passes and is enforced for pull requests, including all 48 integration specifications and the library doctest.
+- `cargo test` passes and is enforced for pull requests, including all 58 integration specifications and the library doctest.
 - `cargo clippy --all-targets --all-features -- -D warnings` passes and is enforced for pull requests.
 - `mdbook build docs` reproducibly generates ignored documentation output from `docs/src/`.
 - English and Burmese READMEs distinguish the available v0.1.11 interpreter from the planned compiled backend platform.
 - Both shipped examples execute against the v0.1.11 interpreter; the donut runs until interrupted.
 - The crate exposes checked file/source execution APIs and typed I/O, parse, and runtime error categories.
+- Checked parsing reports malformed literals, comments, non-ASCII identifiers, and unexpected EOF without panicking or hanging.
 - Lexer, parser, interpreter, type-inference, examples, mdBook documentation, and cargo-dist release assets exist.
 
 ### Quality gaps
@@ -76,7 +77,7 @@ Snapshot: 2026-07-29.
 - [x] Require `cargo clippy --all-targets --all-features -- -D warnings` in pull-request CI.
 - [x] Separate generated mdBook output from sources and define one reproducible documentation build command.
 - [x] Reconcile README commands, branch names, CI claims, supported features, and examples with executable behavior.
-- [ ] Add focused lexer/parser error tests for malformed strings, comments, UTF-8 input, and unexpected EOF.
+- [x] Add focused lexer/parser error tests for malformed strings, comments, UTF-8 input, and unexpected EOF.
 - [ ] Audit release workflow actions, permissions, secrets, installers, and generated cargo-dist configuration.
 - [ ] Add dependency, license, and supply-chain checks appropriate to Rust and future native runtime dependencies.
 

src/lexer.rs

@@ -96,13 +96,20 @@ impl Lexer {
     }
 
     pub fn advance(&mut self) {
-        self.pos += 1;
+        if self.pos < self.input.len() {
+            self.pos += self.at().len_utf8();
+        }
     }
 
     pub fn peek_next_token(&mut self) -> Token {
+        self.peek_next_token_checked()
+            .unwrap_or_else(|message| panic!("{message}"))
+    }
+
+    pub(crate) fn peek_next_token_checked(&mut self) -> Result<Token, String> {
         let current_pos = self.pos;
 
-        let next_token = self.next_token();
+        let next_token = self.next_token_checked();
         self.pos = current_pos;
 
         next_token
@@ -119,24 +126,33 @@ impl Lexer {
     }
 
     pub fn next_token(&mut self) -> Token {
+        self.next_token_checked()
+            .unwrap_or_else(|message| panic!("{message}"))
+    }
+
+    pub(crate) fn next_token_checked(&mut self) -> Result<Token, String> {
         if self.pos >= self.input.len() {
-            return Token::EOF;
+            return Ok(Token::EOF);
         }
 
         let current_char = self.at();
 
-        match current_char {
+        let token = match current_char {
             '/' => {
                 self.advance();
                 if self.at() == '/' {
                     self.advance();
-                    while self.at() != '/' && self.peek_next_char() != '/' {
+
+                    while self.at() != '\0' {
+                        if self.at() == '/' && self.peek_next_char() == '/' {
+                            self.advance();
+                            self.advance();
+                            return self.next_token_checked();
+                        }
                         self.advance();
                     }
-                    self.advance(); // above while miss 1 char
-                    self.advance(); // first /
-                    self.advance(); // second /
-                    self.next_token()
+
+                    return Err("Unterminated comment".to_string());
                 } else {
                     Token::Divide
                 }
@@ -172,44 +188,70 @@ impl Lexer {
             '\"' => {
                 self.advance();
                 let mut str_val = String::new();
-                while self.at() != '\"' && self.at() != '\0' {
+
+                loop {
+                    if self.at() == '\0' {
+                        return Err("Unterminated string literal".to_string());
+                    }
+
+                    if self.at() == '\"' {
+                        self.advance();
+                        break;
+                    }
+
                     if self.at() == '\\' {
                         self.advance();
-                        match self.at() {
+                        let escaped = self.at();
+                        if escaped == '\0' {
+                            return Err("Unterminated string literal".to_string());
+                        }
+                        match escaped {
                             'n' => str_val.push('\n'),
                             't' => str_val.push('\t'),
                             '\\' => str_val.push('\\'),
                             '\"' => str_val.push('\"'),
-                            _ => panic!("Unknown escape sequence"),
+                            _ => return Err(format!("Unknown escape sequence: \\{escaped}")),
                         }
                     } else {
                         str_val.push(self.at());
                     }
                     self.advance();
                 }
-                self.advance(); // Consume closing quote
                 Token::String(str_val)
             }
             '\'' => {
                 self.advance();
                 let mut char_val: char = ' ';
-                while self.at() != '\'' && self.at() != '\0' {
+
+                loop {
+                    if self.at() == '\0' {
+                        return Err("Unterminated character literal".to_string());
+                    }
+
+                    if self.at() == '\'' {
+                        self.advance();
+                        break;
+                    }
+
                     if self.at() == '\\' {
                         self.advance();
-                        match self.at() {
+                        let escaped = self.at();
+                        if escaped == '\0' {
+                            return Err("Unterminated character literal".to_string());
+                        }
+                        match escaped {
                             'n' => char_val = '\n',
                             't' => char_val = '\t',
                             '\\' => char_val = '\\',
                             '\"' => char_val = '\"',
                             '\'' => char_val = '\'',
-                            _ => panic!("Unknown escape sequence"),
+                            _ => return Err(format!("Unknown escape sequence: \\{escaped}")),
                         }
                     } else {
                         char_val = self.at();
                     }
                     self.advance();
                 }
-                self.advance(); // Consume closing quote
                 Token::Char(char_val)
             }
 
@@ -315,7 +357,7 @@ impl Lexer {
             'a'..='z' | 'A'..='Z' | '_' => {
                 let id_str: String = self.input[self.pos..]
                     .chars()
-                    .take_while(|c| c.is_alphanumeric() || *c == '_')
+                    .take_while(|c| c.is_ascii_alphanumeric() || *c == '_')
                     .collect();
                 self.pos += id_str.len();
 
@@ -363,9 +405,11 @@ impl Lexer {
 
             _ if current_char.is_whitespace() => {
                 self.advance();
-                self.next_token() // Skip whitespace and get the next token
+                return self.next_token_checked(); // Skip whitespace and get the next token
             }
-            _ => panic!("Unexpected character: {}", current_char),
-        }
+            _ => return Err(format!("Unexpected character: {current_char}")),
+        };
+
+        Ok(token)
     }
 }

src/parser.rs

@@ -10,6 +10,7 @@ use crate::visitor::ScopedSymbolTable;
 pub struct Parser {
     lexer: Lexer,
     current_token: Token,
+    lexer_error: Option<LangError>,
     defined_types: HashMap<String, Type>,
     current_struct: Option<String>,
 }
@@ -19,6 +20,7 @@ impl Parser {
         let mut parser = Parser {
             lexer,
             current_token: Token::EOF,
+            lexer_error: None,
             defined_types: HashMap::new(),
             current_struct: None,
         };
@@ -28,11 +30,28 @@ impl Parser {
 
     // Advance the lexer and update the current token
     fn advance(&mut self) {
-        self.current_token = self.lexer.next_token();
+        if self.lexer_error.is_some() {
+            self.current_token = Token::EOF;
+            return;
+        }
+
+        match self.lexer.next_token_checked() {
+            Ok(token) => self.current_token = token,
+            Err(message) => {
+                self.current_token = Token::EOF;
+                self.lexer_error = Some(LangError::parse(message));
+            }
+        }
     }
 
     fn peek_token(&mut self) -> Token {
-        self.lexer.peek_next_token()
+        match self.lexer.peek_next_token_checked() {
+            Ok(token) => token,
+            Err(message) => {
+                self.lexer_error = Some(LangError::parse(message));
+                Token::EOF
+            }
+        }
     }
 
     // Check the current token and advance if it matches the expected one
@@ -52,6 +71,11 @@ impl Parser {
         if self.current_token == expected {
             self.advance();
             Ok(())
+        } else if self.current_token == Token::EOF {
+            Err(LangError::parse(format!(
+                "Unexpected end of input, expected: {:?}",
+                expected
+            )))
         } else {
             Err(LangError::parse(format!(
                 "Unexpected token: {:?}, expected: {:?}",
@@ -187,6 +211,9 @@ impl Parser {
                 Ok(Expr::UnaryOp(Token::Minus, Box::new(expr)))
             }
 
+            Token::EOF => Err(LangError::parse(
+                "Unexpected end of input while parsing expression".to_string(),
+            )),
             _ => Err(LangError::parse(format!(
                 "Unexpected token in primary expression: {:?}",
                 self.current_token
@@ -949,15 +976,28 @@ impl Parser {
         while self.current_token != Token::EOF {
             statements.push(self.parse_statement());
         }
+
+        if let Some(error) = self.lexer_error.take() {
+            panic!("{}", error.message);
+        }
+
         statements
     }
 
     pub fn parse_checked(&mut self) -> Result<Vec<Stmt>, LangError> {
-        let mut statements = Vec::new();
-        while self.current_token != Token::EOF {
-            let statement = self.parse_statement_checked()?;
-            statements.push(statement);
+        let parse_result = (|| {
+            let mut statements = Vec::new();
+            while self.current_token != Token::EOF {
+                let statement = self.parse_statement_checked()?;
+                statements.push(statement);
+            }
+            Ok(statements)
+        })();
+
+        if let Some(error) = self.lexer_error.take() {
+            Err(error)
+        } else {
+            parse_result
         }
-        Ok(statements)
     }
 }