From e9925640be47cb90c533351f4c12899eb08e7f1a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 01:08:12 +0000 Subject: [PATCH 1/3] Give a specific lexical error for out-of-range integer literals parse_integer returned None on overflow with no distinction from other unparsable input, so a literal like 99999999999999999999 surfaced the generic "Unparsable context" message instead of explaining that it doesn't fit in a signed 64 bit integer. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01DjJMbX5dSjVuZWXBEuZwKS --- src/lexical_analysis/mod.rs | 8 ++++++++ .../invalid_integer_expression/mod.rs | 3 ++- ..._integer_expression_case_4_lexical_analysis_error.snap | 8 ++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 src/tests/lexical_analysis_error/invalid_integer_expression/snapshots/monkey_interpreter__tests__lexical_analysis_error__invalid_integer_expression__test_invalid_integer_expression_case_4_lexical_analysis_error.snap diff --git a/src/lexical_analysis/mod.rs b/src/lexical_analysis/mod.rs index d954ad84..114c9aa3 100644 --- a/src/lexical_analysis/mod.rs +++ b/src/lexical_analysis/mod.rs @@ -31,6 +31,14 @@ impl LexicalAnalysis<'_> { // When it is a valid keyword and identifier, then it is a keyword. (None, Some(keyword), _) => Ok(keyword), (None, None, Some(identifier)) => Ok(identifier), + (None, None, None) + if context.chars().all(|character| character.is_ascii_digit()) => + { + anyhow::bail!( + "The integer literal {:?} does not fit into a signed 64 bit integer.", + context + ) + } (_, _, _) => { anyhow::bail!("Unparsable context for lexical analysis {:?}.", context) } diff --git a/src/tests/lexical_analysis_error/invalid_integer_expression/mod.rs b/src/tests/lexical_analysis_error/invalid_integer_expression/mod.rs index b72874fd..a2a40e3d 100644 --- a/src/tests/lexical_analysis_error/invalid_integer_expression/mod.rs +++ b/src/tests/lexical_analysis_error/invalid_integer_expression/mod.rs @@ -6,7 +6,8 @@ use super::*; snapshot_name, case("51a23", "invalid_integer_expression_case_1"), case(" 123.0", "invalid_integer_expression_case_2"), - case("\t0xFF", "invalid_integer_expression_case_3") + case("\t0xFF", "invalid_integer_expression_case_3"), + case("99999999999999999999", "invalid_integer_expression_case_4") )] fn invalid_integer_expression_cases(code: &str, snapshot_name: &str) {} diff --git a/src/tests/lexical_analysis_error/invalid_integer_expression/snapshots/monkey_interpreter__tests__lexical_analysis_error__invalid_integer_expression__test_invalid_integer_expression_case_4_lexical_analysis_error.snap b/src/tests/lexical_analysis_error/invalid_integer_expression/snapshots/monkey_interpreter__tests__lexical_analysis_error__invalid_integer_expression__test_invalid_integer_expression_case_4_lexical_analysis_error.snap new file mode 100644 index 00000000..7f9e674f --- /dev/null +++ b/src/tests/lexical_analysis_error/invalid_integer_expression/snapshots/monkey_interpreter__tests__lexical_analysis_error__invalid_integer_expression__test_invalid_integer_expression_case_4_lexical_analysis_error.snap @@ -0,0 +1,8 @@ +--- +source: src/tests/lexical_analysis_error/invalid_integer_expression/mod.rs +assertion_line: 19 +expression: error +--- +Err( + "The integer literal \"99999999999999999999\" does not fit into a signed 64 bit integer.", +) From 21d33ac26b6fd3d8e00ef4b48000c850bb1c0b3f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 09:24:30 +0000 Subject: [PATCH 2/3] Detect out-of-range integer literals via IntErrorKind, not a digit guess parse_integer now returns a ParsedInteger enum so the overflow case is known for certain from ParseIntError::kind() rather than inferred by checking whether the unparsable context happens to be all digits. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01DjJMbX5dSjVuZWXBEuZwKS --- src/lexical_analysis/mod.rs | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/lexical_analysis/mod.rs b/src/lexical_analysis/mod.rs index 114c9aa3..443eea1f 100644 --- a/src/lexical_analysis/mod.rs +++ b/src/lexical_analysis/mod.rs @@ -27,18 +27,14 @@ impl LexicalAnalysis<'_> { parse_keyword(context), parse_identifier(context), ) { - (Some(integer), None, None) => Ok(integer), + (ParsedInteger::Valid(integer), None, None) => Ok(integer), + (ParsedInteger::OutOfRange, None, None) => anyhow::bail!( + "The integer literal {:?} does not fit into a signed 64 bit integer.", + context + ), // When it is a valid keyword and identifier, then it is a keyword. - (None, Some(keyword), _) => Ok(keyword), - (None, None, Some(identifier)) => Ok(identifier), - (None, None, None) - if context.chars().all(|character| character.is_ascii_digit()) => - { - anyhow::bail!( - "The integer literal {:?} does not fit into a signed 64 bit integer.", - context - ) - } + (ParsedInteger::NotAnInteger, Some(keyword), _) => Ok(keyword), + (ParsedInteger::NotAnInteger, None, Some(identifier)) => Ok(identifier), (_, _, _) => { anyhow::bail!("Unparsable context for lexical analysis {:?}.", context) } @@ -177,10 +173,21 @@ fn parse_keyword(parsing: &str) -> Option { } } -fn parse_integer(parsing: &str) -> Option { +enum ParsedInteger { + Valid(Token), + OutOfRange, + NotAnInteger, +} + +fn parse_integer(parsing: &str) -> ParsedInteger { match parsing.parse() { - Ok(integer) => Some(Token::Integer { literal: integer }), - Err(_) => None, + Ok(literal) => ParsedInteger::Valid(Token::Integer { literal }), + Err(error) => match error.kind() { + std::num::IntErrorKind::PosOverflow | std::num::IntErrorKind::NegOverflow => { + ParsedInteger::OutOfRange + } + _ => ParsedInteger::NotAnInteger, + }, } } From 297e402cac7522b215e11f6abee6a8a4c5b5ce6d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 10:47:14 +0000 Subject: [PATCH 3/3] Rename integer parse enum and merge keyword/identifier parsing ParsedInteger becomes IntegerParseOutcome for clarity. parse_keyword and parse_identifier are merged into a single parse_identifier that returns IdentifierParseOutcome (Keyword/Identifier/NotAnIdentifier), moving the keyword-takes-precedence rule into that one function instead of the parse_context match tuple. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01DjJMbX5dSjVuZWXBEuZwKS --- src/lexical_analysis/mod.rs | 82 ++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/src/lexical_analysis/mod.rs b/src/lexical_analysis/mod.rs index 443eea1f..11b80e4a 100644 --- a/src/lexical_analysis/mod.rs +++ b/src/lexical_analysis/mod.rs @@ -22,20 +22,24 @@ impl LexicalAnalysis<'_> { fn parse_source_code(&mut self) -> anyhow::Result> { fn parse_context(context: &str) -> anyhow::Result { - match ( - parse_integer(context), - parse_keyword(context), - parse_identifier(context), - ) { - (ParsedInteger::Valid(integer), None, None) => Ok(integer), - (ParsedInteger::OutOfRange, None, None) => anyhow::bail!( - "The integer literal {:?} does not fit into a signed 64 bit integer.", - context - ), - // When it is a valid keyword and identifier, then it is a keyword. - (ParsedInteger::NotAnInteger, Some(keyword), _) => Ok(keyword), - (ParsedInteger::NotAnInteger, None, Some(identifier)) => Ok(identifier), - (_, _, _) => { + match (parse_integer(context), parse_identifier(context)) { + (IntegerParseOutcome::Valid(integer), IdentifierParseOutcome::NotAnIdentifier) => { + Ok(integer) + } + (IntegerParseOutcome::OutOfRange, IdentifierParseOutcome::NotAnIdentifier) => { + anyhow::bail!( + "The integer literal {:?} does not fit into a signed 64 bit integer.", + context + ) + } + (IntegerParseOutcome::NotAnInteger, IdentifierParseOutcome::Keyword(keyword)) => { + Ok(keyword) + } + ( + IntegerParseOutcome::NotAnInteger, + IdentifierParseOutcome::Identifier(identifier), + ) => Ok(identifier), + (_, _) => { anyhow::bail!("Unparsable context for lexical analysis {:?}.", context) } } @@ -160,44 +164,46 @@ impl LexicalAnalysis<'_> { } } -fn parse_keyword(parsing: &str) -> Option { - match parsing.to_lowercase().as_str() { - "fn" => Some(Token::Function), - "let" => Some(Token::Let), - "true" => Some(Token::True), - "false" => Some(Token::False), - "if" => Some(Token::If), - "else" => Some(Token::Else), - "return" => Some(Token::Return), - _ => None, - } -} - -enum ParsedInteger { +enum IntegerParseOutcome { Valid(Token), OutOfRange, NotAnInteger, } -fn parse_integer(parsing: &str) -> ParsedInteger { +fn parse_integer(parsing: &str) -> IntegerParseOutcome { match parsing.parse() { - Ok(literal) => ParsedInteger::Valid(Token::Integer { literal }), + Ok(literal) => IntegerParseOutcome::Valid(Token::Integer { literal }), Err(error) => match error.kind() { std::num::IntErrorKind::PosOverflow | std::num::IntErrorKind::NegOverflow => { - ParsedInteger::OutOfRange + IntegerParseOutcome::OutOfRange } - _ => ParsedInteger::NotAnInteger, + _ => IntegerParseOutcome::NotAnInteger, }, } } -fn parse_identifier(parsing: &str) -> Option { - if is_valid_identifier(parsing) { - Some(Token::Identifier { +enum IdentifierParseOutcome { + Keyword(Token), + Identifier(Token), + NotAnIdentifier, +} + +fn parse_identifier(parsing: &str) -> IdentifierParseOutcome { + if !is_valid_identifier(parsing) { + return IdentifierParseOutcome::NotAnIdentifier; + } + + match parsing.to_lowercase().as_str() { + "fn" => IdentifierParseOutcome::Keyword(Token::Function), + "let" => IdentifierParseOutcome::Keyword(Token::Let), + "true" => IdentifierParseOutcome::Keyword(Token::True), + "false" => IdentifierParseOutcome::Keyword(Token::False), + "if" => IdentifierParseOutcome::Keyword(Token::If), + "else" => IdentifierParseOutcome::Keyword(Token::Else), + "return" => IdentifierParseOutcome::Keyword(Token::Return), + _ => IdentifierParseOutcome::Identifier(Token::Identifier { literal: parsing.to_string(), - }) - } else { - None + }), } }