From 1bccb70c974dde34ec1cabf94d90484d7e0af54c Mon Sep 17 00:00:00 2001 From: Araozu Date: Sun, 17 Sep 2023 13:58:45 -0500 Subject: [PATCH] Emit DEDENT for each indentation level remaining at EOF --- src/lexic/mod.rs | 85 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/src/lexic/mod.rs b/src/lexic/mod.rs index eae7c9e..64e10cf 100755 --- a/src/lexic/mod.rs +++ b/src/lexic/mod.rs @@ -70,10 +70,13 @@ pub fn get_tokens(input: &String) -> Result, MistiError> { } } - results.push(Token::new_semicolon(0)); - - // TODO: emit DEDENT tokens for each entry in indentation_stack + // emit DEDENT tokens for each entry left in the indentation_stack, + // except the first one (which is 0) + for _ in 0..indentation_stack.len() - 1 { + results.push(Token::new_dedent(current_pos)); + } + // Push EOF results.push(Token::new_eof(0)); Ok(results) } @@ -89,8 +92,7 @@ fn next_token( if at_new_line { return handle_indentation(chars, current_pos, indentation_stack); - } - else if !at_new_line && peek(chars, current_pos) == ' ' { + } else if !at_new_line && peek(chars, current_pos) == ' ' { // Consume whitespace current_pos += 1; while peek(chars, current_pos) == ' ' { @@ -140,8 +142,6 @@ fn handle_indentation( sub_pos += 1; } - // TODO: should emit a DEDENT for every single entry decreased in the stack - // Compare the number of spaces with the top of the stack let top = indentation_stack.last().unwrap_or(&0); @@ -157,28 +157,22 @@ fn handle_indentation( if spaces < *new_top { indentation_stack.pop(); dedent_tokens.push(Token::new_dedent(current_pos)); - } - else if spaces == *new_top { + } else if spaces == *new_top { break; - } - else { + } else { // Illegal state: Indentation error let error = LexError { position: current_pos, reason: format!( "Indentation error: expected {} spaces, found {}", - new_top, - spaces + new_top, spaces ), }; return LexResult::Err(error); } } - return LexResult::Multiple( - dedent_tokens, - current_pos + spaces - ); + return LexResult::Multiple(dedent_tokens, current_pos + spaces); } else { // Same indentation level return next_token(chars, current_pos + spaces, indentation_stack, false); @@ -206,23 +200,23 @@ mod tests { fn should_emit_eof() { let input = String::from(""); let tokens = get_tokens(&input).unwrap(); - // 1 semicolon and 1 EOF token - assert_eq!(2, tokens.len()); - let first = tokens.get(1).unwrap(); + // a EOF token + assert_eq!(1, tokens.len()); + let first = tokens.get(0).unwrap(); assert_eq!(TokenType::EOF, first.token_type); let input = String::from(" "); let tokens = get_tokens(&input).unwrap(); - // 1 semicolon and 1 EOF token - assert_eq!(2, tokens.len()); - let first = tokens.get(1).unwrap(); + // a EOF token + assert_eq!(1, tokens.len()); + let first = tokens.get(0).unwrap(); assert_eq!(TokenType::EOF, first.token_type); let input = String::from(" "); let tokens = get_tokens(&input).unwrap(); - // 1 semicolon and 1 EOF token - assert_eq!(2, tokens.len()); - let first = tokens.get(1).unwrap(); + // a EOF token + assert_eq!(1, tokens.len()); + let first = tokens.get(0).unwrap(); assert_eq!(TokenType::EOF, first.token_type); } @@ -265,8 +259,7 @@ mod tests { assert_eq!("1789e+1", tokens.get(3).unwrap().value); assert_eq!("239.3298e-103", tokens.get(4).unwrap().value); - assert_eq!(TokenType::NewLine, tokens.get(5).unwrap().token_type); - assert_eq!(TokenType::EOF, tokens.get(6).unwrap().token_type); + assert_eq!(TokenType::EOF, tokens.get(5).unwrap().token_type); } #[test] @@ -421,3 +414,39 @@ mod tests { assert_eq!(TokenType::DEDENT, tokens[9].token_type); } } + +#[cfg(test)] +mod indentation_tests { + use super::*; + use token::TokenType; + + #[test] + fn should_emit_dedents_on_eof() { + let input = String::from("1\n 2"); + let tokens = get_tokens(&input).unwrap(); + + assert_eq!(TokenType::Number, tokens[0].token_type); + assert_eq!(TokenType::NewLine, tokens[1].token_type); + assert_eq!(TokenType::INDENT, tokens[2].token_type); + assert_eq!(TokenType::Number, tokens[3].token_type); + assert_eq!(TokenType::DEDENT, tokens[4].token_type); + assert_eq!(TokenType::EOF, tokens[5].token_type); + } + + #[test] + fn should_emit_dedents_on_eof_2() { + let input = String::from("1\n 2\n 3"); + let tokens = get_tokens(&input).unwrap(); + + assert_eq!(TokenType::Number, tokens[0].token_type); + assert_eq!(TokenType::NewLine, tokens[1].token_type); + assert_eq!(TokenType::INDENT, tokens[2].token_type); + assert_eq!(TokenType::Number, tokens[3].token_type); + assert_eq!(TokenType::NewLine, tokens[4].token_type); + assert_eq!(TokenType::INDENT, tokens[5].token_type); + assert_eq!(TokenType::Number, tokens[6].token_type); + assert_eq!(TokenType::DEDENT, tokens[7].token_type); + assert_eq!(TokenType::DEDENT, tokens[8].token_type); + assert_eq!(TokenType::EOF, tokens[9].token_type); + } +}