diff --git a/CHANGELOG.md b/CHANGELOG.md index ca44cba..d498258 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - [x] Begin work on a formal grammar - [x] Simplify/rewrite AST +- [ ] Properly parse expression indentation/dedentation - [x] Define the top level constructs - [ ] Include the original tokens in the AST - [ ] Finish the workflow for a hello world diff --git a/src/syntax/parsers/expression/comparison.rs b/src/syntax/parsers/expression/comparison.rs index 5b4b4e2..a381782 100644 --- a/src/syntax/parsers/expression/comparison.rs +++ b/src/syntax/parsers/expression/comparison.rs @@ -1,4 +1,6 @@ +use crate::lexic::token::TokenType; use crate::{ + handle_dedentation, handle_indentation, lexic::token::Token, syntax::{ast::Expression, ParsingError, ParsingResult}, }; @@ -14,24 +16,36 @@ pub fn try_parse(tokens: &Vec, pos: usize) -> ParsingResult { _ => return Err(ParsingError::Unmatched), }; - parse_many(tokens, next_pos, term) + parse_many(tokens, next_pos, term, 0) } fn parse_many<'a>( tokens: &'a Vec, pos: usize, prev_expr: Expression<'a>, + indentation_level: u32, ) -> ParsingResult<'a, Expression<'a>> { // comparison = term, ((">" | ">=" | "<" | "<="), term)*; - match tokens.get(pos) { + let mut indent_count: u32 = 0; + let mut next_pos = pos; + + // Handle possible indentation before binary operator + handle_indentation!(tokens, next_pos, indent_count, indentation_level); + + let result = match tokens.get(next_pos) { Some(token) if token.value == "<" || token.value == "<=" || token.value == ">" || token.value == ">=" => { - match super::term::try_parse(tokens, pos + 1) { + next_pos += 1; + + // Handle possible indentation after binary operator + handle_indentation!(tokens, next_pos, indent_count, indentation_level); + + match super::term::try_parse(tokens, next_pos) { Ok((expr, next_pos)) => { let expr = Expression::BinaryOperator( Box::new(prev_expr), @@ -39,13 +53,22 @@ fn parse_many<'a>( &token.value, ); - parse_many(tokens, next_pos, expr) + parse_many(tokens, next_pos, expr, indentation_level + indent_count) } - _ => Err(ParsingError::Unmatched), + _ => return Err(ParsingError::Unmatched), } } - _ => Ok((prev_expr, pos)), - } + _ => return Ok((prev_expr, pos)), + }; + + let (new_expr, mut next_pos) = match result { + Ok((e, n)) => (e, n), + _ => return result, + }; + + handle_dedentation!(tokens, next_pos, indent_count); + + Ok((new_expr, next_pos)) } #[cfg(test)] @@ -88,4 +111,80 @@ mod tests { _ => panic!("Expected an Unmatched error"), } } + + #[test] + fn should_parse_indented_1() { + let tokens = get_tokens(&String::from("a\n >= b")).unwrap(); + let (result, next) = try_parse(&tokens, 0).unwrap(); + + assert_eq!(tokens[5].token_type, TokenType::DEDENT); + assert_eq!(next, 6); + + match result { + Expression::BinaryOperator(_, _, op) => { + assert_eq!(op, ">=") + } + _ => panic!("Expected a binary operator"), + } + } + + #[test] + fn should_parse_indented_2() { + let tokens = get_tokens(&String::from("a\n <= b\n <= c")).unwrap(); + let (result, next) = try_parse(&tokens, 0).unwrap(); + assert_eq!(next, 11); + + match result { + Expression::BinaryOperator(_, _, op) => { + assert_eq!(op, "<=") + } + _ => panic!("Expected a binary operator"), + } + } + + #[test] + fn should_parse_indented_3() { + let tokens = get_tokens(&String::from("a\n <= b <= c")).unwrap(); + let (result, next) = try_parse(&tokens, 0).unwrap(); + + assert_eq!(tokens[7].token_type, TokenType::DEDENT); + assert_eq!(next, 8); + + match result { + Expression::BinaryOperator(_, _, op) => { + assert_eq!(op, "<=") + } + _ => panic!("Expected a binary operator"), + } + } + + #[test] + fn should_parse_indented_4() { + let tokens = get_tokens(&String::from("a\n <= b\n <= c")).unwrap(); + let (result, next) = try_parse(&tokens, 0).unwrap(); + + assert_eq!(next, 9); + + match result { + Expression::BinaryOperator(_, _, op) => { + assert_eq!(op, "<=") + } + _ => panic!("Expected a binary operator"), + } + } + + #[test] + fn should_parse_indented_5() { + let tokens = get_tokens(&String::from("a >=\n b")).unwrap(); + let (result, next) = try_parse(&tokens, 0).unwrap(); + + assert_eq!(next, 6); + + match result { + Expression::BinaryOperator(_, _, op) => { + assert_eq!(op, ">=") + } + _ => panic!("Expected a binary operator"), + } + } } diff --git a/src/syntax/parsers/expression/equality.rs b/src/syntax/parsers/expression/equality.rs index f6a6d51..e16bf48 100644 --- a/src/syntax/parsers/expression/equality.rs +++ b/src/syntax/parsers/expression/equality.rs @@ -1,5 +1,7 @@ use crate::{ - handle_dedentation, handle_indentation, lexic::token::{Token, TokenType}, syntax::{ast::Expression, ParsingError, ParsingResult} + handle_dedentation, handle_indentation, + lexic::token::{Token, TokenType}, + syntax::{ast::Expression, ParsingError, ParsingResult}, }; /// Parses a factor expression. @@ -33,7 +35,7 @@ fn parse_many<'a>( let result = match tokens.get(next_pos) { Some(token) if token.value == "==" || token.value == "!=" => { next_pos += 1; - + // Handle possible indentation after binary operator handle_indentation!(tokens, next_pos, indent_count, indentation_level); @@ -52,7 +54,7 @@ fn parse_many<'a>( } _ => return Ok((prev_expr, pos)), }; - + let (new_expr, mut next_pos) = match result { Ok((e, n)) => (e, n), _ => return result, diff --git a/src/syntax/parsers/expression/factor.rs b/src/syntax/parsers/expression/factor.rs index 7e7ef2b..1d452c4 100644 --- a/src/syntax/parsers/expression/factor.rs +++ b/src/syntax/parsers/expression/factor.rs @@ -1,5 +1,7 @@ use crate::{ - handle_dedentation, handle_indentation, lexic::token::{Token, TokenType}, syntax::{ast::Expression, ParsingError, ParsingResult} + handle_dedentation, handle_indentation, + lexic::token::{Token, TokenType}, + syntax::{ast::Expression, ParsingError, ParsingResult}, }; /// Parses a factor expression.