fernando/thp
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Wiki Activity

[Syntax] Refactor binding parsing

Browse Source
This commit is contained in:
Araozu 2023-10-05 20:26:47 -05:00
parent f769a2ec1d
commit ed0e163283
6 changed files with 101 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -16,6 +16,10 @@
- Document code
## v0.0.8
- Parse multiple statements inside a block
## v0.0.7
- Parse minimal function declarations following a grammar

59
src/syntax/binding.rs
View File

@ -1,5 +1,5 @@
use super::ast::var_binding::{Binding, ValBinding, VarBinding};
use super::utils::{try_operator, try_token_type};
use super::utils::{try_operator, try_token_type, parse_token_type};
use super::{expression, ParseResult};
use crate::error_handling::SyntaxError;
use crate::lexic::token::{Token, TokenType};
@ -7,54 +7,36 @@ use crate::utils::Result3;
pub fn try_parse<'a>(tokens: &'a Vec<Token>, pos: usize) -> ParseResult<Binding, ()> {
let mut current_pos = pos;
// Optional datatype annotation
let datatype_annotation = {
match try_token_type(tokens, current_pos, TokenType::Datatype) {
Result3::Ok(t) => {
current_pos += 1;
Some(String::from(&t.value))
}
Result3::Err(_) => None,
Result3::None => return ParseResult::Unmatched,
}
};
/*
* val/var keyword
*/
let (is_val, binding_token) = {
let res1 = try_token_type(tokens, current_pos, TokenType::VAL);
let (is_val, binding_token, next_pos) = {
let res1 = parse_token_type(tokens, current_pos, TokenType::VAL);
match res1 {
Result3::Ok(val_token) => (true, val_token),
ParseResult::Ok(val_token, next) => (true, val_token, next),
_ => {
let res2 = try_token_type(tokens, current_pos, TokenType::VAR);
let res2 = parse_token_type(tokens, current_pos, TokenType::VAR);
match res2 {
Result3::Ok(var_token) => (false, var_token),
// Neither VAL nor VAR were matched, the parser should try
ParseResult::Ok(var_token, next) => (false, var_token, next),
// Neither VAL nor VAR were matched, the caller should try
// other constructs
_ => return ParseResult::Unmatched,
}
}
}
};
current_pos = next_pos;
/*
* identifier
*/
let identifier = match try_token_type(tokens, current_pos + 1, TokenType::Identifier) {
Result3::Ok(t) => t,
Result3::Err(t) => {
let (identifier, next_pos) = match parse_token_type(tokens, current_pos, TokenType::Identifier) {
ParseResult::Ok(t, n) => (t, n),
ParseResult::Err(error) => {
// The parser found a token, but it's not an identifier
return ParseResult::Err(SyntaxError {
reason: format!(
"There should be an identifier after a `{}` token",
if is_val { "val" } else { "var" }
),
error_start: t.position,
error_end: t.get_end_position(),
});
return ParseResult::Err(error);
}
Result3::None => {
_ => {
// The parser didn't find an Identifier after VAL/VAR
return ParseResult::Err(SyntaxError {
reason: format!(
@ -66,11 +48,12 @@ pub fn try_parse<'a>(tokens: &'a Vec<Token>, pos: usize) -> ParseResult<Binding,
});
}
};
current_pos = next_pos;
/*
* Equal (=) operator
*/
let equal_operator: &Token = match try_operator(tokens, current_pos + 2, String::from("=")) {
let equal_operator: &Token = match try_operator(tokens, current_pos, String::from("=")) {
Result3::Ok(t) => t,
Result3::Err(t) => {
// The parser found a token, but it's not the `=` operator
@ -90,7 +73,7 @@ pub fn try_parse<'a>(tokens: &'a Vec<Token>, pos: usize) -> ParseResult<Binding,
}
};
let expression = expression::try_parse(tokens, current_pos + 3);
let expression = expression::try_parse(tokens, current_pos + 1);
if expression.is_none() {
return ParseResult::Err(SyntaxError {
reason: String::from("Expected an expression after the equal `=` operator"),
@ -102,25 +85,25 @@ pub fn try_parse<'a>(tokens: &'a Vec<Token>, pos: usize) -> ParseResult<Binding,
let binding = if is_val {
Binding::Val(ValBinding {
datatype: datatype_annotation,
datatype: None,
identifier: Box::new(identifier.value.clone()),
expression,
})
} else {
Binding::Var(VarBinding {
datatype: datatype_annotation,
datatype: None,
identifier: Box::new(identifier.value.clone()),
expression,
})
};
ParseResult::Ok(binding, current_pos + 4)
ParseResult::Ok(binding, current_pos + 2)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{lexic::get_tokens, syntax::ast::TopLevelDeclaration};
use crate::lexic::get_tokens;
#[test]
fn should_parse_val_binding() {
@ -157,6 +140,7 @@ mod tests {
assert_eq!("=", token.value);
}
/*
#[test]
fn should_parse_binding_with_datatype() {
let tokens = get_tokens(&String::from("Num val identifier = 20")).unwrap();
@ -175,6 +159,7 @@ mod tests {
assert_eq!(Some(String::from("Bool")), binding.datatype);
assert_eq!("identifier", format!("{}", binding.identifier));
}
*/
#[test]
fn should_return_correct_error() {

51
src/syntax/block.rs
View File

@ -21,7 +21,7 @@ pub fn parse_block<'a>(tokens: &'a Vec<Token>, pos: usize) -> ParseResult<Block,
// Parse block statements
let mut statements = Vec::new();
// Only 1 statement for now
// First statement
match super::statement::try_parse(tokens, current_pos) {
ParseResult::Ok(statement, next_pos) => {
current_pos = next_pos;
@ -32,6 +32,23 @@ pub fn parse_block<'a>(tokens: &'a Vec<Token>, pos: usize) -> ParseResult<Block,
ParseResult::Mismatch(_) => {}
}
// More statements separated by new lines
while let Some(t) = tokens.get(current_pos) {
if t.token_type != TokenType::NewLine {
break;
}
current_pos += 1;
match super::statement::try_parse(tokens, current_pos) {
ParseResult::Ok(statement, next_pos) => {
current_pos = next_pos;
statements.push(statement);
}
ParseResult::Err(err) => return ParseResult::Err(err),
_ => break,
}
}
// Parse closing brace
let (_closing_brace, next_pos) =
match parse_token_type(tokens, current_pos, TokenType::RightBrace) {
@ -56,3 +73,35 @@ pub fn parse_block<'a>(tokens: &'a Vec<Token>, pos: usize) -> ParseResult<Block,
ParseResult::Ok(Block { statements }, current_pos)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::lexic::get_tokens;
#[test]
fn test_parse_block() {
let tokens = get_tokens(&String::from("{f()}")).unwrap();
let block = parse_block(&tokens, 0);
let block = match block {
ParseResult::Ok(p, _) => p,
_ => panic!("Expected a block, got: {:?}", block),
};
assert_eq!(block.statements.len(), 1);
}
#[test]
fn test_parse_block_2() {
let tokens = get_tokens(&String::from("{f()\ng()}")).unwrap();
let block = parse_block(&tokens, 0);
let block = match block {
ParseResult::Ok(p, _) => p,
_ => panic!("Expected a block, got: {:?}", block),
};
assert_eq!(block.statements.len(), 2);
}
}

2
src/syntax/mod.rs
View File

@ -5,8 +5,8 @@ mod block;
mod expression;
mod functions;
mod params_list;
mod utils;
mod statement;
mod utils;
pub mod ast;

28
src/syntax/statement.rs
View File

@ -1,22 +1,19 @@
use crate::lexic::token::Token;
use super::{ast::statement::Statement, functions::function_call, ParseResult, binding};
use super::{ast::statement::Statement, binding, functions::function_call, ParseResult};
pub fn try_parse<'a>(tokens: &'a Vec<Token>, pos: usize) -> ParseResult<Statement, ()> {
None
.or_else(
|| match binding::try_parse(tokens, pos) {
ParseResult::Ok(b, next) => Some(ParseResult::Ok(Statement::Binding(b), next)),
ParseResult::Err(err) => Some(ParseResult::Err(err)),
_ => None,
}
)
.or_else(|| match function_call::try_parse(tokens, pos) {
ParseResult::Ok(f, next) => Some(ParseResult::Ok(Statement::FunctionCall(f), next)),
ParseResult::Err(err) => Some(ParseResult::Err(err)),
_ => None,
})
.unwrap_or_else(|| ParseResult::Unmatched)
None.or_else(|| match binding::try_parse(tokens, pos) {
ParseResult::Ok(b, next) => Some(ParseResult::Ok(Statement::Binding(b), next)),
ParseResult::Err(err) => Some(ParseResult::Err(err)),
_ => None,
})
.or_else(|| match function_call::try_parse(tokens, pos) {
ParseResult::Ok(f, next) => Some(ParseResult::Ok(Statement::FunctionCall(f), next)),
ParseResult::Err(err) => Some(ParseResult::Err(err)),
_ => None,
})
.unwrap_or_else(|| ParseResult::Unmatched)
}
#[cfg(test)]
@ -40,7 +37,6 @@ mod tests {
}
}
#[test]
fn should_parse_binding() {
let input = String::from("val identifier = 20");

22
src/syntax/utils.rs
View File

@ -17,6 +17,18 @@ pub fn try_token_type(tokens: &Vec<Token>, pos: usize, token_type: TokenType) ->
}
}
pub fn try_operator(tokens: &Vec<Token>, pos: usize, operator: String) -> Result3<&Token> {
match tokens.get(pos) {
Some(t) if t.token_type == TokenType::Operator && t.value == operator => Result3::Ok(t),
Some(t) if t.token_type == TokenType::NewLine || t.token_type == TokenType::EOF => {
Result3::None
}
Some(t) => Result3::Err(t),
None => Result3::None,
}
}
/// Expects the token at `pos` to be of type `token_type`
pub fn parse_token_type(
tokens: &Vec<Token>,
@ -47,13 +59,3 @@ pub fn parse_token_type(
}
}
pub fn try_operator(tokens: &Vec<Token>, pos: usize, operator: String) -> Result3<&Token> {
match tokens.get(pos) {
Some(t) if t.token_type == TokenType::Operator && t.value == operator => Result3::Ok(t),
Some(t) if t.token_type == TokenType::NewLine || t.token_type == TokenType::EOF => {
Result3::None
}
Some(t) => Result3::Err(t),
None => Result3::None,
}
}