From 69339a955eaa397ea756b5d2235834cb1b3e899c Mon Sep 17 00:00:00 2001 From: Araozu Date: Sun, 28 Jul 2024 18:18:11 -0500 Subject: [PATCH] feat: Add syntax errors to the tokenize command --- CHANGELOG.md | 1 + Cargo.lock | 2 +- src/cli/tokenize.rs | 28 +++++++++++++-- src/syntax/mod.rs | 12 +++++++ src/syntax/parsers/module.rs | 13 ++++++- src/syntax/parsers/statement.rs | 36 +++++++++++++++++++ src/syntax/utils.rs | 62 +++++++++++++++++++++++++++++++++ 7 files changed, 150 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f38391..546cdf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ ## v0.0.15 +- [ ] Include comments in the AST - [ ] Replace all panics with actual errors - [ ] Remove all old codegen - [ ] Test codegen diff --git a/Cargo.lock b/Cargo.lock index c90a38f..2ffbd66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,7 +92,7 @@ dependencies = [ [[package]] name = "thp" -version = "0.0.13" +version = "0.0.14" dependencies = [ "colored", "serde", diff --git a/src/cli/tokenize.rs b/src/cli/tokenize.rs index 8948489..0691e5b 100644 --- a/src/cli/tokenize.rs +++ b/src/cli/tokenize.rs @@ -1,6 +1,19 @@ -use crate::lexic::get_tokens; +use serde::Serialize; + +use crate::{ + error_handling::MistiError, + lexic::{get_tokens, token::Token}, + syntax::build_ast, +}; use std::io::{self, BufRead}; +#[derive(Serialize)] +enum TokenizeResult { + Ok(Vec), + TokensOnly(Vec, MistiError), + Err(MistiError), +} + pub fn tokenize_command(_options: Vec) -> Result<(), ()> { // Get the input from stdin let stdin = io::stdin(); @@ -19,7 +32,18 @@ pub fn tokenize_command(_options: Vec) -> Result<(), ()> { let input_code = lines.join("\n"); let tokens = get_tokens(&input_code); - let json = serde_json::to_string(&tokens).unwrap(); + let result = match tokens { + Ok(tokens) => { + let ast_result = build_ast(&tokens); + match ast_result { + Ok(_) => TokenizeResult::Ok(tokens), + Err(error) => TokenizeResult::TokensOnly(tokens, error), + } + } + Err(error) => TokenizeResult::Err(error), + }; + + let json = serde_json::to_string(&result).unwrap(); println!("{}", json); Ok(()) diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 45f0520..a9c8420 100755 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -70,4 +70,16 @@ mod tests { _ => panic!("Expected a function declaration as first production"), } } + + #[test] + fn should_fail_on_syntax_error() { + let input = String::from("fun gaa {}"); + let tokens = get_tokens(&input).unwrap(); + let ast = build_ast(&tokens); + + match ast { + Ok(_) => panic!("Expected an Err"), + Err(_) => {} + } + } } diff --git a/src/syntax/parsers/module.rs b/src/syntax/parsers/module.rs index 574b43c..9648713 100644 --- a/src/syntax/parsers/module.rs +++ b/src/syntax/parsers/module.rs @@ -98,8 +98,19 @@ mod test { fn should_parse_expression() { let tokens = get_tokens(&String::from("1")).unwrap(); - let (module, next) = ModuleAST::try_parse(&tokens, 0).unwrap(); + let (_, next) = ModuleAST::try_parse(&tokens, 0).unwrap(); assert_eq!(next, 1); } + + #[test] + fn should_fail_on_invalid_expression() { + let tokens = get_tokens(&String::from("function_call(1 2")).unwrap(); + let result = ModuleAST::try_parse(&tokens, 0); + + match result { + Ok(_) => panic!("Expected an error"), + Err(_) => {}, + } + } } diff --git a/src/syntax/parsers/statement.rs b/src/syntax/parsers/statement.rs index 0e8433c..a337cc2 100644 --- a/src/syntax/parsers/statement.rs +++ b/src/syntax/parsers/statement.rs @@ -84,4 +84,40 @@ mod test { } } } + + #[test] + fn should_parse_variable_decl() { + let input = String::from("val x = 322"); + let tokens = get_tokens(&input).unwrap(); + let parsing = Statement::try_parse(&tokens, 0); + + match parsing { + Ok(_) => {} + Err(_) => panic!("Expected parsing to be successful"), + } + } + + #[test] + fn should_not_parse_invalid_variable_decl() { + let input = String::from("val x y"); + let tokens = get_tokens(&input).unwrap(); + let parsing = Statement::try_parse(&tokens, 0); + + match parsing { + Ok(_) => panic!("Expected an Err"), + Err(_) => {} + } + } + + #[test] + fn should_parse_fun_decl() { + let input = String::from("fun name(){}"); + let tokens = get_tokens(&input).unwrap(); + let parsing = Statement::try_parse(&tokens, 0); + + match parsing { + Ok(_) => {} + Err(_) => panic!("Expected parsing to be successful"), + } + } } diff --git a/src/syntax/utils.rs b/src/syntax/utils.rs index b3f627d..00314bb 100644 --- a/src/syntax/utils.rs +++ b/src/syntax/utils.rs @@ -75,3 +75,65 @@ pub fn parse_token_type( None => Err(ParsingError::Unmatched), } } + +#[cfg(test)] +mod tests { + use crate::{ + lexic::{get_tokens, token::TokenType}, + syntax::{parseable::ParsingError, utils::{parse_token_type, Tokenizer}}, + }; + + use super::try_operator; + + #[test] + fn test_1() { + let input = String::from(""); + let tokens = get_tokens(&input).unwrap(); + assert_eq!(1, tokens.len()); + + match try_operator(&tokens, 10, "+".into()) { + Ok(_) => panic!("Expected an error"), + Err(error) => match error { + ParsingError::Unmatched => { + assert!(true); + } + _ => panic!( + "Expected an error due to incorrect position, got {:?}", + error + ), + }, + } + } + + #[test] + fn test_2() { + let input = String::from(""); + let tokens = get_tokens(&input).unwrap(); + assert_eq!(1, tokens.len()); + + match parse_token_type(&tokens, 10, TokenType::Operator) { + Ok(_) => panic!("Expected an error"), + Err(error) => match error { + ParsingError::Unmatched => { + assert!(true); + } + _ => panic!( + "Expected an error due to incorrect position, got {:?}", + error + ), + }, + } + } + + #[test] + fn test_3() { + let input = String::from(""); + let tokens = get_tokens(&input).unwrap(); + assert_eq!(1, tokens.len()); + + match tokens.get_significant(10) { + Some(_) => panic!("Expected a None"), + None => {}, + } + } +}