feat: Add syntax errors to the tokenize command

CHANGELOG.md

@@ -25,6 +25,7 @@
 
 ## v0.0.15
 
+- [ ] Include comments in the AST
 - [ ] Replace all panics with actual errors
 - [ ] Remove all old codegen
 - [ ] Test codegen

Cargo.lock

@@ -92,7 +92,7 @@ dependencies = [
 
 [[package]]
 name = "thp"
-version = "0.0.13"
+version = "0.0.14"
 dependencies = [
  "colored",
  "serde",

src/cli/tokenize.rs

@@ -1,6 +1,19 @@
-use crate::lexic::get_tokens;
+use serde::Serialize;
+
+use crate::{
+    error_handling::MistiError,
+    lexic::{get_tokens, token::Token},
+    syntax::build_ast,
+};
 use std::io::{self, BufRead};
 
+#[derive(Serialize)]
+enum TokenizeResult {
+    Ok(Vec<Token>),
+    TokensOnly(Vec<Token>, MistiError),
+    Err(MistiError),
+}
+
 pub fn tokenize_command(_options: Vec<String>) -> Result<(), ()> {
     // Get the input from stdin
     let stdin = io::stdin();
@@ -19,7 +32,18 @@ pub fn tokenize_command(_options: Vec<String>) -> Result<(), ()> {
     let input_code = lines.join("\n");
     let tokens = get_tokens(&input_code);
 
-    let json = serde_json::to_string(&tokens).unwrap();
+    let result = match tokens {
+        Ok(tokens) => {
+            let ast_result = build_ast(&tokens);
+            match ast_result {
+                Ok(_) => TokenizeResult::Ok(tokens),
+                Err(error) => TokenizeResult::TokensOnly(tokens, error),
+            }
+        }
+        Err(error) => TokenizeResult::Err(error),
+    };
+
+    let json = serde_json::to_string(&result).unwrap();
     println!("{}", json);
 
     Ok(())

src/syntax/mod.rs

@@ -70,4 +70,16 @@ mod tests {
             _ => panic!("Expected a function declaration as first production"),
         }
     }
+
+    #[test]
+    fn should_fail_on_syntax_error() {
+        let input = String::from("fun gaa {}");
+        let tokens = get_tokens(&input).unwrap();
+        let ast = build_ast(&tokens);
+
+        match ast {
+            Ok(_) => panic!("Expected an Err"),
+            Err(_) => {}
+        }
+    }
 }

src/syntax/parsers/module.rs

@@ -98,8 +98,19 @@ mod test {
     fn should_parse_expression() {
         let tokens = get_tokens(&String::from("1")).unwrap();
 
-        let (module, next) = ModuleAST::try_parse(&tokens, 0).unwrap();
+        let (_, next) = ModuleAST::try_parse(&tokens, 0).unwrap();
 
         assert_eq!(next, 1);
     }
+
+    #[test]
+    fn should_fail_on_invalid_expression() {
+        let tokens = get_tokens(&String::from("function_call(1 2")).unwrap();
+        let result = ModuleAST::try_parse(&tokens, 0);
+
+        match result {
+            Ok(_) => panic!("Expected an error"),
+            Err(_) => {},
+        }
+    }
 }

src/syntax/parsers/statement.rs

@@ -84,4 +84,40 @@ mod test {
             }
         }
     }
+
+    #[test]
+    fn should_parse_variable_decl() {
+        let input = String::from("val x = 322");
+        let tokens = get_tokens(&input).unwrap();
+        let parsing = Statement::try_parse(&tokens, 0);
+
+        match parsing {
+            Ok(_) => {}
+            Err(_) => panic!("Expected parsing to be successful"),
+        }
+    }
+
+    #[test]
+    fn should_not_parse_invalid_variable_decl() {
+        let input = String::from("val x y");
+        let tokens = get_tokens(&input).unwrap();
+        let parsing = Statement::try_parse(&tokens, 0);
+
+        match parsing {
+            Ok(_) => panic!("Expected an Err"),
+            Err(_) => {}
+        }
+    }
+
+    #[test]
+    fn should_parse_fun_decl() {
+        let input = String::from("fun name(){}");
+        let tokens = get_tokens(&input).unwrap();
+        let parsing = Statement::try_parse(&tokens, 0);
+
+        match parsing {
+            Ok(_) => {}
+            Err(_) => panic!("Expected parsing to be successful"),
+        }
+    }
 }

src/syntax/utils.rs

@@ -75,3 +75,65 @@ pub fn parse_token_type(
         None => Err(ParsingError::Unmatched),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::{
+        lexic::{get_tokens, token::TokenType},
+        syntax::{parseable::ParsingError, utils::{parse_token_type, Tokenizer}},
+    };
+
+    use super::try_operator;
+
+    #[test]
+    fn test_1() {
+        let input = String::from("");
+        let tokens = get_tokens(&input).unwrap();
+        assert_eq!(1, tokens.len());
+
+        match try_operator(&tokens, 10, "+".into()) {
+            Ok(_) => panic!("Expected an error"),
+            Err(error) => match error {
+                ParsingError::Unmatched => {
+                    assert!(true);
+                }
+                _ => panic!(
+                    "Expected an error due to incorrect position, got {:?}",
+                    error
+                ),
+            },
+        }
+    }
+
+    #[test]
+    fn test_2() {
+        let input = String::from("");
+        let tokens = get_tokens(&input).unwrap();
+        assert_eq!(1, tokens.len());
+
+        match parse_token_type(&tokens, 10, TokenType::Operator) {
+            Ok(_) => panic!("Expected an error"),
+            Err(error) => match error {
+                ParsingError::Unmatched => {
+                    assert!(true);
+                }
+                _ => panic!(
+                    "Expected an error due to incorrect position, got {:?}",
+                    error
+                ),
+            },
+        }
+    }
+
+    #[test]
+    fn test_3() {
+        let input = String::from("");
+        let tokens = get_tokens(&input).unwrap();
+        assert_eq!(1, tokens.len());
+
+        match tokens.get_significant(10) {
+            Some(_) => panic!("Expected a None"),
+            None => {},
+        }
+    }
+}