Refactor token creation

src/file/mod.rs

@@ -1,6 +1,7 @@
 use std::{fs, path::Path};
 
-use crate::{codegen, error_handling::PrintableError, lexic, syntax, token::Token};
+use crate::lexic::token::Token;
+use crate::{codegen, error_handling::PrintableError, lexic, syntax};
 
 pub fn compile_file(input: &String, output: &String) {
     let input_path = Path::new(input);

src/lexic/mod.rs

@@ -1,8 +1,10 @@
 mod scanner;
 mod utils;
 
-use super::token::{self, Token};
+pub mod token;
+
 use crate::error_handling::{LexError, MistiError};
+use token::Token;
 
 type Chars = Vec<char>;
 
@@ -53,8 +55,8 @@ pub fn get_tokens(input: &String) -> Result<Vec<Token>, MistiError> {
         }
     }
 
-    results.push(token::new_semicolon(0));
-    results.push(token::new_eof(0));
+    results.push(Token::new_semicolon(0));
+    results.push(Token::new_eof(0));
     Ok(results)
 }
 

src/lexic/scanner/identifier.rs

@@ -1,7 +1,5 @@
-use crate::{
-    lexic::{token, utils, LexResult},
-    token::TokenType,
-};
+use super::token::TokenType;
+use crate::lexic::{token::Token, utils, LexResult};
 
 /// Checks if a String is a keyword, and returns its TokenType
 fn str_is_keyword(s: &String) -> Option<TokenType> {
@@ -39,17 +37,17 @@ fn scan_impl(chars: &Vec<char>, start_pos: usize, current: String, is_datatype:
             let current_len = current.len();
             if let Some(token_type) = str_is_keyword(&current) {
                 LexResult::Some(
-                    token::new(current, start_pos - current_len, token_type),
+                    Token::new(current, start_pos - current_len, token_type),
                     start_pos,
                 )
             } else if is_datatype {
                 LexResult::Some(
-                    token::new_datatype(current, start_pos - current_len),
+                    Token::new_datatype(current, start_pos - current_len),
                     start_pos,
                 )
             } else {
                 LexResult::Some(
-                    token::new_identifier(current, start_pos - current_len),
+                    Token::new_identifier(current, start_pos - current_len),
                     start_pos,
                 )
             }

src/lexic/scanner/mod.rs

@@ -1,5 +1,5 @@
 use super::{
-    token::{self, TokenType},
+    token::{self, Token, TokenType},
     utils, LexResult,
 };
 
@@ -34,7 +34,7 @@ pub fn grouping_sign(c: char, _: &Vec<char>, start_pos: usize) -> Option<LexResu
         _ => return None,
     };
 
-    let token = token::new(c.to_string(), start_pos, token_type);
+    let token = Token::new(c.to_string(), start_pos, token_type);
     Some(LexResult::Some(token, start_pos + 1))
 }
 

src/lexic/scanner/new_comment.rs

@@ -1,7 +1,5 @@
-use crate::{
-    lexic::{utils, LexResult},
-    token::new_comment,
-};
+use super::token::Token;
+use crate::lexic::{utils, LexResult};
 
 /// Scans a new line.
 ///
@@ -11,7 +9,7 @@ use crate::{
 pub fn scan(chars: &Vec<char>, start_pos: usize) -> LexResult {
     let (comment_content, next_pos) =
         scan_any_except_new_line(chars, start_pos + 2, String::from(""));
-    let token = new_comment(format!("//{}", comment_content), start_pos);
+    let token = Token::new_comment(format!("//{}", comment_content), start_pos);
 
     LexResult::Some(token, next_pos)
 }

src/lexic/scanner/new_line.rs

@@ -1,7 +1,5 @@
-use crate::{
-    lexic::{token, LexResult},
-    token::TokenType,
-};
+use super::token::TokenType;
+use crate::lexic::{token::Token, LexResult};
 
 /// Function to handle new lines
 ///
@@ -17,12 +15,12 @@ pub fn scan(chars: &Vec<char>, start_pos: usize) -> LexResult {
         Some(c) if *c == ' ' => match look_ahead_for_new_line(chars, start_pos + 1) {
             Some(next_pos) => scan(chars, next_pos),
             None => {
-                let token = token::new(String::from(";"), start_pos, TokenType::Semicolon);
+                let token = Token::new(String::from(";"), start_pos, TokenType::Semicolon);
                 LexResult::Some(token, start_pos)
             }
         },
         Some(_) | None => {
-            let token = token::new(String::from(";"), start_pos, TokenType::Semicolon);
+            let token = Token::new(String::from(";"), start_pos, TokenType::Semicolon);
             LexResult::Some(token, start_pos)
         }
     }

src/lexic/scanner/number.rs

@@ -1,8 +1,5 @@
 use crate::error_handling::LexError;
-use crate::lexic::{
-    token::{self, Token},
-    utils, LexResult,
-};
+use crate::lexic::{token::Token, utils, LexResult};
 
 /// Function to scan a number
 ///
@@ -39,7 +36,7 @@ fn scan_decimal(chars: &Vec<char>, start_pos: usize, current: String) -> LexResu
             let current_len = current.len();
 
             LexResult::Some(
-                token::new_number(current, start_pos - current_len),
+                Token::new_number(current, start_pos - current_len),
                 start_pos,
             )
         }
@@ -101,7 +98,7 @@ fn scan_double_impl(chars: &Vec<char>, start_pos: usize, current: String) -> Lex
             let current_len = current.len();
 
             LexResult::Some(
-                token::new_number(current, start_pos - current_len),
+                Token::new_number(current, start_pos - current_len),
                 start_pos,
             )
         }
@@ -147,7 +144,7 @@ fn scan_digits(chars: &Vec<char>, start_pos: usize, current: String) -> (Token,
             let current_len = current.len();
 
             (
-                token::new_number(current, start_pos - current_len),
+                Token::new_number(current, start_pos - current_len),
                 start_pos,
             )
         }
@@ -166,7 +163,7 @@ fn scan_hex_digits(chars: &Vec<char>, start_pos: usize, current: String) -> (Tok
             let current_len = current.len();
 
             (
-                token::new_number(current, start_pos - current_len),
+                Token::new_number(current, start_pos - current_len),
                 start_pos,
             )
         }

src/lexic/scanner/operator.rs

@@ -1,4 +1,4 @@
-use crate::lexic::{token, utils, LexResult};
+use crate::lexic::{token::Token, utils, LexResult};
 
 /// Function to scan an operator
 ///
@@ -18,7 +18,7 @@ pub fn scan_impl(chars: &Vec<char>, start_pos: usize, current: String) -> LexRes
             let current_len = current.len();
 
             LexResult::Some(
-                token::new_operator(current, start_pos - current_len),
+                Token::new_operator(current, start_pos - current_len),
                 start_pos,
             )
         }

src/lexic/scanner/string.rs

@@ -1,5 +1,6 @@
 use crate::error_handling::LexError;
-use crate::lexic::{token, utils, LexResult};
+use crate::lexic::token::Token;
+use crate::lexic::{utils, LexResult};
 
 /// Function to scan a string
 ///
@@ -20,7 +21,7 @@ pub fn scan_impl(chars: &Vec<char>, start_pos: usize, current: String) -> LexRes
 
             let final_str = format!("{}\"", current);
             LexResult::Some(
-                token::new_string(final_str, start_pos - current_len),
+                Token::new_string(final_str, start_pos - current_len),
                 start_pos + 1,
             )
         }

src/main.rs

@@ -4,8 +4,6 @@ use clap::{Parser, Subcommand};
 mod repl;
 // Module to handle file compilation
 mod file;
-// Defines the types of tokens and provides functions to create them
-mod token;
 // Module to handle lexical analysis
 mod syntax;
 // Module to handle syntactic analysis

src/repl/mod.rs

@@ -1,7 +1,7 @@
 use std::io::{self, Write};
 
 use crate::error_handling::PrintableError;
-use crate::token::Token;
+use crate::lexic::token::Token;
 
 use super::codegen;
 use super::lexic;

src/syntax/binding.rs

@@ -1,7 +1,7 @@
 use super::ast_types::{Binding, ValBinding, VarBinding};
 use super::{expression, SyntaxResult};
 use crate::error_handling::SyntaxError;
-use crate::token::{Token, TokenType};
+use crate::lexic::token::{Token, TokenType};
 use crate::utils::Result3;
 
 // TODO: Should return a 3 state value:

src/syntax/expression.rs

@@ -1,5 +1,5 @@
 use super::ast_types::Expression;
-use crate::token::{Token, TokenType};
+use crate::lexic::token::{Token, TokenType};
 
 /// An expression can be:
 ///

src/syntax/mod.rs

@@ -1,11 +1,11 @@
 use crate::ast_types::Binding;
 use crate::error_handling::{MistiError, SyntaxError};
 
-use super::token::Token;
-
 mod binding;
 mod expression;
+
 use super::ast_types;
+use crate::lexic::token::Token;
 
 use ast_types::ModuleAST;
 

src/token.rs

@@ -1,107 +0,0 @@
-#[derive(PartialEq, Debug, Clone)]
-pub enum TokenType {
-    Identifier,
-    Datatype,
-    Number,
-    String,
-    Operator,
-    LeftParen,
-    RightParen,
-    LeftBracket,
-    RightBracket,
-    LeftBrace,
-    RightBrace,
-    Semicolon,
-    Comment,
-    VAR,
-    VAL,
-    EOF,
-}
-
-#[derive(Debug)]
-pub struct Token {
-    pub token_type: TokenType,
-    // The token as a raw string
-    pub value: String,
-    /// The absolute position of this token, from the
-    /// start of the file
-    pub position: usize,
-}
-
-impl Token {
-    pub fn get_end_position(&self) -> usize {
-        self.position + self.value.len()
-    }
-}
-
-pub fn new_eof(position: usize) -> Token {
-    Token {
-        token_type: TokenType::EOF,
-        value: String::from(""),
-        position,
-    }
-}
-
-pub fn new_number(value: String, position: usize) -> Token {
-    Token {
-        token_type: TokenType::Number,
-        value,
-        position,
-    }
-}
-
-pub fn new_operator(value: String, position: usize) -> Token {
-    Token {
-        token_type: TokenType::Operator,
-        value,
-        position,
-    }
-}
-
-pub fn new(value: String, position: usize, token_type: TokenType) -> Token {
-    Token {
-        token_type,
-        value,
-        position,
-    }
-}
-
-pub fn new_identifier(value: String, position: usize) -> Token {
-    Token {
-        token_type: TokenType::Identifier,
-        value,
-        position,
-    }
-}
-
-pub fn new_string(value: String, position: usize) -> Token {
-    Token {
-        token_type: TokenType::String,
-        value,
-        position,
-    }
-}
-
-pub fn new_semicolon(position: usize) -> Token {
-    Token {
-        token_type: TokenType::Semicolon,
-        value: String::from(";"),
-        position,
-    }
-}
-
-pub fn new_datatype(value: String, position: usize) -> Token {
-    Token {
-        token_type: TokenType::Datatype,
-        value,
-        position,
-    }
-}
-
-pub fn new_comment(value: String, position: usize) -> Token {
-    Token {
-        token_type: TokenType::Comment,
-        value,
-        position,
-    }
-}