diff --git a/src/file/mod.rs b/src/file/mod.rs index 5c7cd31..d011745 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -1,6 +1,7 @@ use std::{fs, path::Path}; -use crate::{codegen, error_handling::PrintableError, lexic, syntax, token::Token}; +use crate::lexic::token::Token; +use crate::{codegen, error_handling::PrintableError, lexic, syntax}; pub fn compile_file(input: &String, output: &String) { let input_path = Path::new(input); diff --git a/src/lexic/mod.rs b/src/lexic/mod.rs index ed940b7..be4bf8d 100755 --- a/src/lexic/mod.rs +++ b/src/lexic/mod.rs @@ -1,8 +1,10 @@ mod scanner; mod utils; -use super::token::{self, Token}; +pub mod token; + use crate::error_handling::{LexError, MistiError}; +use token::Token; type Chars = Vec; @@ -53,8 +55,8 @@ pub fn get_tokens(input: &String) -> Result, MistiError> { } } - results.push(token::new_semicolon(0)); - results.push(token::new_eof(0)); + results.push(Token::new_semicolon(0)); + results.push(Token::new_eof(0)); Ok(results) } diff --git a/src/lexic/scanner/identifier.rs b/src/lexic/scanner/identifier.rs index 2bcc5c5..7951867 100755 --- a/src/lexic/scanner/identifier.rs +++ b/src/lexic/scanner/identifier.rs @@ -1,7 +1,5 @@ -use crate::{ - lexic::{token, utils, LexResult}, - token::TokenType, -}; +use super::token::TokenType; +use crate::lexic::{token::Token, utils, LexResult}; /// Checks if a String is a keyword, and returns its TokenType fn str_is_keyword(s: &String) -> Option { @@ -39,17 +37,17 @@ fn scan_impl(chars: &Vec, start_pos: usize, current: String, is_datatype: let current_len = current.len(); if let Some(token_type) = str_is_keyword(¤t) { LexResult::Some( - token::new(current, start_pos - current_len, token_type), + Token::new(current, start_pos - current_len, token_type), start_pos, ) } else if is_datatype { LexResult::Some( - token::new_datatype(current, start_pos - current_len), + Token::new_datatype(current, start_pos - current_len), start_pos, ) } else { LexResult::Some( - token::new_identifier(current, start_pos - current_len), + Token::new_identifier(current, start_pos - current_len), start_pos, ) } diff --git a/src/lexic/scanner/mod.rs b/src/lexic/scanner/mod.rs index 28b7153..c30899e 100755 --- a/src/lexic/scanner/mod.rs +++ b/src/lexic/scanner/mod.rs @@ -1,5 +1,5 @@ use super::{ - token::{self, TokenType}, + token::{self, Token, TokenType}, utils, LexResult, }; @@ -34,7 +34,7 @@ pub fn grouping_sign(c: char, _: &Vec, start_pos: usize) -> Option return None, }; - let token = token::new(c.to_string(), start_pos, token_type); + let token = Token::new(c.to_string(), start_pos, token_type); Some(LexResult::Some(token, start_pos + 1)) } diff --git a/src/lexic/scanner/new_comment.rs b/src/lexic/scanner/new_comment.rs index c60a74d..cb1e327 100644 --- a/src/lexic/scanner/new_comment.rs +++ b/src/lexic/scanner/new_comment.rs @@ -1,7 +1,5 @@ -use crate::{ - lexic::{utils, LexResult}, - token::new_comment, -}; +use super::token::Token; +use crate::lexic::{utils, LexResult}; /// Scans a new line. /// @@ -11,7 +9,7 @@ use crate::{ pub fn scan(chars: &Vec, start_pos: usize) -> LexResult { let (comment_content, next_pos) = scan_any_except_new_line(chars, start_pos + 2, String::from("")); - let token = new_comment(format!("//{}", comment_content), start_pos); + let token = Token::new_comment(format!("//{}", comment_content), start_pos); LexResult::Some(token, next_pos) } diff --git a/src/lexic/scanner/new_line.rs b/src/lexic/scanner/new_line.rs index 6da4a2d..e2cb02e 100644 --- a/src/lexic/scanner/new_line.rs +++ b/src/lexic/scanner/new_line.rs @@ -1,7 +1,5 @@ -use crate::{ - lexic::{token, LexResult}, - token::TokenType, -}; +use super::token::TokenType; +use crate::lexic::{token::Token, LexResult}; /// Function to handle new lines /// @@ -17,12 +15,12 @@ pub fn scan(chars: &Vec, start_pos: usize) -> LexResult { Some(c) if *c == ' ' => match look_ahead_for_new_line(chars, start_pos + 1) { Some(next_pos) => scan(chars, next_pos), None => { - let token = token::new(String::from(";"), start_pos, TokenType::Semicolon); + let token = Token::new(String::from(";"), start_pos, TokenType::Semicolon); LexResult::Some(token, start_pos) } }, Some(_) | None => { - let token = token::new(String::from(";"), start_pos, TokenType::Semicolon); + let token = Token::new(String::from(";"), start_pos, TokenType::Semicolon); LexResult::Some(token, start_pos) } } diff --git a/src/lexic/scanner/number.rs b/src/lexic/scanner/number.rs index d269a38..40059b2 100755 --- a/src/lexic/scanner/number.rs +++ b/src/lexic/scanner/number.rs @@ -1,8 +1,5 @@ use crate::error_handling::LexError; -use crate::lexic::{ - token::{self, Token}, - utils, LexResult, -}; +use crate::lexic::{token::Token, utils, LexResult}; /// Function to scan a number /// @@ -39,7 +36,7 @@ fn scan_decimal(chars: &Vec, start_pos: usize, current: String) -> LexResu let current_len = current.len(); LexResult::Some( - token::new_number(current, start_pos - current_len), + Token::new_number(current, start_pos - current_len), start_pos, ) } @@ -101,7 +98,7 @@ fn scan_double_impl(chars: &Vec, start_pos: usize, current: String) -> Lex let current_len = current.len(); LexResult::Some( - token::new_number(current, start_pos - current_len), + Token::new_number(current, start_pos - current_len), start_pos, ) } @@ -147,7 +144,7 @@ fn scan_digits(chars: &Vec, start_pos: usize, current: String) -> (Token, let current_len = current.len(); ( - token::new_number(current, start_pos - current_len), + Token::new_number(current, start_pos - current_len), start_pos, ) } @@ -166,7 +163,7 @@ fn scan_hex_digits(chars: &Vec, start_pos: usize, current: String) -> (Tok let current_len = current.len(); ( - token::new_number(current, start_pos - current_len), + Token::new_number(current, start_pos - current_len), start_pos, ) } diff --git a/src/lexic/scanner/operator.rs b/src/lexic/scanner/operator.rs index 31f8092..a18d118 100755 --- a/src/lexic/scanner/operator.rs +++ b/src/lexic/scanner/operator.rs @@ -1,4 +1,4 @@ -use crate::lexic::{token, utils, LexResult}; +use crate::lexic::{token::Token, utils, LexResult}; /// Function to scan an operator /// @@ -18,7 +18,7 @@ pub fn scan_impl(chars: &Vec, start_pos: usize, current: String) -> LexRes let current_len = current.len(); LexResult::Some( - token::new_operator(current, start_pos - current_len), + Token::new_operator(current, start_pos - current_len), start_pos, ) } diff --git a/src/lexic/scanner/string.rs b/src/lexic/scanner/string.rs index 75a4a9d..06b4cd4 100755 --- a/src/lexic/scanner/string.rs +++ b/src/lexic/scanner/string.rs @@ -1,5 +1,6 @@ use crate::error_handling::LexError; -use crate::lexic::{token, utils, LexResult}; +use crate::lexic::token::Token; +use crate::lexic::{utils, LexResult}; /// Function to scan a string /// @@ -20,7 +21,7 @@ pub fn scan_impl(chars: &Vec, start_pos: usize, current: String) -> LexRes let final_str = format!("{}\"", current); LexResult::Some( - token::new_string(final_str, start_pos - current_len), + Token::new_string(final_str, start_pos - current_len), start_pos + 1, ) } diff --git a/src/main.rs b/src/main.rs index 7f631ed..a58114d 100755 --- a/src/main.rs +++ b/src/main.rs @@ -4,8 +4,6 @@ use clap::{Parser, Subcommand}; mod repl; // Module to handle file compilation mod file; -// Defines the types of tokens and provides functions to create them -mod token; // Module to handle lexical analysis mod syntax; // Module to handle syntactic analysis diff --git a/src/repl/mod.rs b/src/repl/mod.rs index 0de328a..b09386f 100755 --- a/src/repl/mod.rs +++ b/src/repl/mod.rs @@ -1,7 +1,7 @@ use std::io::{self, Write}; use crate::error_handling::PrintableError; -use crate::token::Token; +use crate::lexic::token::Token; use super::codegen; use super::lexic; diff --git a/src/syntax/binding.rs b/src/syntax/binding.rs index c878ffa..cd33324 100644 --- a/src/syntax/binding.rs +++ b/src/syntax/binding.rs @@ -1,7 +1,7 @@ use super::ast_types::{Binding, ValBinding, VarBinding}; use super::{expression, SyntaxResult}; use crate::error_handling::SyntaxError; -use crate::token::{Token, TokenType}; +use crate::lexic::token::{Token, TokenType}; use crate::utils::Result3; // TODO: Should return a 3 state value: diff --git a/src/syntax/expression.rs b/src/syntax/expression.rs index 2a336c1..6c7018c 100644 --- a/src/syntax/expression.rs +++ b/src/syntax/expression.rs @@ -1,5 +1,5 @@ use super::ast_types::Expression; -use crate::token::{Token, TokenType}; +use crate::lexic::token::{Token, TokenType}; /// An expression can be: /// diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index b9c4fa3..244f872 100755 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -1,11 +1,11 @@ use crate::ast_types::Binding; use crate::error_handling::{MistiError, SyntaxError}; -use super::token::Token; - mod binding; mod expression; + use super::ast_types; +use crate::lexic::token::Token; use ast_types::ModuleAST; diff --git a/src/token.rs b/src/token.rs deleted file mode 100755 index 61648fd..0000000 --- a/src/token.rs +++ /dev/null @@ -1,107 +0,0 @@ -#[derive(PartialEq, Debug, Clone)] -pub enum TokenType { - Identifier, - Datatype, - Number, - String, - Operator, - LeftParen, - RightParen, - LeftBracket, - RightBracket, - LeftBrace, - RightBrace, - Semicolon, - Comment, - VAR, - VAL, - EOF, -} - -#[derive(Debug)] -pub struct Token { - pub token_type: TokenType, - // The token as a raw string - pub value: String, - /// The absolute position of this token, from the - /// start of the file - pub position: usize, -} - -impl Token { - pub fn get_end_position(&self) -> usize { - self.position + self.value.len() - } -} - -pub fn new_eof(position: usize) -> Token { - Token { - token_type: TokenType::EOF, - value: String::from(""), - position, - } -} - -pub fn new_number(value: String, position: usize) -> Token { - Token { - token_type: TokenType::Number, - value, - position, - } -} - -pub fn new_operator(value: String, position: usize) -> Token { - Token { - token_type: TokenType::Operator, - value, - position, - } -} - -pub fn new(value: String, position: usize, token_type: TokenType) -> Token { - Token { - token_type, - value, - position, - } -} - -pub fn new_identifier(value: String, position: usize) -> Token { - Token { - token_type: TokenType::Identifier, - value, - position, - } -} - -pub fn new_string(value: String, position: usize) -> Token { - Token { - token_type: TokenType::String, - value, - position, - } -} - -pub fn new_semicolon(position: usize) -> Token { - Token { - token_type: TokenType::Semicolon, - value: String::from(";"), - position, - } -} - -pub fn new_datatype(value: String, position: usize) -> Token { - Token { - token_type: TokenType::Datatype, - value, - position, - } -} - -pub fn new_comment(value: String, position: usize) -> Token { - Token { - token_type: TokenType::Comment, - value, - position, - } -}