Compare commits

..

7 Commits

20 changed files with 513 additions and 166 deletions

View File

@ -1,7 +1,10 @@
use serde::Serialize; use serde::Serialize;
use crate::{ use crate::{
error_handling::MistiError, lexic::{get_tokens, token::Token}, semantic, syntax::build_ast error_handling::MistiError,
lexic::{get_tokens, token::Token},
semantic,
syntax::build_ast,
}; };
use std::io::{self, BufRead}; use std::io::{self, BufRead};
@ -35,11 +38,9 @@ pub fn tokenize_command(_options: Vec<String>) -> Result<(), ()> {
Ok(tokens) => { Ok(tokens) => {
let ast_result = build_ast(&tokens); let ast_result = build_ast(&tokens);
match ast_result { match ast_result {
Ok(ast) => { Ok(ast) => match semantic::check_semantics(&ast) {
match semantic::check_semantics(&ast) {
Ok(()) => TokenizeResult::Ok(tokens), Ok(()) => TokenizeResult::Ok(tokens),
Err(error) => TokenizeResult::SyntaxOnly(tokens, error) Err(error) => TokenizeResult::SyntaxOnly(tokens, error),
}
}, },
Err(error) => TokenizeResult::TokensOnly(tokens, error), Err(error) => TokenizeResult::TokensOnly(tokens, error),
} }

View File

@ -8,8 +8,3 @@ pub trait Transpilable {
/// Transforms this struct into PHP /// Transforms this struct into PHP
fn transpile(&self) -> String; fn transpile(&self) -> String;
} }
/// Transforms an AST to its representation in PHP
pub fn codegen<'a>(ast: &'a impl Transpilable) -> String {
ast.transpile()
}

View File

@ -1,8 +1,9 @@
use colored::*; use colored::*;
use std::{fs, path::Path}; use std::{fs, path::Path};
use crate::lexic::token::Token; use crate::codegen::Transpilable;
use crate::{codegen, error_handling::PrintableError, lexic, syntax}; use crate::php_ast::transformers::PHPTransformable;
use crate::{error_handling::PrintableError, lexic, syntax};
pub fn compile_file(input: &String) -> Result<(), ()> { pub fn compile_file(input: &String) -> Result<(), ()> {
let input_path = Path::new(input); let input_path = Path::new(input);
@ -58,49 +59,50 @@ pub fn compile_file(input: &String) -> Result<(), ()> {
} }
} }
/// THP source code goes in, PHP code or an error comes out /// Full pipeline from THP source code to PHP output
fn compile(input: &String) -> Result<String, String> { fn compile(input: &String) -> Result<String, String> {
let tokens = lexic::get_tokens(input); //
// Lexical analysis
let tokens = match tokens { //
Ok(tokens) => tokens, let tokens = match lexic::get_tokens(input) {
Ok(t) => t,
Err(error) => { Err(error) => {
let chars: Vec<char> = input.chars().into_iter().collect(); let chars: Vec<char> = input.chars().into_iter().collect();
return Err(format!( return Err(error.get_error_str(&chars));
"{}:\n{}",
"syntax error".on_red(),
error.get_error_str(&chars)
));
} }
}; };
build_ast(input, tokens) //
} // Syntax analysis
//
/// Executes Syntax analysis, and for now, Semantic analysis and Code generation. let ast = match syntax::build_ast(&tokens) {
///
/// Prints the generated code in stdin
fn build_ast(input: &String, tokens: Vec<Token>) -> Result<String, String> {
let ast = syntax::build_ast(&tokens);
let ast = match ast {
Ok(ast) => ast, Ok(ast) => ast,
Err(reason) => { Err(reason) => {
let chars: Vec<char> = input.chars().into_iter().collect(); let chars: Vec<char> = input.chars().into_iter().collect();
let error = format!("{}: {}", "error".on_red(), reason.get_error_str(&chars)); return Err(reason.get_error_str(&chars));
return Err(error);
} }
}; };
match crate::semantic::check_semantics(&ast) { //
// Semantic analysis
//
let res1 = crate::semantic::check_semantics(&ast);
match res1 {
Ok(_) => {} Ok(_) => {}
Err(reason) => { Err(reason) => {
let chars: Vec<char> = input.chars().into_iter().collect(); let chars: Vec<char> = input.chars().into_iter().collect();
let error = format!("{}: {}", "error".on_red(), reason.get_error_str(&chars)); let error = format!("{}: {}", "error".on_red(), reason.get_error_str(&chars));
return Err(error); return Err(error);
} }
}; }
Err("Code generation disabled: rewriting into PHP AST".into()) //
// Ok(codegen::codegen(&ast)) // Intermediate representation (THP -> PHP ast)
//
let php_ast = ast.into_php_ast();
//
// Codegen
//
Ok(php_ast.transpile())
} }

View File

@ -13,15 +13,15 @@ impl<'a> PHPTransformable<'a> for Expression<'_> {
fn into_php_ast(&'a self) -> Self::Item { fn into_php_ast(&'a self) -> Self::Item {
match self { match self {
Expression::String(value) => { Expression::String(value) => {
let expr = PhpPrimaryExpression::StringLiteral(value); let expr = PhpPrimaryExpression::StringLiteral(&value.value);
PhpExpression::Assignment(PhpAssignmentExpression::Primary(expr)) PhpExpression::Assignment(PhpAssignmentExpression::Primary(expr))
} }
Expression::Int(value) => { Expression::Int(value) => {
let expr = PhpPrimaryExpression::IntegerLiteral(value); let expr = PhpPrimaryExpression::IntegerLiteral(&value.value);
PhpExpression::Assignment(PhpAssignmentExpression::Primary(expr)) PhpExpression::Assignment(PhpAssignmentExpression::Primary(expr))
} }
Expression::Float(value) => { Expression::Float(value) => {
let expr = PhpPrimaryExpression::FloatingLiteral(value); let expr = PhpPrimaryExpression::FloatingLiteral(&value.value);
PhpExpression::Assignment(PhpAssignmentExpression::Primary(expr)) PhpExpression::Assignment(PhpAssignmentExpression::Primary(expr))
} }
_ => todo!("transformation for expression: {:?}", self), _ => todo!("transformation for expression: {:?}", self),
@ -32,6 +32,7 @@ impl<'a> PHPTransformable<'a> for Expression<'_> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::{ use crate::{
lexic::token::Token,
php_ast::{ php_ast::{
transformers::PHPTransformable, PhpAssignmentExpression, PhpExpression, transformers::PHPTransformable, PhpAssignmentExpression, PhpExpression,
PhpPrimaryExpression, PhpPrimaryExpression,
@ -41,8 +42,8 @@ mod tests {
#[test] #[test]
fn should_transform_string() { fn should_transform_string() {
let value = String::from("Hello"); let t = Token::new_string("Hello".into(), 0);
let input = Expression::String(&value); let input = Expression::String(&t);
let output = input.into_php_ast(); let output = input.into_php_ast();
match output { match output {
@ -57,8 +58,8 @@ mod tests {
#[test] #[test]
fn should_transform_int() { fn should_transform_int() {
let value = String::from("322"); let binding = Token::new_int(String::from("322"), 0);
let input = Expression::Int(&value); let input = Expression::Int(&binding);
let output = input.into_php_ast(); let output = input.into_php_ast();
match output { match output {
@ -73,8 +74,8 @@ mod tests {
#[test] #[test]
fn should_transform_float() { fn should_transform_float() {
let value = String::from("322.644"); let t = Token::new_float("322.644".into(), 0);
let input = Expression::Float(&value); let input = Expression::Float(&t);
let output = input.into_php_ast(); let output = input.into_php_ast();
match output { match output {

View File

@ -25,7 +25,7 @@ impl<'a> PHPTransformable<'a> for ModuleAST<'_> {
// TODO: This definitely needs refactoring // TODO: This definitely needs refactoring
let function_expr: &Expression = &*fc.function; let function_expr: &Expression = &*fc.function;
match function_expr { match function_expr {
Expression::Identifier(id) if *id == "print" => { Expression::Identifier(id) if id.value == "print" => {
// transform to print() expression // transform to print() expression
// no parameters supported // no parameters supported
@ -37,7 +37,7 @@ impl<'a> PHPTransformable<'a> for ModuleAST<'_> {
match e { match e {
Expression::String(v) => { Expression::String(v) => {
expressions.push( expressions.push(
PhpExpression::Assignment(PhpAssignmentExpression::Primary(PhpPrimaryExpression::StringLiteral(v))) PhpExpression::Assignment(PhpAssignmentExpression::Primary(PhpPrimaryExpression::StringLiteral(&v.value)))
) )
}, },
_ => todo!("Non string expressions not supported") _ => todo!("Non string expressions not supported")
@ -54,21 +54,21 @@ impl<'a> PHPTransformable<'a> for ModuleAST<'_> {
Expression::Int(value) => { Expression::Int(value) => {
php_statements.push(PhpStatement::PhpExpressionStatement( php_statements.push(PhpStatement::PhpExpressionStatement(
PhpExpression::Assignment(PhpAssignmentExpression::Primary( PhpExpression::Assignment(PhpAssignmentExpression::Primary(
PhpPrimaryExpression::IntegerLiteral(value), PhpPrimaryExpression::IntegerLiteral(&value.value),
)), )),
)); ));
} }
Expression::Float(value) => { Expression::Float(value) => {
php_statements.push(PhpStatement::PhpExpressionStatement( php_statements.push(PhpStatement::PhpExpressionStatement(
PhpExpression::Assignment(PhpAssignmentExpression::Primary( PhpExpression::Assignment(PhpAssignmentExpression::Primary(
PhpPrimaryExpression::FloatingLiteral(value), PhpPrimaryExpression::FloatingLiteral(&value.value),
)), )),
)); ));
} }
Expression::String(value) => { Expression::String(value) => {
php_statements.push(PhpStatement::PhpExpressionStatement( php_statements.push(PhpStatement::PhpExpressionStatement(
PhpExpression::Assignment(PhpAssignmentExpression::Primary( PhpExpression::Assignment(PhpAssignmentExpression::Primary(
PhpPrimaryExpression::StringLiteral(value), PhpPrimaryExpression::StringLiteral(&value.value),
)), )),
)); ));
} }

View File

@ -58,8 +58,9 @@ mod tests {
value: "name".into(), value: "name".into(),
position: 0, position: 0,
}; };
let expr_value = String::from("Hello");
let expression = Expression::String(&expr_value); let t = Token::new_string("Hello".into(), 0);
let expression = Expression::String(&t);
let binding = Statement::Binding(VariableBinding { let binding = Statement::Binding(VariableBinding {
datatype: None, datatype: None,
identifier: &identifier_token, identifier: &identifier_token,

View File

@ -0,0 +1,223 @@
use crate::{
error_handling::{semantic_error::SemanticError, MistiError},
semantic::{
impls::SemanticCheck,
symbol_table::SymbolTable,
types::{Type, Typed},
},
syntax::ast::{Expression, Positionable},
};
impl SemanticCheck for Expression<'_> {
fn check_semantics(&self, scope: &SymbolTable) -> Result<(), MistiError> {
match self {
Expression::FunctionCall(f) => {
let fun = &*f.function;
let arguments = &*f.arguments.arguments;
let function_datatype = fun.get_type(scope)?;
match function_datatype {
Type::Function(parameters, _return_type) => {
// Check parameters length
if parameters.len() != arguments.len() {
let (error_start, error_end) = f.arguments.get_position();
return Err(MistiError::Semantic(SemanticError {
error_start,
error_end,
reason: format!(
"Expected {} arguments, got {}",
parameters.len(),
arguments.len(),
),
}));
}
// Check that each argument matches the required datatype
for i in 0..parameters.len() {
let parameter = &parameters[i];
let argument = &arguments[i];
let argument_datatype = argument.get_type(scope)?;
if !argument_datatype.is_value(parameter) {
// The argument and the parameter have diferent types
let (error_start, error_end) = argument.get_position();
return Err(MistiError::Semantic(SemanticError {
// TODO: fix
error_start,
error_end,
reason: format!(
"Expected a {}, got {:?}",
parameter, argument_datatype
),
}));
}
}
}
_ => {
let (error_start, error_end) = fun.get_position();
return Err(MistiError::Semantic(SemanticError {
error_start,
error_end,
reason: format!(
"Expected a function type, got {:?}",
function_datatype
),
}));
}
}
}
Expression::Int(_) => {}
Expression::Float(_) => {}
Expression::String(_) => {}
Expression::Boolean(_) => {}
_ => todo!("Check semantics for expression other than function call and primitive"),
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use crate::{
error_handling::MistiError,
lexic::token::Token,
semantic::{impls::SemanticCheck, std::populate, symbol_table::SymbolTable},
syntax::ast::{
functions::{ArgumentsList, FunctionCall},
Expression,
},
};
#[test]
fn should_error_on_undefined_symbol() {
// source code: `print()`
let expr_token = Token::new_identifier("print".into(), 0);
let expr_function = Expression::Identifier(&expr_token);
let arguments = ArgumentsList {
arguments: vec![],
paren_open_pos: 5,
paren_close_pos: 7,
};
let expr = Expression::FunctionCall(FunctionCall {
function: Box::new(expr_function),
arguments: Box::new(arguments),
});
let scope = SymbolTable::new();
let output = expr.check_semantics(&scope);
match output {
Ok(_) => panic!("Expected an error"),
Err(MistiError::Semantic(err)) => {
assert_eq!(err.reason, "Cannot find `print` in this scope.");
assert_eq!(err.error_start, 0);
assert_eq!(err.error_end, 5);
}
Err(e) => panic!("Expected a Semantic error, got {:?}", e),
}
}
#[test]
fn should_error_on_invalid_function_argument() {
// source code: `print(322)`
let mut scope = SymbolTable::new();
populate(&mut scope);
let expr_token = Token::new_identifier("print".into(), 0);
let expr_function = Expression::Identifier(&expr_token);
let arg_t = Token::new_int(String::from("322"), 6);
let arg_1 = Expression::Int(&arg_t);
let arguments = ArgumentsList {
arguments: vec![arg_1],
paren_open_pos: 5,
paren_close_pos: 10,
};
let expr = Expression::FunctionCall(FunctionCall {
function: Box::new(expr_function),
arguments: Box::new(arguments),
});
match expr.check_semantics(&scope) {
Ok(_) => panic!("Expected semantic error, got ok"),
Err(MistiError::Semantic(e)) => {
assert_eq!(e.reason, "Expected a String, got Value(\"Int\")");
assert_eq!(e.error_start, 6);
assert_eq!(e.error_end, 9);
}
Err(e) => panic!("Expected semantic error, got {:?}", e),
}
}
#[test]
fn should_error_on_invalid_function_argument_count() {
// source code: `print()`
let mut scope = SymbolTable::new();
populate(&mut scope);
let expr_token = Token::new_identifier("print".into(), 0);
let expr_function = Expression::Identifier(&expr_token);
let arguments = ArgumentsList {
arguments: vec![],
paren_open_pos: 5,
paren_close_pos: 7,
};
let expr = Expression::FunctionCall(FunctionCall {
function: Box::new(expr_function),
arguments: Box::new(arguments),
});
match expr.check_semantics(&scope) {
Ok(_) => panic!("Expected semantic error, got ok"),
Err(MistiError::Semantic(e)) => {
assert_eq!(e.reason, "Expected 1 arguments, got 0");
assert_eq!(e.error_start, 5);
assert_eq!(e.error_end, 7);
}
Err(e) => panic!("Expected semantic error, got {:?}", e),
}
}
#[test]
fn should_error_on_invalid_function_argument_2() {
// source code: `print(322, 644)`
let mut scope = SymbolTable::new();
populate(&mut scope);
let expr_token = Token::new_identifier("print".into(), 0);
let expr_function = Expression::Identifier(&expr_token);
let arg_t = Token::new_int(String::from("322"), 6);
let arg_1 = Expression::Int(&arg_t);
let arg_t_2 = Token::new_int(String::from("644"), 11);
let arg_2 = Expression::Int(&arg_t_2);
let arguments = ArgumentsList {
arguments: vec![arg_1, arg_2],
paren_open_pos: 5,
paren_close_pos: 15,
};
let expr = Expression::FunctionCall(FunctionCall {
function: Box::new(expr_function),
arguments: Box::new(arguments),
});
match expr.check_semantics(&scope) {
Ok(_) => panic!("Expected semantic error, got ok"),
Err(MistiError::Semantic(e)) => {
assert_eq!(e.reason, "Expected 1 arguments, got 2");
assert_eq!(e.error_start, 5);
assert_eq!(e.error_end, 15);
}
Err(e) => panic!("Expected semantic error, got {:?}", e),
}
}
}

View File

@ -1,3 +1,4 @@
pub mod binding; pub mod binding;
pub mod expression;
pub mod function_declaration; pub mod function_declaration;
pub mod top_level_declaration; pub mod top_level_declaration;

View File

@ -29,72 +29,3 @@ impl SemanticCheck for Statement<'_> {
} }
} }
} }
// TODO: Move to its own file when it grows
impl SemanticCheck for Expression<'_> {
fn check_semantics(&self, scope: &SymbolTable) -> Result<(), MistiError> {
match self {
Expression::FunctionCall(f) => {
let fun = &*f.function;
let arguments = &*f.arguments.arguments;
let function_datatype = fun.get_type(scope)?;
match function_datatype {
Type::Function(parameters, _return_type) => {
// Check parameters length
if parameters.len() != arguments.len() {
return Err(MistiError::Semantic(SemanticError {
// TODO: fix
error_start: 0,
error_end: 1,
reason: format!(
"Expected {} arguments, found {}",
parameters.len(),
arguments.len(),
),
}));
}
// Check that each argument matches the required datatype
for i in 0..parameters.len() {
let parameter = &parameters[i];
let argument = &arguments[i];
let argument_datatype = argument.get_type(scope)?;
if !argument_datatype.is_value(parameter) {
// The argument and the parameter have diferent types
return Err(MistiError::Semantic(SemanticError {
// TODO: fix
error_start: 0,
error_end: 1,
reason: format!(
"Expected datatype {}, got {:?}",
parameter, argument
),
}));
}
}
}
_ => {
return Err(MistiError::Semantic(SemanticError {
// TODO: fix
error_start: 0,
error_end: 1,
reason: format!(
"Expected a function type, got {:?}",
function_datatype
),
}));
}
}
}
Expression::Int(_) => {}
Expression::Float(_) => {}
Expression::String(_) => {}
Expression::Boolean(_) => {}
_ => todo!("Check semantics for expression other than function call and primitive"),
}
Ok(())
}
}

View File

@ -16,13 +16,13 @@ impl Typed for Expression<'_> {
Expression::Boolean(_) => Ok(Type::Value("Bool".into())), Expression::Boolean(_) => Ok(Type::Value("Bool".into())),
Expression::Identifier(identifier) => { Expression::Identifier(identifier) => {
// Attempt to get the datatype of the identifier in the current scope // Attempt to get the datatype of the identifier in the current scope
let datatype = match scope.get_type(identifier) { let datatype = match scope.get_type(&identifier.value) {
Some(x) => x, Some(x) => x,
None => { None => {
return Err(MistiError::Semantic(SemanticError { return Err(MistiError::Semantic(SemanticError {
error_start: 0, error_start: identifier.position,
error_end: 1, error_end: identifier.get_end_position(),
reason: format!("The identifier {} does not exist.", identifier), reason: format!("Cannot find `{}` in this scope.", identifier.value),
})) }))
} }
}; };
@ -35,17 +35,26 @@ impl Typed for Expression<'_> {
// for this to work with any arbitrary expression. // for this to work with any arbitrary expression.
// for now it justs expects an identifier // for now it justs expects an identifier
// TODO: Should this check that the type signature is correct?
// or is this done elsewhere?
match &*f.function { match &*f.function {
Expression::Identifier(id) => { Expression::Identifier(id) => {
match scope.get_type(id) { match scope.get_type(&id.value) {
Some(t) => Ok(t), Some(Type::Function(_, return_type)) => {
// Return the return type of the function,
// not the function itself
Ok(Type::Value(return_type))
}
Some(_) => Err(MistiError::Semantic(SemanticError {
error_start: id.position,
error_end: id.get_end_position(),
reason: format!("Expected `{}` to be a function", &id.value),
})),
None => Err(MistiError::Semantic(SemanticError { None => Err(MistiError::Semantic(SemanticError {
// TODO: Actually find the start and end position error_start: id.position,
// this requires the token to be stored, rather than error_end: id.get_end_position(),
// just the string value reason: format!("Cannot find `{}` in this scope.", id.value),
error_start: 0,
error_end: 1,
reason: format!("Type not found for symbol {}", id),
})), })),
} }
} }
@ -114,3 +123,143 @@ impl Typed for Expression<'_> {
} }
} }
} }
#[cfg(test)]
mod tests {
use crate::{
error_handling::MistiError,
lexic::token::Token,
semantic::{
std::populate,
symbol_table::SymbolTable,
types::{Type, Typed},
},
syntax::ast::{
functions::{ArgumentsList, FunctionCall},
Expression,
},
};
#[test]
fn should_get_global_print_type() {
let mut scope = SymbolTable::new();
populate(&mut scope);
let identifier_token = Token::new_identifier("print".into(), 0);
let expr = Expression::Identifier(&identifier_token);
match expr.get_type(&scope) {
Ok(Type::Function(params, return_type)) => {
assert_eq!(params.len(), 1);
assert_eq!(params[0], "String");
assert_eq!(return_type, "Void");
}
Ok(t) => panic!("Expected a Function, got {:?}", t),
Err(e) => panic!("Expected Ok, got Err: {:?}", e),
}
}
#[test]
fn should_error_on_invalid_identifier() {
let identifier_token = Token::new_identifier("print".into(), 0);
let expr = Expression::Identifier(&identifier_token);
let scope = SymbolTable::new();
let expr_type = expr.get_type(&scope);
match expr_type {
Ok(_) => panic!("Expected an error"),
Err(MistiError::Semantic(err)) => {
assert_eq!(err.error_start, 0);
assert_eq!(err.error_end, 5);
assert_eq!(err.reason, "Cannot find `print` in this scope.");
}
Err(e) => panic!("Expected a semantic error, got {:?}", e),
}
}
#[test]
fn should_get_type_from_function_call() {
let mut scope = SymbolTable::new();
populate(&mut scope);
let id_token = Token::new_identifier("print".into(), 0);
let fn_expr = Expression::Identifier(&id_token);
let args = ArgumentsList {
arguments: vec![],
paren_open_pos: 5,
paren_close_pos: 7,
};
let fn_call = Expression::FunctionCall(FunctionCall {
function: Box::new(fn_expr),
arguments: Box::new(args),
});
match fn_call.get_type(&scope) {
Ok(Type::Value(v)) => assert_eq!(v, "Void"),
Ok(v) => panic!("Expected a value, got {:?}", v),
Err(e) => panic!("Expected a value, got Err {:?}", e),
}
}
#[test]
fn should_fail_if_a_function_is_expected() {
let scope = SymbolTable::new();
// Add `print` as a Int
scope.insert("print".into(), Type::Value("Int".into()));
let id_token = Token::new_identifier("print".into(), 0);
let fn_expr = Expression::Identifier(&id_token);
let args = ArgumentsList {
arguments: vec![],
paren_open_pos: 5,
paren_close_pos: 7,
};
let fn_call = Expression::FunctionCall(FunctionCall {
function: Box::new(fn_expr),
arguments: Box::new(args),
});
match fn_call.get_type(&scope) {
Ok(v) => panic!("Expected an error, got {:?}", v),
Err(MistiError::Semantic(e)) => {
assert_eq!(e.error_start, 0);
assert_eq!(e.error_end, 5);
assert_eq!(e.reason, "Expected `print` to be a function");
}
Err(e) => panic!("Expected a semantic error, got {:?}", e),
}
}
#[test]
fn should_fail_if_a_function_is_not_defined() {
let scope = SymbolTable::new();
let id_token = Token::new_identifier("print".into(), 0);
let fn_expr = Expression::Identifier(&id_token);
let args = ArgumentsList {
arguments: vec![],
paren_open_pos: 5,
paren_close_pos: 7,
};
let fn_call = Expression::FunctionCall(FunctionCall {
function: Box::new(fn_expr),
arguments: Box::new(args),
});
match fn_call.get_type(&scope) {
Ok(v) => panic!("Expected an error, got {:?}", v),
Err(MistiError::Semantic(e)) => {
assert_eq!(e.error_start, 0);
assert_eq!(e.error_end, 5);
assert_eq!(e.reason, "Cannot find `print` in this scope.");
}
Err(e) => panic!("Expected a semantic error, got {:?}", e),
}
}
}

View File

@ -1,4 +1,4 @@
use super::Expression; use super::{Expression, Positionable};
#[derive(Debug)] #[derive(Debug)]
pub struct FunctionCall<'a> { pub struct FunctionCall<'a> {
@ -9,4 +9,13 @@ pub struct FunctionCall<'a> {
#[derive(Debug)] #[derive(Debug)]
pub struct ArgumentsList<'a> { pub struct ArgumentsList<'a> {
pub arguments: Vec<Expression<'a>>, pub arguments: Vec<Expression<'a>>,
pub paren_open_pos: usize,
/// This is after the paren is closed
pub paren_close_pos: usize,
}
impl Positionable for ArgumentsList<'_> {
fn get_position(&self) -> (usize, usize) {
(self.paren_open_pos, self.paren_close_pos)
}
} }

View File

@ -6,6 +6,12 @@ use var_binding::VariableBinding;
pub mod functions; pub mod functions;
pub mod var_binding; pub mod var_binding;
/// Trait that allows nodes to inform
/// on where they start and end on the source code
pub trait Positionable {
fn get_position(&self) -> (usize, usize);
}
/// The AST for a whole THP file /// The AST for a whole THP file
#[derive(Debug)] #[derive(Debug)]
pub struct ModuleAST<'a> { pub struct ModuleAST<'a> {
@ -60,12 +66,29 @@ pub struct Parameter<'a> {
#[derive(Debug)] #[derive(Debug)]
pub enum Expression<'a> { pub enum Expression<'a> {
Int(&'a String), Int(&'a Token),
Float(&'a String), Float(&'a Token),
String(&'a String), String(&'a Token),
Boolean(bool), Boolean(&'a Token),
Identifier(&'a String), Identifier(&'a Token),
FunctionCall(FunctionCall<'a>), FunctionCall(FunctionCall<'a>),
UnaryOperator(&'a String, Box<Expression<'a>>), UnaryOperator(&'a String, Box<Expression<'a>>),
BinaryOperator(Box<Expression<'a>>, Box<Expression<'a>>, &'a String), BinaryOperator(Box<Expression<'a>>, Box<Expression<'a>>, &'a String),
} }
impl Positionable for Expression<'_> {
/// Returns the absolute start and end position
/// of this expression
fn get_position(&self) -> (usize, usize) {
match self {
Expression::Identifier(id) => (id.position, id.get_end_position()),
Expression::Int(id) => (id.position, id.get_end_position()),
Expression::Float(id) => (id.position, id.get_end_position()),
Expression::String(id) => (id.position, id.get_end_position()),
Expression::Boolean(id) => (id.position, id.get_end_position()),
Expression::FunctionCall(_) => (0, 1),
Expression::UnaryOperator(_, _) => (0, 1),
Expression::BinaryOperator(_, _, _) => (0, 1),
}
}
}

View File

@ -53,7 +53,7 @@ pub fn try_parse<'a>(tokens: &'a Vec<Token>, pos: usize) -> ParsingResult<Argume
} }
// Parse closing paren // Parse closing paren
let (_closing_paren, next_pos) = let (closing_paren, next_pos) =
match parse_token_type(tokens, current_pos, TokenType::RightParen) { match parse_token_type(tokens, current_pos, TokenType::RightParen) {
Ok((t, next)) => (t, next), Ok((t, next)) => (t, next),
Err(ParsingError::Err(err)) => return Err(ParsingError::Err(err)), Err(ParsingError::Err(err)) => return Err(ParsingError::Err(err)),
@ -74,7 +74,14 @@ pub fn try_parse<'a>(tokens: &'a Vec<Token>, pos: usize) -> ParsingResult<Argume
}; };
current_pos = next_pos; current_pos = next_pos;
Ok((ArgumentsList { arguments }, current_pos)) Ok((
ArgumentsList {
arguments,
paren_open_pos: opening_paren.position,
paren_close_pos: closing_paren.get_end_position(),
},
current_pos,
))
} }
#[cfg(test)] #[cfg(test)]

View File

@ -62,8 +62,8 @@ mod tests {
Expression::BinaryOperator(exp1, exp2, op) => { Expression::BinaryOperator(exp1, exp2, op) => {
match (*exp1, *exp2) { match (*exp1, *exp2) {
(Expression::Identifier(id1), Expression::Identifier(id2)) => { (Expression::Identifier(id1), Expression::Identifier(id2)) => {
assert_eq!("a", id1); assert_eq!("a", id1.value);
assert_eq!("b", id2); assert_eq!("b", id2.value);
} }
_ => panic!("Expected 2 identifiers"), _ => panic!("Expected 2 identifiers"),
} }

View File

@ -61,8 +61,8 @@ mod tests {
Expression::BinaryOperator(exp1, exp2, op) => { Expression::BinaryOperator(exp1, exp2, op) => {
match (*exp1, *exp2) { match (*exp1, *exp2) {
(Expression::Identifier(id1), Expression::Identifier(id2)) => { (Expression::Identifier(id1), Expression::Identifier(id2)) => {
assert_eq!("a", id1); assert_eq!("a", id1.value);
assert_eq!("b", id2); assert_eq!("b", id2.value);
} }
_ => panic!("Expected 2 identifiers"), _ => panic!("Expected 2 identifiers"),
} }

View File

@ -65,8 +65,8 @@ mod tests {
Expression::BinaryOperator(exp1, exp2, op) => { Expression::BinaryOperator(exp1, exp2, op) => {
match (*exp1, *exp2) { match (*exp1, *exp2) {
(Expression::Identifier(id1), Expression::Identifier(id2)) => { (Expression::Identifier(id1), Expression::Identifier(id2)) => {
assert_eq!("a", id1); assert_eq!("a", id1.value);
assert_eq!("b", id2); assert_eq!("b", id2.value);
} }
_ => panic!("Expected 2 identifiers"), _ => panic!("Expected 2 identifiers"),
} }

View File

@ -13,13 +13,13 @@ use crate::{
pub fn try_parse(tokens: &Vec<Token>, pos: usize) -> ParsingResult<Expression> { pub fn try_parse(tokens: &Vec<Token>, pos: usize) -> ParsingResult<Expression> {
match tokens.get_significant(pos) { match tokens.get_significant(pos) {
Some((token, token_pos)) => match token.token_type { Some((token, token_pos)) => match token.token_type {
TokenType::Int => Ok((Expression::Int(&token.value), token_pos + 1)), TokenType::Int => Ok((Expression::Int(&token), token_pos + 1)),
TokenType::Float => Ok((Expression::Float(&token.value), token_pos + 1)), TokenType::Float => Ok((Expression::Float(&token), token_pos + 1)),
TokenType::String => Ok((Expression::String(&token.value), token_pos + 1)), TokenType::String => Ok((Expression::String(&token), token_pos + 1)),
TokenType::Identifier if token.value == "true" || token.value == "false" => { TokenType::Identifier if token.value == "true" || token.value == "false" => {
Ok((Expression::Boolean(token.value == "true"), token_pos + 1)) Ok((Expression::Boolean(&token), token_pos + 1))
} }
TokenType::Identifier => Ok((Expression::Identifier(&token.value), token_pos + 1)), TokenType::Identifier => Ok((Expression::Identifier(&token), token_pos + 1)),
TokenType::LeftParen => parse_parenthesized_expression(tokens, token_pos), TokenType::LeftParen => parse_parenthesized_expression(tokens, token_pos),
_ => Err(ParsingError::Unmatched), _ => Err(ParsingError::Unmatched),
}, },
@ -53,7 +53,7 @@ mod tests {
match expression { match expression {
Ok((Expression::Int(value), _)) => { Ok((Expression::Int(value), _)) => {
assert_eq!("40", format!("{}", value)) assert_eq!("40", format!("{}", value.value))
} }
_ => panic!(), _ => panic!(),
} }
@ -66,7 +66,7 @@ mod tests {
match expression { match expression {
Ok((Expression::String(value), _)) => { Ok((Expression::String(value), _)) => {
assert_eq!("Hello", format!("{}", value)) assert_eq!("Hello", format!("{}", value.value))
} }
_ => panic!(), _ => panic!(),
} }
@ -78,7 +78,7 @@ mod tests {
let expression = try_parse(&tokens, 0); let expression = try_parse(&tokens, 0);
match expression { match expression {
Ok((Expression::Boolean(value), _)) => assert!(value), Ok((Expression::Boolean(value), _)) => assert_eq!(value.value, "true"),
_ => panic!(), _ => panic!(),
} }
} }
@ -90,7 +90,7 @@ mod tests {
match expression { match expression {
Ok((Expression::Identifier(value), _)) => { Ok((Expression::Identifier(value), _)) => {
assert_eq!("someIdentifier", format!("{}", value)) assert_eq!("someIdentifier", format!("{}", value.value))
} }
_ => panic!(), _ => panic!(),
} }
@ -103,7 +103,7 @@ mod tests {
match expression { match expression {
Ok((Expression::Identifier(value), _)) => { Ok((Expression::Identifier(value), _)) => {
assert_eq!("identifier", format!("{}", value)) assert_eq!("identifier", format!("{}", value.value))
} }
_ => panic!(), _ => panic!(),
} }

View File

@ -66,8 +66,8 @@ mod tests {
Expression::BinaryOperator(exp1, exp2, op) => { Expression::BinaryOperator(exp1, exp2, op) => {
match (*exp1, *exp2) { match (*exp1, *exp2) {
(Expression::Identifier(id1), Expression::Identifier(id2)) => { (Expression::Identifier(id1), Expression::Identifier(id2)) => {
assert_eq!("a", id1); assert_eq!("a", id1.value);
assert_eq!("b", id2); assert_eq!("b", id2.value);
} }
_ => panic!("Expected 2 identifiers"), _ => panic!("Expected 2 identifiers"),
} }

View File

@ -38,7 +38,7 @@ mod tests {
match expression { match expression {
Ok((Expression::Identifier(value), _)) => { Ok((Expression::Identifier(value), _)) => {
assert_eq!("identifier", format!("{}", value)) assert_eq!("identifier", format!("{}", value.value))
} }
_ => panic!(), _ => panic!(),
} }
@ -54,7 +54,7 @@ mod tests {
match (operator, *expression) { match (operator, *expression) {
(op, Expression::Int(value)) => { (op, Expression::Int(value)) => {
assert_eq!(*op, "-"); assert_eq!(*op, "-");
assert_eq!(*value, "10"); assert_eq!(value.value, "10");
} }
_ => panic!("unexpected values"), _ => panic!("unexpected values"),
} }

View File

@ -11,13 +11,15 @@ impl Tokenizer for Vec<Token> {
fn get_significant<'a>(&'a self, index: usize) -> Option<(&'a Token, usize)> { fn get_significant<'a>(&'a self, index: usize) -> Option<(&'a Token, usize)> {
let mut current_pos = index; let mut current_pos = index;
// Ignore all whitespace and newlines // Ignore all whitespace, newlines and comments
loop { loop {
match self.get(current_pos) { match self.get(current_pos) {
Some(token) => { Some(token) => {
if token.token_type == TokenType::INDENT if token.token_type == TokenType::INDENT
|| token.token_type == TokenType::DEDENT || token.token_type == TokenType::DEDENT
|| token.token_type == TokenType::NewLine || token.token_type == TokenType::NewLine
|| token.token_type == TokenType::Comment
|| token.token_type == TokenType::MultilineComment
{ {
current_pos += 1; current_pos += 1;
} else { } else {
@ -44,7 +46,7 @@ pub fn try_operator(tokens: &Vec<Token>, pos: usize, operator: String) -> Parsin
/// Expects the token at `pos` to be of type `token_type`, and returns the token and the next position. /// Expects the token at `pos` to be of type `token_type`, and returns the token and the next position.
/// ///
/// Ignores all whitespace and newlines. /// Ignores all whitespace, newlines and comments.
/// ///
/// Only returns: Ok, Unmatched, Mismatched /// Only returns: Ok, Unmatched, Mismatched
pub fn parse_token_type( pub fn parse_token_type(
@ -59,6 +61,8 @@ pub fn parse_token_type(
if t.token_type == TokenType::INDENT if t.token_type == TokenType::INDENT
|| t.token_type == TokenType::DEDENT || t.token_type == TokenType::DEDENT
|| t.token_type == TokenType::NewLine || t.token_type == TokenType::NewLine
|| t.token_type == TokenType::Comment
|| t.token_type == TokenType::MultilineComment
{ {
current_pos += 1; current_pos += 1;
} else { } else {