CHANGELOG.md

@@ -26,7 +26,7 @@
 ## v0.0.15
 
 - [x] Multiline comments
-- [x] Nested multiline comments
+- [ ] Nested multiline comments
 - [ ] Include comments in the AST
 - [ ] Replace all panics with actual errors
 - [ ] Remove all old codegen

src/codegen/binding.rs

@@ -0,0 +1,41 @@
+use super::Transpilable;
+use crate::syntax::ast::var_binding::VariableBinding;
+
+impl Transpilable for VariableBinding<'_> {
+    /// Transpiles val and var bindings into PHP.
+    fn transpile(&self) -> String {
+        let expression_str = self.expression.transpile();
+
+        format!("${} = {}", self.identifier.value, expression_str)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        lexic::token::{Token, TokenType},
+        syntax::ast::{var_binding::VariableBinding, Expression},
+    };
+
+    #[test]
+    fn binding_should_transpile() {
+        let id = String::from("identifier");
+        let id_token = Token {
+            token_type: TokenType::Identifier,
+            value: id,
+            position: 0,
+        };
+        let value = String::from("322");
+        let binding = VariableBinding {
+            datatype: None,
+            identifier: &id_token,
+            expression: Expression::Int(&value),
+            is_mutable: false,
+        };
+
+        let result = binding.transpile();
+
+        assert_eq!("$identifier = 322", result);
+    }
+}

src/codegen/block.rs

@@ -0,0 +1,17 @@
+use crate::syntax::ast::Block;
+
+use super::Transpilable;
+
+impl Transpilable for Block<'_> {
+    fn transpile(&self) -> String {
+        // TODO: Handle indentation
+        todo!("transpilation for block");
+        /*
+        self.members
+            .iter()
+            .map(|x| x.transpile())
+            .collect::<Vec<_>>()
+            .join("\n")
+            */
+    }
+}

src/codegen/expression.rs

@@ -0,0 +1,76 @@
+use super::Transpilable;
+use crate::syntax::ast::Expression;
+
+impl Transpilable for Expression<'_> {
+    /// Transpiles an Expression to PHP
+    ///
+    /// Right now the expressions in the grammar are:
+    /// - Number
+    /// - String
+    /// - Boolean
+    /// - Identifier
+    fn transpile(&self) -> String {
+        match self {
+            Expression::Int(value) => format!("{}", value),
+            Expression::Float(value) => format!("{}", value),
+            Expression::String(value) => {
+                format!("{}", *value)
+            }
+            Expression::Boolean(value) => String::from(if *value { "true" } else { "false" }),
+            Expression::Identifier(value) => format!("{}", *value),
+            Expression::FunctionCall(f) => f.transpile(),
+            Expression::BinaryOperator(left_expr, right_expr, operator) => {
+                format!(
+                    "{}{}{}",
+                    left_expr.transpile(),
+                    operator,
+                    right_expr.transpile()
+                )
+            }
+            Expression::UnaryOperator(operator, expression) => {
+                format!("{}{}", operator, expression.transpile())
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::syntax::ast::Expression;
+
+    #[test]
+    fn should_transpile_number() {
+        let str = String::from("42");
+        let exp = Expression::Int(&str);
+        let result = exp.transpile();
+
+        assert_eq!("42", result);
+    }
+
+    #[test]
+    fn should_transpile_string() {
+        let str = String::from("\"Hello world\"");
+        let exp = Expression::String(&str);
+        let result = exp.transpile();
+
+        assert_eq!("\"Hello world\"", result);
+    }
+
+    #[test]
+    fn should_transpile_boolean() {
+        let exp = Expression::Boolean(true);
+        let result = exp.transpile();
+
+        assert_eq!("true", result);
+    }
+
+    #[test]
+    fn should_transpile_identifier() {
+        let s = String::from("newValue");
+        let exp = Expression::Identifier(&s);
+        let result = exp.transpile();
+
+        assert_eq!("newValue", result);
+    }
+}

src/codegen/function_call.rs

@@ -0,0 +1,17 @@
+use crate::syntax::ast::functions::FunctionCall;
+
+use super::Transpilable;
+
+impl Transpilable for FunctionCall<'_> {
+    fn transpile(&self) -> String {
+        let parameters = &self
+            .arguments
+            .arguments
+            .iter()
+            .map(|expr| expr.transpile())
+            .collect::<Vec<_>>()
+            .join(", ");
+
+        format!("{}({})", self.function.transpile(), parameters)
+    }
+}

src/codegen/function_declaration.rs

@@ -0,0 +1,43 @@
+use crate::syntax::ast::FunctionDeclaration;
+
+use super::Transpilable;
+
+impl Transpilable for FunctionDeclaration<'_> {
+    fn transpile(&self) -> String {
+        format!(
+            "function {}() {{\n{}\n}}",
+            self.identifier.value,
+            self.block.transpile()
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        lexic::get_tokens,
+        syntax::{
+            ast::{ModuleMembers, Statement},
+            build_ast,
+        },
+    };
+
+    /* TODO: reimplement
+    #[test]
+    fn should_transpile() {
+        let tokens = get_tokens(&String::from("fun id() {}")).unwrap();
+        let result = build_ast(&tokens).unwrap();
+
+        let fun_dec = result.productions.get(0).unwrap();
+
+        match fun_dec {
+            ModuleMembers::Stmt(Statement::FnDecl(fun_decl)) => {
+                let transpiled = fun_decl.transpile();
+
+                assert_eq!("function id() {\n\n}", transpiled);
+            }
+            _ => panic!("Expected a function declaration"),
+        }
+    }*/
+}

src/codegen/mod.rs

@@ -1,5 +1,13 @@
 // TODO: These are for the THP AST. Eventually replace this
 // with the PHP AST
+mod binding;
+mod block;
+mod expression;
+mod function_call;
+mod function_declaration;
+mod module_ast;
+mod statement;
+mod top_level_construct;
 
 mod php;
 

src/codegen/module_ast.rs

@@ -0,0 +1,50 @@
+use super::Transpilable;
+use crate::syntax::ast::ModuleAST;
+
+impl Transpilable for ModuleAST<'_> {
+    /// Transpiles the whole AST into PHP, using this same trait on the
+    /// nodes and leaves of the AST
+    fn transpile(&self) -> String {
+        let bindings_str: Vec<String> = self
+            .productions
+            .iter()
+            .map(|binding| binding.transpile())
+            .collect();
+
+        format!("<?php\n\n{}\n", bindings_str.join("\n"))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{
+        lexic::token::{Token, TokenType},
+        syntax::ast::{var_binding::VariableBinding, Expression, ModuleMembers, Statement},
+    };
+
+    #[test]
+    fn module_ast_should_transpile() {
+        let id = String::from("identifier");
+        let id_token = Token {
+            token_type: TokenType::Identifier,
+            value: id,
+            position: 0,
+        };
+        let value = String::from("322");
+        let binding = VariableBinding {
+            datatype: None,
+            identifier: &id_token,
+            expression: Expression::Int(&value),
+            is_mutable: false,
+        };
+
+        let module = ModuleAST {
+            productions: vec![ModuleMembers::Stmt(Statement::Binding(binding))],
+        };
+
+        let result = module.transpile();
+
+        assert_eq!("<?php\n\n$identifier = 322\n", result);
+    }
+}

src/codegen/php/expression/mod.rs

@@ -1 +0,0 @@
-mod primary_expression;

src/codegen/php/expression/primary_expression.rs

@@ -1,61 +0,0 @@
-use crate::{codegen::Transpilable, php_ast::PhpPrimaryExpression};
-
-impl Transpilable for PhpPrimaryExpression<'_> {
-    fn transpile(&self) -> String {
-        match self {
-            PhpPrimaryExpression::IntegerLiteral(value) => value.to_string(),
-            PhpPrimaryExpression::FloatingLiteral(value) => value.to_string(),
-            PhpPrimaryExpression::StringLiteral(value) => format!("\"{}\"", value),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::{codegen::Transpilable, php_ast::PhpPrimaryExpression};
-
-    #[test]
-    fn should_transpile_empty_string() {
-        let input = String::from("");
-        let ast = PhpPrimaryExpression::StringLiteral(&input);
-        let output = ast.transpile();
-
-        assert_eq!("\"\"", output)
-    }
-
-    #[test]
-    fn should_transpile_string() {
-        let input = String::from("abc");
-        let ast = PhpPrimaryExpression::StringLiteral(&input);
-        let output = ast.transpile();
-
-        assert_eq!("\"abc\"", output)
-    }
-
-    #[test]
-    fn should_transpile_string_with_quotes() {
-        let input = String::from("a\\\"b\\\"c");
-        let ast = PhpPrimaryExpression::StringLiteral(&input);
-        let output = ast.transpile();
-
-        assert_eq!("\"a\\\"b\\\"c\"", output)
-    }
-
-    #[test]
-    fn should_transpile_int() {
-        let input = String::from("322");
-        let ast = PhpPrimaryExpression::IntegerLiteral(&input);
-        let output = ast.transpile();
-
-        assert_eq!("322", output)
-    }
-
-    #[test]
-    fn should_transpile_floating() {
-        let input = String::from("322.644");
-        let ast = PhpPrimaryExpression::FloatingLiteral(&input);
-        let output = ast.transpile();
-
-        assert_eq!("322.644", output)
-    }
-}

src/codegen/php/mod.rs

@@ -1,14 +1,50 @@
-use super::Transpilable;
+use std::os::linux::raw::stat;
-use crate::php_ast::PhpExpression;
 
-pub mod statement;
+use crate::php_ast::{PhpAst, PhpExpression, PhpStatement};
-pub mod statement_list;
+
-mod expression;
+use super::Transpilable;
+
+impl Transpilable for PhpAst<'_> {
+    fn transpile(&self) -> String {
+        let mut fragments = vec![String::from("<?php\n")];
+        
+        for statement in self.statements.iter() {
+            fragments.push(statement.transpile());
+        }
+
+        fragments.join("")
+    }
+}
+
+impl Transpilable for PhpStatement<'_> {
+    fn transpile(&self) -> String {
+        match self {
+            PhpStatement::PhpEchoStatement(expr_list) => {
+                let expressions_vec = expr_list.expressions
+                    .iter()
+                    .map(|e| e.transpile())
+                    .collect::<Vec<_>>();
+
+                let expressions_str = if expressions_vec.is_empty() {
+                    "\"\"".into()
+                } else {
+                    expressions_vec.join(", ")
+                };
+
+                format!("echo {};", expressions_str)
+            }
+        }
+    }
+}
 
 impl Transpilable for PhpExpression<'_> {
     fn transpile(&self) -> String {
         match self {
-            PhpExpression::PrimaryExpression(p) => p.transpile(),
+            PhpExpression::String(value) => {
+                format!("{}", value)
+            }
         }
     }
 }
+
+

src/codegen/php/statement/mod.rs

@@ -1,77 +0,0 @@
-use crate::{codegen::Transpilable, php_ast::PhpStatement};
-
-mod echo_statement;
-
-impl Transpilable for PhpStatement<'_> {
-    fn transpile(&self) -> String {
-        match self {
-            PhpStatement::PhpEchoStatement(expr_list) => {
-                let expressions_vec = expr_list
-                    .expressions
-                    .iter()
-                    .map(|e| e.transpile())
-                    .collect::<Vec<_>>();
-
-                let expressions_str = if expressions_vec.is_empty() {
-                    "\"\"".into()
-                } else {
-                    expressions_vec.join(", ")
-                };
-
-                format!("echo {};", expressions_str)
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::{
-        codegen::Transpilable,
-        php_ast::{PhpExpression, PhpExpressionList, PhpPrimaryExpression, PhpStatement},
-    };
-
-    #[test]
-    fn should_gen_empty_echo_statement() {
-        let expressions = PhpExpressionList {
-            expressions: vec![],
-        };
-        let ast = PhpStatement::PhpEchoStatement(expressions);
-        let output = ast.transpile();
-
-        assert_eq!("echo \"\";", output)
-    }
-
-    #[test]
-    fn should_gen_echo_with_expr() {
-        let input = String::from("322");
-        let exp_1 = PhpPrimaryExpression::FloatingLiteral(&input);
-        let expressions = PhpExpressionList {
-            expressions: vec![PhpExpression::PrimaryExpression(exp_1)],
-        };
-        let ast = PhpStatement::PhpEchoStatement(expressions);
-        let output = ast.transpile();
-
-        assert_eq!("echo 322;", output)
-    }
-
-    #[test]
-    fn should_gen_echo_with_multiple_expr() {
-        let input = String::from("322");
-        let exp_1 = PhpPrimaryExpression::FloatingLiteral(&input);
-
-        let input = String::from("Hai world");
-        let exp_2 = PhpPrimaryExpression::StringLiteral(&input);
-
-        let expressions = PhpExpressionList {
-            expressions: vec![
-                PhpExpression::PrimaryExpression(exp_1),
-                PhpExpression::PrimaryExpression(exp_2),
-            ],
-        };
-        let ast = PhpStatement::PhpEchoStatement(expressions);
-        let output = ast.transpile();
-
-        assert_eq!("echo 322, \"Hai world\";", output)
-    }
-}

src/codegen/php/statement_list.rs

@@ -1,27 +0,0 @@
-use crate::{codegen::Transpilable, php_ast::PhpAst};
-
-impl Transpilable for PhpAst<'_> {
-    fn transpile(&self) -> String {
-        let mut fragments = vec![String::from("<?php\n")];
-
-        for statement in self.statements.iter() {
-            fragments.push(statement.transpile());
-        }
-
-        fragments.join("")
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::{codegen::Transpilable, php_ast::PhpAst};
-
-    #[test]
-    fn should_transpile_empty_file() {
-        let ast = PhpAst {statements: vec![]};
-        let output = ast.transpile();
-
-        assert_eq!("<?php\n", output);
-    }
-}
-

src/codegen/statement.rs

@@ -0,0 +1,14 @@
+use crate::syntax::ast::Statement;
+
+use super::Transpilable;
+
+impl Transpilable for Statement<'_> {
+    fn transpile(&self) -> String {
+        let stmt = match self {
+            Statement::FnDecl(f) => f.transpile(),
+            Statement::Binding(b) => b.transpile(),
+        };
+
+        format!("{stmt};")
+    }
+}

src/codegen/top_level_construct.rs

@@ -0,0 +1,13 @@
+use crate::syntax::ast::{ModuleMembers, Statement};
+
+use super::Transpilable;
+
+impl Transpilable for ModuleMembers<'_> {
+    fn transpile(&self) -> String {
+        match self {
+            ModuleMembers::Stmt(Statement::Binding(b)) => b.transpile(),
+            ModuleMembers::Stmt(Statement::FnDecl(f)) => f.transpile(),
+            _ => todo!("Not implemented: Transpilable for Expression"),
+        }
+    }
+}

src/error_handling/mod.rs

@@ -22,7 +22,6 @@ pub enum MistiError {
 pub struct LexError {
     pub position: usize,
     // TODO: Add and end position
-    pub end_position: usize,
     pub reason: String,
 }
 

src/file/mod.rs

@@ -101,6 +101,5 @@ fn build_ast(input: &String, tokens: Vec<Token>) -> Result<String, String> {
         }
     };
 
-    Err("Code generation disabled: rewriting into PHP AST".into())
+    Ok(codegen::codegen(&ast))
-    // Ok(codegen::codegen(&ast))
 }

src/lexic/mod.rs

@@ -152,7 +152,6 @@ fn next_token(
         .unwrap_or_else(|| {
             let error = LexError {
                 position: current_pos,
-                end_position: current_pos + 1,
                 reason: format!(
                     "Illegal character `{}` (escaped: {})",
                     next_char,
@@ -197,7 +196,6 @@ fn handle_indentation(
                 // Illegal state: Indentation error
                 let error = LexError {
                     position: current_pos,
-                    end_position: current_pos + 1,
                     reason: format!(
                         "Indentation error: expected {} spaces, found {}",
                         new_top, spaces

src/lexic/scanner/new_comment.rs

@@ -34,42 +34,38 @@ fn scan_any_except_new_line(
 /// and the character at `start_pos + 1` is '*'
 pub fn scan_multiline(chars: &Vec<char>, start_pos: usize) -> LexResult {
     match multiline_impl(chars, start_pos + 2) {
-        Ok((value, next_position)) => LexResult::Some(
+        Some((value, next_position)) => LexResult::Some(
             Token::new_multiline_comment(value.iter().collect(), start_pos),
             next_position,
         ),
-        Err(last_position) => {
+        None => {
             // Throw an error: Incomplete multiline comment
             LexResult::Err(LexError {
                 position: start_pos,
                 // TODO: add an end_position
-                end_position: last_position,
                 reason: "Unfinished multiline commend".into(),
             })
         }
     }
 }
 
-/// Implementation that scans the multiline comment.
+fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Option<(Vec<char>, usize)> {
-/// 
-/// May only error if EOF is found before the comment is finished.
-/// If Err, returns the last position where a char was available.
-fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Result<(Vec<char>, usize), usize> {
     let mut current_position = start_pos;
     let mut result = Vec::<char>::new();
 
     loop {
         match chars.get(current_position) {
             Some('/') => {
+
                 match chars.get(current_position + 1) {
                     Some('*') => {
                         // Scan nested comment
-                        let (mut nested, next_position) =
+                        let (mut nested, next_position) = match multiline_impl(chars, current_position + 2)
-                            match multiline_impl(chars, current_position + 2) {
+                        {
-                                Ok(v) => v,
+                            Some(v) => v,
-                                Err(pos) => {
+                            None => {
                                 // The nested comment is not closed.
-                                    return Err(pos);
+                                return None;
                             }
                         };
                         result.push('/');
@@ -84,7 +80,7 @@ fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Result<(Vec<char>, usi
                         result.push('/');
                         result.push(*c);
                     }
-                    None => return Err(current_position),
+                    None => return None,
                 }
             }
             Some('*') => {
@@ -93,7 +89,7 @@ fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Result<(Vec<char>, usi
                     Some('/') => {
                         // Create and return the token,
                         // ignoring the `*/`
-                        return Ok((result, current_position + 2));
+                        return Some((result, current_position + 2));
                     }
                     Some(c) => {
                         // Append both and continue
@@ -103,7 +99,7 @@ fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Result<(Vec<char>, usi
                     }
                     None => {
                         // Throw an error
-                        return Err(current_position);
+                        return None;
                     }
                 }
             }
@@ -113,7 +109,10 @@ fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Result<(Vec<char>, usi
                 current_position += 1;
             }
             None => {
-                return Err(current_position);
+                // TODO: Also return the position where this token ends,
+                // to display better error messages.
+                // Requires LexError to implement an end_position field
+                return None;
             }
         }
     }

src/lexic/scanner/number.rs

@@ -53,7 +53,6 @@ fn scan_hex(chars: &Vec<char>, start_pos: usize, current: String) -> LexResult {
         }
         _ => LexResult::Err(LexError {
             position: start_pos,
-            end_position: start_pos + 1,
             reason: String::from("Tried to scan an incomplete hex value"),
         }),
     }
@@ -70,14 +69,12 @@ fn scan_double(chars: &Vec<char>, start_pos: usize, current: String) -> LexResul
         Some(c) if utils::is_digit(*c) => scan_double_impl(chars, start_pos, current),
         Some(_) => LexResult::Err(LexError {
             position: start_pos,
-            end_position: start_pos + 1,
             reason: String::from(
                 "The character after the dot when scanning a double is not a number.",
             ),
         }),
         _ => LexResult::Err(LexError {
             position: start_pos,
-            end_position: start_pos + 1,
             reason: String::from("EOF when scanning a double number."),
         }),
     }
@@ -125,7 +122,6 @@ fn scan_scientific(chars: &Vec<char>, start_pos: usize, current: String) -> LexR
         }
         _ => LexResult::Err(LexError {
             position: start_pos,
-            end_position: start_pos + 1,
             reason: String::from(
                 "The characters after 'e' are not + or -, or are not followed by a number",
             ),

src/lexic/scanner/string.rs

@@ -7,11 +7,9 @@ use crate::lexic::{utils, LexResult};
 /// This function assumes that `start_pos` is after the first double quote,
 /// e.g. if the input is `"hello"`, `start_pos == 1`
 pub fn scan(chars: &Vec<char>, start_pos: usize) -> LexResult {
-    scan_impl(chars, start_pos, String::from(""))
+    scan_impl(chars, start_pos, String::from("\""))
 }
 
-// TODO: This can be iterative instead of recursive
-
 /// Recursive function that does the scanning
 pub fn scan_impl(chars: &Vec<char>, start_pos: usize, current: String) -> LexResult {
     match chars.get(start_pos) {
@@ -19,16 +17,16 @@ pub fn scan_impl(chars: &Vec<char>, start_pos: usize, current: String) -> LexRes
             // start_pos is the position where the token ENDS, not where it STARTS,
             // so this is used to retrieve the original START position of the token
             // 1 is added to account for the opening `"`
-            let current_len = current.len() + 1;
+            let current_len = current.len();
 
+            let final_str = format!("{}\"", current);
             LexResult::Some(
-                Token::new_string(current, start_pos - current_len),
+                Token::new_string(final_str, start_pos - current_len),
                 start_pos + 1,
             )
         }
         Some(c) if *c == '\n' => LexResult::Err(LexError {
             position: start_pos,
-            end_position: start_pos + 1,
             reason: String::from("Unexpected new line inside a string."),
         }),
         Some(c) if *c == '\\' => {
@@ -42,7 +40,6 @@ pub fn scan_impl(chars: &Vec<char>, start_pos: usize, current: String) -> LexRes
         Some(c) => scan_impl(chars, start_pos + 1, utils::str_append(current, *c)),
         None => LexResult::Err(LexError {
             position: start_pos,
-            end_position: start_pos + 1,
             reason: String::from("Incomplete string found"),
         }),
     }
@@ -82,7 +79,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(2, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("", token.value);
+            assert_eq!("\"\"", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -96,7 +93,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(15, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("Hello, world!", token.value);
+            assert_eq!("\"Hello, world!\"", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -121,7 +118,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("Sample\\ntext", token.value);
+            assert_eq!("\"Sample\\ntext\"", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -132,7 +129,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("Sample\\\"text", token.value);
+            assert_eq!("\"Sample\\\"text\"", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -143,7 +140,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("Sample\\rtext", token.value);
+            assert_eq!("\"Sample\\rtext\"", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -154,7 +151,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("Sample\\\\text", token.value);
+            assert_eq!("\"Sample\\\\text\"", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -165,7 +162,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("Sample\\ttext", token.value);
+            assert_eq!("\"Sample\\ttext\"", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -176,7 +173,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("Sample\\ text", token.value);
+            assert_eq!("\"Sample\\ text\"", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -190,7 +187,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("Sample\\atext", token.value);
+            assert_eq!("\"Sample\\atext\"", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()

src/lexic/token.rs

@@ -38,15 +38,7 @@ pub struct Token {
 
 impl Token {
     pub fn get_end_position(&self) -> usize {
-        match self.token_type {
+        self.position + self.value.len()
-            // 4 extra characters for /* and */
-            TokenType::MultilineComment => self.position + self.value.len() + 4,
-            // 2 extra characters for //
-            TokenType::Comment => self.position + self.value.len() + 2,
-            // 2 extra characters for ""
-            TokenType::String => self.position + self.value.len() + 2,
-            _ => self.position + self.value.len()
-        }
     }
 }
 

src/php_ast/mod.rs

@@ -1,23 +1,12 @@
-/// This AST implements a subset of the PHP AST as defined
+// Follows https://phplang.org/spec/19-grammar.html#syntactic-grammar
-/// by https://phplang.org/spec/19-grammar.html#syntactic-grammar
-/// 
-/// This subset only includes nodes that can be generated by
-/// THP
 
 pub mod transformers;
 
-/// Represents `statement-list` on the grammar,
+/// Represents `statement-list` on the grammar
-/// and thus a whole PHP source file
 pub struct PhpAst<'a> {
     pub statements: Vec<PhpStatement<'a>>,
 }
 
-/// https://phplang.org/spec/19-grammar.html#grammar-statement
-/// 
-/// Not fully implemented
-/// 
-/// statement:
-///   echo-statement
 pub enum PhpStatement<'a> {
     PhpEchoStatement(PhpExpressionList<'a>),
 }
@@ -27,16 +16,5 @@ pub struct PhpExpressionList<'a> {
 }
 
 pub enum PhpExpression<'a> {
-    PrimaryExpression(PhpPrimaryExpression<'a>),
+    String(&'a String),
 }
-
-/// https://phplang.org/spec/19-grammar.html#grammar-primary-expression
-/// 
-/// primary-expression:
-///     literal
-pub enum PhpPrimaryExpression<'a> {
-    IntegerLiteral(&'a String),
-    FloatingLiteral(&'a String),
-    StringLiteral(&'a String),
-}
-

src/php_ast/transformers/expression.rs

@@ -1,5 +1,5 @@
 use super::super::PhpExpression;
-use crate::{php_ast::PhpPrimaryExpression, syntax::ast::Expression};
+use crate::syntax::ast::Expression;
 
 use super::PHPTransformable;
 
@@ -9,10 +9,7 @@ impl<'a> PHPTransformable<'a> for Expression<'_> {
 
     fn into_php_ast(&'a self) -> Self::Item {
         match self {
-            Expression::String(value) => {
+            Expression::String(value) => PhpExpression::String(value),
-                let expr = PhpPrimaryExpression::StringLiteral(value);
-                PhpExpression::PrimaryExpression(expr)
-            },
             _ => todo!("transformation for expression: {:?}", self),
         }
     }

src/php_ast/transformers/module_ast.rs

@@ -1,5 +1,5 @@
 use super::super::PhpAst;
-use crate::php_ast::{PhpExpression, PhpExpressionList, PhpPrimaryExpression, PhpStatement};
+use crate::php_ast::{PhpExpression, PhpExpressionList, PhpStatement};
 use crate::syntax::ast::{Expression, ModuleAST, ModuleMembers};
 
 use super::PHPTransformable;
@@ -33,9 +33,7 @@ impl<'a> PHPTransformable<'a> for ModuleAST<'_> {
                                     for e in fc.arguments.arguments.iter() {
                                         match e {
                                             Expression::String(v) => {
-                                                expressions.push(
+                                                expressions.push(PhpExpression::String(v))
-                                                    PhpExpression::PrimaryExpression(PhpPrimaryExpression::StringLiteral(v.clone()))
-                                                )
                                             },
                                             _ => panic!("Non string expressions not supported")
                                         }

src/syntax/parsers/expression/primary.rs

@@ -66,7 +66,7 @@ mod tests {
 
         match expression {
             Ok((Expression::String(value), _)) => {
-                assert_eq!("Hello", format!("{}", value))
+                assert_eq!("\"Hello\"", format!("{}", value))
             }
             _ => panic!(),
         }

src/syntax/parsers/module.rs

@@ -110,7 +110,7 @@ mod test {
 
         match result {
             Ok(_) => panic!("Expected an error"),
-            Err(_) => {}
+            Err(_) => {},
         }
     }
 }

src/syntax/utils.rs

@@ -80,10 +80,7 @@ pub fn parse_token_type(
 mod tests {
     use crate::{
         lexic::{get_tokens, token::TokenType},
-        syntax::{
+        syntax::{parseable::ParsingError, utils::{parse_token_type, Tokenizer}},
-            parseable::ParsingError,
-            utils::{parse_token_type, Tokenizer},
-        },
     };
 
     use super::try_operator;
@@ -136,7 +133,7 @@ mod tests {
 
         match tokens.get_significant(10) {
             Some(_) => panic!("Expected a None"),
-            None => {}
+            None => {},
         }
     }
 }