refactor: Dont store quotes inside string tokens

CHANGELOG.md

@@ -26,7 +26,7 @@
 ## v0.0.15
 
 - [x] Multiline comments
-- [ ] Nested multiline comments
+- [x] Nested multiline comments
 - [ ] Include comments in the AST
 - [ ] Replace all panics with actual errors
 - [ ] Remove all old codegen

src/codegen/binding.rs

@@ -1,41 +0,0 @@
-use super::Transpilable;
-use crate::syntax::ast::var_binding::VariableBinding;
-
-impl Transpilable for VariableBinding<'_> {
-    /// Transpiles val and var bindings into PHP.
-    fn transpile(&self) -> String {
-        let expression_str = self.expression.transpile();
-
-        format!("${} = {}", self.identifier.value, expression_str)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        lexic::token::{Token, TokenType},
-        syntax::ast::{var_binding::VariableBinding, Expression},
-    };
-
-    #[test]
-    fn binding_should_transpile() {
-        let id = String::from("identifier");
-        let id_token = Token {
-            token_type: TokenType::Identifier,
-            value: id,
-            position: 0,
-        };
-        let value = String::from("322");
-        let binding = VariableBinding {
-            datatype: None,
-            identifier: &id_token,
-            expression: Expression::Int(&value),
-            is_mutable: false,
-        };
-
-        let result = binding.transpile();
-
-        assert_eq!("$identifier = 322", result);
-    }
-}

src/codegen/block.rs

@@ -1,17 +0,0 @@
-use crate::syntax::ast::Block;
-
-use super::Transpilable;
-
-impl Transpilable for Block<'_> {
-    fn transpile(&self) -> String {
-        // TODO: Handle indentation
-        todo!("transpilation for block");
-        /*
-        self.members
-            .iter()
-            .map(|x| x.transpile())
-            .collect::<Vec<_>>()
-            .join("\n")
-            */
-    }
-}

src/codegen/expression.rs

@@ -1,76 +0,0 @@
-use super::Transpilable;
-use crate::syntax::ast::Expression;
-
-impl Transpilable for Expression<'_> {
-    /// Transpiles an Expression to PHP
-    ///
-    /// Right now the expressions in the grammar are:
-    /// - Number
-    /// - String
-    /// - Boolean
-    /// - Identifier
-    fn transpile(&self) -> String {
-        match self {
-            Expression::Int(value) => format!("{}", value),
-            Expression::Float(value) => format!("{}", value),
-            Expression::String(value) => {
-                format!("{}", *value)
-            }
-            Expression::Boolean(value) => String::from(if *value { "true" } else { "false" }),
-            Expression::Identifier(value) => format!("{}", *value),
-            Expression::FunctionCall(f) => f.transpile(),
-            Expression::BinaryOperator(left_expr, right_expr, operator) => {
-                format!(
-                    "{}{}{}",
-                    left_expr.transpile(),
-                    operator,
-                    right_expr.transpile()
-                )
-            }
-            Expression::UnaryOperator(operator, expression) => {
-                format!("{}{}", operator, expression.transpile())
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::syntax::ast::Expression;
-
-    #[test]
-    fn should_transpile_number() {
-        let str = String::from("42");
-        let exp = Expression::Int(&str);
-        let result = exp.transpile();
-
-        assert_eq!("42", result);
-    }
-
-    #[test]
-    fn should_transpile_string() {
-        let str = String::from("\"Hello world\"");
-        let exp = Expression::String(&str);
-        let result = exp.transpile();
-
-        assert_eq!("\"Hello world\"", result);
-    }
-
-    #[test]
-    fn should_transpile_boolean() {
-        let exp = Expression::Boolean(true);
-        let result = exp.transpile();
-
-        assert_eq!("true", result);
-    }
-
-    #[test]
-    fn should_transpile_identifier() {
-        let s = String::from("newValue");
-        let exp = Expression::Identifier(&s);
-        let result = exp.transpile();
-
-        assert_eq!("newValue", result);
-    }
-}

src/codegen/function_call.rs

@@ -1,17 +0,0 @@
-use crate::syntax::ast::functions::FunctionCall;
-
-use super::Transpilable;
-
-impl Transpilable for FunctionCall<'_> {
-    fn transpile(&self) -> String {
-        let parameters = &self
-            .arguments
-            .arguments
-            .iter()
-            .map(|expr| expr.transpile())
-            .collect::<Vec<_>>()
-            .join(", ");
-
-        format!("{}({})", self.function.transpile(), parameters)
-    }
-}

src/codegen/function_declaration.rs

@@ -1,43 +0,0 @@
-use crate::syntax::ast::FunctionDeclaration;
-
-use super::Transpilable;
-
-impl Transpilable for FunctionDeclaration<'_> {
-    fn transpile(&self) -> String {
-        format!(
-            "function {}() {{\n{}\n}}",
-            self.identifier.value,
-            self.block.transpile()
-        )
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        lexic::get_tokens,
-        syntax::{
-            ast::{ModuleMembers, Statement},
-            build_ast,
-        },
-    };
-
-    /* TODO: reimplement
-    #[test]
-    fn should_transpile() {
-        let tokens = get_tokens(&String::from("fun id() {}")).unwrap();
-        let result = build_ast(&tokens).unwrap();
-
-        let fun_dec = result.productions.get(0).unwrap();
-
-        match fun_dec {
-            ModuleMembers::Stmt(Statement::FnDecl(fun_decl)) => {
-                let transpiled = fun_decl.transpile();
-
-                assert_eq!("function id() {\n\n}", transpiled);
-            }
-            _ => panic!("Expected a function declaration"),
-        }
-    }*/
-}

src/codegen/mod.rs

@@ -1,13 +1,5 @@
 // TODO: These are for the THP AST. Eventually replace this
 // with the PHP AST
-mod binding;
-mod block;
-mod expression;
-mod function_call;
-mod function_declaration;
-mod module_ast;
-mod statement;
-mod top_level_construct;
 
 mod php;
 

src/codegen/module_ast.rs

@@ -1,50 +0,0 @@
-use super::Transpilable;
-use crate::syntax::ast::ModuleAST;
-
-impl Transpilable for ModuleAST<'_> {
-    /// Transpiles the whole AST into PHP, using this same trait on the
-    /// nodes and leaves of the AST
-    fn transpile(&self) -> String {
-        let bindings_str: Vec<String> = self
-            .productions
-            .iter()
-            .map(|binding| binding.transpile())
-            .collect();
-
-        format!("<?php\n\n{}\n", bindings_str.join("\n"))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{
-        lexic::token::{Token, TokenType},
-        syntax::ast::{var_binding::VariableBinding, Expression, ModuleMembers, Statement},
-    };
-
-    #[test]
-    fn module_ast_should_transpile() {
-        let id = String::from("identifier");
-        let id_token = Token {
-            token_type: TokenType::Identifier,
-            value: id,
-            position: 0,
-        };
-        let value = String::from("322");
-        let binding = VariableBinding {
-            datatype: None,
-            identifier: &id_token,
-            expression: Expression::Int(&value),
-            is_mutable: false,
-        };
-
-        let module = ModuleAST {
-            productions: vec![ModuleMembers::Stmt(Statement::Binding(binding))],
-        };
-
-        let result = module.transpile();
-
-        assert_eq!("<?php\n\n$identifier = 322\n", result);
-    }
-}

src/codegen/php/expression/mod.rs

@@ -0,0 +1 @@
+mod primary_expression;

src/codegen/php/expression/primary_expression.rs

@@ -0,0 +1,61 @@
+use crate::{codegen::Transpilable, php_ast::PhpPrimaryExpression};
+
+impl Transpilable for PhpPrimaryExpression<'_> {
+    fn transpile(&self) -> String {
+        match self {
+            PhpPrimaryExpression::IntegerLiteral(value) => value.to_string(),
+            PhpPrimaryExpression::FloatingLiteral(value) => value.to_string(),
+            PhpPrimaryExpression::StringLiteral(value) => format!("\"{}\"", value),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{codegen::Transpilable, php_ast::PhpPrimaryExpression};
+
+    #[test]
+    fn should_transpile_empty_string() {
+        let input = String::from("");
+        let ast = PhpPrimaryExpression::StringLiteral(&input);
+        let output = ast.transpile();
+
+        assert_eq!("\"\"", output)
+    }
+
+    #[test]
+    fn should_transpile_string() {
+        let input = String::from("abc");
+        let ast = PhpPrimaryExpression::StringLiteral(&input);
+        let output = ast.transpile();
+
+        assert_eq!("\"abc\"", output)
+    }
+
+    #[test]
+    fn should_transpile_string_with_quotes() {
+        let input = String::from("a\\\"b\\\"c");
+        let ast = PhpPrimaryExpression::StringLiteral(&input);
+        let output = ast.transpile();
+
+        assert_eq!("\"a\\\"b\\\"c\"", output)
+    }
+
+    #[test]
+    fn should_transpile_int() {
+        let input = String::from("322");
+        let ast = PhpPrimaryExpression::IntegerLiteral(&input);
+        let output = ast.transpile();
+
+        assert_eq!("322", output)
+    }
+
+    #[test]
+    fn should_transpile_floating() {
+        let input = String::from("322.644");
+        let ast = PhpPrimaryExpression::FloatingLiteral(&input);
+        let output = ast.transpile();
+
+        assert_eq!("322.644", output)
+    }
+}

src/codegen/php/mod.rs

@@ -1,50 +1,14 @@
-use std::os::linux::raw::stat;
-
-use crate::php_ast::{PhpAst, PhpExpression, PhpStatement};
-
 use super::Transpilable;
+use crate::php_ast::PhpExpression;
 
-impl Transpilable for PhpAst<'_> {
-    fn transpile(&self) -> String {
-        let mut fragments = vec![String::from("<?php\n")];
-        
-        for statement in self.statements.iter() {
-            fragments.push(statement.transpile());
-        }
-
-        fragments.join("")
-    }
-}
-
-impl Transpilable for PhpStatement<'_> {
-    fn transpile(&self) -> String {
-        match self {
-            PhpStatement::PhpEchoStatement(expr_list) => {
-                let expressions_vec = expr_list.expressions
-                    .iter()
-                    .map(|e| e.transpile())
-                    .collect::<Vec<_>>();
-
-                let expressions_str = if expressions_vec.is_empty() {
-                    "\"\"".into()
-                } else {
-                    expressions_vec.join(", ")
-                };
-
-                format!("echo {};", expressions_str)
-            }
-        }
-    }
-}
+pub mod statement;
+pub mod statement_list;
+mod expression;
 
 impl Transpilable for PhpExpression<'_> {
     fn transpile(&self) -> String {
         match self {
-            PhpExpression::String(value) => {
-                format!("{}", value)
-            }
+            PhpExpression::PrimaryExpression(p) => p.transpile(),
         }
     }
 }
-
-

src/codegen/php/statement/mod.rs

@@ -0,0 +1,77 @@
+use crate::{codegen::Transpilable, php_ast::PhpStatement};
+
+mod echo_statement;
+
+impl Transpilable for PhpStatement<'_> {
+    fn transpile(&self) -> String {
+        match self {
+            PhpStatement::PhpEchoStatement(expr_list) => {
+                let expressions_vec = expr_list
+                    .expressions
+                    .iter()
+                    .map(|e| e.transpile())
+                    .collect::<Vec<_>>();
+
+                let expressions_str = if expressions_vec.is_empty() {
+                    "\"\"".into()
+                } else {
+                    expressions_vec.join(", ")
+                };
+
+                format!("echo {};", expressions_str)
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{
+        codegen::Transpilable,
+        php_ast::{PhpExpression, PhpExpressionList, PhpPrimaryExpression, PhpStatement},
+    };
+
+    #[test]
+    fn should_gen_empty_echo_statement() {
+        let expressions = PhpExpressionList {
+            expressions: vec![],
+        };
+        let ast = PhpStatement::PhpEchoStatement(expressions);
+        let output = ast.transpile();
+
+        assert_eq!("echo \"\";", output)
+    }
+
+    #[test]
+    fn should_gen_echo_with_expr() {
+        let input = String::from("322");
+        let exp_1 = PhpPrimaryExpression::FloatingLiteral(&input);
+        let expressions = PhpExpressionList {
+            expressions: vec![PhpExpression::PrimaryExpression(exp_1)],
+        };
+        let ast = PhpStatement::PhpEchoStatement(expressions);
+        let output = ast.transpile();
+
+        assert_eq!("echo 322;", output)
+    }
+
+    #[test]
+    fn should_gen_echo_with_multiple_expr() {
+        let input = String::from("322");
+        let exp_1 = PhpPrimaryExpression::FloatingLiteral(&input);
+
+        let input = String::from("Hai world");
+        let exp_2 = PhpPrimaryExpression::StringLiteral(&input);
+
+        let expressions = PhpExpressionList {
+            expressions: vec![
+                PhpExpression::PrimaryExpression(exp_1),
+                PhpExpression::PrimaryExpression(exp_2),
+            ],
+        };
+        let ast = PhpStatement::PhpEchoStatement(expressions);
+        let output = ast.transpile();
+
+        assert_eq!("echo 322, \"Hai world\";", output)
+    }
+}

src/codegen/php/statement_list.rs

@@ -0,0 +1,27 @@
+use crate::{codegen::Transpilable, php_ast::PhpAst};
+
+impl Transpilable for PhpAst<'_> {
+    fn transpile(&self) -> String {
+        let mut fragments = vec![String::from("<?php\n")];
+
+        for statement in self.statements.iter() {
+            fragments.push(statement.transpile());
+        }
+
+        fragments.join("")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{codegen::Transpilable, php_ast::PhpAst};
+
+    #[test]
+    fn should_transpile_empty_file() {
+        let ast = PhpAst {statements: vec![]};
+        let output = ast.transpile();
+
+        assert_eq!("<?php\n", output);
+    }
+}
+

src/codegen/statement.rs

@@ -1,14 +0,0 @@
-use crate::syntax::ast::Statement;
-
-use super::Transpilable;
-
-impl Transpilable for Statement<'_> {
-    fn transpile(&self) -> String {
-        let stmt = match self {
-            Statement::FnDecl(f) => f.transpile(),
-            Statement::Binding(b) => b.transpile(),
-        };
-
-        format!("{stmt};")
-    }
-}

src/codegen/top_level_construct.rs

@@ -1,13 +0,0 @@
-use crate::syntax::ast::{ModuleMembers, Statement};
-
-use super::Transpilable;
-
-impl Transpilable for ModuleMembers<'_> {
-    fn transpile(&self) -> String {
-        match self {
-            ModuleMembers::Stmt(Statement::Binding(b)) => b.transpile(),
-            ModuleMembers::Stmt(Statement::FnDecl(f)) => f.transpile(),
-            _ => todo!("Not implemented: Transpilable for Expression"),
-        }
-    }
-}

src/error_handling/mod.rs

@@ -22,6 +22,7 @@ pub enum MistiError {
 pub struct LexError {
     pub position: usize,
     // TODO: Add and end position
+    pub end_position: usize,
     pub reason: String,
 }
 

src/file/mod.rs

@@ -101,5 +101,6 @@ fn build_ast(input: &String, tokens: Vec<Token>) -> Result<String, String> {
         }
     };
 
-    Ok(codegen::codegen(&ast))
+    Err("Code generation disabled: rewriting into PHP AST".into())
+    // Ok(codegen::codegen(&ast))
 }

src/lexic/mod.rs

@@ -152,6 +152,7 @@ fn next_token(
         .unwrap_or_else(|| {
             let error = LexError {
                 position: current_pos,
+                end_position: current_pos + 1,
                 reason: format!(
                     "Illegal character `{}` (escaped: {})",
                     next_char,
@@ -196,6 +197,7 @@ fn handle_indentation(
                 // Illegal state: Indentation error
                 let error = LexError {
                     position: current_pos,
+                    end_position: current_pos + 1,
                     reason: format!(
                         "Indentation error: expected {} spaces, found {}",
                         new_top, spaces

src/lexic/scanner/new_comment.rs

@@ -34,40 +34,44 @@ fn scan_any_except_new_line(
 /// and the character at `start_pos + 1` is '*'
 pub fn scan_multiline(chars: &Vec<char>, start_pos: usize) -> LexResult {
     match multiline_impl(chars, start_pos + 2) {
-        Some((value, next_position)) => LexResult::Some(
+        Ok((value, next_position)) => LexResult::Some(
             Token::new_multiline_comment(value.iter().collect(), start_pos),
             next_position,
         ),
-        None => {
+        Err(last_position) => {
             // Throw an error: Incomplete multiline comment
             LexResult::Err(LexError {
                 position: start_pos,
                 // TODO: add an end_position
+                end_position: last_position,
                 reason: "Unfinished multiline commend".into(),
             })
         }
     }
 }
 
-fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Option<(Vec<char>, usize)> {
+/// Implementation that scans the multiline comment.
+/// 
+/// May only error if EOF is found before the comment is finished.
+/// If Err, returns the last position where a char was available.
+fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Result<(Vec<char>, usize), usize> {
     let mut current_position = start_pos;
     let mut result = Vec::<char>::new();
 
     loop {
         match chars.get(current_position) {
             Some('/') => {
-
                 match chars.get(current_position + 1) {
                     Some('*') => {
                         // Scan nested comment
-                        let (mut nested, next_position) = match multiline_impl(chars, current_position + 2)
-                        {
-                            Some(v) => v,
-                            None => {
-                                // The nested comment is not closed.
-                                return None;
-                            }
-                        };
+                        let (mut nested, next_position) =
+                            match multiline_impl(chars, current_position + 2) {
+                                Ok(v) => v,
+                                Err(pos) => {
+                                    // The nested comment is not closed.
+                                    return Err(pos);
+                                }
+                            };
                         result.push('/');
                         result.push('*');
                         result.append(&mut nested);
@@ -80,7 +84,7 @@ fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Option<(Vec<char>, usi
                         result.push('/');
                         result.push(*c);
                     }
-                    None => return None,
+                    None => return Err(current_position),
                 }
             }
             Some('*') => {
@@ -89,7 +93,7 @@ fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Option<(Vec<char>, usi
                     Some('/') => {
                         // Create and return the token,
                         // ignoring the `*/`
-                        return Some((result, current_position + 2));
+                        return Ok((result, current_position + 2));
                     }
                     Some(c) => {
                         // Append both and continue
@@ -99,7 +103,7 @@ fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Option<(Vec<char>, usi
                     }
                     None => {
                         // Throw an error
-                        return None;
+                        return Err(current_position);
                     }
                 }
             }
@@ -109,10 +113,7 @@ fn multiline_impl(chars: &Vec<char>, start_pos: usize) -> Option<(Vec<char>, usi
                 current_position += 1;
             }
             None => {
-                // TODO: Also return the position where this token ends,
-                // to display better error messages.
-                // Requires LexError to implement an end_position field
-                return None;
+                return Err(current_position);
             }
         }
     }

src/lexic/scanner/number.rs

@@ -53,6 +53,7 @@ fn scan_hex(chars: &Vec<char>, start_pos: usize, current: String) -> LexResult {
         }
         _ => LexResult::Err(LexError {
             position: start_pos,
+            end_position: start_pos + 1,
             reason: String::from("Tried to scan an incomplete hex value"),
         }),
     }
@@ -69,12 +70,14 @@ fn scan_double(chars: &Vec<char>, start_pos: usize, current: String) -> LexResul
         Some(c) if utils::is_digit(*c) => scan_double_impl(chars, start_pos, current),
         Some(_) => LexResult::Err(LexError {
             position: start_pos,
+            end_position: start_pos + 1,
             reason: String::from(
                 "The character after the dot when scanning a double is not a number.",
             ),
         }),
         _ => LexResult::Err(LexError {
             position: start_pos,
+            end_position: start_pos + 1,
             reason: String::from("EOF when scanning a double number."),
         }),
     }
@@ -122,6 +125,7 @@ fn scan_scientific(chars: &Vec<char>, start_pos: usize, current: String) -> LexR
         }
         _ => LexResult::Err(LexError {
             position: start_pos,
+            end_position: start_pos + 1,
             reason: String::from(
                 "The characters after 'e' are not + or -, or are not followed by a number",
             ),

src/lexic/scanner/string.rs

@@ -7,9 +7,11 @@ use crate::lexic::{utils, LexResult};
 /// This function assumes that `start_pos` is after the first double quote,
 /// e.g. if the input is `"hello"`, `start_pos == 1`
 pub fn scan(chars: &Vec<char>, start_pos: usize) -> LexResult {
-    scan_impl(chars, start_pos, String::from("\""))
+    scan_impl(chars, start_pos, String::from(""))
 }
 
+// TODO: This can be iterative instead of recursive
+
 /// Recursive function that does the scanning
 pub fn scan_impl(chars: &Vec<char>, start_pos: usize, current: String) -> LexResult {
     match chars.get(start_pos) {
@@ -17,16 +19,16 @@ pub fn scan_impl(chars: &Vec<char>, start_pos: usize, current: String) -> LexRes
             // start_pos is the position where the token ENDS, not where it STARTS,
             // so this is used to retrieve the original START position of the token
             // 1 is added to account for the opening `"`
-            let current_len = current.len();
+            let current_len = current.len() + 1;
 
-            let final_str = format!("{}\"", current);
             LexResult::Some(
-                Token::new_string(final_str, start_pos - current_len),
+                Token::new_string(current, start_pos - current_len),
                 start_pos + 1,
             )
         }
         Some(c) if *c == '\n' => LexResult::Err(LexError {
             position: start_pos,
+            end_position: start_pos + 1,
             reason: String::from("Unexpected new line inside a string."),
         }),
         Some(c) if *c == '\\' => {
@@ -40,6 +42,7 @@ pub fn scan_impl(chars: &Vec<char>, start_pos: usize, current: String) -> LexRes
         Some(c) => scan_impl(chars, start_pos + 1, utils::str_append(current, *c)),
         None => LexResult::Err(LexError {
             position: start_pos,
+            end_position: start_pos + 1,
             reason: String::from("Incomplete string found"),
         }),
     }
@@ -79,7 +82,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(2, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("\"\"", token.value);
+            assert_eq!("", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -93,7 +96,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(15, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("\"Hello, world!\"", token.value);
+            assert_eq!("Hello, world!", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -118,7 +121,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("\"Sample\\ntext\"", token.value);
+            assert_eq!("Sample\\ntext", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -129,7 +132,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("\"Sample\\\"text\"", token.value);
+            assert_eq!("Sample\\\"text", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -140,7 +143,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("\"Sample\\rtext\"", token.value);
+            assert_eq!("Sample\\rtext", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -151,7 +154,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("\"Sample\\\\text\"", token.value);
+            assert_eq!("Sample\\\\text", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -162,7 +165,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("\"Sample\\ttext\"", token.value);
+            assert_eq!("Sample\\ttext", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -173,7 +176,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("\"Sample\\ text\"", token.value);
+            assert_eq!("Sample\\ text", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()
@@ -187,7 +190,7 @@ mod tests {
         if let LexResult::Some(token, next) = scan(&input, start_pos) {
             assert_eq!(14, next);
             assert_eq!(TokenType::String, token.token_type);
-            assert_eq!("\"Sample\\atext\"", token.value);
+            assert_eq!("Sample\\atext", token.value);
             assert_eq!(0, token.position);
         } else {
             panic!()

src/lexic/token.rs

@@ -38,7 +38,15 @@ pub struct Token {
 
 impl Token {
     pub fn get_end_position(&self) -> usize {
-        self.position + self.value.len()
+        match self.token_type {
+            // 4 extra characters for /* and */
+            TokenType::MultilineComment => self.position + self.value.len() + 4,
+            // 2 extra characters for //
+            TokenType::Comment => self.position + self.value.len() + 2,
+            // 2 extra characters for ""
+            TokenType::String => self.position + self.value.len() + 2,
+            _ => self.position + self.value.len()
+        }
     }
 }
 

src/php_ast/mod.rs

@@ -1,12 +1,23 @@
-// Follows https://phplang.org/spec/19-grammar.html#syntactic-grammar
+/// This AST implements a subset of the PHP AST as defined
+/// by https://phplang.org/spec/19-grammar.html#syntactic-grammar
+/// 
+/// This subset only includes nodes that can be generated by
+/// THP
 
 pub mod transformers;
 
-/// Represents `statement-list` on the grammar
+/// Represents `statement-list` on the grammar,
+/// and thus a whole PHP source file
 pub struct PhpAst<'a> {
     pub statements: Vec<PhpStatement<'a>>,
 }
 
+/// https://phplang.org/spec/19-grammar.html#grammar-statement
+/// 
+/// Not fully implemented
+/// 
+/// statement:
+///   echo-statement
 pub enum PhpStatement<'a> {
     PhpEchoStatement(PhpExpressionList<'a>),
 }
@@ -16,5 +27,16 @@ pub struct PhpExpressionList<'a> {
 }
 
 pub enum PhpExpression<'a> {
-    String(&'a String),
+    PrimaryExpression(PhpPrimaryExpression<'a>),
 }
+
+/// https://phplang.org/spec/19-grammar.html#grammar-primary-expression
+/// 
+/// primary-expression:
+///     literal
+pub enum PhpPrimaryExpression<'a> {
+    IntegerLiteral(&'a String),
+    FloatingLiteral(&'a String),
+    StringLiteral(&'a String),
+}
+

src/php_ast/transformers/expression.rs

@@ -1,5 +1,5 @@
 use super::super::PhpExpression;
-use crate::syntax::ast::Expression;
+use crate::{php_ast::PhpPrimaryExpression, syntax::ast::Expression};
 
 use super::PHPTransformable;
 
@@ -9,7 +9,10 @@ impl<'a> PHPTransformable<'a> for Expression<'_> {
 
     fn into_php_ast(&'a self) -> Self::Item {
         match self {
-            Expression::String(value) => PhpExpression::String(value),
+            Expression::String(value) => {
+                let expr = PhpPrimaryExpression::StringLiteral(value);
+                PhpExpression::PrimaryExpression(expr)
+            },
             _ => todo!("transformation for expression: {:?}", self),
         }
     }

src/php_ast/transformers/module_ast.rs

@@ -1,5 +1,5 @@
 use super::super::PhpAst;
-use crate::php_ast::{PhpExpression, PhpExpressionList, PhpStatement};
+use crate::php_ast::{PhpExpression, PhpExpressionList, PhpPrimaryExpression, PhpStatement};
 use crate::syntax::ast::{Expression, ModuleAST, ModuleMembers};
 
 use super::PHPTransformable;
@@ -33,7 +33,9 @@ impl<'a> PHPTransformable<'a> for ModuleAST<'_> {
                                     for e in fc.arguments.arguments.iter() {
                                         match e {
                                             Expression::String(v) => {
-                                                expressions.push(PhpExpression::String(v))
+                                                expressions.push(
+                                                    PhpExpression::PrimaryExpression(PhpPrimaryExpression::StringLiteral(v.clone()))
+                                                )
                                             },
                                             _ => panic!("Non string expressions not supported")
                                         }

src/syntax/parsers/expression/primary.rs

@@ -66,7 +66,7 @@ mod tests {
 
         match expression {
             Ok((Expression::String(value), _)) => {
-                assert_eq!("\"Hello\"", format!("{}", value))
+                assert_eq!("Hello", format!("{}", value))
             }
             _ => panic!(),
         }

src/syntax/parsers/module.rs

@@ -110,7 +110,7 @@ mod test {
 
         match result {
             Ok(_) => panic!("Expected an error"),
-            Err(_) => {},
+            Err(_) => {}
         }
     }
 }

src/syntax/utils.rs

@@ -80,7 +80,10 @@ pub fn parse_token_type(
 mod tests {
     use crate::{
         lexic::{get_tokens, token::TokenType},
-        syntax::{parseable::ParsingError, utils::{parse_token_type, Tokenizer}},
+        syntax::{
+            parseable::ParsingError,
+            utils::{parse_token_type, Tokenizer},
+        },
     };
 
     use super::try_operator;
@@ -133,7 +136,7 @@ mod tests {
 
         match tokens.get_significant(10) {
             Some(_) => panic!("Expected a None"),
-            None => {},
+            None => {}
         }
     }
 }