From 1849e11ebb1aa0e1e7013bb7bff25d20afc1bc1c Mon Sep 17 00:00:00 2001
From: Araozu <faraoz@unsa.edu.pe>
Date: Tue, 14 Feb 2023 15:32:45 -0500
Subject: [PATCH] v0.0.3 - token stream always ends with Semicolon & EOF

---
 CHANGELOG.md     |  1 +
 Cargo.lock       |  2 +-
 Cargo.toml       |  2 +-
 src/lexic/mod.rs | 19 ++++++++++++-------
 src/token.rs     |  8 ++++++++
 5 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1a1873b..1b1b205 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,7 @@
 - Get datatype of an identifier from the symbol table
 - Improve documentation of the code
 - Simple ASI: insert semicolon after a single or series of new lines
+- The token stream now always ends with a Semicolon and EOF token, regardless of input
 
 ## v0.0.2
 
diff --git a/Cargo.lock b/Cargo.lock
index c3d9de2..b70f350 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -169,7 +169,7 @@ dependencies = [
 
 [[package]]
 name = "misti"
-version = "0.0.2"
+version = "0.0.3"
 dependencies = [
  "chrono",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index e904f1a..f16e7c6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "misti"
-version = "0.0.2"
+version = "0.0.3"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
diff --git a/src/lexic/mod.rs b/src/lexic/mod.rs
index 7d643be..81338f6 100755
--- a/src/lexic/mod.rs
+++ b/src/lexic/mod.rs
@@ -52,6 +52,7 @@ pub fn get_tokens(input: &String) -> Result<Vec<Token>, LexError> {
         }
     }
 
+    results.push(token::new_semicolon(0));
     results.push(token::new_eof(0));
     Ok(results)
 }
@@ -110,20 +111,23 @@ mod tests {
     fn test1() {
         let input = String::from("");
         let tokens = get_tokens(&input).unwrap();
-        assert_eq!(1, tokens.len());
-        let first = tokens.get(0).unwrap();
+        // 1 semicolon and 1 EOF token
+        assert_eq!(2, tokens.len());
+        let first = tokens.get(1).unwrap();
         assert_eq!(TokenType::EOF, first.token_type);
 
         let input = String::from("  ");
         let tokens = get_tokens(&input).unwrap();
-        assert_eq!(1, tokens.len());
-        let first = tokens.get(0).unwrap();
+        // 1 semicolon and 1 EOF token
+        assert_eq!(2, tokens.len());
+        let first = tokens.get(1).unwrap();
         assert_eq!(TokenType::EOF, first.token_type);
 
         let input = String::from("    ");
         let tokens = get_tokens(&input).unwrap();
-        assert_eq!(1, tokens.len());
-        let first = tokens.get(0).unwrap();
+        // 1 semicolon and 1 EOF token
+        assert_eq!(2, tokens.len());
+        let first = tokens.get(1).unwrap();
         assert_eq!(TokenType::EOF, first.token_type);
     }
 
@@ -165,7 +169,8 @@ mod tests {
         
         assert_eq!("1789e+1", tokens.get(3).unwrap().value);
         assert_eq!("239.3298e-103", tokens.get(4).unwrap().value);
-        assert_eq!(TokenType::EOF, tokens.get(5).unwrap().token_type);
+        assert_eq!(TokenType::Semicolon, tokens.get(5).unwrap().token_type);
+        assert_eq!(TokenType::EOF, tokens.get(6).unwrap().token_type);
     }
 
     #[test]
diff --git a/src/token.rs b/src/token.rs
index d25f236..0a03426 100755
--- a/src/token.rs
+++ b/src/token.rs
@@ -68,3 +68,11 @@ pub fn new_string(value: String, position: i32) -> Token {
         _position: position,
     }
 }
+
+pub fn new_semicolon(position: i32) -> Token {
+    Token {
+        token_type: TokenType::Semicolon,
+        value: String::from(";"),
+        _position: position,
+    }
+}