v0.0.3 - token stream always ends with Semicolon & EOF

CHANGELOG.md

@@ -16,6 +16,7 @@
 - Get datatype of an identifier from the symbol table
 - Improve documentation of the code
 - Simple ASI: insert semicolon after a single or series of new lines
+- The token stream now always ends with a Semicolon and EOF token, regardless of input
 
 ## v0.0.2
 

Cargo.lock

@@ -169,7 +169,7 @@ dependencies = [
 
 [[package]]
 name = "misti"
-version = "0.0.2"
+version = "0.0.3"
 dependencies = [
  "chrono",
 ]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "misti"
-version = "0.0.2"
+version = "0.0.3"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

src/lexic/mod.rs

@@ -52,6 +52,7 @@ pub fn get_tokens(input: &String) -> Result<Vec<Token>, LexError> {
         }
     }
 
+    results.push(token::new_semicolon(0));
     results.push(token::new_eof(0));
     Ok(results)
 }
@@ -110,20 +111,23 @@ mod tests {
     fn test1() {
         let input = String::from("");
         let tokens = get_tokens(&input).unwrap();
-        assert_eq!(1, tokens.len());
+        // 1 semicolon and 1 EOF token
-        let first = tokens.get(0).unwrap();
+        assert_eq!(2, tokens.len());
+        let first = tokens.get(1).unwrap();
         assert_eq!(TokenType::EOF, first.token_type);
 
         let input = String::from("  ");
         let tokens = get_tokens(&input).unwrap();
-        assert_eq!(1, tokens.len());
+        // 1 semicolon and 1 EOF token
-        let first = tokens.get(0).unwrap();
+        assert_eq!(2, tokens.len());
+        let first = tokens.get(1).unwrap();
         assert_eq!(TokenType::EOF, first.token_type);
 
         let input = String::from("    ");
         let tokens = get_tokens(&input).unwrap();
-        assert_eq!(1, tokens.len());
+        // 1 semicolon and 1 EOF token
-        let first = tokens.get(0).unwrap();
+        assert_eq!(2, tokens.len());
+        let first = tokens.get(1).unwrap();
         assert_eq!(TokenType::EOF, first.token_type);
     }
 
@@ -165,7 +169,8 @@ mod tests {
         
         assert_eq!("1789e+1", tokens.get(3).unwrap().value);
         assert_eq!("239.3298e-103", tokens.get(4).unwrap().value);
-        assert_eq!(TokenType::EOF, tokens.get(5).unwrap().token_type);
+        assert_eq!(TokenType::Semicolon, tokens.get(5).unwrap().token_type);
+        assert_eq!(TokenType::EOF, tokens.get(6).unwrap().token_type);
     }
 
     #[test]

src/token.rs

@@ -68,3 +68,11 @@ pub fn new_string(value: String, position: i32) -> Token {
         _position: position,
     }
 }
+
+pub fn new_semicolon(position: i32) -> Token {
+    Token {
+        token_type: TokenType::Semicolon,
+        value: String::from(";"),
+        _position: position,
+    }
+}