diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a1873b..1b1b205 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - Get datatype of an identifier from the symbol table - Improve documentation of the code - Simple ASI: insert semicolon after a single or series of new lines +- The token stream now always ends with a Semicolon and EOF token, regardless of input ## v0.0.2 diff --git a/Cargo.lock b/Cargo.lock index c3d9de2..b70f350 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,7 +169,7 @@ dependencies = [ [[package]] name = "misti" -version = "0.0.2" +version = "0.0.3" dependencies = [ "chrono", ] diff --git a/Cargo.toml b/Cargo.toml index e904f1a..f16e7c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "misti" -version = "0.0.2" +version = "0.0.3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/lexic/mod.rs b/src/lexic/mod.rs index 7d643be..81338f6 100755 --- a/src/lexic/mod.rs +++ b/src/lexic/mod.rs @@ -52,6 +52,7 @@ pub fn get_tokens(input: &String) -> Result, LexError> { } } + results.push(token::new_semicolon(0)); results.push(token::new_eof(0)); Ok(results) } @@ -110,20 +111,23 @@ mod tests { fn test1() { let input = String::from(""); let tokens = get_tokens(&input).unwrap(); - assert_eq!(1, tokens.len()); - let first = tokens.get(0).unwrap(); + // 1 semicolon and 1 EOF token + assert_eq!(2, tokens.len()); + let first = tokens.get(1).unwrap(); assert_eq!(TokenType::EOF, first.token_type); let input = String::from(" "); let tokens = get_tokens(&input).unwrap(); - assert_eq!(1, tokens.len()); - let first = tokens.get(0).unwrap(); + // 1 semicolon and 1 EOF token + assert_eq!(2, tokens.len()); + let first = tokens.get(1).unwrap(); assert_eq!(TokenType::EOF, first.token_type); let input = String::from(" "); let tokens = get_tokens(&input).unwrap(); - assert_eq!(1, tokens.len()); - let first = tokens.get(0).unwrap(); + // 1 semicolon and 1 EOF token + assert_eq!(2, tokens.len()); + let first = tokens.get(1).unwrap(); assert_eq!(TokenType::EOF, first.token_type); } @@ -165,7 +169,8 @@ mod tests { assert_eq!("1789e+1", tokens.get(3).unwrap().value); assert_eq!("239.3298e-103", tokens.get(4).unwrap().value); - assert_eq!(TokenType::EOF, tokens.get(5).unwrap().token_type); + assert_eq!(TokenType::Semicolon, tokens.get(5).unwrap().token_type); + assert_eq!(TokenType::EOF, tokens.get(6).unwrap().token_type); } #[test] diff --git a/src/token.rs b/src/token.rs index d25f236..0a03426 100755 --- a/src/token.rs +++ b/src/token.rs @@ -68,3 +68,11 @@ pub fn new_string(value: String, position: i32) -> Token { _position: position, } } + +pub fn new_semicolon(position: i32) -> Token { + Token { + token_type: TokenType::Semicolon, + value: String::from(";"), + _position: position, + } +}