From 2e93df0fd85650728ce71389fd3ca018cb02cf08 Mon Sep 17 00:00:00 2001 From: Araozu Date: Wed, 27 Mar 2024 08:18:31 -0500 Subject: [PATCH] scan datatypes --- lexer/identifier_lexer.test.ts | 7 +++++++ lexer/identifier_lexer.ts | 13 +++++++++++-- lexer/lexer.ts | 15 ++++++++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lexer/identifier_lexer.test.ts b/lexer/identifier_lexer.test.ts index 6016882..8e210f6 100644 --- a/lexer/identifier_lexer.test.ts +++ b/lexer/identifier_lexer.test.ts @@ -44,5 +44,12 @@ describe("Identifier Lexer", () => { expect(token).toEqual([{ v: "val", token_type: "keyword" }, 3]); }); + + test("should scan a datatype", () => { + const code = "Int"; + const token = scan_identifier(code, 0, true); + + expect(token).toEqual([{ v: "Int", token_type: "class-name" }, 3]); + }); }); diff --git a/lexer/identifier_lexer.ts b/lexer/identifier_lexer.ts index 086740e..29f34da 100644 --- a/lexer/identifier_lexer.ts +++ b/lexer/identifier_lexer.ts @@ -4,8 +4,12 @@ import { is_identifier_char } from "./utils.ts"; /** * Scans an identifier, at the given position in the input string. * This function assumes that the character at the given position is a letter. + * + * @param input the input string + * @param starting_position the position to start scanning from + * @param is_datatype whether the identifier is a datatype */ -export function scan_identifier(input: string, starting_position: number): [Token, number] { +export function scan_identifier(input: string, starting_position: number, is_datatype = false): [Token, number] { let value = input[starting_position]; let pos = starting_position + 1; @@ -21,7 +25,12 @@ export function scan_identifier(input: string, starting_position: number): [Toke } } - return [{ v: value, token_type: check_keyword(value) }, pos]; + if (is_datatype) { + return [{ v: value, token_type: "class-name" }, pos]; + } + else { + return [{ v: value, token_type: check_keyword(value) }, pos]; + } } function check_keyword(value: string): string { diff --git a/lexer/lexer.ts b/lexer/lexer.ts index a62cb57..ff1de07 100644 --- a/lexer/lexer.ts +++ b/lexer/lexer.ts @@ -1,6 +1,6 @@ import { scan_identifier } from "./identifier_lexer.ts"; import { scan_number } from "./number_lexer.ts"; -import { is_digit, is_lowercase } from "./utils.ts"; +import { is_digit, is_lowercase, is_uppercase } from "./utils.ts"; export type Token = { v: string, @@ -65,6 +65,19 @@ export function lex(code: string): Array { tokens.push(token); continue; } + // try to scan a datatype + else if (is_uppercase(c)) { + // if the current default token is not empty, push it to the tokens array + if (current_default_token !== "") { + tokens.push({ v: current_default_token, token_type: "" }); + current_default_token = ""; + } + + const [token, next] = scan_identifier(code, current_pos, true); + current_pos = next; + tokens.push(token); + continue; + } // here, check if a token was found if (next_token !== null && next_position !== null) {