From 4e1b2c3cabc89c8c1151648465d863dcb3219a6c Mon Sep 17 00:00:00 2001
From: Araozu <fernando@araozu.dev>
Date: Mon, 18 Nov 2024 21:14:09 -0500
Subject: [PATCH] feat: integrate identifier/datatype lexer

---
 src/01_lexic/datatype.zig   | 12 ++++++++++++
 src/01_lexic/identifier.zig | 12 ++++++++++++
 src/01_lexic/root.zig       | 27 ++++++++++++++++++++++++---
 3 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/src/01_lexic/datatype.zig b/src/01_lexic/datatype.zig
index a05433b..e81f96b 100644
--- a/src/01_lexic/datatype.zig
+++ b/src/01_lexic/datatype.zig
@@ -45,6 +45,18 @@ test "should lex datatype" {
     }
 }
 
+test "should lex datatype 2" {
+    const input = "MyTypeWith322";
+    const output = try lex(input, 0);
+
+    if (output) |tuple| {
+        const t = tuple[0];
+        try std.testing.expectEqualDeep("MyTypeWith322", t.value);
+    } else {
+        try std.testing.expect(false);
+    }
+}
+
 test "shouldnt lex identifier" {
     const input = "myDatatype";
     const output = try lex(input, 0);
diff --git a/src/01_lexic/identifier.zig b/src/01_lexic/identifier.zig
index a5509c8..019eb37 100644
--- a/src/01_lexic/identifier.zig
+++ b/src/01_lexic/identifier.zig
@@ -93,6 +93,18 @@ test "should lex identifier 3" {
     }
 }
 
+test "should lex identifier 4" {
+    const input = "identifier_number_3";
+    const output = try lex(input, 0);
+
+    if (output) |tuple| {
+        const t = tuple[0];
+        try std.testing.expectEqualDeep("identifier_number_3", t.value);
+    } else {
+        try std.testing.expect(false);
+    }
+}
+
 test "shouldnt lex datatype" {
     const input = "MyDatatype";
     const output = try lex(input, 0);
diff --git a/src/01_lexic/root.zig b/src/01_lexic/root.zig
index 8334fbb..5732ba2 100644
--- a/src/01_lexic/root.zig
+++ b/src/01_lexic/root.zig
@@ -1,5 +1,7 @@
 const std = @import("std");
 const number = @import("./number.zig");
+const identifier = @import("./identifier.zig");
+const datatype = @import("./datatype.zig");
 const token = @import("./token.zig");
 
 const TokenType = token.TokenType;
@@ -15,13 +17,32 @@ pub fn tokenize(input: []const u8, alloc: std.mem.Allocator) !void {
     while (current_pos < input_len) {
         const actual_next_pos = ignore_whitespace(input, current_pos);
 
-        const next_token = try number.lex(input, input_len, actual_next_pos);
-        if (next_token) |tuple| {
+        // attempt to lex a number
+        if (try number.lex(input, input_len, actual_next_pos)) |tuple| {
             const t = tuple[0];
             current_pos = tuple[1];
 
             try tokens.append(t);
-        } else {
+        }
+        // attempt to lex an identifier
+        else if (try identifier.lex(input, actual_next_pos)) |tuple| {
+            const t = tuple[0];
+            current_pos = tuple[1];
+
+            try tokens.append(t);
+        }
+        // attempt to lex a datatype
+        else if (try datatype.lex(input, actual_next_pos)) |tuple| {
+            const t = tuple[0];
+            current_pos = tuple[1];
+
+            try tokens.append(t);
+        }
+        // nothing was matched. fail
+        // TODO: instead of failing add an error, ignore all chars
+        // until next whitespace, and continue lexing
+        // TODO: check if this is a good error recovery strategy
+        else {
             // no lexer matched
             std.debug.print("unmatched args: anytype:c\n", .{});
             break;