diff --git a/src/01_lexic/root.zig b/src/01_lexic/root.zig index a7c33bd..0fd9c27 100644 --- a/src/01_lexic/root.zig +++ b/src/01_lexic/root.zig @@ -24,7 +24,7 @@ const LexError = token.LexError; pub fn tokenize( input: []const u8, alloc: std.mem.Allocator, - err_arrl: *std.ArrayList(*errors.ErrorData), + err_arrl: *std.ArrayList(errors.ErrorData), ) !std.ArrayList(Token) { const input_len = input.len; var current_pos: usize = 0; @@ -32,9 +32,6 @@ pub fn tokenize( var tokens = std.ArrayList(Token).init(alloc); errdefer tokens.deinit(); - var current_error = try alloc.create(errors.ErrorData); - defer alloc.destroy(current_error); - while (current_pos < input_len) { const actual_next_pos = ignore_whitespace(input, current_pos); assert(current_pos <= actual_next_pos); @@ -45,13 +42,12 @@ pub fn tokenize( } // attempt to lex a number - const number_lex = number.lex(input, input_len, actual_next_pos, current_error) catch |e| switch (e) { + var current_error: errors.ErrorData = undefined; + const number_lex = number.lex(input, input_len, actual_next_pos, ¤t_error) catch |e| switch (e) { // recoverable errors LexError.Incomplete => { // add to list of errors try err_arrl.append(current_error); - // refresh the previous error pointer - current_error = try alloc.create(errors.ErrorData); // ignore everything until whitespace and loop current_pos = ignore_until_whitespace(input, actual_next_pos); @@ -127,13 +123,11 @@ pub fn tokenize( } // nothing was matched. fail - // TODO: instead of failing add an error, ignore all chars - // until next whitespace, and continue lexing - // TODO: check if this is a good error recovery strategy else { // Create an error "nothing matched" and continue lexing // after the whitespace current_error.init("Unrecognized character", actual_next_pos, actual_next_pos + 1); + try err_arrl.append(current_error); current_pos = ignore_until_whitespace(input, actual_next_pos); continue; } @@ -177,7 +171,7 @@ test { test "should insert 1 item" { const input = "322"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const arrl = try tokenize(input, std.testing.allocator, &error_list); arrl.deinit(); @@ -185,7 +179,7 @@ test "should insert 1 item" { test "should insert 2 item" { const input = "322 644"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const arrl = try tokenize(input, std.testing.allocator, &error_list); arrl.deinit(); @@ -193,7 +187,7 @@ test "should insert 2 item" { test "should insert an item, fail, and not leak" { const input = "322 \"hello"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const arrl = tokenize(input, std.testing.allocator, &error_list) catch |e| switch (e) { error.IncompleteString => { @@ -207,3 +201,26 @@ test "should insert an item, fail, and not leak" { try std.testing.expect(false); arrl.deinit(); } + +test "shouldnt leak" { + const input = ""; + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); + defer error_list.deinit(); + const arrl = try tokenize(input, std.testing.allocator, &error_list); + arrl.deinit(); +} + +test "should handle recoverable errors" { + const input = "322 0b 644"; + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); + defer error_list.deinit(); + const arrl = try tokenize(input, std.testing.allocator, &error_list); + defer arrl.deinit(); + + try std.testing.expectEqual(@as(usize, 1), error_list.items.len); + try std.testing.expectEqual(@as(usize, 2), arrl.items.len); + + try std.testing.expectEqualStrings("Invalid prefix passed to `prefixed` function.", error_list.items[0].reason); + try std.testing.expectEqual(@as(usize, 4), error_list.items[0].start_position); + try std.testing.expectEqual(@as(usize, 6), error_list.items[0].end_position); +} diff --git a/src/02_syntax/expression.zig b/src/02_syntax/expression.zig index 45ac339..1e58a62 100644 --- a/src/02_syntax/expression.zig +++ b/src/02_syntax/expression.zig @@ -27,7 +27,7 @@ pub const Expression = union(enum) { test "should parse expression" { const input = "322"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); @@ -40,7 +40,7 @@ test "should parse expression" { test "should fail on non expression" { const input = "identifier"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); diff --git a/src/02_syntax/root.zig b/src/02_syntax/root.zig index 99bde53..979adb8 100644 --- a/src/02_syntax/root.zig +++ b/src/02_syntax/root.zig @@ -84,7 +84,7 @@ test { test "should parse a single statement" { const input = "var my_variable = 322"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); @@ -100,7 +100,7 @@ test "should parse a single statement" { test "should clean memory if a statement parsing fails after one item has been inserted" { const input = "var my_variable = 322 unrelated()"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); diff --git a/src/02_syntax/statement.zig b/src/02_syntax/statement.zig index a4082f4..ff8b449 100644 --- a/src/02_syntax/statement.zig +++ b/src/02_syntax/statement.zig @@ -59,7 +59,7 @@ pub const Statement = struct { test "should parse a variable declaration statement" { const input = "var my_variable = 322"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); @@ -78,7 +78,7 @@ test "should parse a variable declaration statement" { test "should fail on other constructs" { const input = "a_function_call(322)"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); diff --git a/src/02_syntax/variable.zig b/src/02_syntax/variable.zig index 710de74..7f02f99 100644 --- a/src/02_syntax/variable.zig +++ b/src/02_syntax/variable.zig @@ -71,7 +71,7 @@ pub const VariableBinding = struct { test "should parse a minimal var" { const input = "var my_variable = 322"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); @@ -93,7 +93,7 @@ test "should parse a minimal var" { test "should fail is it doesnt start with var" { const input = "different_token_stream()"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); @@ -109,7 +109,7 @@ test "should fail is it doesnt start with var" { test "should fail if the identifier is missing" { const input = "var "; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); @@ -125,7 +125,7 @@ test "should fail if the identifier is missing" { test "should fail if there is not an identifier after var" { const input = "var 322"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); @@ -141,7 +141,7 @@ test "should fail if there is not an identifier after var" { test "should fail if the equal sign is missing" { const input = "var my_id "; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); @@ -157,7 +157,7 @@ test "should fail if the equal sign is missing" { test "should fail if the equal sign is not found" { const input = "var my_id is string"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); @@ -173,7 +173,7 @@ test "should fail if the equal sign is not found" { test "should fail if the expression parsing fails" { const input = "var my_id = ehhh"; - var error_list = std.ArrayList(*errors.ErrorData).init(std.testing.allocator); + var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator); defer error_list.deinit(); const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list); defer tokens.deinit(); diff --git a/src/main.zig b/src/main.zig index 0449f9f..7c6a253 100644 --- a/src/main.zig +++ b/src/main.zig @@ -51,19 +51,16 @@ fn repl() !void { // // Tokenize // - const tokens = lexic.tokenize(line, alloc) catch |e| switch (e) { + var error_array = std.ArrayList(errors.ErrorData).init(alloc); + defer error_array.deinit(); + + const tokens = lexic.tokenize(line, alloc, &error_array) catch |e| switch (e) { error.OutOfMemory => { try stdout.print("FATAL ERROR: System Out of Memory!", .{}); try bw.flush(); return e; }, - else => { - // TODO: implement error handling in the lexer, - // and print those errors here - try stdout.print("Unknown error while lexing :c\n", .{}); - try bw.flush(); - continue; - }, + else => return e, }; defer tokens.deinit(); @@ -78,6 +75,12 @@ fn repl() !void { } } + // Print errors + for (error_array.items) |err| { + try stdout.print("Lex error: {s} at pos {d}\n", .{ err.reason, err.start_position }); + try bw.flush(); + } + // next repl line }