fernando/thp-zig
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: tests of lexer and parser

Browse Source
This commit is contained in:
Fernando Araoz 2025-01-30 20:26:36 -05:00
parent d600c575f3
commit 8425f621eb
5 changed files with 65 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/01_lexic/root.zig
View File

@ -24,13 +24,12 @@ const LexError = token.LexError;
/// found while lexing. The caller is responsible for freeing it.
pub fn tokenize(
input: []const u8,
alloc: std.mem.Allocator,
ctx: *context.CompilerContext,
) !std.ArrayList(Token) {
const input_len = input.len;
var current_pos: usize = 0;
var tokens = std.ArrayList(Token).init(alloc);
var tokens = std.ArrayList(Token).init(ctx.allocator);
errdefer tokens.deinit();
while (current_pos < input_len) {
@ -189,30 +188,27 @@ test {
}
test "should insert 1 item" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "322";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const arrl = try tokenize(input, std.testing.allocator, &error_list);
const arrl = try tokenize(input, &ctx);
arrl.deinit();
}
test "should insert 2 item" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "322 644";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const arrl = try tokenize(input, std.testing.allocator, &error_list);
const arrl = try tokenize(input, &ctx);
arrl.deinit();
}
test "should insert an item, fail, and not leak" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "322 \"hello";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
defer for (error_list.items) |*i| {
i.deinit();
};
const arrl = tokenize(input, std.testing.allocator, &error_list) catch |e| switch (e) {
const arrl = tokenize(input, &ctx) catch |e| switch (e) {
else => {
try std.testing.expect(false);
return;
@ -222,25 +218,25 @@ test "should insert an item, fail, and not leak" {
}
test "shouldnt leak" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const arrl = try tokenize(input, std.testing.allocator, &error_list);
const arrl = try tokenize(input, &ctx);
arrl.deinit();
}
test "should handle recoverable errors" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "322 0b 644";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
defer for (error_list.items) |*err| err.deinit();
const arrl = try tokenize(input, std.testing.allocator, &error_list);
const arrl = try tokenize(input, &ctx);
defer arrl.deinit();
try std.testing.expectEqual(@as(usize, 1), error_list.items.len);
try std.testing.expectEqual(@as(usize, 1), ctx.errors.items.len);
try std.testing.expectEqual(@as(usize, 2), arrl.items.len);
try std.testing.expectEqualStrings("Incomplete number", error_list.items[0].reason);
try std.testing.expectEqual(@as(usize, 4), error_list.items[0].start_position);
try std.testing.expectEqual(@as(usize, 6), error_list.items[0].end_position);
try std.testing.expectEqualStrings("Incomplete number", ctx.errors.items[0].reason);
try std.testing.expectEqual(@as(usize, 4), ctx.errors.items[0].start_position);
try std.testing.expectEqual(@as(usize, 6), ctx.errors.items[0].end_position);
}

10
src/02_syntax/expression.zig
View File

@ -1,6 +1,8 @@
const std = @import("std");
const lexic = @import("lexic");
const errors = @import("errors");
const context = @import("context");
const Token = lexic.Token;
const TokenType = lexic.TokenType;
@ -27,10 +29,12 @@ pub const Expression = union(enum) {
};
test "should parse expression" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "322";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var expr: Expression = undefined;
@ -43,10 +47,12 @@ test "should parse expression" {
}
test "should fail on non expression" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "identifier";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var expr: Expression = undefined;

9
src/02_syntax/root.zig
View File

@ -1,6 +1,7 @@
const std = @import("std");
const lexic = @import("lexic");
const errors = @import("errors");
const context = @import("context");
const expression = @import("./expression.zig");
const variable = @import("./variable.zig");
@ -92,10 +93,12 @@ test {
}
test "should parse a single statement" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "var my_variable = 322";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var module: Module = undefined;
@ -105,10 +108,12 @@ test "should parse a single statement" {
}
test "should clean memory if a statement parsing fails after one item has been inserted" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "var my_variable = 322 unrelated()";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var module: Module = undefined;

9
src/02_syntax/statement.zig
View File

@ -5,6 +5,7 @@ const types = @import("./types.zig");
const utils = @import("./utils.zig");
const variable = @import("./variable.zig");
const errors = @import("errors");
const context = @import("context");
const TokenStream = types.TokenStream;
const ParseError = types.ParseError;
@ -55,10 +56,12 @@ pub const Statement = struct {
};
test "should parse a variable declaration statement" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "var my_variable = 322";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var statement: Statement = undefined;
@ -74,10 +77,12 @@ test "should parse a variable declaration statement" {
}
test "should fail on other constructs" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "a_function_call(322)";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var statement: Statement = undefined;

29
src/02_syntax/variable.zig
View File

@ -4,6 +4,7 @@ const expression = @import("expression.zig");
const types = @import("./types.zig");
const utils = @import("./utils.zig");
const errors = @import("errors");
const context = @import("context");
const TokenStream = types.TokenStream;
const ParseError = types.ParseError;
@ -88,10 +89,12 @@ pub const VariableBinding = struct {
};
test "should parse a minimal var" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "var my_variable = 322";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var binding: VariableBinding = undefined;
@ -110,10 +113,12 @@ test "should parse a minimal var" {
}
test "should return null if stream doesnt start with var" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "different_token_stream()";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var binding: VariableBinding = undefined;
@ -123,10 +128,12 @@ test "should return null if stream doesnt start with var" {
}
test "should fail if the identifier is missing" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "var ";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var error_data: errors.ErrorData = undefined;
@ -148,10 +155,12 @@ test "should fail if the identifier is missing" {
}
test "should fail if there is not an identifier after var" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "var 322";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var binding: VariableBinding = undefined;
@ -164,10 +173,12 @@ test "should fail if there is not an identifier after var" {
}
test "should fail if the equal sign is missing" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "var my_id ";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var binding: VariableBinding = undefined;
@ -180,10 +191,12 @@ test "should fail if the equal sign is missing" {
}
test "should fail if the equal sign is not found" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "var my_id is string";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var binding: VariableBinding = undefined;
@ -196,10 +209,12 @@ test "should fail if the equal sign is not found" {
}
test "should fail if the expression parsing fails" {
var ctx = context.CompilerContext.init(std.testing.allocator);
defer ctx.deinit();
const input = "var my_id = ehhh";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
const tokens = try lexic.tokenize(input, std.testing.allocator, &error_list);
const tokens = try lexic.tokenize(input, &ctx);
defer tokens.deinit();
var binding: VariableBinding = undefined;