test: fix error in the lexer, after ignoring whitespace

This commit is contained in:
Fernando Araoz 2024-12-12 07:06:25 -05:00
parent 55231e986e
commit fd0459aef1
2 changed files with 35 additions and 1 deletions

View File

@ -25,6 +25,11 @@ pub fn tokenize(input: []const u8, alloc: std.mem.Allocator) !std.ArrayList(Toke
const actual_next_pos = ignore_whitespace(input, current_pos);
assert(current_pos <= actual_next_pos);
// if after processing whitespace we reach eof, exit
if (actual_next_pos == input_len) {
break;
}
// attempt to lex a number
if (try number.lex(input, input_len, actual_next_pos)) |tuple| {
assert(tuple[1] > current_pos);

View File

@ -14,6 +14,7 @@ const VariableBinding = struct {
expression: *expression.Expression,
alloc: std.mem.Allocator,
/// Parses a variable binding
fn init(target: *VariableBinding, tokens: *const TokenStream, pos: usize, allocator: std.mem.Allocator) ParseError!void {
std.debug.assert(pos < tokens.items.len);
@ -57,7 +58,7 @@ const VariableBinding = struct {
};
}
fn deinit(self: *@This()) void {
fn deinit(self: *VariableBinding) void {
self.alloc.destroy(self.expression);
}
};
@ -73,3 +74,31 @@ test "should parse a minimal var" {
try std.testing.expectEqual(true, binding.is_mutable);
}
test "should fail is it doesnt start with var" {
const input = "different_token_stream()";
const tokens = try lexic.tokenize(input, std.testing.allocator);
defer tokens.deinit();
var binding: VariableBinding = undefined;
binding.init(&tokens, 0, std.testing.allocator) catch |err| {
try std.testing.expectEqual(ParseError.Unmatched, err);
return;
};
try std.testing.expect(false);
}
test "should fail if the idenfier is missing" {
const input = "var ";
const tokens = try lexic.tokenize(input, std.testing.allocator);
defer tokens.deinit();
var binding: VariableBinding = undefined;
binding.init(&tokens, 0, std.testing.allocator) catch |err| {
try std.testing.expectEqual(ParseError.Error, err);
return;
};
try std.testing.expect(false);
}