fernando/thp-zig
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: fernando:9d89f2fad04d1f1f2f4eb090b3784463d1298f68
Branches Tags
fernando:main
..
compare: fernando:c969830a421c6cf149a1a2d891b6d7d539a51d95
Branches Tags
fernando:main

No commits in common. "9d89f2fad04d1f1f2f4eb090b3784463d1298f68" and "c969830a421c6cf149a1a2d891b6d7d539a51d95" have entirely different histories.
9d89f2fad0 ... c969830a42

3 changed files with 27 additions and 71 deletions
Show Stats Expand all files Collapse all files

12
src/01_lexic/number.zig
View File

@ -264,7 +264,7 @@ fn scientific(
test "int lexer 1" {
const input = "322 ";
const result = try integer(input, input.len, 0, undefined, std.heap.page_allocator);
const result = try integer(input, input.len, 0);
if (result) |tuple| {
const r = tuple[0];
@ -276,7 +276,7 @@ test "int lexer 1" {
test "int lexer 2" {
const input = " 644 ";
const result = try integer(input, input.len, 3, undefined, std.heap.page_allocator);
const result = try integer(input, input.len, 3);
if (result) |tuple| {
const r = tuple[0];
@ -288,7 +288,7 @@ test "int lexer 2" {
test "int lexer 3" {
const input = "4";
const result = try integer(input, input.len, 0, undefined, std.heap.page_allocator);
const result = try integer(input, input.len, 0);
if (result) |tuple| {
const r = tuple[0];
@ -300,7 +300,7 @@ test "int lexer 3" {
test "should return null if not an integer" {
const input = "prosor prosor";
const result = try integer(input, input.len, 0, undefined, std.heap.page_allocator);
const result = try integer(input, input.len, 0);
try std.testing.expect(result == null);
}
@ -323,7 +323,6 @@ test "should fail on integer with leading zero" {
defer std.testing.allocator.destroy(errdata);
const result = lex(input, input.len, 0, errdata, std.testing.allocator) catch |err| {
try std.testing.expect(err == token.LexError.LeadingZero);
defer errdata.deinit();
return;
};
@ -509,7 +508,6 @@ test "should fail on incomplete fp number" {
defer std.testing.allocator.destroy(errdata);
const result = lex(input, input.len, 0, errdata, std.testing.allocator) catch |err| {
try std.testing.expect(err == token.LexError.IncompleteFloatingNumber);
errdata.deinit();
return;
};
@ -541,7 +539,6 @@ test "should fail on incomplete scientific number" {
defer std.testing.allocator.destroy(errdata);
const result = lex(input, input.len, 0, errdata, std.testing.allocator) catch |err| {
try std.testing.expect(err == token.LexError.IncompleteScientificNumber);
defer errdata.deinit();
return;
};
@ -561,7 +558,6 @@ test "should fail on incomplete scientific number 2" {
defer std.testing.allocator.destroy(errdata);
const result = lex(input, input.len, 0, errdata, std.testing.allocator) catch |err| {
try std.testing.expect(err == token.LexError.IncompleteScientificNumber);
defer errdata.deinit();
return;
};

34
src/01_lexic/root.zig
View File

@ -41,9 +41,8 @@ pub fn tokenize(
break;
}
var current_error: errors.ErrorData = undefined;
// attempt to lex a number
var current_error: errors.ErrorData = undefined;
const number_lex = number.lex(input, input_len, actual_next_pos, &current_error, alloc) catch |e| switch (e) {
// recoverable errors
LexError.Incomplete, LexError.LeadingZero, LexError.IncompleteFloatingNumber, LexError.IncompleteScientificNumber => {
@ -52,6 +51,7 @@ pub fn tokenize(
// ignore everything until whitespace and loop
current_pos = ignore_until_whitespace(input, actual_next_pos);
assert(current_pos > actual_next_pos);
continue;
},
// just throw unrecoverable errors
@ -66,36 +66,24 @@ pub fn tokenize(
continue;
}
// attempt to lex an identifier. identifier parsing has no errors
if (try identifier.lex(input, actual_next_pos)) |tuple| {
// attempt to lex an identifier
else if (try identifier.lex(input, actual_next_pos)) |tuple| {
assert(tuple[1] > current_pos);
const t = tuple[0];
current_pos = tuple[1];
try tokens.append(t);
continue;
}
// attempt to lex a string
const str_lex = string.lex(input, actual_next_pos, &current_error, alloc) catch |e| switch (e) {
LexError.IncompleteString => {
try err_arrl.append(current_error);
current_pos = ignore_until_whitespace(input, actual_next_pos);
continue;
},
else => return e,
};
if (str_lex) |tuple| {
else if (try string.lex(input, actual_next_pos)) |tuple| {
assert(tuple[1] > current_pos);
const t = tuple[0];
current_pos = tuple[1];
try tokens.append(t);
continue;
}
// attempt to lex a datatype
if (try datatype.lex(input, actual_next_pos)) |tuple| {
else if (try datatype.lex(input, actual_next_pos)) |tuple| {
assert(tuple[1] > current_pos);
const t = tuple[0];
current_pos = tuple[1];
@ -202,17 +190,17 @@ test "should insert an item, fail, and not leak" {
const input = "322 \"hello";
var error_list = std.ArrayList(errors.ErrorData).init(std.testing.allocator);
defer error_list.deinit();
defer for (error_list.items) |*i| {
i.deinit();
};
const arrl = tokenize(input, std.testing.allocator, &error_list) catch |e| switch (e) {
error.IncompleteString => {
return;
},
else => {
try std.testing.expect(false);
return;
},
};
defer arrl.deinit();
try std.testing.expect(false);
arrl.deinit();
}
test "shouldnt leak" {

52
src/01_lexic/string.zig
View File

@ -2,19 +2,13 @@ const std = @import("std");
const assert = std.debug.assert;
const token = @import("./token.zig");
const utils = @import("./utils.zig");
const errors = @import("errors");
const Token = token.Token;
const TokenType = token.TokenType;
const LexError = token.LexError;
const LexReturn = token.LexReturn;
pub fn lex(
input: []const u8,
start: usize,
err: *errors.ErrorData,
alloc: std.mem.Allocator,
) LexError!?LexReturn {
pub fn lex(input: []const u8, start: usize) LexError!?LexReturn {
const cap = input.len;
assert(start < cap);
@ -34,28 +28,18 @@ pub fn lex(
current_pos + 1,
};
}
// new line, initialize and return error
// new line, return error
else if (next_char == '\n') {
try err.init("Incomplete String", current_pos, current_pos + 1, alloc);
try err.add_label("Found a new line here", current_pos, current_pos + 1);
err.set_help("Strings must always end on the same line that they start.");
return LexError.IncompleteString;
}
// lex escape characters
else if (next_char == '\\') {
// if next char is EOF, return error
if (current_pos + 1 == cap) {
try err.init("Incomplete String", current_pos, current_pos + 1, alloc);
try err.add_label("Found EOF here", current_pos, current_pos + 1);
err.set_help("Strings must always end on the same line that they start.");
return LexError.IncompleteString;
}
// if next char is newline, return error
else if (input[current_pos + 1] == '\n') {
try err.init("Incomplete String", current_pos, current_pos + 1, alloc);
try err.add_label("Found a new line here", current_pos, current_pos + 1);
err.set_help("Strings must always end on the same line that they start.");
return LexError.IncompleteString;
}
// here just consume whatever char is after
@ -68,16 +52,12 @@ pub fn lex(
}
// this could only reach when EOF is hit, return error
try err.init("Incomplete String", current_pos, current_pos + 1, alloc);
try err.add_label("Found EOF here", current_pos, current_pos + 1);
err.set_help("Strings must always end on the same line that they start.");
return LexError.IncompleteString;
}
test "should lex empty string" {
const input = "\"\"";
const output = try lex(input, 0, undefined, std.testing.allocator);
const output = try lex(input, 0);
if (output) |tuple| {
const t = tuple[0];
@ -90,7 +70,7 @@ test "should lex empty string" {
test "should lex string with 1 char" {
const input = "\"a\"";
const output = try lex(input, 0, undefined, std.testing.allocator);
const output = try lex(input, 0);
if (output) |tuple| {
const t = tuple[0];
@ -103,7 +83,7 @@ test "should lex string with 1 char" {
test "should lex string with unicode" {
const input = "\"😭\"";
const output = try lex(input, 0, undefined, std.testing.allocator);
const output = try lex(input, 0);
if (output) |tuple| {
const t = tuple[0];
@ -116,16 +96,14 @@ test "should lex string with unicode" {
test "shouldnt lex other things" {
const input = "322";
const output = try lex(input, 0, undefined, std.testing.allocator);
const output = try lex(input, 0);
try std.testing.expect(output == null);
}
test "should fail on EOF before closing string" {
const input = "\"hello";
var errdata: errors.ErrorData = undefined;
_ = lex(input, 0, &errdata, std.testing.allocator) catch |err| {
_ = lex(input, 0) catch |err| {
try std.testing.expectEqual(LexError.IncompleteString, err);
defer errdata.deinit();
return;
};
@ -134,10 +112,8 @@ test "should fail on EOF before closing string" {
test "should fail on newline before closing string" {
const input = "\"hello\n";
var errdata: errors.ErrorData = undefined;
_ = lex(input, 0, &errdata, std.testing.allocator) catch |err| {
_ = lex(input, 0) catch |err| {
try std.testing.expectEqual(LexError.IncompleteString, err);
defer errdata.deinit();
return;
};
@ -146,7 +122,7 @@ test "should fail on newline before closing string" {
test "should lex string with escape character 1" {
const input = "\"test\\\"string\"";
const output = try lex(input, 0, undefined, std.testing.allocator);
const output = try lex(input, 0);
if (output) |tuple| {
const t = tuple[0];
@ -159,7 +135,7 @@ test "should lex string with escape character 1" {
test "should lex string with escape character 2" {
const input = "\"test\\\\string\"";
const output = try lex(input, 0, undefined, std.testing.allocator);
const output = try lex(input, 0);
if (output) |tuple| {
const t = tuple[0];
@ -172,9 +148,7 @@ test "should lex string with escape character 2" {
test "should fail on EOF after backslash" {
const input = "\"hello \\";
var errdata: errors.ErrorData = undefined;
_ = lex(input, 0, &errdata, std.testing.allocator) catch |err| {
defer errdata.deinit();
_ = lex(input, 0) catch |err| {
try std.testing.expectEqual(LexError.IncompleteString, err);
return;
};
@ -184,9 +158,7 @@ test "should fail on EOF after backslash" {
test "should fail on newline after backslash" {
const input = "\"hello \\\n";
var errdata: errors.ErrorData = undefined;
_ = lex(input, 0, &errdata, std.testing.allocator) catch |err| {
defer errdata.deinit();
_ = lex(input, 0) catch |err| {
try std.testing.expectEqual(LexError.IncompleteString, err);
return;
};