refactor: lexer serialization

src/cli.zig

@@ -2,62 +2,37 @@ const std = @import("std");
 const errors = @import("errors");
 const lexic = @import("lexic");
 
-const LexResult = struct {
-    tokens: std.ArrayList(lexic.Token),
-    error_array: std.ArrayList(errors.ErrorData),
-};
-
 pub fn tokenize_to_json() !void {
-    // setup stdin, stdout and allocators
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
-    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer arena.deinit();
-    const alloc = gpa.allocator();
+    const alloc = arena.allocator();
 
+    // Setup buffered stdout once
     const stdout_file = std.io.getStdOut().writer();
     var bw = std.io.bufferedWriter(stdout_file);
     const stdout = bw.writer();
 
-    // read up to 8192 bytes from stdin until EOF
+    // Read all stdin
-    var buffer: [8192]u8 = undefined;
+    var stdin_buf = std.ArrayList(u8).init(alloc);
-    const stdin = std.io.getStdIn();
+    defer stdin_buf.deinit();
-    const bytes_read = try stdin.readAll(&buffer);
+    // 16MB, why would anyone ever have source code bigger than that??
+    const max_file_size = 16 * 1024 * 1024;
+    try std.io.getStdIn().reader().readAllArrayList(&stdin_buf, max_file_size);
 
-    const bytes = buffer[0..bytes_read];
+    // Tokenize
-
-    // tokenize
     var error_array = std.ArrayList(errors.ErrorData).init(alloc);
     defer error_array.deinit();
-
+    const tokens = try lexic.tokenize(stdin_buf.items, alloc, &error_array);
-    const tokens = try lexic.tokenize(bytes, alloc, &error_array);
     defer tokens.deinit();
 
-    // serialize & print json to stdout
+    // Write JSON directly to stdout
-    var json_arrl = std.ArrayList(u8).init(alloc);
+    try stdout.writeAll("{\"errors\":[");
-    defer json_arrl.deinit();
-    var json_writer = json_arrl.writer();
-
-    try json_writer.writeAll("{\"errors\":[");
-
-    const errors_len = error_array.items.len - 1;
     for (error_array.items, 0..) |err, idx| {
-        try err.write_json(alloc, json_writer);
+        try err.write_json(alloc, stdout);
-
+        if (idx < error_array.items.len - 1) try stdout.writeAll(",");
-        // write a comma only if there are items left
-        if (idx < errors_len) {
-            try json_writer.writeAll(",");
-        }
     }
-    try json_writer.writeAll("],\"tokens\":");
+    try stdout.writeAll("],\"tokens\":");
-
+    try std.json.stringify(tokens.items, .{}, stdout);
-    // write tokens as JSON
+    try stdout.writeAll("}");
-    const tokens_json = try std.json.stringifyAlloc(alloc, tokens.items, .{});
-    defer alloc.free(tokens_json);
-
-    try json_writer.writeAll(tokens_json);
-
-    try json_writer.writeAll("}");
-
-    try stdout.print("{s}", .{json_arrl.items});
     try bw.flush();
-
-    // the end
 }

src/errors/root.zig

@@ -154,7 +154,7 @@ pub const ErrorData = struct {
     // - Display message
 
     /// Writes this error as a JSON to the writer
-    pub fn write_json(self: ErrorData, alloc: std.mem.Allocator, writer: std.ArrayList(u8).Writer) !void {
+    pub fn write_json(self: ErrorData, alloc: std.mem.Allocator, writer: anytype) !void {
         // get this as JSON
         const json_str = try std.json.stringifyAlloc(alloc, .{
             .reason = self.reason,