feat: serialize into buffers

src/cli.zig

@@ -0,0 +1,42 @@
+const std = @import("std");
+const errors = @import("errors");
+const lexic = @import("lexic");
+
+const LexResult = struct {
+    tokens: std.ArrayList(lexic.Token),
+    error_array: std.ArrayList(errors.ErrorData),
+};
+
+pub fn tokenize_to_json() !void {
+    // setup stdin, stdout and allocators
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    const alloc = gpa.allocator();
+
+    const stdout_file = std.io.getStdOut().writer();
+    var bw = std.io.bufferedWriter(stdout_file);
+    const stdout = bw.writer();
+
+    // read up to 8192 bytes from stdin until EOF
+    var buffer: [8192]u8 = undefined;
+    const stdin = std.io.getStdIn();
+    const bytes_read = try stdin.readAll(&buffer);
+
+    const bytes = buffer[0..bytes_read];
+
+    // tokenize
+    var error_array = std.ArrayList(errors.ErrorData).init(alloc);
+    defer error_array.deinit();
+
+    const tokens = try lexic.tokenize(bytes, alloc, &error_array);
+    defer tokens.deinit();
+
+    // serialize & print json to stdout
+    const json_out = try std.json.stringifyAlloc(alloc, .{
+        .errors = error_array.items,
+    }, .{});
+    defer alloc.free(json_out);
+    try stdout.print("{s}", .{json_out});
+    try bw.flush();
+
+    // the end
+}

src/errors/error_label.zig

@@ -4,58 +4,4 @@ pub const ErrorLabel = struct {
     message: []const u8,
     start: usize,
     end: usize,
-
-    /// Converts this struct into JSON
-    pub fn json(self: ErrorLabel, alloc: std.mem.Allocator) ![]u8 {
-        return try std.json.stringifyAlloc(alloc, .{
-            .message = self.message,
-            .start = self.start,
-            .end = self.end,
-        }, .{});
-    }
 };
-
-test "should serialize" {
-    const label = ErrorLabel{
-        .message = "Error",
-        .start = 5,
-        .end = 6,
-    };
-    const json_str = try label.json(std.testing.allocator);
-    defer std.testing.allocator.free(json_str);
-
-    const expected =
-        \\{"message":"Error","start":5,"end":6}
-    ;
-    try std.testing.expectEqualStrings(expected, json_str);
-}
-
-test "should handle special characters" {
-    const label = ErrorLabel{
-        .message = "Error\"with\"quotes",
-        .start = 0,
-        .end = 1,
-    };
-    const json_str = try label.json(std.testing.allocator);
-    defer std.testing.allocator.free(json_str);
-
-    const expected =
-        \\{"message":"Error\"with\"quotes","start":0,"end":1}
-    ;
-    try std.testing.expectEqualStrings(expected, json_str);
-}
-
-test "should serialize empty message" {
-    const label = ErrorLabel{
-        .message = "",
-        .start = 0,
-        .end = 0,
-    };
-    const json_str = try label.json(std.testing.allocator);
-    defer std.testing.allocator.free(json_str);
-
-    const expected =
-        \\{"message":"","start":0,"end":0}
-    ;
-    try std.testing.expectEqualStrings(expected, json_str);
-}

src/errors/root.zig

@@ -153,10 +153,20 @@ pub const ErrorData = struct {
     // - Get previous, current and next line
     // - Display message
 
-    /// Transform this error into a JSON
-    pub fn json(alloc: std.mem.Allocator) void {
-        _ = alloc;
-        std.debug.panic(":c");
+    /// Writes this error as a JSON to the writer
+    pub fn write_json(self: ErrorData, alloc: std.mem.Allocator, writer: std.ArrayList(u8).Writer) !void {
+        // get this as JSON
+        const json_str = try std.json.stringifyAlloc(alloc, .{
+            .reason = self.reason,
+            .help = self.help,
+            .start_position = self.start_position,
+            .end_position = self.end_position,
+            .labels = self.labels.items,
+        }, .{});
+        defer alloc.free(json_str);
+
+        // write the JSON to the writer
+        try writer.writeAll(json_str);
     }
 
     pub fn deinit(self: *@This()) void {
@@ -332,6 +342,27 @@ test "should gen error message with label and help" {
     , out);
 }
 
+test "should serialize a minimal error" {
+    var err = ErrorData{
+        .reason = "Invalid identifier",
+        .help = null,
+        .start_position = 6,
+        .end_position = 9,
+        .labels = std.ArrayList(ErrorLabel).init(std.testing.allocator),
+        .alloc = std.testing.allocator,
+    };
+    defer err.deinit();
+    var out_writer = std.ArrayList(u8).init(std.testing.allocator);
+    defer out_writer.deinit();
+
+    try err.write_json(std.testing.allocator, out_writer.writer());
+
+    const expected =
+        \\{"reason":"Invalid identifier","help":null,"start_position":6,"end_position":9,"labels":[]}
+    ;
+    try std.testing.expectEqualStrings(expected, out_writer.items);
+}
+
 // TODO: add more tests:
 // - when the error has len=1
 // - when the error has len=0

src/main.zig

@@ -3,6 +3,8 @@ const lexic = @import("lexic");
 const syntax = @import("syntax");
 const errors = @import("errors");
 
+const cli = @import("cli.zig");
+
 const tracing = @import("config").tracing;
 
 const thp_version: []const u8 = "0.0.1";
@@ -12,6 +14,19 @@ pub fn main() !void {
 }
 
 fn repl() !void {
+    // first check to see if we are serializing tokens
+    var args = std.process.args();
+    defer args.deinit();
+
+    // ignore executable
+    _ = args.next();
+    if (args.next()) |arg| {
+        if (std.mem.eql(u8, "lex", arg)) {
+            try cli.tokenize_to_json();
+            return;
+        }
+    }
+
     const stdout_file = std.io.getStdOut().writer();
     var bw = std.io.bufferedWriter(stdout_file);
     const stdout = bw.writer();