Refactor syntax highlighter to new spec from the compiler

src/components/Code.astro

@@ -1,10 +1,17 @@
 ---
 import { native_highlighter } from "../lexer/highlighter";
+import CodeError from "./docs/CodeError.astro";
 
 const { thpcode } = Astro.props;
 
-const native_html = await native_highlighter(thpcode);
+const [native_html, error_type, error_message] = await native_highlighter(thpcode);
 ---
 
 <pre
-    class="language-thp"><code class="language-thp" set:html={native_html} /><span class="absolute top-2 right-2 inline-block text-sm select-none opacity-75">thp</span></pre>
+    class="language-thp"><code class="language-thp" set:html={native_html} /><span class="absolute top-2 right-2 inline-block text-sm select-none opacity-75">thp
+    </span></pre>
+{
+    error_message !== null && (
+        <CodeError error_type={error_type}>{error_message}</CodeError>
+    )
+}

src/components/docs/CodeError.astro

@@ -0,0 +1,8 @@
+---
+const { error_type = "Unknown" } = Astro.props;
+---
+
+<div class="px-4 py-2 rounded bg-red-200 dark:bg-red-950">
+    <span class="inline-block font-bold">{error_type} error:</span>
+    <slot />
+</div>

src/layouts/SpecLayout.astro

@@ -4,7 +4,7 @@ import PagesLayout from "./PagesLayout.astro";
 const { frontmatter, headings } = Astro.props;
 
 const posts = await Astro.glob("../pages/spec/**/*.{md,mdx}");
-const indexSubpath = `/spec/index.md`;
+const indexSubpath = `/spec/index.mdx`;
 ---
 
 <PagesLayout

src/lexer/highlighter.ts

@@ -1,11 +1,6 @@
 import { spawn } from "node:child_process";
 import { leftTrimDedent } from "../components/utils";
 
-export interface LexResult {
-    Ok?: Token[]
-    Err?: Err
-}
-
 export interface Token {
     token_type: TokenType
     value: string
@@ -36,7 +31,8 @@ type TokenType =
     "FUN";
 
 export interface Err {
-    Lex: LexError
+    Lex?: LexError
+    Syntax?: SyntaxError
 }
 
 export interface LexError {
@@ -44,23 +40,72 @@ export interface LexError {
     reason: string
 }
 
+export interface SyntaxError {
+    error_start: number
+    error_end: number
+    reason: string
+}
 
-export async function native_highlighter(code: string): Promise<string> {
+export interface TokenizeResult {
+    Ok?: Token[],
+    TokensOnly?: [Token[], Err],
+    Err?: Err,
+}
+
+
+export async function native_highlighter(code: string): Promise<[string, string, string | null]> {
     let formatted_code = leftTrimDedent(code).join("\n");
 
     const result = await native_lex(formatted_code);
 
     if (result.Err) {
-        throw new Error(JSON.stringify(result.Err.Lex) + "\n" + code);
+        return lex_error_highlighter(formatted_code, result.Err!.Lex!);
+    }
+    else if (result.TokensOnly) {
+        // TODO
+        const [tokens, error] = result.TokensOnly!;
+        return syntax_error_highlighter(formatted_code, tokens, error.Syntax!);
     }
 
     const tokens = result.Ok!;
 
-    const input_chars = formatted_code.split("");
+    const output = highlight_tokens(formatted_code, tokens);
+
+    return [output, "", null];
+}
+
+
+/**
+ * Highlights code that has a lexic error
+ */
+function lex_error_highlighter(code: string, error: LexError): [string, string, string] {
+    // Create a single error token
+
+    const err_pos = error.position;
+    const before_err = code.substring(0, err_pos);
+    const err_str = code[err_pos];
+    const after_err = code.substring(err_pos + 1);
+
+    const token = `<span class="token underline decoration-wavy decoration-red-500">${err_str}</span>`;
+
+    const all = `${before_err}${token}${after_err}`;
+
+    // TODO: Transform absolute posijion (error.position) into line:column
+    return [all, "Lexical", error.reason + " at position " + error.position]
+}
+
+function syntax_error_highlighter(code: string, tokens: Array<Token>, error: SyntaxError): [string, string, string] {
+    const highlighted = highlight_tokens(code, tokens);
+
+    const error_message = `${error.reason} from position ${error.error_start} to ${error.error_end}`;
+    return [highlighted, "Syntax", error_message];
+}
+
+function highlight_tokens(input: string, tokens: Array<Token>): string {
+    const input_chars = input.split("");
     let output = "";
 
     let current_pos = 0;
-
     for (let i = 0; i < tokens.length; i += 1) {
         const t = tokens[i]!;
         const token_start = t.position;
@@ -85,6 +130,7 @@ export async function native_highlighter(code: string): Promise<string> {
     return output;
 }
 
+
 function translate_token_type(tt: TokenType, value: string): string {
     const keywords = ["throws", "extends", "constructor", "case", "static", "const",
         "enum", "union", "loop", "use", "break", "catch", "continue", "as", "do",
@@ -120,7 +166,7 @@ function translate_token_type(tt: TokenType, value: string): string {
     }
 }
 
-const native_lex = (code: string) => new Promise<LexResult>((resolve, reject) => {
+const native_lex = (code: string) => new Promise<TokenizeResult>((resolve, reject) => {
     // Get binary path from .env
     const binary = import.meta.env.THP_BINARY;
     if (!binary) {

src/pages/learn/classes/readonly.mdx

@@ -5,3 +5,11 @@ title: Readonly
 import Code from "../../../components/Code.astro"
 
 # Readonly
+
+<Code thpcode={`
+class Caño
+{
+}
+`} />
+
+

src/pages/learn/templating/control-flow.mdx

@@ -57,11 +57,11 @@ fun UserDetail(User user) -> HTML
 {
     <div>
         @match user.type
-        case ::Admin
+        @case ::Admin
         {
             <button>Delete resource</button>
         }
-        case ::User
+        @case ::User
         {
             <button disable>Not allowed</button>
         }

src/pages/spec/index.mdx

@@ -20,6 +20,7 @@ pagesLayout:
   - path: ast
   - path: expression
 ---
+import Code from "../../components/Code.astro"
 
 
 # The THP Language Specification
@@ -100,11 +101,11 @@ greater than before, it emits a Indent token. If it's lower, emits a Dedent toke
 if it's the same it does nothing.
 
 
-```thp
+<Code thpcode={`
 1 + 2
     + 3
     + 4
-```
+`} />
 
 The previous code would emit the following tokens: `1` `+` `2` `NewLine` `Indent` `+` `3` `NewLine`
 `+` `4` `Dedent`
@@ -114,12 +115,12 @@ Additionaly, it is a lexical error to have wrong indentation. The lexer stores a
 previous indentation levels in a stack, and reports an error if a decrease in indentation
 doesn't match a previous level.
 
-```thp
+<Code thpcode={`
 if true {   // 0 indentation
-    print() // 4 indentation
-  print()   // 2 indentation. Error. There is no 2-indentation level
+//    print() // 4 indentation
+//  print()   // 2 indentation. Error. There is no 2-indentation level
 }
-```
+`} />
 
 All productions of the grammar ignore whitespace/indentation, except those involved in
 semicolon inference.
@@ -134,26 +135,26 @@ Statements in THP end when a new line is encountered:
 
 
 
-```thp
+<Code thpcode={`
 // The statement ends         | here, on the newline
 val value = (123 + 456) * 0.75
-```
+`} />
 
-```thp
+<Code thpcode={`
 // Each line contains a different statement. They all end on their new lines
 
 var a = 1 + 2   // a = 3
-+ 3             // this is not part of `a`, this is a different statement
-```
++ 3             // this is not part of \`a\`, this is a different statement
+`} />
 
 This is true even if the line ends with an operator:
 
-```thp
+<Code thpcode={`
 // These are still different statements
 
 var a = 1 + 2 +     // This is now a compile error, there is a hanging `+`
 3                   // This is still a different statement
-```
+`} />
 
 
 ### Parenthesis
@@ -161,16 +162,16 @@ var a = 1 + 2 +     // This is now a compile error, there is a hanging `+`
 Exception 1: When a parenthesis is open, all following whitespace is ignored
 until the closing parenthesis.
 
-```thp
+<Code thpcode={`
 // open parenthesis found, all whitespace is ignored until the closing
 name.contains(
 "weird"
     )
-```
+`} />
 
 However, for a parenthesis to begin to act, it needs to be open on the same line.
 
-```thp
+<Code thpcode={`
 // Still 2 statements, because the parenthesis is in a new line
 print
 (
@@ -181,7 +182,7 @@ print
 print(
     "hello"
 )
-```
+`} />
 
 ### Indented binary operator
 
@@ -189,22 +190,22 @@ Exception 2:
 
 - When a binary operator is followed by indentation:
 
-```thp
+<Code thpcode={`
 val sum = 1 + 2 +   // The line ends with a binary operator
     3               // There is indentation
-```
+`} />
 
 - Or when indentation is followed by a binary operator:
 
-```thp
+<Code thpcode={`
 val sum = 1 + 2
     + 3             // Indentation and a binary operator
-```
+`} />
 
 In theses cases, all whitespace will be ignored
 until the indentation returns to the initial level.
 
-```thp
+<Code thpcode={`
 // This method chain is a single statement because of the indentation
 val person = PersonBuilder()
     .set_name("john")
@@ -215,6 +216,6 @@ val person = PersonBuilder()
 
 // Here indentation returns, and a new statement begins
 print(person)
-```
+`} />
 
 

src/pages/spec/tokens/comments.mdx

@@ -2,6 +2,7 @@
 layout: ../../../layouts/SpecLayout.astro
 title: Comment
 ---
+import Code from "../../../components/Code.astro"
 
 # Comment
 
@@ -9,8 +10,8 @@ title: Comment
 Comment = "//", any_except_new_line
 ```
 
-```thp
+<Code thpcode={`
 // This is a comment
 // 
 // Another // comment
-```
+`} />

src/pages/spec/tokens/identifier.mdx

@@ -2,6 +2,7 @@
 layout: ../../../layouts/SpecLayout.astro
 title: Identifiers & Datatypes
 ---
+import Code from "../../../components/Code.astro"
 
 # Identifiers & Datatypes
 
@@ -18,13 +19,13 @@ Identifier        = (underscore | lowercase_letter), identifier_letter*
 identifier_letter = underscore | lowercase_letter | uppercase_letter | decimal_digit
 ```
 
-```thp
+<Code thpcode={`
 identifier
 _identifier
 _123
 _many_letters
 camelCase
-```
+`} />
 
 
 ## Datatype
@@ -33,20 +34,20 @@ camelCase
 Datatype = uppercase_letter, indentifier_letter*
 ```
 
-```thp
+<Code thpcode={`
 Datatype
 PDO
 WEIRD_DATATYPE
-```
+`} />
 
 
 ## Keywords
 
 The following are (currently) THP keywords:
 
-```thp
+<Code thpcode={`
 val     var     fun
-```
+`} />
 
 Keywords are scanned first as identifiers, then transformed
 to their respective tokens.

src/pages/spec/tokens/numbers.mdx

@@ -2,6 +2,7 @@
 layout: ../../../layouts/SpecLayout.astro
 title: Numbers
 ---
+import Code from "../../../components/Code.astro"
 
 # Numbers
 
@@ -15,12 +16,12 @@ hexadecimal_number = "0", ("x" | "X"), hexadecimal_digit+
 decimal_number     = decimal_digit+
 ```
 
-```thp
+<Code thpcode={`
 12345
 01234   // This is a decimal number, not an octal number
 0xff25
 0XFfaA
-```
+`} />
 
 `TODO`: Implement octal `0o777` and binary `0b0110`.
 
@@ -36,14 +37,14 @@ Float = decimal_number, ".", decimal_number+, scientific_notation?
 scientific_notation = "e", ("+" | "-"), decimal_number
 ```
 
-```thp
+<Code thpcode={`
 123.456
 123.456e+4
 123.456e-2
 
 123e+10
 123e-3
-```
+`} />
 
 
 All floating point numbers must start with at least 1 digit.

src/pages/spec/tokens/operator.mdx

@@ -2,6 +2,7 @@
 layout: ../../../layouts/SpecLayout.astro
 title: Operator
 ---
+import Code from "../../../components/Code.astro"
 
 # Operator
 
@@ -14,9 +15,9 @@ operator_char = "+" | "-" | "=" | "*" | "!" | "/" | "|"
               | "<" | ">" | "^" | "." | ":"
 ```
 
-```thp
+<Code thpcode={`
 + - / * % < > <= >= -> =>
-```
+`} />
 
 These are all the characters that can make an operator.
 

src/pages/spec/tokens/string.mdx

@@ -2,6 +2,7 @@
 layout: ../../../layouts/SpecLayout.astro
 title: String
 ---
+import Code from "../../../components/Code.astro"
 
 # String
 
@@ -19,11 +20,11 @@ escape_seq   = "\n"
 string_char  = any_unicode_except_newline_and_double_quote
 ```
 
-```thp
+<Code thpcode={`
 "hello"
 ""
 "it's me"
-"\"Mario\""
-```
+"\\"Mario\\""
+`} />
 
 `TODO`: String interpolation