Complete minimal flow for editor highlighting

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 static/css/out.css
 static/learn
+static/js/highlighter.js

lexer/highlighter.ts

@@ -0,0 +1,21 @@
+import {lex} from "./lexer.ts";
+import { CodeJar } from "codejar"
+
+function thp_highlighter(editor: any) {
+    let code: string = editor.textContent;
+
+    let tokens = lex(code);
+
+    let highlighted_code = "";
+
+    for (let token of tokens) {
+        highlighted_code += `<span class="token ${token.token_type}">${token.v}</span>`;
+    }
+
+    editor.innerHTML = highlighted_code;
+}
+
+// @ts-ignore
+window.thp_highlighter = thp_highlighter;
+// @ts-ignore
+window.CodeJar = CodeJar;

lexer/lexer.test.ts

@@ -11,34 +11,34 @@ describe("Lexer", () => {
     test("program with whitespace should return a single token", () => {
         const code = " ";
         const tokens = lex(code);
-        expect(tokens).toEqual([{v: " "}]);
+        expect(tokens).toEqual([{v: " ", token_type: ""}]);
     })
 
     test("program with newlines should return a single token", () => {
         const code = "\n";
         const tokens = lex(code);
-        expect(tokens).toEqual([{v: "\n"}]);
+        expect(tokens).toEqual([{v: "\n", token_type: ""}]);
     });
 
     test("program with random unicode should return the same unicode", () => {
         const code = "🍕";
         const tokens = lex(code);
-        expect(tokens).toEqual([{v: "🍕"}]);
+        expect(tokens).toEqual([{v: "🍕", token_type: ""}]);
     });
 
     test("should scan integers", () => {
         const code = "12345";
         const tokens = lex(code);
-        expect(tokens).toEqual([{v: "12345"}]);
+        expect(tokens).toEqual([{v: "12345", token_type: "number"}]);
     });
 
     test("should scan integers and whitespace around", () => {
         const code = "   12345  \n  ";
         const tokens = lex(code);
         expect(tokens).toEqual([
-            {v: "   "},
-            {v: "12345"},
-            {v: "  \n  "}
+            {v: "   ", token_type: ""},
+            {v: "12345", token_type: "number"},
+            {v: "  \n  ", token_type: ""},
         ]);
     });
 });

lexer/lexer.ts

@@ -1,7 +1,10 @@
 import { lex_number } from "./number_lexer.ts";
 import { is_digit } from "./utils.ts";
 
-export type Token = { v: string };
+export type Token = {
+    v: string,
+    token_type: string,
+};
 
 /**
  * Lexes a string of THP code, and returns an array of tokens. Unlike a regular
@@ -37,7 +40,7 @@ export function lex(code: string): Array<Token> {
         if (is_digit(c)) {
             // if the current default token is not empty, push it to the tokens array
             if (current_default_token !== "") {
-                tokens.push({ v: current_default_token });
+                tokens.push({ v: current_default_token, token_type: "" });
                 current_default_token = "";
             }
 
@@ -52,7 +55,7 @@ export function lex(code: string): Array<Token> {
         if (next_token !== null && next_position !== null) {
             // if there was a default token, push it to the tokens array
             if (current_default_token !== "") {
-                tokens.push({ v: current_default_token });
+                tokens.push({ v: current_default_token, token_type: "" });
                 current_default_token = "";
             }
 
@@ -70,7 +73,7 @@ export function lex(code: string): Array<Token> {
 
     // if there was a default token, push it to the tokens array
     if (current_default_token !== "") {
-        tokens.push({ v: current_default_token });
+        tokens.push({ v: current_default_token, token_type: "" });
         current_default_token = "";
     }
 

lexer/number_lexer.test.ts

@@ -6,14 +6,14 @@ describe("Number Lexer", () => {
         const code = "1";
         const token = lex_number(code, 0);
 
-        expect(token).toEqual([{ v: "1" }, 1]);
+        expect(token).toEqual([{ v: "1", token_type: "number" }, 1]);
     });
 
     test("should return a whole number token pt 2", () => {
         const code = "12345";
         const token = lex_number(code, 0);
 
-        expect(token).toEqual([{ v: "12345" }, 5]);
+        expect(token).toEqual([{ v: "12345", token_type: "number" }, 5]);
     });
 });
 

lexer/number_lexer.ts

@@ -13,7 +13,7 @@ import { is_digit } from "./utils.ts";
 export function lex_number(input: string, pos: number): [Token, number] {
     const [token_value, next] = scan_decimal(input, pos);
 
-    return [{ v: token_value }, next];
+    return [{ v: token_value, token_type: "number" }, next];
 }
 
 function scan_decimal(input: string, starting_position: number): [string, number] {

package.json

@@ -5,7 +5,7 @@
   "main": "index.js",
   "scripts": {
     "generate": "md-docs",
-    "bundle": "bun build ./lexer/lexer.ts --outdir ./static/js/ --format esm --minify",
+    "bundle": "bun build ./lexer/highlighter.ts --outdir ./static/js/ --format esm --minify",
     "dev": "concurrently -k \"tailwindcss -i ./tailwind.css -o ./static/css/out.css --watch\" \"serve ./static/ -l 3333\"",
     "codemirror": "esbuild --bundle ./static/js/codemirror.js  --outfile=./static/js/codemirror.min.js --minify --sourcemap",
     "tailwind:watch": "tailwindcss -i ./tailwind.css -o ./static/css/out.css --watch",

static/index.html

@@ -43,19 +43,13 @@
                 <br>
                 <br>
                 <div class="text-center">
-                    <a 
-            class="inline-block font-display text-lg border-2 border-pink-400 hover:bg-pink-400 transition-colors
-                hover:text-c-bg py-3 px-8 mx-6 rounded"
-            href="/learn/"
-        >
+                    <a class="inline-block font-display text-lg border-2 border-pink-400 hover:bg-pink-400 transition-colors
+                hover:text-c-bg py-3 px-8 mx-6 rounded" href="/learn/">
                         Learn
                     </a>
 
-        <a 
-            class="inline-block font-display text-lg border-2 border-sky-400 py-3 px-8 mx-6 rounded
-                transition-colors hover:text-black hover:bg-sky-400" 
-            href="/install/"
-        >
+                    <a class="inline-block font-display text-lg border-2 border-sky-400 py-3 px-8 mx-6 rounded
+                transition-colors hover:text-black hover:bg-sky-400" href="/install/">
                         Install
                     </a>
                 </div>
@@ -76,23 +70,41 @@
                 </g>
             </svg>
             <div class="h-1"></div>
-            
-        </div>
-    </div>
-
-    <div class="container mx-auto">
             <div id="editor" class="font-mono language-thp"></div>
         </div>
+    </div>
 
     <script src="/js/prism.min.js"></script>
     <script src="/js/prism.thp.js"></script>
-    <script src="/js/codemirror.min.js"></script>
-    <script src="https://unpkg.com/codeflask/build/codeflask.min.js"></script>
-    <!--
+    <script src="/js/highlighter.js"></script>
     <script>
-        const flask = new CodeFlask('#editor', { language: 'js' });
+        let jar = CodeJar(document.getElementById("editor"), thp_highlighter, {
+            tab: "    ",
+        });
+
+        jar.updateCode(
+            `// Actual generics & sum types 322 644
+fun find_person(Int person_id) -> Result[String, String] {
+    // Easy, explicit error bubbling
+    try Person::find_by_id(person_id)
+}
+
+val id = POST::get("person_id") ?? exit("Null person_id")
+// Errors as values
+val person = try find_person(person_id: id) with error {
+    eprint("Error: {error}")
+    exit("Person not found")
+}
+
+// First class HTML-like templates & components
+print(
+    <a href="/person/reports/{person.id}">
+        welcome, {person.name}
+    </a>
+)
+// And more!`
+        )
     </script>
-    -->
 </body>
 
 </html>

static/js/codemirror.js

@@ -1,58 +0,0 @@
-import { CodeJar } from "codejar"
-
-// Custom highlighter for THP, based on regex for now.
-// It'll implement a lexer & parser in the future.
-const thpHighlighter = (editor, pos) => {
-    let code = editor.textContent;
-
-    // Highlighting rules
-
-    // Identifier regex
-    const identifier = /[a-z][a-z0-9_]*/g;
-
-    // Datatype regex
-    const datatype = /[A-Z][a-z0-9_]*/g;
-
-    
-
-    // example
-    code = code.replace(
-        /\((\w+?)(\b)/g,
-        '(<font color="#8a2be2">$1</font>$2'
-    );
-    editor.innerHTML = code;
-
-    console.log("running highlighter...", pos);
-    code = code.replace(
-        /\((\w+?)(\b)/g,
-        '(<font color="#8a2be2">$1</font>$2'
-    );
-    editor.innerHTML = code;
-};
-
-let jar = CodeJar(document.getElementById("editor"), thpHighlighter, {
-    tab: "    ",
-});
-
-jar.updateCode(
-    `// Actual generics & sum types
-fun find_person(Int person_id) -> Result[String, String] {
-    // Easy, explicit error bubbling
-    try Person::find_by_id(person_id)
-}
-
-val id = POST::get("person_id") ?? exit("Null person_id")
-// Errors as values
-val person = try find_person(person_id: id) with error {
-    eprint("Error: {error}")
-    exit("Person not found")
-}
-
-// First class HTML-like templates & components
-print(
-    <a href="/person/reports/{person.id}">
-        welcome, {person.name}
-    </a>
-)
-// And more!`
-)

static/js/lexer.js

@@ -1 +0,0 @@
-console.log(Bun.version);