write a custom highlighter for codejar

lexer/lexer.test.ts

@@ -0,0 +1,45 @@
+import { expect, test, describe } from "bun:test";
+import { lex } from "./lexer";
+
+describe("Lexer", () => {
+    test("empty program should return no tokens", () => {
+        const code = "";
+        const tokens = lex(code);
+        expect(tokens).toEqual([]);
+    });
+
+    test("program with whitespace should return a single token", () => {
+        const code = " ";
+        const tokens = lex(code);
+        expect(tokens).toEqual([{v: " "}]);
+    })
+
+    test("program with newlines should return a single token", () => {
+        const code = "\n";
+        const tokens = lex(code);
+        expect(tokens).toEqual([{v: "\n"}]);
+    });
+
+    test("program with random unicode should return the same unicode", () => {
+        const code = "🍕";
+        const tokens = lex(code);
+        expect(tokens).toEqual([{v: "🍕"}]);
+    });
+
+    test("should scan integers", () => {
+        const code = "12345";
+        const tokens = lex(code);
+        expect(tokens).toEqual([{v: "12345"}]);
+    });
+
+    test("should scan integers and whitespace around", () => {
+        const code = "   12345  \n  ";
+        const tokens = lex(code);
+        expect(tokens).toEqual([
+            {v: "   "},
+            {v: "12345"},
+            {v: "  \n  "}
+        ]);
+    });
+});
+

lexer/lexer.ts

@@ -0,0 +1,80 @@
+import { lex_number } from "./number_lexer.ts";
+import { is_digit } from "./utils.ts";
+
+export type Token = { v: string };
+
+/**
+ * Lexes a string of THP code, and returns an array of tokens. Unlike a regular
+ * lexer, whitespace and other characters are not ignored, and are instead treated
+ * as a default token.
+ * 
+ * This lexer implements a subset of the grammar defined in the THP language specification,
+ * only recognizing the following tokens:
+ * - Identifier
+ * - Datatype
+ * - String
+ * - Number
+ * - Single line comment
+ * - Multi line comment
+ * - Keywords
+ * 
+ * @param code Code to lex
+ * @returns An array of all the tokens found
+ */
+export function lex(code: string): Array<Token> {
+    const code_len = code.length;
+    const tokens: Array<Token> = [];
+
+    let current_pos = 0;
+    let current_default_token = "";
+
+    while (current_pos < code_len) {
+        const c = code[current_pos];
+
+        let next_token: Token | null = null;
+        let next_position: number | null = null;
+
+        if (is_digit(c)) {
+            // if the current default token is not empty, push it to the tokens array
+            if (current_default_token !== "") {
+                tokens.push({ v: current_default_token });
+                current_default_token = "";
+            }
+
+            // lex a number
+            const [token, next] = lex_number(code, current_pos);
+            current_pos = next;
+            tokens.push(token);
+            continue;
+        }
+
+        // here, check if a token was found
+        if (next_token !== null && next_position !== null) {
+            // if there was a default token, push it to the tokens array
+            if (current_default_token !== "") {
+                tokens.push({ v: current_default_token });
+                current_default_token = "";
+            }
+
+            // then push the new token found
+            tokens.push(next_token);
+            current_pos = next_position;
+            continue;
+        }
+        // otherwise, add the current character to the default token
+        else {
+            current_default_token += c;
+            current_pos++;
+        }
+    }
+
+    // if there was a default token, push it to the tokens array
+    if (current_default_token !== "") {
+        tokens.push({ v: current_default_token });
+        current_default_token = "";
+    }
+
+    return tokens;
+}
+
+

lexer/number_lexer.test.ts

@@ -0,0 +1,19 @@
+import { expect, test, describe } from "bun:test";
+import { lex_number } from "./number_lexer";
+
+describe("Number Lexer", () => {
+    test("should return a whole number token", () => {
+        const code = "1";
+        const token = lex_number(code, 0);
+
+        expect(token).toEqual([{ v: "1" }, 1]);
+    });
+
+    test("should return a whole number token pt 2", () => {
+        const code = "12345";
+        const token = lex_number(code, 0);
+
+        expect(token).toEqual([{ v: "12345" }, 5]);
+    });
+});
+

lexer/number_lexer.ts

@@ -0,0 +1,47 @@
+import type { Token } from "./lexer.ts";
+import { is_digit } from "./utils.ts";
+
+/**
+ * Scans a number, at the given position in the input string.
+ * This function assumes that the character at the given position is a digit.
+ * It follows this grammar:
+ * 
+ * @param input the input string
+ * @param pos the position to start scanning from
+ * @returns 
+ */
+export function lex_number(input: string, pos: number): [Token, number] {
+    const [token_value, next] = scan_decimal(input, pos);
+
+    return [{ v: token_value }, next];
+}
+
+function scan_decimal(input: string, starting_position: number): [string, number] {
+    let current_value = "";
+    let pos = starting_position;
+
+    while (pos < input.length) {
+        const c = input[pos];
+
+        if (c === ".") {
+            // todo
+            throw new Error("Not implemented");
+        }
+        else if (c == "e" || c == "E") {
+            // todo
+            throw new Error("Not implemented");
+        }
+        else if (is_digit(c)) {
+            current_value += c;
+            pos += 1;
+        }
+        else {
+            break;
+        }
+
+    }
+
+    return [current_value, pos];
+}
+
+

lexer/utils.ts

@@ -0,0 +1,3 @@
+export function is_digit(c: string): boolean {
+    return c >= '0' && c <= '9';
+}

package.json

@@ -5,7 +5,9 @@
   "main": "index.js",
   "scripts": {
     "generate": "md-docs",
-    "dev": "concurrently -k \"pnpm tailwind:watch\" \"serve ./static/ -l 3333\"",
+    "bundle": "bun build ./lexer/lexer.ts --outdir ./static/js/ --format esm --minify",
+    "dev": "concurrently -k \"tailwindcss -i ./tailwind.css -o ./static/css/out.css --watch\" \"serve ./static/ -l 3333\"",
+    "codemirror": "esbuild --bundle ./static/js/codemirror.js  --outfile=./static/js/codemirror.min.js --minify --sourcemap",
     "tailwind:watch": "tailwindcss -i ./tailwind.css -o ./static/css/out.css --watch",
     "tailwind:build": "tailwindcss -i ./tailwind.css -o ./static/css/out.css --minify"
   },
@@ -13,10 +15,16 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "@types/bun": "^1.0.10",
+    "codejar": "^4.2.0",
     "tailwindcss": "^3.2.7"
   },
   "devDependencies": {
     "concurrently": "^8.2.0",
-    "serve": "^14.2.0"
+    "serve": "^14.2.1",
+    "bun-types": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
   }
-}
+}

static/index.html

@@ -13,7 +13,7 @@
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
     <link
-        href="https://fonts.googleapis.com/css2?family=Fira+Sans:wght@400;500;600;700;800;900&family=Fira+Code&family=Inter:ital,wght@0,400;1,700&display=swap"
+        href="https://fonts.googleapis.com/css2?family=Fira+Sans:wght@400;500;600;700;800;900&family=Fira+Code&display=swap"
         rel="stylesheet">
 </head>
 
@@ -26,7 +26,7 @@
         </div>
     </nav>
 
-    <div class="container mx-auto py-16 grid grid-cols-[5fr_4fr] gap-4 px-10">
+    <div class="container mx-auto py-16 grid grid-cols-[auto_32rem] gap-4 px-10">
         <div class="pl-10 table">
             <div class="table-cell align-middle">
                 <h1 class="font-display font-bold text-5xl leading-tight">
@@ -76,35 +76,23 @@
                 </g>
             </svg>
             <div class="h-1"></div>
-            <pre style="padding: 0 !important; border: none !important;">
+            
-                <code class="language-thp">
-                    // Actual generics & sum types
-                    fun find_person(Int person_id) -> Result[String, String] {
-                        // Easy, explicit error bubbling
-                        try Person::find_by_id(person_id)
-                    }
-
-                    val id = POST::get("person_id") ?? exit("Null person_id")
-                    // Errors as values
-                    val person = try find_person(person_id: id) with error {
-                        eprint("Error: {error}")
-                        exit("Person not found")
-                    }
-
-                    // First class HTML-like templates & components
-                    print(
-                        &lt;a href="/person/reports/{person.id}"&gt;
-                            welcome, {person.name}
-                        &lt;/a&gt;
-                    )
-                    // And more!
-                </code>
-            </pre>
         </div>
     </div>
 
+    <div class="container mx-auto">
+        <div id="editor" class="font-mono language-thp"></div>
+    </div>
+
     <script src="/js/prism.min.js"></script>
     <script src="/js/prism.thp.js"></script>
+    <script src="/js/codemirror.min.js"></script>
+    <script src="https://unpkg.com/codeflask/build/codeflask.min.js"></script>
+    <!--
+    <script>
+        const flask = new CodeFlask('#editor', { language: 'js' });
+    </script>
+    -->
 </body>
 
 </html>

static/js/codemirror.js

@@ -0,0 +1,58 @@
+import { CodeJar } from "codejar"
+
+// Custom highlighter for THP, based on regex for now.
+// It'll implement a lexer & parser in the future.
+const thpHighlighter = (editor, pos) => {
+    let code = editor.textContent;
+
+    // Highlighting rules
+
+    // Identifier regex
+    const identifier = /[a-z][a-z0-9_]*/g;
+
+    // Datatype regex
+    const datatype = /[A-Z][a-z0-9_]*/g;
+
+    
+
+    // example
+    code = code.replace(
+        /\((\w+?)(\b)/g,
+        '(<font color="#8a2be2">$1</font>$2'
+    );
+    editor.innerHTML = code;
+
+    console.log("running highlighter...", pos);
+    code = code.replace(
+        /\((\w+?)(\b)/g,
+        '(<font color="#8a2be2">$1</font>$2'
+    );
+    editor.innerHTML = code;
+};
+
+let jar = CodeJar(document.getElementById("editor"), thpHighlighter, {
+    tab: "    ",
+});
+
+jar.updateCode(
+    `// Actual generics & sum types
+fun find_person(Int person_id) -> Result[String, String] {
+    // Easy, explicit error bubbling
+    try Person::find_by_id(person_id)
+}
+
+val id = POST::get("person_id") ?? exit("Null person_id")
+// Errors as values
+val person = try find_person(person_id: id) with error {
+    eprint("Error: {error}")
+    exit("Person not found")
+}
+
+// First class HTML-like templates & components
+print(
+    <a href="/person/reports/{person.id}">
+        welcome, {person.name}
+    </a>
+)
+// And more!`
+)

static/js/lexer.js

@@ -0,0 +1 @@
+console.log(Bun.version);

tailwind.config.js

@@ -17,7 +17,7 @@ module.exports = {
             }
         },
         fontFamily: {
-            "mono": ["'Fira Code'", "Inconsolata", "Iosevka", "monospace"],
+            "mono": ["Iosevka", "monospace"],
             "display": ["Inter", "'Josefin Sans'", "'Fugaz One'", "sans-serif"],
             "body": ["'Fira Sans'", "Inter", "sans-serif"],
         },

tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noPropertyAccessFromIndexSignature": true
+  }
+}