diff --git a/.gitignore b/.gitignore index 597f9e4..8b01b9b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules static/css/out.css static/learn +static/js/highlighter.js diff --git a/lexer/highlighter.ts b/lexer/highlighter.ts new file mode 100644 index 0000000..c1b612b --- /dev/null +++ b/lexer/highlighter.ts @@ -0,0 +1,21 @@ +import {lex} from "./lexer.ts"; +import { CodeJar } from "codejar" + +function thp_highlighter(editor: any) { + let code: string = editor.textContent; + + let tokens = lex(code); + + let highlighted_code = ""; + + for (let token of tokens) { + highlighted_code += `${token.v}`; + } + + editor.innerHTML = highlighted_code; +} + +// @ts-ignore +window.thp_highlighter = thp_highlighter; +// @ts-ignore +window.CodeJar = CodeJar; diff --git a/lexer/lexer.test.ts b/lexer/lexer.test.ts index c45c59d..db3af6e 100644 --- a/lexer/lexer.test.ts +++ b/lexer/lexer.test.ts @@ -11,34 +11,34 @@ describe("Lexer", () => { test("program with whitespace should return a single token", () => { const code = " "; const tokens = lex(code); - expect(tokens).toEqual([{v: " "}]); + expect(tokens).toEqual([{v: " ", token_type: ""}]); }) test("program with newlines should return a single token", () => { const code = "\n"; const tokens = lex(code); - expect(tokens).toEqual([{v: "\n"}]); + expect(tokens).toEqual([{v: "\n", token_type: ""}]); }); test("program with random unicode should return the same unicode", () => { const code = "🍕"; const tokens = lex(code); - expect(tokens).toEqual([{v: "🍕"}]); + expect(tokens).toEqual([{v: "🍕", token_type: ""}]); }); test("should scan integers", () => { const code = "12345"; const tokens = lex(code); - expect(tokens).toEqual([{v: "12345"}]); + expect(tokens).toEqual([{v: "12345", token_type: "number"}]); }); test("should scan integers and whitespace around", () => { const code = " 12345 \n "; const tokens = lex(code); expect(tokens).toEqual([ - {v: " "}, - {v: "12345"}, - {v: " \n "} + {v: " ", token_type: ""}, + {v: "12345", token_type: "number"}, + {v: " \n ", token_type: ""}, ]); }); }); diff --git a/lexer/lexer.ts b/lexer/lexer.ts index 58f898a..aa3f2af 100644 --- a/lexer/lexer.ts +++ b/lexer/lexer.ts @@ -1,7 +1,10 @@ import { lex_number } from "./number_lexer.ts"; import { is_digit } from "./utils.ts"; -export type Token = { v: string }; +export type Token = { + v: string, + token_type: string, +}; /** * Lexes a string of THP code, and returns an array of tokens. Unlike a regular @@ -37,7 +40,7 @@ export function lex(code: string): Array { if (is_digit(c)) { // if the current default token is not empty, push it to the tokens array if (current_default_token !== "") { - tokens.push({ v: current_default_token }); + tokens.push({ v: current_default_token, token_type: "" }); current_default_token = ""; } @@ -52,7 +55,7 @@ export function lex(code: string): Array { if (next_token !== null && next_position !== null) { // if there was a default token, push it to the tokens array if (current_default_token !== "") { - tokens.push({ v: current_default_token }); + tokens.push({ v: current_default_token, token_type: "" }); current_default_token = ""; } @@ -70,7 +73,7 @@ export function lex(code: string): Array { // if there was a default token, push it to the tokens array if (current_default_token !== "") { - tokens.push({ v: current_default_token }); + tokens.push({ v: current_default_token, token_type: "" }); current_default_token = ""; } diff --git a/lexer/number_lexer.test.ts b/lexer/number_lexer.test.ts index 17b3b6f..4c1da1d 100644 --- a/lexer/number_lexer.test.ts +++ b/lexer/number_lexer.test.ts @@ -6,14 +6,14 @@ describe("Number Lexer", () => { const code = "1"; const token = lex_number(code, 0); - expect(token).toEqual([{ v: "1" }, 1]); + expect(token).toEqual([{ v: "1", token_type: "number" }, 1]); }); test("should return a whole number token pt 2", () => { const code = "12345"; const token = lex_number(code, 0); - expect(token).toEqual([{ v: "12345" }, 5]); + expect(token).toEqual([{ v: "12345", token_type: "number" }, 5]); }); }); diff --git a/lexer/number_lexer.ts b/lexer/number_lexer.ts index 8e2ae95..6f86e9e 100644 --- a/lexer/number_lexer.ts +++ b/lexer/number_lexer.ts @@ -13,7 +13,7 @@ import { is_digit } from "./utils.ts"; export function lex_number(input: string, pos: number): [Token, number] { const [token_value, next] = scan_decimal(input, pos); - return [{ v: token_value }, next]; + return [{ v: token_value, token_type: "number" }, next]; } function scan_decimal(input: string, starting_position: number): [string, number] { diff --git a/package.json b/package.json index 590e491..bde3e3c 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.js", "scripts": { "generate": "md-docs", - "bundle": "bun build ./lexer/lexer.ts --outdir ./static/js/ --format esm --minify", + "bundle": "bun build ./lexer/highlighter.ts --outdir ./static/js/ --format esm --minify", "dev": "concurrently -k \"tailwindcss -i ./tailwind.css -o ./static/css/out.css --watch\" \"serve ./static/ -l 3333\"", "codemirror": "esbuild --bundle ./static/js/codemirror.js --outfile=./static/js/codemirror.min.js --minify --sourcemap", "tailwind:watch": "tailwindcss -i ./tailwind.css -o ./static/css/out.css --watch", diff --git a/static/index.html b/static/index.html index 4d0f7ff..6feb3d7 100644 --- a/static/index.html +++ b/static/index.html @@ -43,21 +43,15 @@

- - Learn - + + Learn + - - Install - + + Install +
@@ -76,23 +70,41 @@
- +
-
-
-
- - - - \ No newline at end of file diff --git a/static/js/codemirror.js b/static/js/codemirror.js deleted file mode 100644 index e88d9b4..0000000 --- a/static/js/codemirror.js +++ /dev/null @@ -1,58 +0,0 @@ -import { CodeJar } from "codejar" - -// Custom highlighter for THP, based on regex for now. -// It'll implement a lexer & parser in the future. -const thpHighlighter = (editor, pos) => { - let code = editor.textContent; - - // Highlighting rules - - // Identifier regex - const identifier = /[a-z][a-z0-9_]*/g; - - // Datatype regex - const datatype = /[A-Z][a-z0-9_]*/g; - - - - // example - code = code.replace( - /\((\w+?)(\b)/g, - '($1$2' - ); - editor.innerHTML = code; - - console.log("running highlighter...", pos); - code = code.replace( - /\((\w+?)(\b)/g, - '($1$2' - ); - editor.innerHTML = code; -}; - -let jar = CodeJar(document.getElementById("editor"), thpHighlighter, { - tab: " ", -}); - -jar.updateCode( - `// Actual generics & sum types -fun find_person(Int person_id) -> Result[String, String] { - // Easy, explicit error bubbling - try Person::find_by_id(person_id) -} - -val id = POST::get("person_id") ?? exit("Null person_id") -// Errors as values -val person = try find_person(person_id: id) with error { - eprint("Error: {error}") - exit("Person not found") -} - -// First class HTML-like templates & components -print( - - welcome, {person.name} - -) -// And more!` -) diff --git a/static/js/lexer.js b/static/js/lexer.js deleted file mode 100644 index 2a27078..0000000 --- a/static/js/lexer.js +++ /dev/null @@ -1 +0,0 @@ -console.log(Bun.version);