Fix highlighter. Add docs on templating

src/components/CodeEditor.astro

@@ -6,6 +6,8 @@ const {thpcode} = Astro.props;
 
 <div class="flex h-full py-8 items-center">
     <div class="p-4 w-full">
-            <pre class="language-thp" data-language="thp"><code class="language-thp">{trimAndDedent(thpcode).join("\n")}</code></pre>
+        <pre
+            class="language-thp"
+            data-language="thp"><code class="language-thp">{trimAndDedent(thpcode).join("\n")}</code></pre>
     </div>
 </div>

src/components/InteractiveCode.astro

@@ -5,19 +5,20 @@ import { parse_str } from "../thp_machine/machine_parser";
 import { trimAndDedent } from "./utils";
 const { code, steps } = Astro.props;
 
-
 function highlightCode(lines: Array<string>): string {
     let outLines: Array<string> = [];
 
     for (const [idx, line] of lines.entries()) {
         const tokens = lex(line);
         const lineArray = [
-            `<div class=\"inline-block w-full\" :class=\"line === ${idx + 1}? 'bg-green-200 dark:bg-green-900': ''\">`
+            `<div class=\"inline-block w-full\" :class=\"line === ${idx + 1}? 'bg-green-200 dark:bg-green-900': ''\">`,
         ];
 
         for (const token of tokens) {
             if (token.token_type !== "") {
-                lineArray.push(`<span class="token ${token.token_type}">${token.v}</span>`);
+                lineArray.push(
+                    `<span class="token ${token.token_type}">${token.v}</span>`,
+                );
             } else {
                 lineArray.push(token.v);
             }
@@ -42,7 +43,8 @@ try {
 const serialized_inst = JSON.stringify(instructionSet);
 ---
 
-<div class="bg-black text-white rounded px-1"
+<div
+    class="bg-black text-white rounded px-1"
     x-data={`{
         line: 0,
         stdout: "",
@@ -52,45 +54,62 @@ const serialized_inst = JSON.stringify(instructionSet);
         state: {},
     }`}
 >
-    <span class="inline-block bg-[var(--code-theme-bg-acolor)] px-2 rounded-tl rounded-tr font-mono text-sm">thp code</span>
+    <span
-    <pre class="language-thp" style="margin: 0;" data-disabled><code set:html={codeHtml}></code></pre>
+        class="inline-block bg-[var(--code-theme-bg-acolor)] px-2 rounded-tl rounded-tr font-mono text-sm"
+        >thp code</span
+    >
+    <pre
+        class="language-thp"
+        style="margin: 0;"
+        data-disabled><code set:html={codeHtml} /></pre>
     <div class="grid grid-cols-2 font-mono text-sm">
         <div>
             <div class="p-1 border-b border-r border-white">stdout</div>
             <div class="h-24 p-1 border-r border-white">
-                <pre><code class="bg-black" x-text="stdout"></code></pre>
+                <pre><code class="bg-black" x-text="stdout" /></pre>
             </div>
         </div>
         <div>
             <div class="p-1 border-b border-white">state</div>
             <div class="h-24 p-1 overflow-y-scroll">
                 <template x-for="(value, key) in state">
-                    <div x-text="key.replaceAll(' ', '&nbsp;') + ' = ' + value"></div>
+                    <div x-text="key.replaceAll(' ', ' ') + ' = ' + value">
+                    </div>
                 </template>
             </div>
         </div>
     </div>
     <div class="border-t border-white p-1">
-        <button class="font-mono px-1 rounded bg-pink-200 dark:bg-pink-950 text-black dark:text-white disabled:opacity-50 disabled:cursor-not-allowed" @click="alpineNext($data)" :disabled="done && 'true'">
+        <button
+            class="font-mono px-1 rounded bg-pink-200 dark:bg-pink-950 text-black dark:text-white disabled:opacity-50 disabled:cursor-not-allowed"
+            @click="alpineNext($data)"
+            :disabled="done && 'true'"
+        >
             Step: <span x-text="ip"></span>
         </button>
-        <button class="font-mono px-1 rounded bg-pink-200 dark:bg-pink-950 text-black dark:text-white" @click="alpineReset($data)">
+        <button
+            class="font-mono px-1 rounded bg-pink-200 dark:bg-pink-950 text-black dark:text-white"
+            @click="alpineReset($data)"
+        >
             Reset
         </button>
     </div>
 </div>
 
 <script>
-    import { InstructionType, type Instruction } from "../thp_machine/machine_parser";
+    import {
+        InstructionType,
+        type Instruction,
+    } from "../thp_machine/machine_parser";
 
     type AlpineState = {
-        line: number,
+        line: number;
-        stdout: string,
+        stdout: string;
-        ip: number,
+        ip: number;
-        inst: Array<Array<Instruction>>
+        inst: Array<Array<Instruction>>;
-        done: boolean,
+        done: boolean;
-        state: {[key: string]: string},
+        state: { [key: string]: string };
-    }
+    };
 
     /// Executes the instruction following the state of the machine.
     function alpineNext(data: AlpineState) {
@@ -108,7 +127,7 @@ const serialized_inst = JSON.stringify(instructionSet);
                     break;
                 }
                 case InstructionType.Out: {
-                    data.stdout += i.v0.slice(1, -1) + "\n"
+                    data.stdout += i.v0.slice(1, -1) + "\n";
                     break;
                 }
                 case InstructionType.Set: {

src/layouts/thpHighlighter.ts

@@ -17,7 +17,10 @@ export function highlightOnDom() {
 
         const pre_parent = el.parentElement!;
         const new_div = document.createElement("div");
-        const code = el.innerText;
+        let code = el.innerHTML;
+
+        // Replace all &lt; with < and all &gt; with >
+        code = code.replace(/&lt;/g, "<").replace(/&gt;/g, ">");
 
         el.parentElement!.className = "language-thp";
         pre_parent.removeChild(el);

src/lexer/lexer.ts

@@ -26,11 +26,11 @@ export type Token = {
  * @param code Code to lex
  * @returns An array of all the tokens found
  */
-export function lex(code: string): Array<Token> {
+export function lex(code: string, start = 0): Array<Token> {
     const code_len = code.length;
     const tokens: Array<Token> = [];
 
-    let current_pos = 0;
+    let current_pos = start;
     let current_default_token = "";
 
     while (current_pos < code_len) {
@@ -115,6 +115,34 @@ export function lex(code: string): Array<Token> {
             current_pos = pos;
             continue;
         }
+        // try to scan a multiline comment
+        else if (c === "/" && code[current_pos + 1] === "*") {
+            // if the current default token is not empty, push it to the tokens array
+            if (current_default_token !== "") {
+                tokens.push({ v: current_default_token, token_type: "" });
+                current_default_token = "";
+            }
+
+            let comment = "";
+            let pos = current_pos;
+
+            while (pos < code_len) {
+                const char = code[pos];
+
+                if (char === "*" && code[pos + 1] === "/") {
+                    pos += 2;
+                    comment += "*/";
+                    break;
+                }
+
+                comment += char;
+                pos++;
+            }
+
+            tokens.push({ v: comment, token_type: "comment" });
+            current_pos = pos;
+            continue;
+        }
         // replace < with &lt;
         else if (c === "<") {
             current_default_token += "&lt;";

src/pages/index.astro

@@ -1,7 +1,6 @@
 ---
 import BaseLayout from "../layouts/BaseLayout.astro";
 import Navbar from "../components/Navbar.astro";
-import CodeEditor from "../components/CodeEditor.astro";
 import HeroSection from "../components/HeroSection.astro";
 ---
 
@@ -145,6 +144,7 @@ import HeroSection from "../components/HeroSection.astro";
     }
 
     val root = DirEntry::Dir("/")
+    val test_file = DirEntry::File("test.txt")
     `}
     >
         Make invalid state irrepresentable.
@@ -164,7 +164,7 @@ import HeroSection from "../components/HeroSection.astro";
     {
         print(filename)
     }
-    else
+    case _
     {
         print("A valid file was not found")
     }
@@ -181,13 +181,18 @@ import HeroSection from "../components/HeroSection.astro";
         thpcode={`
     String? username = Globals::Post::get("username")
     
-    if username? {
+    if username?
+    {
         // username is a \`String\` here
         print("Hello, {username}")
     }
     `}
     >
-        Nulls are explicit and require handling.
+        Nulls are explicit and require handling. Types are nullable, and they
+        are used everywhere they are needed.
+        <br>
+        <br>
+        Nullable types must be explicitly checked before using them.
     </HeroSection>
 
     <script>
@@ -213,7 +218,8 @@ case Dog(name)
 { 
 	print("{name} has 1 live ")
 }
-case Cat(name, lives) {
+case Cat(name, lives)
+{
 	print("{name} has {lives}")
 }`,
         );

src/pages/learn/basics/comments.md

@@ -5,17 +5,27 @@ title: Comments
 
 # Comments
 
-Only these two:
+THP supports single and multi line comments:
+
 
 ## Single line
 
+Begin with double slash `//` and continue until the end of the line.
+
 ```thp
 // This is a single line comment
+print("hello!")
+
+print("the result is {5 + 5}") // This will print 10
 ```
 
 
 ## Multi line
 
+These begin with `/*` and end with `*/`. Everything in between is ignored.
+
+Multi line comments can be nested in THP.
+
 ```thp
 /*
     This is a
@@ -30,5 +40,7 @@ Only these two:
 */
 ```
 
-TBD: Doc comments use triple slash `///`? or `/** */`?
+## Documentation comments
+
+TBD: Should doc comments use triple slash `///`? or `/** */`?
 

src/pages/learn/basics/hello-world.mdx

@@ -6,11 +6,39 @@ import InteractiveCode from "../../../components/InteractiveCode.astro";
 
 # Hello, world!
 
-Create a file named `hello.thp` with the contents:
+## THP source code
+
+Unlike PHP, THP code is written directly. There is no need to use any `<?php` tags,
+just write the code like any other programming language.
+
+As a consequence of this, HTML templates are defined in other ways, which will be
+detailed later on.
+
+To write a hello world program write the following code in a file:
 
 ```thp
 print("Hello, world!")
 ```
 
-Then run `thp hello.thp`
+Then run `thp hello.thp` from your terminal.
+
+
+## Instruction separation
+
+THP uses whitespace to determine when a statement is over. In short,
+where PHP uses a semicolon `;`, THP uses a newline.
+
+```php
+echo("A");
+echo("B");
+echo("C");
+```
+
+```thp
+print("A")
+print("B")
+print("C")
+```
+
+As a consequence of this, there can only be 1 statement per line.
 

src/pages/learn/basics/variables.md

@@ -6,12 +6,18 @@ title: Variables
 
 # Variables
 
-thp distinguishes between mutable and immutable variables.
+THP distinguishes between mutable and immutable variables.
 
-Variables have to be declared in THP, to avoid issues with scopes.
+Variables must be declared in THP to avoid issues with scoping and
-It is a compile error to use undeclared variables.
+to know if they are mutable/immutable.
+It's a compile error to use undeclared variables.
 
-Variable names **must** begin with a lowercase letter.
+Variable names **don't** start with a dollar sign `$`.
+
+Variable names **must** begin with a lowercase letter or an underscore.
+Then they may contain lowercase/uppercase letters, numbers and underscores.
+
+As a regex: `[a-z_][a-zA-Z0-9_]*`
 
 ## Immutable variables
 
@@ -39,7 +45,7 @@ String surname = "Doe"
 Int year_of_birth = 1984
 ```
 
-This means that if a variable has only a datatype, it is immutable.
+This means that if a variable only has a datatype, it is immutable.
 
 
 

src/pages/learn/index.mdx

@@ -9,9 +9,9 @@ pagesLayout:
   title: Basics
   children:
   - path: hello-world
+  - path: comments
   - path: variables
   - path: datatypes
-  - path: comments
   - path: operators
 - path: flow-control
   title: Flow control
@@ -48,6 +48,10 @@ pagesLayout:
   - path: interfaces
   - path: anonymous
   - path: magic
+- path: templating
+  title: Templating
+  children:
+  - path: intro
 ---
 import InteractiveCode from "../../components/InteractiveCode.astro";
 
@@ -106,19 +110,17 @@ THP program!
 
 ## Goals
 
-- Bring static typing to PHP: Not just type hints, not just `mixed` for everything
+- Bring static typing to PHP: Generics, type checks at compile and runtime.
-    that isn't a primitive type.
+- Reduce implicit type conversion to a minimum.
-- Generics & ADTs
-- Avoid implicit type conversion.
 - Remove the inconsistencies in the language.
-- Organize the stdlib into modules.
+- Organize the PHP stdlib.
-- Differentiate between Arrays, Tuples, Maps and Sets.
+- Have clear distinctions between Arrays, Tuples, Maps and Sets.
+- Implement Union types
 - Create a **consistent** language.
 - Have typings for popular libraries (like TS's `.d.ts`).
 - Have a simple instalation and configuration (requiring just Composer).
 - Ship a fast, native binary (written in Rust) (why use PHP when we can go **_blazingly fast_**?).
 - Support in-place compilation.
-- (Try to) emit readable PHP.
 - Implement an LSP server.
 - Implement a formatter.
 
@@ -128,8 +130,7 @@ THP program!
 These are **not** aspects that THP looks to solve or implement.
 
 - Be what TypeScript is for JavaScript (PHP with types).
-- Use PHP syntax/conventions.
+- Strictly adhere to PHP syntax/conventions.
-- Be familiar for PHP developers.
 
 THP **_intentionally_** uses a different syntax from PHP to signal
 that it is a different language, and has different semantics.

src/pages/learn/install.md

@@ -5,6 +5,8 @@ title: Install
 
 # Install
 
+**THP is not available for public use. These are goals.**
+
 ## From scratch
 
 Also install php (through XAMPP in windows/mac, php in linux) and Composer.

src/pages/learn/templating/intro.md

@@ -0,0 +1,104 @@
+---
+layout: ../../../layouts/DocsLayout.astro
+title: Introduction
+---
+
+# THP templating
+
+React changed the way HTML is rendered with the introduction of JSX.
+Having access and being able to create and manipulate HTML elements
+from code led to the pattern of components.
+
+On the other hand, PHP is characterized for its ability to easily render
+HTML, to the point that PHP code is conceptually written inside an
+HTML tag. However, this model doesn't play well with the components
+model pioneered by React and that is so common nowadays on front-end
+development.
+
+Creating small, reusable templates is hard with pure PHP. Instead you
+need to rely on external templating libraries, and even still, all your
+HTML is written outside the PHP file, and communicates in one way
+with weak, fragile data passing.
+
+For example, using the Plates PHP library you may create a Button component
+like so:
+
+```php
+// button.php
+function render_button(string $name) {
+    $templates = new League\Plates\Engine("./");
+    $button_html = $templates->render("button.template", ["name" => $name]);
+    return $button_html;
+}
+```
+
+```php
+// button.template.php
+<button class="some tailwind classes">
+    Hello <?= $this->e($name) ?>!
+</button>
+```
+
+This approach has many problems:
+
+- You have to create a new file for every new component,
+    polluting the file system and the project. This may be significant
+    for some web hosting providers that establish a fixed inode limit.
+- Data is passed dinamically, via strings. If either the render
+    function or the template change, the component will behave
+    incorrectly without any warning.
+- It's hard to include components inside components. Every new
+    one requires a change in 2 files.
+
+Maybe for these (and other) reasons components are not used with
+templating libraries. Instead people use sections, layouts, etc.
+
+We believe that by inverting the PHP model we can improve
+the experience of writing HTML. By having HTML inside of code,
+not code inside of HTML, we can easily create many small components
+and compose them.
+
+The following would be the equivalent in THP:
+
+```thp
+fun Button(String name) -> Html {
+    <button class="some tailwind classes">
+        Hello {name}!
+    </button>
+}
+```
+
+It is very similar to React. The HTML is inside the THP code, not the other
+way around, so you can have arbitrary logic in the component.
+
+```thp
+fun User(String name) {
+    // Get info from the database
+    val user = try Model::get_user(name)
+    else {
+        return <div>User not found!</div>
+    }
+
+    // Run other logic
+
+    <div class="some tailwind classes">
+        Hello {user.name}!
+        <br>
+        Here are your transactions:
+        #for t in user.transactions {
+            <TransactionItem t={t} />
+        }
+    </div>
+}
+
+fun TransactionItem(Transaction t) {
+    <li>
+        {t.date} - {t.name} ({t.price})
+    </li>
+}
+```
+
+
+
+
+