From 25a28bd8eba155449cdbf52688ed90460edc473b Mon Sep 17 00:00:00 2001 From: Araozu Date: Sat, 1 Apr 2023 16:17:12 -0500 Subject: [PATCH] [Web] Use compiler to color Datatype tokens in inline-code --- compiler/Cargo.toml | 5 + compiler/src/lib.rs | 13 + compiler/src/token.rs | 2 + doc-generator/Cargo.lock | 346 ++++++++++++++++++++- doc-generator/Cargo.toml | 1 + doc-generator/src/generator/code.rs | 6 +- doc-generator/src/generator/inline_code.rs | 59 +++- doc-generator/static/tailwind.config.js | 3 - 8 files changed, 426 insertions(+), 9 deletions(-) create mode 100644 compiler/src/lib.rs diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index dc680e0..c3976d8 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -3,6 +3,11 @@ name = "misti" version = "0.0.4" edition = "2021" + +[lib] +name = "misti" +path = "src/lib.rs" + # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs new file mode 100644 index 0000000..c335b9f --- /dev/null +++ b/compiler/src/lib.rs @@ -0,0 +1,13 @@ + +mod lexic; +mod token; +mod error_handling; + +use token::Token; +use error_handling::MistiError; + +pub use token::TokenType; + +pub fn tokenize(input: &String) -> Result, MistiError> { + lexic::get_tokens(input) +} diff --git a/compiler/src/token.rs b/compiler/src/token.rs index ee35d04..a4fd259 100755 --- a/compiler/src/token.rs +++ b/compiler/src/token.rs @@ -17,6 +17,8 @@ pub enum TokenType { EOF, } + +#[derive(Debug)] pub struct Token { pub token_type: TokenType, // The token as a raw string diff --git a/doc-generator/Cargo.lock b/doc-generator/Cargo.lock index f524b3c..fd55291 100644 --- a/doc-generator/Cargo.lock +++ b/doc-generator/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.2.6" @@ -42,18 +51,62 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + [[package]] name = "cc" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "time", + "wasm-bindgen", + "winapi", +] + [[package]] name = "clap" version = "4.2.0" @@ -87,7 +140,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.11", ] [[package]] @@ -96,6 +149,27 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "colored" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" +dependencies = [ + "atty", + "lazy_static", + "winapi", +] + [[package]] name = "concolor-override" version = "1.0.0" @@ -111,12 +185,63 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cxx" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.11", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.11", +] + [[package]] name = "doc-generator" version = "0.1.0" dependencies = [ "clap", "markdown", + "misti", ] [[package]] @@ -146,19 +271,52 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +[[package]] +name = "iana-time-zone" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "io-lifetimes" version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.1", "libc", "windows-sys", ] @@ -169,24 +327,57 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8687c819457e979cc940d09cb16e42a1bf70aa6b60a549de6d3a62a0ee90c69e" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.1", "io-lifetimes", "rustix", "windows-sys", ] +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + [[package]] name = "linux-raw-sys" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + [[package]] name = "markdown" version = "1.0.0-alpha.7" @@ -196,6 +387,34 @@ dependencies = [ "unicode-id", ] +[[package]] +name = "misti" +version = "0.0.4" +dependencies = [ + "chrono", + "clap", + "colored", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.17.1" @@ -234,12 +453,29 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "scratch" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.11" @@ -251,6 +487,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi", + "winapi", +] + [[package]] name = "unicode-id" version = "0.3.3" @@ -263,12 +519,78 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + [[package]] name = "winapi" version = "0.3.9" @@ -285,12 +607,30 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.45.0" diff --git a/doc-generator/Cargo.toml b/doc-generator/Cargo.toml index f60d60e..86574fa 100644 --- a/doc-generator/Cargo.toml +++ b/doc-generator/Cargo.toml @@ -6,5 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +misti = { path = "../compiler"} clap = { version = "4.2.0", features = ["derive"] } markdown = "1.0.0-alpha.7" diff --git a/doc-generator/src/generator/code.rs b/doc-generator/src/generator/code.rs index 2e673da..48ccf93 100644 --- a/doc-generator/src/generator/code.rs +++ b/doc-generator/src/generator/code.rs @@ -4,7 +4,11 @@ use super::Printable; impl Printable for Code { fn to_html(&self) -> String { - format!("
{}
", self.value) + if let Some(lang) = &self.lang { + format!("
{}
", lang, self.value) + } else { + format!("
{}
", self.value) + } } fn get_text(&self) -> String { diff --git a/doc-generator/src/generator/inline_code.rs b/doc-generator/src/generator/inline_code.rs index 8d3b189..ef2e637 100644 --- a/doc-generator/src/generator/inline_code.rs +++ b/doc-generator/src/generator/inline_code.rs @@ -1,17 +1,72 @@ use markdown::mdast::InlineCode; use super::Printable; +use misti::TokenType; impl Printable for InlineCode { fn to_html(&self) -> String { + /* + let tokens = misti::tokenize(&self.value); + println!("INLINE CODE ==== tokens ====\n\n{:?}\n\n==== code ====\n\n{}\n\n", tokens, self.value); + let s = self.value .replace("<", "<") .replace(">", ">"); - - format!("{}", s) + */ + + format!("{}", highlight(&self.value)) } fn get_text(&self) -> String { self.value.clone() } } + +fn highlight(input: &String) -> String { + let tokens = misti::tokenize(&input); + + if tokens.is_err() { + eprintln!( + "Found a lexical error processing an inline-code with input {}", + input + ); + return input.clone(); + } + + let mut output = input.clone(); + + for token in tokens.unwrap() { + match &token.token_type { + TokenType::Datatype => { + let start_pos = token.position; + let end_pos = token.get_end_position(); + + let range = start_pos..end_pos; + let html = format!("{}", token.value); + + output.replace_range(range, html.as_str()); + } + _ => {} + } + } + + output +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn should_return_simple_string() { + assert_eq!("sample", highlight(&String::from("sample"))) + } + + #[test] + fn should_highlight_datatype() { + assert_eq!( + "Num", + highlight(&String::from("Num")) + ) + } +} diff --git a/doc-generator/static/tailwind.config.js b/doc-generator/static/tailwind.config.js index 3cc5810..d2ec805 100644 --- a/doc-generator/static/tailwind.config.js +++ b/doc-generator/static/tailwind.config.js @@ -42,9 +42,6 @@ module.exports = { "c5-on-primary-container": "var(--c5-on-primary-container)", "bg-color": "var(--bg-color)", }, - borderWidth: { - '1': '1px' - }, extend: {}, }, plugins: [],