From 9a3460d176365b133a16408e5e3f5e7b1a7d31a2 Mon Sep 17 00:00:00 2001 From: Araozu Date: Thu, 13 Apr 2023 20:21:03 -0500 Subject: [PATCH] [Web] Replace TOML with YAML for markdown indexing --- doc-generator/Cargo.lock | 4 +- .../markdown/en/docs/latest/index.yaml | 35 ++- .../markdown/en/stdlib/latest/index.yaml | 9 +- doc-generator/src/pages/md_compiler.rs | 65 +++++ doc-generator/src/pages/mod.rs | 66 ++++-- doc-generator/src/processor.rs | 222 ++++-------------- doc-generator/src/utils.rs | 39 +++ doc-generator/static/index.html | 13 +- doc-generator/static/styles/global.css | 9 - doc-generator/static/template.html | 12 + 10 files changed, 269 insertions(+), 205 deletions(-) create mode 100644 doc-generator/src/pages/md_compiler.rs diff --git a/doc-generator/Cargo.lock b/doc-generator/Cargo.lock index 0f42418..8f453a6 100644 --- a/doc-generator/Cargo.lock +++ b/doc-generator/Cargo.lock @@ -491,9 +491,9 @@ checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "serde" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" [[package]] name = "serde_spanned" diff --git a/doc-generator/markdown/en/docs/latest/index.yaml b/doc-generator/markdown/en/docs/latest/index.yaml index e1c5e98..37c6e77 100644 --- a/doc-generator/markdown/en/docs/latest/index.yaml +++ b/doc-generator/markdown/en/docs/latest/index.yaml @@ -4,10 +4,41 @@ has_index: true children: - path: index +- path: basics + name: Basics + children: + - path: variables-and-constants + - path: simple-datatypes + - path: function-calls + - path: operators + - path: tuples + - path: indentation-rules + - path: flow-control name: Flow control - has_index: true children: - - path: arrays - path: conditionals + - path: arrays + - path: loops +- path: functions + name: Functions + children: + - path: definition + - path: lambdas + - path: parameters + +- path: objects + name: Objects + children: + - path: definition + +- path: classes + name: Classes + children: + - path: definition + +- path: modules + name: Modules + children: + - path: import diff --git a/doc-generator/markdown/en/stdlib/latest/index.yaml b/doc-generator/markdown/en/stdlib/latest/index.yaml index 1a68e74..2ada17c 100644 --- a/doc-generator/markdown/en/stdlib/latest/index.yaml +++ b/doc-generator/markdown/en/stdlib/latest/index.yaml @@ -1 +1,8 @@ -path: "" \ No newline at end of file +path: "" +name: "" +children: +- path: prelude + name: Prelude + children: + - path: String + - path: Console diff --git a/doc-generator/src/pages/md_compiler.rs b/doc-generator/src/pages/md_compiler.rs new file mode 100644 index 0000000..80d1acc --- /dev/null +++ b/doc-generator/src/pages/md_compiler.rs @@ -0,0 +1,65 @@ +use std::{ + fs::{self, File}, + io::Write, + path::{Path, PathBuf}, +}; + +use crate::{generator::Printable, sidebar::SidebarGenerator}; + +/// ## Parameters +/// +/// - `file`: Path to the MD file to compile +/// - `input_folder`: Path to the input folder passed as parameter of the program +/// - `output_folder`: Path to the output folder passed as parameter of the program +/// - `file_tree_html`: HTML code of the file tree to be inserted into the generated HTML +pub fn compile(file: &PathBuf, input_folder: &Path, output_folder: &Path, file_tree_html: &String) { + // /home/fernando/misti/docs/markdown + let input_folder = input_folder.canonicalize().unwrap(); + + // /home/fernando/misti/docs/markdown/en/docs/latest/index.md + let input_file = file + .canonicalize() + .expect(format!("Expected file {:?} to exist", file).as_str()); + + // /home/fernando/misti/docs/static + let output_folder = output_folder.canonicalize().unwrap(); + + // en/docs/latests/index.md + let relative_input_file = input_file.strip_prefix(input_folder).unwrap(); + + let mut output_file = output_folder.clone(); + output_file.push(relative_input_file); + output_file.set_extension("html"); + + // + // Compilation + // + let file_content_bytes = fs::read(&input_file).unwrap(); + let markdown_text = String::from_utf8(file_content_bytes).unwrap(); + + // let html_text = to_html(markdown_text.as_str()); + let md_ast = markdown::to_mdast(&markdown_text, &markdown::ParseOptions::gfm()).unwrap(); + let html_text = md_ast.to_html(); + let sidebar_html = md_ast.generate_sidebar(); + + // Read template.html + let mut template_path = output_folder.clone(); + template_path.push("template.html"); + + let template_contents = fs::read(template_path).unwrap(); + let template_contents = String::from_utf8(template_contents).unwrap(); + + // Insert the markdown, sidebar and file tree into the template + let final_output = template_contents + .replace("{{markdown}}", &html_text) + .replace("{{sidebar}}", &sidebar_html) + .replace("{{pages}}", &file_tree_html); + + // + // Write to disk + // + File::create(&output_file) + .expect(format!("MD: Output file should be valid {:?}", &output_file).as_str()) + .write_all(final_output.as_bytes()) + .unwrap(); +} diff --git a/doc-generator/src/pages/mod.rs b/doc-generator/src/pages/mod.rs index d56827b..25b7738 100644 --- a/doc-generator/src/pages/mod.rs +++ b/doc-generator/src/pages/mod.rs @@ -1,6 +1,10 @@ -use std::{fs, path::Path}; +use std::path::Path; -use yaml_rust::{Yaml, YamlLoader}; +use yaml_rust::Yaml; + +use crate::utils; + +mod md_compiler; pub enum Node<'a> { File(File<'a>), @@ -30,7 +34,7 @@ macro_rules! y_str { }; } -fn generate_pages_tree(values: &Yaml) -> Node { +pub fn parse_yaml(values: &Yaml) -> Node { let Yaml::Hash(table) = values else {panic!("YAML: input MUST be an object")}; @@ -61,7 +65,7 @@ fn generate_pages_tree(values: &Yaml) -> Node { let children_nodes: Vec = children .into_iter() - .map(|values| generate_pages_tree(values)) + .map(|values| parse_yaml(values)) .collect(); Node::Folder(Folder { @@ -77,7 +81,7 @@ fn generate_pages_tree(values: &Yaml) -> Node { } } -fn generate_pages_html(file_tree: &Node, current_path: &Path) -> String { +pub fn generate_pages_html(file_tree: &Node, current_path: &Path) -> String { match file_tree { Node::File(file) => { if file.path == "index" { @@ -127,21 +131,43 @@ fn generate_pages_html(file_tree: &Node, current_path: &Path) -> String { } } -pub fn generate_pages(yaml_folder: &Path, input_folder: &Path) -> String { - let mut yaml_path = yaml_folder.canonicalize().unwrap(); - yaml_path.push("index.yaml"); +pub fn compile_md_to_html( + file_tree: &Node, + current_path: &Path, + input_folder: &Path, + output_folder: &Path, + file_tree_html: &String, +) { + match file_tree { + Node::File(file) if file.path != "" => { + let mut file_path = current_path.canonicalize().unwrap(); + file_path.push(format!("{}.md", file.path)); - let yaml_bytes = fs::read(yaml_path).expect("File index.yaml MUST exist"); - let yaml = String::from_utf8(yaml_bytes).expect("YAML index file MUST be valid UTF-8"); + md_compiler::compile(&file_path, input_folder, output_folder, file_tree_html); + } + Node::File(_) => { + panic!("YAML: A file cannot have an empty `path` key") + } + Node::Folder(folder) if folder.path != "" => { + let mut new_path = current_path.canonicalize().unwrap(); + new_path.push(folder.path); + utils::ensure_folder_exists(&new_path, input_folder, output_folder) + .expect("SHOULD be able to create folder"); - let yaml_docs = - YamlLoader::load_from_str(yaml.as_str()).expect("YAML file MUST contain valid YAML"); - let yaml = &yaml_docs[0]; - - let input_folder = input_folder.canonicalize().unwrap(); - let yaml_folder_2 = yaml_folder.canonicalize().unwrap(); - let web_absolute_path = yaml_folder_2.strip_prefix(input_folder).unwrap(); - - let root_node = generate_pages_tree(yaml); - generate_pages_html(&root_node, web_absolute_path) + for node in folder.children.iter() { + compile_md_to_html(node, &new_path, input_folder, output_folder, file_tree_html); + } + } + Node::Folder(folder) => { + for node in folder.children.iter() { + compile_md_to_html( + node, + ¤t_path, + input_folder, + output_folder, + file_tree_html, + ); + } + } + } } diff --git a/doc-generator/src/processor.rs b/doc-generator/src/processor.rs index 6d6896f..20d9785 100644 --- a/doc-generator/src/processor.rs +++ b/doc-generator/src/processor.rs @@ -1,21 +1,20 @@ -use super::generator::Printable; -use crate::sidebar::SidebarGenerator; -use std::io::Write; +use crate::pages::{compile_md_to_html, generate_pages_html, parse_yaml}; +use crate::utils; use std::{ - fs::{self, File}, - path::{Path, PathBuf}, + fs, + path::Path, }; -use toml::{Table, Value}; +use yaml_rust::YamlLoader; enum EntryFound { - TomlFile, + YamlFile, OtherFile, None, } -// Traverses the current path searching for a TOML file +// Traverses the current path searching for a YAML file pub fn search_config_file(current_path: &Path, input_folder: &Path, output_folder: &Path) { - // Iterate over all the files searching for a TOML file + // Iterate over all the files searching for a YAML file let result = current_path .read_dir() .unwrap() @@ -25,25 +24,25 @@ pub fn search_config_file(current_path: &Path, input_folder: &Path, output_folde let ext = p.extension(); match (acc, is_file, ext) { - (EntryFound::TomlFile, true, Some(x)) if x == "toml" => { - panic!("FOUND A SECOND TOML FILE!!!") + (EntryFound::YamlFile, true, Some(x)) if x == "yaml" => { + panic!("FOUND A SECOND YAML FILE!!!") } - (EntryFound::TomlFile, _, _) => acc, - (EntryFound::OtherFile, true, Some(x)) if x == "toml" => &EntryFound::TomlFile, - (EntryFound::None, true, Some(x)) if x == "toml" => &EntryFound::TomlFile, + (EntryFound::YamlFile, _, _) => acc, + (EntryFound::OtherFile, true, Some(x)) if x == "yaml" => &EntryFound::YamlFile, + (EntryFound::None, true, Some(x)) if x == "yaml" => &EntryFound::YamlFile, (EntryFound::None, true, Some(_)) => &EntryFound::OtherFile, _ => acc, } }); match result { - // If a file other than a TOML file is found, panic + // If a file other than a YAML file is found, panic EntryFound::OtherFile => panic!( - "Found an orphan file without a TOML parent at {:?}", + "Found an orphan file without a YAML parent at {:?}", current_path ), - // Process the TOML file - EntryFound::TomlFile => process_toml(current_path, input_folder, output_folder), + // Process the YAML file + EntryFound::YamlFile => process_yaml(current_path, input_folder, output_folder), // No files found, recursively read children folders EntryFound::None => { for entry in current_path.read_dir().unwrap() { @@ -51,168 +50,51 @@ pub fn search_config_file(current_path: &Path, input_folder: &Path, output_folde let x = entry.unwrap(); let path = x.path(); - ensure_folder_exists(&path, input_folder, output_folder).unwrap(); + utils::ensure_folder_exists(&path, input_folder, output_folder).unwrap(); search_config_file(&path, input_folder, output_folder); } } }; } -fn process_toml(current_path: &Path, input_folder: &Path, output_folder: &Path) { - let file_tree_html = crate::pages::generate_pages(current_path, input_folder); +fn process_yaml(current_path: &Path, input_folder: &Path, output_folder: &Path) { + // + // Read YAML file + // + let mut yaml_path = current_path.canonicalize().unwrap(); + yaml_path.push("index.yaml"); - let mut toml_file_path = current_path.canonicalize().unwrap(); - toml_file_path.push("index.toml"); + let yaml_bytes = fs::read(yaml_path).expect("File index.yaml MUST exist"); + let yaml = String::from_utf8(yaml_bytes).expect("YAML index file MUST be valid UTF-8"); - // Read TOML file - let toml_bytes = fs::read(toml_file_path.clone()) - .expect(format!("index.toml MUST exist ({:?})", toml_file_path).as_str()); - let toml_file = String::from_utf8(toml_bytes).expect("index.toml MUST be UTF-8"); + let yaml_docs = + YamlLoader::load_from_str(yaml.as_str()).expect("YAML file MUST contain valid YAML"); + let yaml = &yaml_docs[0]; - // Parse TOML file - let toml_table = toml_file - .parse::() - .expect("index.toml MUST contain valid TOML"); + // + // Parse YAML + // + let file_tree = parse_yaml(&yaml); - // Process MD files indicated in TOML file - // Expect a key named entry-point and compile it - let Value::String(entry_point) = toml_table.get("entry-point").expect("TOML: key entry-point MUST exist") - else {panic!("TOML: entry-point MUST be a String")}; + // + // Generate File Tree HTML + // + let tree_html = { + let input_folder = input_folder.canonicalize().unwrap(); + let yaml_folder_temp = current_path.canonicalize().unwrap(); + let web_absolute_path = yaml_folder_temp.strip_prefix(input_folder).unwrap(); - let mut file = current_path.canonicalize().unwrap(); - file.push(format!("{}.md", entry_point)); + generate_pages_html(&file_tree, web_absolute_path) + }; - compile_md_file(&file, input_folder, output_folder, &file_tree_html) - .expect("FS: entry-point file MUST point to a valid file"); - - // Subsequent keys should have schema: - // [key] - // section-name = "Section name" - // children = ["file1", "file2", "file3"] - for (key, value) in toml_table.into_iter() { - if key == "entry-point" { - continue; - } - - match value { - Value::Table(t) => { - let Value::String(_section_name) = t.get("section-name").expect(format!("TOML: table {} MUST have a key section-name", key).as_str()) - else {panic!("TOML: key section-name of table {} MUST be a String", key)}; - - let Value::Array(children) = t.get("children").expect(format!("TOML: table {} MUST have a key children", key).as_str()) - else {panic!("TOML: in table {} > children MUST be an Array", key)}; - - // Ensure folder exists - let mut folder_path = current_path.canonicalize().unwrap(); - folder_path.push(key.clone()); - ensure_folder_exists(&folder_path, input_folder, output_folder).unwrap(); - - for file_name in children { - let Value::String(file_name) = file_name - else {panic!("TOML: in table {} > children's value MUST be Strings (found {:?})", key, file_name)}; - - let mut file = current_path.canonicalize().unwrap(); - file.push(key.clone()); - file.push(format!("{}.md", file_name)); - - compile_md_file(&file, input_folder, output_folder, &file_tree_html) - .expect(format!("Error compiling file {}", file.display()).as_str()); - } - } - _ => panic!("TOML: key {} MUST be a table", key), - } - } -} - -fn compile_md_file( - file: &PathBuf, - input_folder: &Path, - output_folder: &Path, - file_tree_html: &String, -) -> Result<(), String> { - // /home/fernando/misti/docs/markdown - let input_folder = input_folder.canonicalize().unwrap(); - - // /home/fernando/misti/docs/markdown/en/docs/latest/index.md - let input_file = file.canonicalize().unwrap(); - - // /home/fernando/misti/docs/static - let output_folder = output_folder.canonicalize().unwrap(); - - // en/docs/latests/index.md - let relative_input_file = input_file.strip_prefix(input_folder).unwrap(); - - let mut output_file = output_folder.clone(); - output_file.push(relative_input_file); - output_file.set_extension("html"); - - // - // Compilation - // - let file_content_bytes = fs::read(&input_file).unwrap(); - let markdown_text = String::from_utf8(file_content_bytes).unwrap(); - - // let html_text = to_html(markdown_text.as_str()); - let md_ast = markdown::to_mdast(&markdown_text, &markdown::ParseOptions::gfm()).unwrap(); - let html_text = md_ast.to_html(); - let sidebar_html = md_ast.generate_sidebar(); - - // Read template.html - let mut template_path = output_folder.clone(); - template_path.push("template.html"); - - let template_contents = fs::read(template_path).unwrap(); - let template_contents = String::from_utf8(template_contents).unwrap(); - - let final_output = template_contents - .replace("{{markdown}}", &html_text) - .replace("{{sidebar}}", &sidebar_html) - .replace("{{pages}}", &file_tree_html); - - // - // Write to disk - // - let _ = File::create(&output_file) - .expect(format!("MD: Output file should be valid {:?}", &output_file).as_str()) - .write_all(final_output.as_bytes()) - .unwrap(); - - Ok(()) -} - -fn ensure_folder_exists( - folder: &Path, - input_folder: &Path, - output_folder: &Path, -) -> Result<(), String> { - // /home/fernando/misti/docs/markdown - let input_folder = input_folder.canonicalize().unwrap(); - - // /home/fernando/misti/docs/static - let output_folder = output_folder.canonicalize().unwrap(); - - // /home/fernando/misti/docs/markdown/en/ - let full_input_folder = folder.canonicalize().unwrap(); - - let relative_new_folder = full_input_folder.strip_prefix(input_folder).unwrap(); - - let mut full_output_folder = output_folder.clone(); - full_output_folder.push(relative_new_folder); - - // println!("Ensuring that folder exists:\n{:?}", full_output_folder); - - // If this is a "top-level" folder, remove all its contents, if it exists - if full_output_folder.is_dir() { - // println!("| Removing..."); - let _ = fs::remove_dir_all(&full_output_folder); - } - - // Create folder - match fs::create_dir(&full_output_folder) { - Ok(_) => { - // println!("| done\n\n"); - Ok(()) - } - Err(_) => Err(format!("Error creating folder {:?}", full_output_folder)), - } + // + // Compile MD to HTML + // + compile_md_to_html( + &file_tree, + current_path, + input_folder, + output_folder, + &tree_html, + ); } diff --git a/doc-generator/src/utils.rs b/doc-generator/src/utils.rs index d180325..ac2720e 100644 --- a/doc-generator/src/utils.rs +++ b/doc-generator/src/utils.rs @@ -1,3 +1,5 @@ +use std::{fs, path::Path}; + use markdown::mdast::Node; use crate::generator::Printable; @@ -25,3 +27,40 @@ pub fn collect_children_text(vec: &Vec) -> String { result.join("-") } + +pub fn ensure_folder_exists( + folder: &Path, + input_folder: &Path, + output_folder: &Path, +) -> Result<(), String> { + // /home/fernando/misti/docs/markdown + let input_folder = input_folder.canonicalize().unwrap(); + + // /home/fernando/misti/docs/static + let output_folder = output_folder.canonicalize().unwrap(); + + // /home/fernando/misti/docs/markdown/en/ + let full_input_folder = folder.canonicalize().unwrap(); + + let relative_new_folder = full_input_folder.strip_prefix(input_folder).unwrap(); + + let mut full_output_folder = output_folder.clone(); + full_output_folder.push(relative_new_folder); + + // println!("Ensuring that folder exists:\n{:?}", full_output_folder); + + // If this is a "top-level" folder, remove all its contents, if it exists + if full_output_folder.is_dir() { + // println!("| Removing..."); + let _ = fs::remove_dir_all(&full_output_folder); + } + + // Create folder + match fs::create_dir(&full_output_folder) { + Ok(_) => { + // println!("| done\n\n"); + Ok(()) + } + Err(_) => Err(format!("Error creating folder {:?}", full_output_folder)), + } +} diff --git a/doc-generator/static/index.html b/doc-generator/static/index.html index 5a65391..3946487 100644 --- a/doc-generator/static/index.html +++ b/doc-generator/static/index.html @@ -28,7 +28,7 @@
@@ -42,6 +42,17 @@

Misti

 + +

+ Docs +

+ +

+ Stdlib +

+ +
+

diff --git a/doc-generator/static/styles/global.css b/doc-generator/static/styles/global.css index b831d28..55091aa 100644 --- a/doc-generator/static/styles/global.css +++ b/doc-generator/static/styles/global.css @@ -1,8 +1,3 @@ -body ::selection { - background-color: var(--js-color); - color: var(--dark-color); -} - :root { --js-color: #FFE70B; --dark-color: #0f0f0f; @@ -92,10 +87,6 @@ body ::selection { } - body ::selection { - color: var(--js-color); - background-color: var(--dark-color); - } } html { diff --git a/doc-generator/static/template.html b/doc-generator/static/template.html index 54f3557..31762d2 100644 --- a/doc-generator/static/template.html +++ b/doc-generator/static/template.html @@ -39,12 +39,24 @@

Misti

+

+ Docs +

+ +

+ Stdlib +

+ +
+