From 5ec34369e93968575881b578c35743f3a62f3654 Mon Sep 17 00:00:00 2001 From: Araozu Date: Wed, 12 Apr 2023 21:15:00 -0500 Subject: [PATCH] [Web] Use a TOML file to index the markdown files --- doc-generator/README.md | 42 ++-- .../en/docs/latest/basics/function-calls.md | 10 - .../en/docs/latest/basics/simple-datatypes.md | 11 - .../latest/basics/variables-and-constants.md | 10 - .../markdown/en/docs/latest/index.md | 8 - .../markdown/en/docs/latest/index.toml | 29 +++ .../markdown/en/stdlib/latest/index.md | 1 + .../markdown/en/stdlib/latest/index.toml | 1 + doc-generator/src/generator/code.rs | 5 +- doc-generator/src/generator/heading.rs | 2 +- doc-generator/src/main.rs | 118 +--------- doc-generator/src/processor.rs | 214 ++++++++++++++++++ doc-generator/static/styles/global.css | 2 +- doc-generator/static/template.html | 4 +- 14 files changed, 282 insertions(+), 175 deletions(-) create mode 100644 doc-generator/markdown/en/docs/latest/index.toml create mode 100644 doc-generator/markdown/en/stdlib/latest/index.md create mode 100644 doc-generator/markdown/en/stdlib/latest/index.toml create mode 100644 doc-generator/src/processor.rs diff --git a/doc-generator/README.md b/doc-generator/README.md index d001b70..7cfe443 100644 --- a/doc-generator/README.md +++ b/doc-generator/README.md @@ -18,10 +18,34 @@ generator --input /path/to/markdown/folder/ --output /path/to/static/folder/ Contains the Markdown. All files inside are expected to be UTF-8 encoded markdown, and have the `.md` file extension. +### Indexing + +`doc-generator` will not search for `.md` files. Instead, it will search for +`.toml` files, which index all the markdown files. This is used to generate +the file hierarchy and previous/next links in the documentation. + +This file must be named `index.toml`, must be the only TOML file in its folder, +and must follow the following schema: + +```toml +# Tipically index +entry-point = "file-without-extension" + +[folder-1] +section-name = "Display name for the folder" +# Markdown files, without the .md extension +children = [ + "file1", + "file2", +] +``` + +### Markdown extensions + The markdown follows the CommonMark specification, but certain code blocks contain custom behaviour: -### `meta` +#### `meta` A code block with language `meta` contains text in TOML format that indicates metadata for the current page. @@ -36,22 +60,6 @@ description: "Description of the page" - title: Used to create the title of the page with the format `{title} - Misti` - description: The description of the page, placed in a `` element in the `` -### `nav` - -Used to link to the previous/next page. - -````toml -```nav -[previous] -href = "./relative/path/to/previous.html" -title = "Title of previous page" - -[next] -href = "./relative/path/to/previous.html" -title = "Title of previous page" -``` -```` - ## `static` folder diff --git a/doc-generator/markdown/en/docs/latest/basics/function-calls.md b/doc-generator/markdown/en/docs/latest/basics/function-calls.md index 585ac33..cf1a0ae 100755 --- a/doc-generator/markdown/en/docs/latest/basics/function-calls.md +++ b/doc-generator/markdown/en/docs/latest/basics/function-calls.md @@ -15,13 +15,3 @@ val total = add(60, -30, 90) ``` - -```nav -[previous] -href = "./simple-datatypes.html" -title = "Datatypes" - -[next] -href = "./operators.html" -title = "Operators" -``` diff --git a/doc-generator/markdown/en/docs/latest/basics/simple-datatypes.md b/doc-generator/markdown/en/docs/latest/basics/simple-datatypes.md index 6397794..8be0c8a 100755 --- a/doc-generator/markdown/en/docs/latest/basics/simple-datatypes.md +++ b/doc-generator/markdown/en/docs/latest/basics/simple-datatypes.md @@ -33,14 +33,3 @@ True and false true false ``` - -```nav -[previous] -href = "./variables-and-constants.html" -title = "Variables and constants" - -[next] -href = "./function-calls.html" -title = "Function calls" -``` - diff --git a/doc-generator/markdown/en/docs/latest/basics/variables-and-constants.md b/doc-generator/markdown/en/docs/latest/basics/variables-and-constants.md index 018a75b..7b638ab 100755 --- a/doc-generator/markdown/en/docs/latest/basics/variables-and-constants.md +++ b/doc-generator/markdown/en/docs/latest/basics/variables-and-constants.md @@ -107,13 +107,3 @@ val roi = income / investment // This will be the value of `roi` ``` - -```nav -[previous] -href = "../" -title = "Welcome" - -[next] -href = "./simple-datatypes.html" -title = "Datatypes" -``` diff --git a/doc-generator/markdown/en/docs/latest/index.md b/doc-generator/markdown/en/docs/latest/index.md index b20455f..5c4db5f 100755 --- a/doc-generator/markdown/en/docs/latest/index.md +++ b/doc-generator/markdown/en/docs/latest/index.md @@ -372,12 +372,4 @@ val list = items.map fun (item, count) {<li key={count}>{item}</li>} -```nav -[next] -href = "./basics/variables-and-constants.html" -title = "Variables and constants" -``` - - - diff --git a/doc-generator/markdown/en/docs/latest/index.toml b/doc-generator/markdown/en/docs/latest/index.toml new file mode 100644 index 0000000..c1eb5c8 --- /dev/null +++ b/doc-generator/markdown/en/docs/latest/index.toml @@ -0,0 +1,29 @@ + +entry-point = "index" + +[basics] +section-name = "Basics" +children = [ + "variables-and-constants", + "simple-datatypes", + "function-calls", + "operators", + "tuples", + "indentation-rules", +] + +[flow-control] +section-name = "Flow control" +children = [ + "conditionals", + "arrays", + "loops", +] + +[functions] +section-name = "Functions" +children = [ + "definition", + "lambdas", + "parameters", +] diff --git a/doc-generator/markdown/en/stdlib/latest/index.md b/doc-generator/markdown/en/stdlib/latest/index.md new file mode 100644 index 0000000..feaadf0 --- /dev/null +++ b/doc-generator/markdown/en/stdlib/latest/index.md @@ -0,0 +1 @@ +# Stdlib index diff --git a/doc-generator/markdown/en/stdlib/latest/index.toml b/doc-generator/markdown/en/stdlib/latest/index.toml new file mode 100644 index 0000000..8666146 --- /dev/null +++ b/doc-generator/markdown/en/stdlib/latest/index.toml @@ -0,0 +1 @@ +entry-point = "index" \ No newline at end of file diff --git a/doc-generator/src/generator/code.rs b/doc-generator/src/generator/code.rs index d256027..de2e8bf 100644 --- a/doc-generator/src/generator/code.rs +++ b/doc-generator/src/generator/code.rs @@ -81,5 +81,8 @@ fn generate_nav_html(data: &String) -> String { _ => String::from("
"), }; - format!("
{}{}
", previous, next) + format!( + "
{}{}
", + previous, next + ) } diff --git a/doc-generator/src/generator/heading.rs b/doc-generator/src/generator/heading.rs index aacda2f..44e7409 100644 --- a/doc-generator/src/generator/heading.rs +++ b/doc-generator/src/generator/heading.rs @@ -18,7 +18,7 @@ impl Printable for Heading { let html_fragment_text = utils::to_html_fragment(&self.get_text()); format!( - "{}", + "{}", self.depth, html_fragment_text, html_fragment_text, text, self.depth ) } else { diff --git a/doc-generator/src/main.rs b/doc-generator/src/main.rs index 0fa5ee2..b577ecb 100644 --- a/doc-generator/src/main.rs +++ b/doc-generator/src/main.rs @@ -1,11 +1,8 @@ use clap::Parser; -use generator::Printable; -use sidebar::SidebarGenerator; -use std::fs::File; -use std::io::Write; -use std::{fs, path::Path}; +use std::path::Path; mod generator; +mod processor; mod sidebar; mod utils; @@ -27,116 +24,9 @@ fn main() { let output_folder = Path::new(&args.output); if input_folder.is_dir() && output_folder.is_dir() { - process_folder(&input_folder, input_folder, output_folder); + processor::search_config_file(&input_folder, input_folder, output_folder); + // process_folder(&input_folder, input_folder, output_folder); } else { eprint!("Input folder is not a valid path to a folder") } } - -fn process_folder(path: &Path, input_folder: &Path, output_folder: &Path) { - for entry in path.read_dir().unwrap() { - match entry { - Ok(entry) if entry.path().is_dir() => { - let path = entry.path(); - - match ensure_folder_exists(&entry.path(), input_folder, output_folder) { - Ok(_) => { - process_folder(&path, input_folder, output_folder); - } - Err(reason) => panic!("{}", reason), - } - } - Ok(entry) if entry.path().is_file() => { - let _ = process_markdown(&entry.path(), input_folder, output_folder); - } - _ => panic!(), - } - } -} - -fn ensure_folder_exists( - folder: &Path, - input_folder: &Path, - output_folder: &Path, -) -> Result<(), String> { - // /home/fernando/misti/docs/markdown - let input_folder = input_folder.canonicalize().unwrap(); - - // /home/fernando/misti/docs/static - let output_folder = output_folder.canonicalize().unwrap(); - - // /home/fernando/misti/docs/markdown/en/ - let full_input_folder = folder.canonicalize().unwrap(); - - let relative_new_folder = full_input_folder.strip_prefix(input_folder).unwrap(); - - let mut full_output_folder = output_folder.clone(); - full_output_folder.push(relative_new_folder); - - // println!("Ensuring that folder exists:\n{:?}", full_output_folder); - - // If this is a "top-level" folder, remove all its contents, if it exists - if full_output_folder.is_dir() { - // println!("| Removing..."); - let _ = fs::remove_dir_all(&full_output_folder); - } - - // Create folder - match fs::create_dir(&full_output_folder) { - Ok(_) => { - // println!("| done\n\n"); - Ok(()) - } - Err(_) => Err(format!("Error creating folder {:?}", full_output_folder)), - } -} - -fn process_markdown(file: &Path, input_folder: &Path, output_folder: &Path) -> Result<(), String> { - // /home/fernando/misti/docs/markdown - let input_folder = input_folder.canonicalize().unwrap(); - - // /home/fernando/misti/docs/markdown/en/docs/latest/index.md - let input_file = file.canonicalize().unwrap(); - - // /home/fernando/misti/docs/static - let output_folder = output_folder.canonicalize().unwrap(); - - // en/docs/latests/index.md - let relative_input_file = input_file.strip_prefix(input_folder).unwrap(); - - let mut output_file = output_folder.clone(); - output_file.push(relative_input_file); - output_file.set_extension("html"); - - // - // Compilation - // - let file_content_bytes = fs::read(&input_file).unwrap(); - let markdown_text = String::from_utf8(file_content_bytes).unwrap(); - - // let html_text = to_html(markdown_text.as_str()); - let md_ast = markdown::to_mdast(&markdown_text, &markdown::ParseOptions::gfm()).unwrap(); - let html_text = md_ast.to_html(); - let sidebar_html = md_ast.generate_sidebar(); - - // Read template.html - let mut template_path = output_folder.clone(); - template_path.push("template.html"); - - let template_contents = fs::read(template_path).unwrap(); - let template_contents = String::from_utf8(template_contents).unwrap(); - - let final_output = template_contents - .replace("{{markdown}}", &html_text) - .replace("{{sidebar}}", &sidebar_html); - - // - // Write to disk - // - let _ = File::create(&output_file) - .unwrap() - .write_all(final_output.as_bytes()) - .unwrap(); - - Ok(()) -} diff --git a/doc-generator/src/processor.rs b/doc-generator/src/processor.rs new file mode 100644 index 0000000..508fbe3 --- /dev/null +++ b/doc-generator/src/processor.rs @@ -0,0 +1,214 @@ +use super::generator::Printable; +use crate::sidebar::SidebarGenerator; +use std::io::Write; +use std::{ + fs::{self, File}, + path::{Path, PathBuf}, +}; +use toml::{Table, Value}; + +enum EntryFound { + TomlFile, + OtherFile, + None, +} + +// Traverses the current path searching for a TOML file +pub fn search_config_file(current_path: &Path, input_folder: &Path, output_folder: &Path) { + // Iterate over all the files searching for a TOML file + let result = current_path + .read_dir() + .unwrap() + .fold(&EntryFound::None, |acc, next| { + let p = next.unwrap().path(); + let is_file = p.is_file(); + let ext = p.extension(); + + match (acc, is_file, ext) { + (EntryFound::TomlFile, true, Some(x)) if x == "toml" => { + panic!("FOUND A SECOND TOML FILE!!!") + } + (EntryFound::TomlFile, _, _) => acc, + (EntryFound::OtherFile, true, Some(x)) if x == "toml" => &EntryFound::TomlFile, + (EntryFound::None, true, Some(x)) if x == "toml" => &EntryFound::TomlFile, + (EntryFound::None, true, Some(_)) => &EntryFound::OtherFile, + _ => acc, + } + }); + + match result { + // If a file other than a TOML file is found, panic + EntryFound::OtherFile => panic!( + "Found an orphan file without a TOML parent at {:?}", + current_path + ), + // Process the TOML file + EntryFound::TomlFile => process_toml(current_path, input_folder, output_folder), + // No files found, recursively read children folders + EntryFound::None => { + for entry in current_path.read_dir().unwrap() { + // Should always succeed, and countain a folder + let x = entry.unwrap(); + let path = x.path(); + + ensure_folder_exists(&path, input_folder, output_folder).unwrap(); + search_config_file(&path, input_folder, output_folder); + } + } + }; +} + +fn process_toml(current_path: &Path, input_folder: &Path, output_folder: &Path) { + let mut toml_file_path = current_path.canonicalize().unwrap(); + toml_file_path.push("index.toml"); + + // Read TOML file + let toml_bytes = fs::read(toml_file_path.clone()) + .expect(format!("index.toml MUST exist ({:?})", toml_file_path).as_str()); + let toml_file = String::from_utf8(toml_bytes).expect("index.toml MUST be UTF-8"); + + // Parse TOML file + let toml_table = toml_file + .parse::() + .expect("index.toml MUST contain valid TOML"); + + // Process MD files indicated in TOML file + // Expect a key named entry-point and compile it + let Value::String(entry_point) = toml_table.get("entry-point").expect("TOML: key entry-point MUST exist") + else {panic!("TOML: entry-point MUST be a String")}; + + let mut file = current_path.canonicalize().unwrap(); + file.push(format!("{}.md", entry_point)); + + compile_md_file(&file, input_folder, output_folder) + .expect("FS: entry-point file MUST point to a valid file"); + + // Subsequent keys should have schema: + // [key] + // section-name = "Section name" + // children = ["file1", "file2", "file3"] + for (key, value) in toml_table.into_iter() { + if key == "entry-point" { + continue; + } + + match value { + Value::Table(t) => { + let Value::String(_section_name) = t.get("section-name").expect(format!("TOML: table {} MUST have a key section-name", key).as_str()) + else {panic!("TOML: key section-name of table {} MUST be a String", key)}; + + let Value::Array(children) = t.get("children").expect(format!("TOML: table {} MUST have a key children", key).as_str()) + else {panic!("TOML: in table {} > children MUST be an Array", key)}; + + // Ensure folder exists + let mut folder_path = current_path.canonicalize().unwrap(); + folder_path.push(key.clone()); + ensure_folder_exists(&folder_path, input_folder, output_folder).unwrap(); + + for file_name in children { + let Value::String(file_name) = file_name + else {panic!("TOML: in table {} > children's value MUST be Strings (found {:?})", key, file_name)}; + + let mut file = current_path.canonicalize().unwrap(); + file.push(key.clone()); + file.push(format!("{}.md", file_name)); + + compile_md_file(&file, input_folder, output_folder) + .expect(format!("Error compiling file {}", file.display()).as_str()); + } + } + _ => panic!("TOML: key {} MUST be a table", key), + } + } +} + +fn compile_md_file( + file: &PathBuf, + input_folder: &Path, + output_folder: &Path, +) -> Result<(), String> { + // /home/fernando/misti/docs/markdown + let input_folder = input_folder.canonicalize().unwrap(); + + // /home/fernando/misti/docs/markdown/en/docs/latest/index.md + let input_file = file.canonicalize().unwrap(); + + // /home/fernando/misti/docs/static + let output_folder = output_folder.canonicalize().unwrap(); + + // en/docs/latests/index.md + let relative_input_file = input_file.strip_prefix(input_folder).unwrap(); + + let mut output_file = output_folder.clone(); + output_file.push(relative_input_file); + output_file.set_extension("html"); + + // + // Compilation + // + let file_content_bytes = fs::read(&input_file).unwrap(); + let markdown_text = String::from_utf8(file_content_bytes).unwrap(); + + // let html_text = to_html(markdown_text.as_str()); + let md_ast = markdown::to_mdast(&markdown_text, &markdown::ParseOptions::gfm()).unwrap(); + let html_text = md_ast.to_html(); + let sidebar_html = md_ast.generate_sidebar(); + + // Read template.html + let mut template_path = output_folder.clone(); + template_path.push("template.html"); + + let template_contents = fs::read(template_path).unwrap(); + let template_contents = String::from_utf8(template_contents).unwrap(); + + let final_output = template_contents + .replace("{{markdown}}", &html_text) + .replace("{{sidebar}}", &sidebar_html); + + // + // Write to disk + // + let _ = File::create(&output_file) + .expect(format!("MD: Output file should be valid {:?}", &output_file).as_str()) + .write_all(final_output.as_bytes()) + .unwrap(); + + Ok(()) +} + +fn ensure_folder_exists( + folder: &Path, + input_folder: &Path, + output_folder: &Path, +) -> Result<(), String> { + // /home/fernando/misti/docs/markdown + let input_folder = input_folder.canonicalize().unwrap(); + + // /home/fernando/misti/docs/static + let output_folder = output_folder.canonicalize().unwrap(); + + // /home/fernando/misti/docs/markdown/en/ + let full_input_folder = folder.canonicalize().unwrap(); + + let relative_new_folder = full_input_folder.strip_prefix(input_folder).unwrap(); + + let mut full_output_folder = output_folder.clone(); + full_output_folder.push(relative_new_folder); + + // println!("Ensuring that folder exists:\n{:?}", full_output_folder); + + // If this is a "top-level" folder, remove all its contents, if it exists + if full_output_folder.is_dir() { + // println!("| Removing..."); + let _ = fs::remove_dir_all(&full_output_folder); + } + + // Create folder + match fs::create_dir(&full_output_folder) { + Ok(_) => { + // println!("| done\n\n"); + Ok(()) + } + Err(_) => Err(format!("Error creating folder {:?}", full_output_folder)), + } +} diff --git a/doc-generator/static/styles/global.css b/doc-generator/static/styles/global.css index e8c74f3..b831d28 100644 --- a/doc-generator/static/styles/global.css +++ b/doc-generator/static/styles/global.css @@ -307,7 +307,7 @@ code, pre { /* Used by headers generated at src/generator/heading.rs */ .heading-linked :hover::after{ - color: var(--c1); + color: var(--c2-primary); content: "#"; display: inline-block; font-size: 0.7em; diff --git a/doc-generator/static/template.html b/doc-generator/static/template.html index 7e79f67..09f3d7c 100644 --- a/doc-generator/static/template.html +++ b/doc-generator/static/template.html @@ -59,11 +59,11 @@