fernando
/
thp
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Wiki Activity

[Web] Replace TOML with YAML for markdown indexing

master
Araozu 2023-04-13 20:21:03 -05:00
parent da8277f5dd
commit 9a3460d176
10 changed files with 269 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc-generator/Cargo.lock generated
View File

@ -491,9 +491,9 @@ checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1"
[[package]]
name = "serde"
version = "1.0.159"
version = "1.0.160"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c"
[[package]]
name = "serde_spanned"

35
doc-generator/markdown/en/docs/latest/index.yaml
View File

@ -4,10 +4,41 @@ has_index: true
children:
- path: index
- path: basics
name: Basics
children:
- path: variables-and-constants
- path: simple-datatypes
- path: function-calls
- path: operators
- path: tuples
- path: indentation-rules
- path: flow-control
name: Flow control
has_index: true
children:
- path: arrays
- path: conditionals
- path: arrays
- path: loops
- path: functions
name: Functions
children:
- path: definition
- path: lambdas
- path: parameters
- path: objects
name: Objects
children:
- path: definition
- path: classes
name: Classes
children:
- path: definition
- path: modules
name: Modules
children:
- path: import

7
doc-generator/markdown/en/stdlib/latest/index.yaml
View File

@ -1 +1,8 @@
path: ""
name: ""
children:
- path: prelude
name: Prelude
children:
- path: String
- path: Console

65
doc-generator/src/pages/md_compiler.rs Normal file
View File

@ -0,0 +1,65 @@
use std::{
fs::{self, File},
io::Write,
path::{Path, PathBuf},
};
use crate::{generator::Printable, sidebar::SidebarGenerator};
/// ## Parameters
///
/// - `file`: Path to the MD file to compile
/// - `input_folder`: Path to the input folder passed as parameter of the program
/// - `output_folder`: Path to the output folder passed as parameter of the program
/// - `file_tree_html`: HTML code of the file tree to be inserted into the generated HTML
pub fn compile(file: &PathBuf, input_folder: &Path, output_folder: &Path, file_tree_html: &String) {
// /home/fernando/misti/docs/markdown
let input_folder = input_folder.canonicalize().unwrap();
// /home/fernando/misti/docs/markdown/en/docs/latest/index.md
let input_file = file
.canonicalize()
.expect(format!("Expected file {:?} to exist", file).as_str());
// /home/fernando/misti/docs/static
let output_folder = output_folder.canonicalize().unwrap();
// en/docs/latests/index.md
let relative_input_file = input_file.strip_prefix(input_folder).unwrap();
let mut output_file = output_folder.clone();
output_file.push(relative_input_file);
output_file.set_extension("html");
//
// Compilation
//
let file_content_bytes = fs::read(&input_file).unwrap();
let markdown_text = String::from_utf8(file_content_bytes).unwrap();
// let html_text = to_html(markdown_text.as_str());
let md_ast = markdown::to_mdast(&markdown_text, &markdown::ParseOptions::gfm()).unwrap();
let html_text = md_ast.to_html();
let sidebar_html = md_ast.generate_sidebar();
// Read template.html
let mut template_path = output_folder.clone();
template_path.push("template.html");
let template_contents = fs::read(template_path).unwrap();
let template_contents = String::from_utf8(template_contents).unwrap();
// Insert the markdown, sidebar and file tree into the template
let final_output = template_contents
.replace("{{markdown}}", &html_text)
.replace("{{sidebar}}", &sidebar_html)
.replace("{{pages}}", &file_tree_html);
//
// Write to disk
//
File::create(&output_file)
.expect(format!("MD: Output file should be valid {:?}", &output_file).as_str())
.write_all(final_output.as_bytes())
.unwrap();
}

66
doc-generator/src/pages/mod.rs
View File

@ -1,6 +1,10 @@
use std::{fs, path::Path};
use std::path::Path;
use yaml_rust::{Yaml, YamlLoader};
use yaml_rust::Yaml;
use crate::utils;
mod md_compiler;
pub enum Node<'a> {
File(File<'a>),
@ -30,7 +34,7 @@ macro_rules! y_str {
};
}
fn generate_pages_tree(values: &Yaml) -> Node {
pub fn parse_yaml(values: &Yaml) -> Node {
let Yaml::Hash(table) = values
else {panic!("YAML: input MUST be an object")};
@ -61,7 +65,7 @@ fn generate_pages_tree(values: &Yaml) -> Node {
let children_nodes: Vec<Node> = children
.into_iter()
.map(|values| generate_pages_tree(values))
.map(|values| parse_yaml(values))
.collect();
Node::Folder(Folder {
@ -77,7 +81,7 @@ fn generate_pages_tree(values: &Yaml) -> Node {
}
}
fn generate_pages_html(file_tree: &Node, current_path: &Path) -> String {
pub fn generate_pages_html(file_tree: &Node, current_path: &Path) -> String {
match file_tree {
Node::File(file) => {
if file.path == "index" {
@ -127,21 +131,43 @@ fn generate_pages_html(file_tree: &Node, current_path: &Path) -> String {
}
}
pub fn generate_pages(yaml_folder: &Path, input_folder: &Path) -> String {
let mut yaml_path = yaml_folder.canonicalize().unwrap();
yaml_path.push("index.yaml");
pub fn compile_md_to_html(
file_tree: &Node,
current_path: &Path,
input_folder: &Path,
output_folder: &Path,
file_tree_html: &String,
) {
match file_tree {
Node::File(file) if file.path != "" => {
let mut file_path = current_path.canonicalize().unwrap();
file_path.push(format!("{}.md", file.path));
let yaml_bytes = fs::read(yaml_path).expect("File index.yaml MUST exist");
let yaml = String::from_utf8(yaml_bytes).expect("YAML index file MUST be valid UTF-8");
md_compiler::compile(&file_path, input_folder, output_folder, file_tree_html);
}
Node::File(_) => {
panic!("YAML: A file cannot have an empty `path` key")
}
Node::Folder(folder) if folder.path != "" => {
let mut new_path = current_path.canonicalize().unwrap();
new_path.push(folder.path);
utils::ensure_folder_exists(&new_path, input_folder, output_folder)
.expect("SHOULD be able to create folder");
let yaml_docs =
YamlLoader::load_from_str(yaml.as_str()).expect("YAML file MUST contain valid YAML");
let yaml = &yaml_docs[0];
let input_folder = input_folder.canonicalize().unwrap();
let yaml_folder_2 = yaml_folder.canonicalize().unwrap();
let web_absolute_path = yaml_folder_2.strip_prefix(input_folder).unwrap();
let root_node = generate_pages_tree(yaml);
generate_pages_html(&root_node, web_absolute_path)
for node in folder.children.iter() {
compile_md_to_html(node, &new_path, input_folder, output_folder, file_tree_html);
}
}
Node::Folder(folder) => {
for node in folder.children.iter() {
compile_md_to_html(
node,
&current_path,
input_folder,
output_folder,
file_tree_html,
);
}
}
}
}

222
doc-generator/src/processor.rs
View File

@ -1,21 +1,20 @@
use super::generator::Printable;
use crate::sidebar::SidebarGenerator;
use std::io::Write;
use crate::pages::{compile_md_to_html, generate_pages_html, parse_yaml};
use crate::utils;
use std::{
fs::{self, File},
path::{Path, PathBuf},
fs,
path::Path,
};
use toml::{Table, Value};
use yaml_rust::YamlLoader;
enum EntryFound {
TomlFile,
YamlFile,
OtherFile,
None,
}
// Traverses the current path searching for a TOML file
// Traverses the current path searching for a YAML file
pub fn search_config_file(current_path: &Path, input_folder: &Path, output_folder: &Path) {
// Iterate over all the files searching for a TOML file
// Iterate over all the files searching for a YAML file
let result = current_path
.read_dir()
.unwrap()
@ -25,25 +24,25 @@ pub fn search_config_file(current_path: &Path, input_folder: &Path, output_folde
let ext = p.extension();
match (acc, is_file, ext) {
(EntryFound::TomlFile, true, Some(x)) if x == "toml" => {
panic!("FOUND A SECOND TOML FILE!!!")
(EntryFound::YamlFile, true, Some(x)) if x == "yaml" => {
panic!("FOUND A SECOND YAML FILE!!!")
}
(EntryFound::TomlFile, _, _) => acc,
(EntryFound::OtherFile, true, Some(x)) if x == "toml" => &EntryFound::TomlFile,
(EntryFound::None, true, Some(x)) if x == "toml" => &EntryFound::TomlFile,
(EntryFound::YamlFile, _, _) => acc,
(EntryFound::OtherFile, true, Some(x)) if x == "yaml" => &EntryFound::YamlFile,
(EntryFound::None, true, Some(x)) if x == "yaml" => &EntryFound::YamlFile,
(EntryFound::None, true, Some(_)) => &EntryFound::OtherFile,
_ => acc,
}
});
match result {
// If a file other than a TOML file is found, panic
// If a file other than a YAML file is found, panic
EntryFound::OtherFile => panic!(
"Found an orphan file without a TOML parent at {:?}",
"Found an orphan file without a YAML parent at {:?}",
current_path
),
// Process the TOML file
EntryFound::TomlFile => process_toml(current_path, input_folder, output_folder),
// Process the YAML file
EntryFound::YamlFile => process_yaml(current_path, input_folder, output_folder),
// No files found, recursively read children folders
EntryFound::None => {
for entry in current_path.read_dir().unwrap() {
@ -51,168 +50,51 @@ pub fn search_config_file(current_path: &Path, input_folder: &Path, output_folde
let x = entry.unwrap();
let path = x.path();
ensure_folder_exists(&path, input_folder, output_folder).unwrap();
utils::ensure_folder_exists(&path, input_folder, output_folder).unwrap();
search_config_file(&path, input_folder, output_folder);
}
}
};
}
fn process_toml(current_path: &Path, input_folder: &Path, output_folder: &Path) {
let file_tree_html = crate::pages::generate_pages(current_path, input_folder);
fn process_yaml(current_path: &Path, input_folder: &Path, output_folder: &Path) {
//
// Read YAML file
//
let mut yaml_path = current_path.canonicalize().unwrap();
yaml_path.push("index.yaml");
let mut toml_file_path = current_path.canonicalize().unwrap();
toml_file_path.push("index.toml");
let yaml_bytes = fs::read(yaml_path).expect("File index.yaml MUST exist");
let yaml = String::from_utf8(yaml_bytes).expect("YAML index file MUST be valid UTF-8");
// Read TOML file
let toml_bytes = fs::read(toml_file_path.clone())
.expect(format!("index.toml MUST exist ({:?})", toml_file_path).as_str());
let toml_file = String::from_utf8(toml_bytes).expect("index.toml MUST be UTF-8");
let yaml_docs =
YamlLoader::load_from_str(yaml.as_str()).expect("YAML file MUST contain valid YAML");
let yaml = &yaml_docs[0];
// Parse TOML file
let toml_table = toml_file
.parse::<Table>()
.expect("index.toml MUST contain valid TOML");
//
// Parse YAML
//
let file_tree = parse_yaml(&yaml);
// Process MD files indicated in TOML file
// Expect a key named entry-point and compile it
let Value::String(entry_point) = toml_table.get("entry-point").expect("TOML: key entry-point MUST exist")
else {panic!("TOML: entry-point MUST be a String")};
//
// Generate File Tree HTML
//
let tree_html = {
let input_folder = input_folder.canonicalize().unwrap();
let yaml_folder_temp = current_path.canonicalize().unwrap();
let web_absolute_path = yaml_folder_temp.strip_prefix(input_folder).unwrap();
let mut file = current_path.canonicalize().unwrap();
file.push(format!("{}.md", entry_point));
generate_pages_html(&file_tree, web_absolute_path)
};
compile_md_file(&file, input_folder, output_folder, &file_tree_html)
.expect("FS: entry-point file MUST point to a valid file");
// Subsequent keys should have schema:
// [key]
// section-name = "Section name"
// children = ["file1", "file2", "file3"]
for (key, value) in toml_table.into_iter() {
if key == "entry-point" {
continue;
}
match value {
Value::Table(t) => {
let Value::String(_section_name) = t.get("section-name").expect(format!("TOML: table {} MUST have a key section-name", key).as_str())
else {panic!("TOML: key section-name of table {} MUST be a String", key)};
let Value::Array(children) = t.get("children").expect(format!("TOML: table {} MUST have a key children", key).as_str())
else {panic!("TOML: in table {} > children MUST be an Array", key)};
// Ensure folder exists
let mut folder_path = current_path.canonicalize().unwrap();
folder_path.push(key.clone());
ensure_folder_exists(&folder_path, input_folder, output_folder).unwrap();
for file_name in children {
let Value::String(file_name) = file_name
else {panic!("TOML: in table {} > children's value MUST be Strings (found {:?})", key, file_name)};
let mut file = current_path.canonicalize().unwrap();
file.push(key.clone());
file.push(format!("{}.md", file_name));
compile_md_file(&file, input_folder, output_folder, &file_tree_html)
.expect(format!("Error compiling file {}", file.display()).as_str());
}
}
_ => panic!("TOML: key {} MUST be a table", key),
}
}
}
fn compile_md_file(
file: &PathBuf,
input_folder: &Path,
output_folder: &Path,
file_tree_html: &String,
) -> Result<(), String> {
// /home/fernando/misti/docs/markdown
let input_folder = input_folder.canonicalize().unwrap();
// /home/fernando/misti/docs/markdown/en/docs/latest/index.md
let input_file = file.canonicalize().unwrap();
// /home/fernando/misti/docs/static
let output_folder = output_folder.canonicalize().unwrap();
// en/docs/latests/index.md
let relative_input_file = input_file.strip_prefix(input_folder).unwrap();
let mut output_file = output_folder.clone();
output_file.push(relative_input_file);
output_file.set_extension("html");
//
// Compilation
//
let file_content_bytes = fs::read(&input_file).unwrap();
let markdown_text = String::from_utf8(file_content_bytes).unwrap();
// let html_text = to_html(markdown_text.as_str());
let md_ast = markdown::to_mdast(&markdown_text, &markdown::ParseOptions::gfm()).unwrap();
let html_text = md_ast.to_html();
let sidebar_html = md_ast.generate_sidebar();
// Read template.html
let mut template_path = output_folder.clone();
template_path.push("template.html");
let template_contents = fs::read(template_path).unwrap();
let template_contents = String::from_utf8(template_contents).unwrap();
let final_output = template_contents
.replace("{{markdown}}", &html_text)
.replace("{{sidebar}}", &sidebar_html)
.replace("{{pages}}", &file_tree_html);
//
// Write to disk
//
let _ = File::create(&output_file)
.expect(format!("MD: Output file should be valid {:?}", &output_file).as_str())
.write_all(final_output.as_bytes())
.unwrap();
Ok(())
}
fn ensure_folder_exists(
folder: &Path,
input_folder: &Path,
output_folder: &Path,
) -> Result<(), String> {
// /home/fernando/misti/docs/markdown
let input_folder = input_folder.canonicalize().unwrap();
// /home/fernando/misti/docs/static
let output_folder = output_folder.canonicalize().unwrap();
// /home/fernando/misti/docs/markdown/en/
let full_input_folder = folder.canonicalize().unwrap();
let relative_new_folder = full_input_folder.strip_prefix(input_folder).unwrap();
let mut full_output_folder = output_folder.clone();
full_output_folder.push(relative_new_folder);
// println!("Ensuring that folder exists:\n{:?}", full_output_folder);
// If this is a "top-level" folder, remove all its contents, if it exists
if full_output_folder.is_dir() {
// println!("| Removing...");
let _ = fs::remove_dir_all(&full_output_folder);
}
// Create folder
match fs::create_dir(&full_output_folder) {
Ok(_) => {
// println!("| done\n\n");
Ok(())
}
Err(_) => Err(format!("Error creating folder {:?}", full_output_folder)),
}
//
// Compile MD to HTML
//
compile_md_to_html(
&file_tree,
current_path,
input_folder,
output_folder,
&tree_html,
);
}

39
doc-generator/src/utils.rs
View File

@ -1,3 +1,5 @@
use std::{fs, path::Path};
use markdown::mdast::Node;
use crate::generator::Printable;
@ -25,3 +27,40 @@ pub fn collect_children_text(vec: &Vec<Node>) -> String {
result.join("-")
}
pub fn ensure_folder_exists(
folder: &Path,
input_folder: &Path,
output_folder: &Path,
) -> Result<(), String> {
// /home/fernando/misti/docs/markdown
let input_folder = input_folder.canonicalize().unwrap();
// /home/fernando/misti/docs/static
let output_folder = output_folder.canonicalize().unwrap();
// /home/fernando/misti/docs/markdown/en/
let full_input_folder = folder.canonicalize().unwrap();
let relative_new_folder = full_input_folder.strip_prefix(input_folder).unwrap();
let mut full_output_folder = output_folder.clone();
full_output_folder.push(relative_new_folder);
// println!("Ensuring that folder exists:\n{:?}", full_output_folder);
// If this is a "top-level" folder, remove all its contents, if it exists
if full_output_folder.is_dir() {
// println!("| Removing...");
let _ = fs::remove_dir_all(&full_output_folder);
}
// Create folder
match fs::create_dir(&full_output_folder) {
Ok(_) => {
// println!("| done\n\n");
Ok(())
}
Err(_) => Err(format!("Error creating folder {:?}", full_output_folder)),
}
}

13
doc-generator/static/index.html
View File

@ -28,7 +28,7 @@
<div class="text-sm p-4 h-screen py-8 sticky top-0 overflow-y-scroll overflow-x-hidden border-r border-border-color">
<a
href="/"
class="inline-block border border-transparent
class="inline-block border border-transparent mb-8
hover:border-c2-primary hover:text-c2-primary transition-colors rounded-md"
>
<svg style="width: 100%" viewBox="0 0 10240 10240">
@ -42,6 +42,17 @@
<p class="font-extrabold text-6xl text-center">Misti</p>
</a>
<p>
<a href="/en/docs/latest/">Docs</a>
</p>
<p class="mb-8">
<a href="/en/stdlib/latest">Stdlib</a>
</p>
<hr>
</div>
<div class="px-8 max-w-screen-lg">
<h2 class="text-6xl px-4 py-6 font-bold leading-snug">

9
doc-generator/static/styles/global.css
View File

@ -1,8 +1,3 @@
body ::selection {
background-color: var(--js-color);
color: var(--dark-color);
}
:root {
--js-color: #FFE70B;
--dark-color: #0f0f0f;
@ -92,10 +87,6 @@ body ::selection {
}
body ::selection {
color: var(--js-color);
background-color: var(--dark-color);
}
}
html {

12
doc-generator/static/template.html
View File

@ -39,12 +39,24 @@
<p class="font-extrabold text-6xl text-center">Misti</p>
</a>
<p>
<a href="/en/docs/latest/">Docs</a>
</p>
<p class="mb-8">
<a href="/en/stdlib/latest">Stdlib</a>
</p>
<hr>
<nav id="secondary-nav">
{{pages}}
</nav>
</div>
<nav id="primary-nav" class="text-sm p-4 h-screen py-12 sticky top-0 overflow-y-scroll overflow-x-hidden border-r border-border-color">
<div class="mx-2 my-4 uppercase opacity-80 font-semibold">On this page</div>
{{sidebar}}
</nav>
<div class="marked px-8 max-w-screen-lg">