diff --git a/backend/.env.example b/backend/.env.example index d25237e..7e61bf8 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,5 +1,12 @@ +# URL de la base de datos. Se espera que sea Mysql DATABASE_URL=mysql://user:password@localhost:3306/database -RENIEC_API=B8RT6dKlN5DF408cD5vds +# Clave API de Reniec +RENIEC_API=abcdefgh +# URL de la plataforma de aula virtual CLASSROOM_URL=https://testing.aulavirtual.eegsac.com +# Usuario de la plataforma de aula virtual CLASSROOM_USER=user +# Contraseña de la plataforma de aula virtual CLASSROOM_PASSWORD=password +# Nivel de log de la aplicación +RUST_LOG=debug diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..711edc3 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,114 @@ +# eeg-system: Backend + +Programa escrito en Rust que contiene toda la lógica de negocios. +Expose únicamente rutas bajo `/api/`, todas se comunican mediante JSON. + +Contiene el Modelo y Controlador de `MVC`. La vista está en la carpeta `frontend`. + +## Lanzamiento + +Para lanzar se requiere lo siguiente en el entorno: + +- Sistema operativo Linux (no se desarrolló ni se probó el programa en windows/mac/bsd) +- Libreria Imagemagick instalada y accesible mediante el binario `convert`, + capaz de convertir archivos a PDF (en *buntu puede haber problemas de conversión, + ver [link en StackOverflow](https://stackoverflow.com/questions/52998331/imagemagick-security-policy-pdf-blocking-conversion)) +- Una carpeta `ESCANEOS` accesible, con permisos de lectura y escritura para + el usuario que ejecuta el backend. La ruta de esta carpeta está definida en + `/src/controller/scans/mod.rs`, en la variable `SCAN_PATH` +- Un archivo `.env` con las variables de entorno necesarias. Los nombres de las + claves están en el archivo `.env.example` de este repositorio + + + + +### Lanzamiento mediante Jenkins + +Si se quiere lanzar mediante Jenkins ya hay un `Jenkinsfile` una carpeta arriba. +Modificar lo que sea necesario para ejecutar en el entorno objetivo. + + +## Funcionamiento + +Básicamente, al iniciar el backend realiza lo siguiente: + +- Cargar variables de entorno +- Inicializar el logger +- Conectar a la base de datos y crear un `pool` de conexiones +- Inicializar las rutas + +Luego, las rutas brindan la siguiente funcionalidad: + +- Buscar y crear personas +- CRUD de certificados +- Crear y gestionar accesos a aula virtual +- Automatizar escaneo de certificados + + +## Componentes del sistema + + +### `controller` + +Los controladores, definen las rutas, reciben y envían datos al frontend + +### `model` + +Contiene todas las operaciones referentes a la base de datos. + +El SQL se escribe manualmente, se usa la librería [sqlx](https://github.com/launchbadge/sqlx). + +Sqxl brinda los sig. beneficios: + +- Verificar tipos de datos correctos +- Verificar las consultas SQL +- Si hay algún problema en los tipos de datos o las consultas SQL, `sqlx` lanzará errores de compilación. +De este modo no es posible lanzar el backend si hay errores en el SQL. + +Por estas razones no se usa un ORM (se intento SeaORM, es demasiado complicado y no brinda ningún beneficio +importante frente a sqlx). + + +### `online_classroom` + +Contiene código que realiza scrapping del aula virtual de EEGSAC. + + +## Componentes dentro de `controller` + +### `classroom` + +Operaciones respecto al aula virtual. Se comunica con el aula virtual mediante scrapping. + +### `course` + +Devuelve una lista con todos los cursos del sistema + +### `custom_label` + +Devuelve una lista con las denominaciones de los certificados de maquinarias. + +### `person` + +CRUD de personas. La operación más importante es buscar una persona por DNI: Busca en la base de datos, si no encuentra +busca en API de RENIEC. + +### `register` + +CRUD de certificados. + +### `scans` + +Automatiza el escaneo de certificados. Realiza lo siguiente: + +- Lee los archivos de la carpeta `ESCANEOS` de forma no recursiva. +- Cambia de nombre los archivos a `eeg_.jpg`. Esto se hace para que solo 1 persona pueda escanear + a la vez. +- Abre cada escaneo y busca el código QR: recorta la imagen a la sección donde se encuentra el QR, aplica [thresholding](https://en.wikipedia.org/wiki/Thresholding_%28image_processing%29) y detecta el QR. +- Según los datos del QR, busca en la base de datos nombres, apellidos y nombre del curso del escaneo +- Convierte a PDF, rota y renombra los escaneos. + +--- + +Las operaciones a más detalles están detalladas en el código. + diff --git a/backend/src/controller/scans/mod.rs b/backend/src/controller/scans/mod.rs index 4cb7941..f840225 100644 --- a/backend/src/controller/scans/mod.rs +++ b/backend/src/controller/scans/mod.rs @@ -9,9 +9,13 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +// TODO: Move to ENV const SCAN_PATH: &str = "/srv/srv/shares/eegsac/ESCANEOS/"; -/// Represents the result of parsing an eegsac URL +/// Represents the result of parsing a QR code, and look for an eegsac URL. +/// +/// When the frontend requests the list of files available for conversion, this +/// is sent. It contains essentially a unix timestamp #[derive(Serialize, Deserialize)] pub enum ScanInfo { /// The url has both DNI & id. @@ -28,6 +32,7 @@ pub enum ScanInfo { Error(String), } +/// Reads files from the `ESCANEOS` folder, and returns those that follow the rules #[get("/scans/detect")] pub async fn detect_scans() -> (Status, Json>>) { let files = match get_valid_files() { @@ -82,6 +87,7 @@ fn get_valid_files() -> Result, String> { let file_path = p.path(); + // Ignore directories if file_path.is_dir() { continue; } @@ -94,6 +100,7 @@ fn get_valid_files() -> Result, String> { } }; + // This may fail because filenames may contain non UTF-8 characters let filename = match filename.to_str() { Some(p) => p, None => { @@ -102,6 +109,7 @@ fn get_valid_files() -> Result, String> { } }; + // The rules for the filenames if filename.starts_with("eeg") && filename.ends_with(".jpg") { result.push(file_path); }; @@ -115,7 +123,38 @@ fn get_details_from_paths(paths: Vec) -> Vec { paths.into_iter().map(|path| get_image_info(path)).collect() } +/// Actually detects the QR data from a file. +/// Reads the file, looks for a QR & return info about this process +/// +/// Here 3 things may happen: +/// +/// - QR found +/// +/// The URL is parsed to get the person DNI & certificate id. +/// +/// The file in disk is renamed to `eeg_.jpg` +/// +/// The timestamp, DNI & cert id (if found) are sent to FE +/// +/// - QR not found +/// +/// The file in disk is renamed to `eeg_.jpg`, and this timestamp is sent to FE +/// +/// - Error finding QR +/// +/// An error is returned +/// +/// ## File renaming +/// +/// The processed files are renamed to `eeg_.jpg` and sent to the FE. +/// +/// Later on, the FE will send back this information, and another function in this +/// file will use the timestamps to read the JPGs, and do the conversion to PDF. +/// +/// This is done so that there's no race conditions. If 2 persons call this function at +/// the same time, only the last one will have the valid timestamps. fn get_image_info(path: PathBuf) -> ScanInfo { + // Open the image let img = image::open(&path).unwrap(); // get unix timestamp now in ms @@ -133,6 +172,7 @@ fn get_image_info(path: PathBuf) -> ScanInfo { let width = img.width(); let height = img.height(); + // Calculate where to crop the image, such that only 1/8 is used let third_point_height = (height / 4) * 3; let mid_point_width = width / 2; let remaining_height = height - third_point_height; @@ -140,17 +180,17 @@ fn get_image_info(path: PathBuf) -> ScanInfo { // crop image let cropped_img = img.crop_imm(0, third_point_height, mid_point_width, remaining_height); - // - // TODO: threshold cropped image before getting qr - // + // Apply thresholding let thresh_img = cropped_img.to_luma8(); let thresholded_image = imageproc::contrast::threshold(&thresh_img, 128); - // convert to image let thresholded_image = image::DynamicImage::ImageLuma8(thresholded_image); - // get qr from cropped image + // get qr from thresholded image. + // if the image is not thresholded, it may fail more often let results = bardecoder::default_decoder().decode(&thresholded_image); + + // If no QR is detected, only rename if results.is_empty() { log::info!("QR not found"); @@ -165,7 +205,7 @@ fn get_image_info(path: PathBuf) -> ScanInfo { ScanInfo::Error("Error renombrando archivo.".into()) } - } + }; } let url = match &results[0] { @@ -179,8 +219,11 @@ fn get_image_info(path: PathBuf) -> ScanInfo { // URL must have format `https://eegsac.com/certificado/?iid=` // or `https://eegsac.com/certificado/` + // Try to get DNI & iid match url.find('?') { Some(p) => { + // Here, an iid is found. Extract it & return + let dni_length = p - 31; let equals_pos = match url.find('=') { Some(p) => p, @@ -213,6 +256,8 @@ fn get_image_info(path: PathBuf) -> ScanInfo { } } None => { + // Here the URL has only a DNI, no iid + // Rename file let mut new_path = path.clone(); new_path.set_file_name(format!("eeg_{}.jpg", current_ms)); @@ -229,8 +274,12 @@ fn get_image_info(path: PathBuf) -> ScanInfo { } } -// TODO: this should return a list of files that succeeded & failed + +/// Converts a list of files into PDFs. +/// +/// Uses the timestamps inside `data` to read the correct JPG files and convert them. async fn convert_scans_from_data(data: &Vec) -> Result<(), String> { + // Get a tuple with all the DNIs & iids. let (ids, dnis) = data.iter().fold( (vec!["-1".to_string()], vec!["''".to_string()]), |mut acc, next| { @@ -244,12 +293,16 @@ async fn convert_scans_from_data(data: &Vec) -> Result<(), String> { }, ); + // Concatenate the iids & dnis let register_id_list = ids.join(","); let person_dni_list = dnis.join(","); log::info!("register_id_list: {}", register_id_list); log::info!("person_dni_list: {}", person_dni_list); + // This method expects the concatenated dnis & iids to build a single SQL query, + // instead of making a request for each file. + // Returns a list of information about each cert: person names, cert name let persons_info = Register::get_course_and_person(register_id_list, person_dni_list).await; let persons_info = match persons_info { @@ -287,7 +340,7 @@ async fn convert_scans_from_data(data: &Vec) -> Result<(), String> { } } ScanInfo::Empty(timestamp) => { - // This is dummy data, to convert files without QR + // This is dummy data, to convert files without a QR let scan_data = ScanData { course_name: "pendiente".into(), person_names: "EEG".into(), @@ -315,6 +368,7 @@ async fn convert_scans_from_data(data: &Vec) -> Result<(), String> { }; let current_ms = current_time.as_millis(); + // Create the resulting filename let filename = format!( "{} {} {} - {} [{}][{:X}].pdf", person_info.person_names, @@ -349,6 +403,7 @@ async fn convert_scans_from_data(data: &Vec) -> Result<(), String> { Ok(()) } +/// Converts a file to PDF using Imagemagick fn convert_to_pdf(image_path: &PathBuf, output_path: &PathBuf) -> Result<(), String> { // Just use imagemagick to convert the image to pdf // -interlace Plane -gaussian-blur 0.05 -quality 75% -rotate 270 doc.jpg doc.jpg.pdf diff --git a/backend/src/cors.rs b/backend/src/cors.rs index f9cc04f..6f84294 100644 --- a/backend/src/cors.rs +++ b/backend/src/cors.rs @@ -3,6 +3,7 @@ use rocket::http::Header; pub struct Cors; +/// This runs every request and adds the CORS headers to the response. #[rocket::async_trait] impl Fairing for Cors { fn info(&self) -> Info { diff --git a/backend/src/json_result.rs b/backend/src/json_result.rs index 78bc211..5ddb0f7 100644 --- a/backend/src/json_result.rs +++ b/backend/src/json_result.rs @@ -1,6 +1,8 @@ use rocket::serde::json::Json; use serde::{Deserialize, Serialize}; +/// Represents a JSON response that can be either an error or a success +/// Is used as a helper to return JSON responses from the backend #[derive(Debug, Serialize, Deserialize)] pub enum JsonResult { Ok(A), diff --git a/backend/src/main.rs b/backend/src/main.rs index aba6677..1e8cd33 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -16,6 +16,9 @@ pub mod json_result; static DB: OnceCell> = OnceCell::new(); +/// Returns a global reference to the database pool +/// This MUST be called after the DB pool has been initialized, +/// otherwise it will panic pub fn db() -> &'static Pool { DB.get().expect("DB not initialized") } @@ -26,7 +29,7 @@ async fn rocket() -> _ { env_logger::init(); /* - Init DB + Init DB and set it as a global variable */ let db_url = env::var("DATABASE_URL").expect("DATABASE_URL is not set in .env file"); let pool = MySqlPoolOptions::new() @@ -36,7 +39,7 @@ async fn rocket() -> _ { match pool { Ok(pool) => DB.set(pool).expect("Failed to set DB pool"), - Err(e) => println!("Error connecting to DB: {}", e), + Err(e) => log::error!("Error connecting to DB: {}", e), } /* Init Rocket */ diff --git a/backend/src/model/course.rs b/backend/src/model/course.rs index 227ed47..ca84a54 100644 --- a/backend/src/model/course.rs +++ b/backend/src/model/course.rs @@ -2,6 +2,7 @@ use rocket::serde::Serialize; use crate::db; +/// Information about the available courses of EEGSAC #[derive(Serialize, Clone)] #[serde(crate = "rocket::serde")] pub struct Course { diff --git a/backend/src/model/person.rs b/backend/src/model/person.rs index 7c81aec..416681e 100644 --- a/backend/src/model/person.rs +++ b/backend/src/model/person.rs @@ -3,11 +3,12 @@ use serde::{Deserialize, Serialize}; use crate::db; +/// Information about a person registered in DB #[derive(Serialize, Clone)] pub struct Person { /// Internal id pub person_id: i32, - /// Country-specific id. For now only supports Peru's DNI. + /// Country-specific id. /// /// Example: `74185293` pub person_dni: String, @@ -24,6 +25,7 @@ pub struct Person { /// Example: `Gomez` pub person_maternal_surname: String, /// Id of the online classroom user id linked to this user + /// This is the user id found in the online classroom pub person_classroom_id: Option, pub person_classroom_username: Option, } @@ -80,6 +82,7 @@ pub struct PersonLink { } impl PersonLink { + /// Links a person to a user in the online classroom pub async fn insert(&self) -> Result<(), String> { let db = db(); diff --git a/backend/src/model/register.rs b/backend/src/model/register.rs index d089fa1..0f78cdb 100644 --- a/backend/src/model/register.rs +++ b/backend/src/model/register.rs @@ -30,6 +30,7 @@ pub struct RegisterCreate { } impl RegisterCreate { + /// Registers a new certificate pub async fn create(&self) -> Result<(), sqlx::Error> { let db = db(); @@ -49,6 +50,8 @@ impl RegisterCreate { } }; + // Each certificate has its own list of codes (for some reason), + // this method gets the next code let next_register_code = Self::get_next_register_code(self.course_id).await?; // Current date in YYYY-MM-DD format diff --git a/backend/src/model/reniec_person.rs b/backend/src/model/reniec_person.rs index 447104e..b5dadaa 100644 --- a/backend/src/model/reniec_person.rs +++ b/backend/src/model/reniec_person.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; +/// This is the data returned from the fake RENIEC API #[allow(non_snake_case)] #[derive(Deserialize, Serialize, Clone)] pub struct ReniecPerson { diff --git a/backend/src/online_classroom/create_user.rs b/backend/src/online_classroom/create_user.rs index 44fecbb..c8b4de4 100644 --- a/backend/src/online_classroom/create_user.rs +++ b/backend/src/online_classroom/create_user.rs @@ -61,6 +61,11 @@ pub async fn create(data: &ClassroomPersonCreate) -> Result Err("Expected empty body, found something else".into()) } + +/// Makes a request to the user creation form, and gets the +/// security token. +/// +/// This security token must be present for CSRF reasons async fn get_form_sec_token() -> Result { let creation_form = request("/main/admin/user_add.php".into()).await?; @@ -86,6 +91,8 @@ async fn get_form_sec_token() -> Result { Ok(sec_token_value.into()) } + +/// Creates a request body data to send to the classroom fn get_request_body( surnames: &String, names: &String, diff --git a/backend/src/online_classroom/get_courses.rs b/backend/src/online_classroom/get_courses.rs index 0c13d00..e76a396 100644 --- a/backend/src/online_classroom/get_courses.rs +++ b/backend/src/online_classroom/get_courses.rs @@ -7,6 +7,7 @@ pub struct ClassroomCourse { name: String, } +/// Returns all the courses the user is enrolled in pub async fn get_courses(user_id: i32) -> Result, String> { let html = request(format!( "/main/admin/user_information.php?user_id={}", diff --git a/backend/src/online_classroom/mod.rs b/backend/src/online_classroom/mod.rs index 047b1b0..75ba722 100644 --- a/backend/src/online_classroom/mod.rs +++ b/backend/src/online_classroom/mod.rs @@ -12,7 +12,14 @@ mod session; pub mod update_expiration_date; pub mod user; -/// Tries to connect to the online classroom, and get a session cookie + +/// Tries to connect to the online classroom, and gets a session cookie +/// if neccesary. +/// +/// Mantains a global session cookie for ~20 minutes. +/// +/// Every time a request is made the session cookie is checked. If more than +/// 20 minutes have passed, a new session cookie is requested. #[get("/classroom/connect")] pub async fn connection() -> (Status, Json>) { match ensure_session().await { diff --git a/backend/src/online_classroom/register_course.rs b/backend/src/online_classroom/register_course.rs index 2b095dd..771e05b 100644 --- a/backend/src/online_classroom/register_course.rs +++ b/backend/src/online_classroom/register_course.rs @@ -1,5 +1,6 @@ use scraper::{Html, Selector}; +/// Enrolls a user in a list of courses pub async fn register_course( user_id: i32, surname_first_letter: &String, diff --git a/backend/src/online_classroom/session.rs b/backend/src/online_classroom/session.rs index 4e18217..a408094 100644 --- a/backend/src/online_classroom/session.rs +++ b/backend/src/online_classroom/session.rs @@ -13,7 +13,7 @@ struct CookieHold { lazy_static! { /// Stores a client with a persistent cookie store static ref SESSION_COOKIE: RwLock = RwLock::new(CookieHold { jar: CookieJar::new()}); - /// Stores the last time a request was made, in seconds since UNIX epoch + /// Stores the last time a cookie was saved, in seconds since UNIX epoch static ref SESSION_TIME: RwLock = RwLock::new(0); } @@ -165,7 +165,9 @@ pub async fn register_courses_request(url: String, body: String) -> Result Result<(), String> { let last_usage_time = SESSION_TIME.read().unwrap().clone(); @@ -236,6 +238,9 @@ async fn login() -> Result<(), String> { } } + +/// Helper function to log the responses recieved from the online classroom, +/// for debugging purposes pub fn log_html(html: &String) { // Get current time and date in iso let now: DateTime = Local::now(); diff --git a/backend/src/online_classroom/update_expiration_date.rs b/backend/src/online_classroom/update_expiration_date.rs index 735f1f4..0eed3c4 100644 --- a/backend/src/online_classroom/update_expiration_date.rs +++ b/backend/src/online_classroom/update_expiration_date.rs @@ -2,6 +2,7 @@ use scraper::{Html, Selector}; use super::session::{classroom_post_redirect, request}; +/// Sets the expiration date of the user account pub async fn update_expiration_date( user_id: i32, new_expiration_date: String,