From 495c60c323ccc6c1c04d1292f9bce13d7fb357c5 Mon Sep 17 00:00:00 2001 From: Araozu Date: Sat, 30 Sep 2023 11:16:20 -0500 Subject: [PATCH] [BE] Broken scrappin with reqwest --- backend/Cargo.lock | 45 ++++++++- backend/Cargo.toml | 1 + backend/src/online_classroom/session.rs | 94 ++++++++++++------- frontend/src/OnlineClassroom/index.tsx | 35 +++++-- .../src/types/ClassroomRegistrationUser.ts | 6 ++ 5 files changed, 139 insertions(+), 42 deletions(-) create mode 100644 frontend/src/types/ClassroomRegistrationUser.ts diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 6b08d73..cb263f6 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -125,6 +125,7 @@ dependencies = [ "scraper", "serde", "sqlx", + "ureq", ] [[package]] @@ -317,6 +318,15 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-queue" version = "0.3.8" @@ -551,6 +561,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "flate2" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.10.14" @@ -1967,6 +1987,7 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" dependencies = [ + "log", "ring", "rustls-webpki", "sct", @@ -2335,7 +2356,7 @@ dependencies = [ "tokio-stream", "tracing", "url", - "webpki-roots", + "webpki-roots 0.24.0", ] [[package]] @@ -2928,6 +2949,22 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5ccd538d4a604753ebc2f17cd9946e89b77bf87f6a8e2309667c6f2e87855e3" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-webpki", + "url", + "webpki-roots 0.25.2", +] + [[package]] name = "url" version = "2.4.0" @@ -3078,6 +3115,12 @@ dependencies = [ "rustls-webpki", ] +[[package]] +name = "webpki-roots" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" + [[package]] name = "whoami" version = "1.4.1" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2ef0663..7285a46 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -14,3 +14,4 @@ dotenvy = "0.15.7" serde = "1.0.188" chrono = "0.4.27" scraper = "0.17.1" +ureq = "2.8.0" diff --git a/backend/src/online_classroom/session.rs b/backend/src/online_classroom/session.rs index b1efd10..f4d3fe6 100644 --- a/backend/src/online_classroom/session.rs +++ b/backend/src/online_classroom/session.rs @@ -1,10 +1,12 @@ use std::time::{SystemTime, UNIX_EPOCH}; use once_cell::sync::OnceCell; -use reqwest::{Client, cookie::Jar, Url}; +use reqwest::{cookie::Jar, Client, Url}; -/// Stores the ch_sid cookie value -static SESSION_COOKIE: OnceCell = OnceCell::new(); +use std::io::prelude::*; + +/// Stores a client with a persistent cookie store +static SESSION_COOKIE: OnceCell = OnceCell::new(); /// Stores the last time a request was made, in seconds since UNIX epoch static SESSION_TIME: OnceCell = OnceCell::new(); @@ -14,37 +16,34 @@ pub async fn request(url: String) -> Result { ensure_session().await?; - // Create a client & set cookies - let cookie = SESSION_COOKIE.get().expect("SESSION_COOKIE was not set, even after calling ensure_session"); - let cookie = format!("ch_sid={};", cookie); - let cookie_url = format!("{}", classroom_url).parse::().expect("Error parsing CLASSROOM_URL into a url"); - - let jar = Jar::default(); - jar.add_cookie_str(cookie.as_str(), &cookie_url); - - let client = reqwest::Client::builder() - .cookie_provider(jar.into()) - .build(); - - let client = match client { - Ok(c) => c, - Err(error) => return Err(format!("Error creating client: {:?}", error)) - }; + // Get the stored client + let client = SESSION_COOKIE + .get() + .expect("SESSION_COOKIE was not set, even after calling ensure_session"); // Do the request - let response = client + let req_builder = client .get(format!("{}{}", classroom_url, url)) - .send() - .await; + .build() + .unwrap(); + + println!("{:?}", req_builder); + + let response = client.execute(req_builder).await; let response = match response { Ok(r) => r, - Err(err) => return Err(format!("Error sending request: {:?}", err)) + Err(err) => return Err(format!("Error sending request: {:?}", err)), }; + // Check if there's a new cookie + if let Some(session_cookie) = response.cookies().find(|c| c.name() == "ch_sid") { + println!("new cookie: {}", session_cookie.value()); + } + match response.text().await { Ok(t) => Ok(t), - Err(err) => Err(format!("Error getting text from response: {:?}", err)) + Err(err) => Err(format!("Error getting text from response: {:?}", err)), } } @@ -77,33 +76,60 @@ pub async fn ensure_session() -> Result<(), String> { /// Logins to the online classroom, and sets the session cookie async fn login() -> Result<(), String> { let classroom_url = std::env::var("CLASSROOM_URL").expect("CLASSROOM_URL env var is not set!"); - let clasroom_user = + let classroom_user = std::env::var("CLASSROOM_USER").expect("CLASSROOM_USER env var is not set!"); - let clasroom_password = + let classroom_password = std::env::var("CLASSROOM_PASSWORD").expect("CLASSROOM_PASSWORD env var is not set!"); let params = [ - ("login", clasroom_user), - ("password", clasroom_password), + ("login", classroom_user), + ("password", classroom_password), ("submitAuth", "".into()), ("_qf__formLogin", "".into()), ]; - let client = Client::new(); - let result = client + let jar = Jar::default(); + + + let client = Client::builder() + .cookie_store(true) + .cookie_provider(jar.into()) + .build() + .unwrap(); + // let client = Client::new(); + + + let req = client .post(format!("{}/index.php", classroom_url)) .form(¶ms) - .send() + .build().unwrap(); + + println!("{:?}\n", req); + let body_bytes= req.body().unwrap().as_bytes().unwrap(); + println!("{:?}\n", std::str::from_utf8(body_bytes)); + + let result = client.execute(req) .await; match result { Ok(response) => { + println!("{:?}\n\n", response); + let Some(session_cookie) = response.cookies().find(|c| c.name() == "ch_sid") else { - return Err("Response succeeded, but no session cookie was foun".into()); + return Err("Response succeeded, but no session cookie was found".into()); }; - match SESSION_COOKIE.set(session_cookie.value().into()) { + // Save the client with the session cookie + println!("Got a cookie: {}", session_cookie.value()); + + let text = response.text().await.unwrap(); + + // save text to file + let mut f = std::fs::File::create("scraps/test.html").unwrap(); + f.write_all(text.as_bytes()).unwrap(); + + match SESSION_COOKIE.set(client) { Ok(_) => Ok(()), - Err(error) => Err(format!("Error setting session cookie: {:?}", error)), + Err(error) => Err(format!("Error saving client: {:?}", error)), } } Err(error) => Err(format!("Error connecting to online classroom: {:?}", error)), diff --git a/frontend/src/OnlineClassroom/index.tsx b/frontend/src/OnlineClassroom/index.tsx index f2685d6..e06ad66 100644 --- a/frontend/src/OnlineClassroom/index.tsx +++ b/frontend/src/OnlineClassroom/index.tsx @@ -1,9 +1,10 @@ -import { Show, createSignal } from "solid-js"; +import { For, Show, createSignal, onMount } from "solid-js"; import { Search } from "../certs/Search"; import { Person } from "../types/Person"; import { FilledCard } from "../components/FilledCard"; import { LinkIcon } from "../icons/LinkIcon"; import { ClassroomUserCreation } from "./ClassroomUserCreation"; +import { ClassroomRegistrationUser } from "../types/ClassroomRegistrationUser"; type TabType = "Vinculate" | "Create"; @@ -15,7 +16,9 @@ export function OnlineClassroom() {
- + + +
@@ -37,7 +40,9 @@ function ClassroomUser(props: {person: Person}) {
- + @@ -49,15 +54,31 @@ function ClassroomUser(props: {person: Person}) { ); } -function ClassroomVinculation() { +function ClassroomVinculation(props: {person_surname: string}) { + const [classroomUsers, setClassroomUsers] = createSignal([]); + + const loadUsers = async() => { + const response = await fetch(`${import.meta.env.VITE_BACKEND_URL}/api/classroom/users/${encodeURIComponent(props.person_surname)}`); + const json = await response.json(); + + if (response.ok) { + setClassroomUsers(json); + } else { + console.error("Error loading users", json); + } + + }; + + onMount(loadUsers); + return (

Vincule un usuario existente:

- - - + + {(_) => } +
); } diff --git a/frontend/src/types/ClassroomRegistrationUser.ts b/frontend/src/types/ClassroomRegistrationUser.ts new file mode 100644 index 0000000..571c391 --- /dev/null +++ b/frontend/src/types/ClassroomRegistrationUser.ts @@ -0,0 +1,6 @@ +export type ClassroomRegistrationUser = { + name: string, + surname: string, + username: string, + user_id: string, +}