[BE] Broken scrappin with reqwest

backend/Cargo.lock

@@ -125,6 +125,7 @@ dependencies = [
  "scraper",
  "serde",
  "sqlx",
+ "ureq",
 ]
 
 [[package]]
@@ -317,6 +318,15 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484"
 
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crossbeam-queue"
 version = "0.3.8"
@@ -551,6 +561,16 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "flate2"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "flume"
 version = "0.10.14"
@@ -1967,6 +1987,7 @@ version = "0.21.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36"
 dependencies = [
+ "log",
  "ring",
  "rustls-webpki",
  "sct",
@@ -2335,7 +2356,7 @@ dependencies = [
  "tokio-stream",
  "tracing",
  "url",
- "webpki-roots",
+ "webpki-roots 0.24.0",
 ]
 
 [[package]]
@@ -2928,6 +2949,22 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
 
+[[package]]
+name = "ureq"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5ccd538d4a604753ebc2f17cd9946e89b77bf87f6a8e2309667c6f2e87855e3"
+dependencies = [
+ "base64",
+ "flate2",
+ "log",
+ "once_cell",
+ "rustls",
+ "rustls-webpki",
+ "url",
+ "webpki-roots 0.25.2",
+]
+
 [[package]]
 name = "url"
 version = "2.4.0"
@@ -3078,6 +3115,12 @@ dependencies = [
  "rustls-webpki",
 ]
 
+[[package]]
+name = "webpki-roots"
+version = "0.25.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
+
 [[package]]
 name = "whoami"
 version = "1.4.1"

backend/Cargo.toml

@@ -14,3 +14,4 @@ dotenvy = "0.15.7"
 serde = "1.0.188"
 chrono = "0.4.27"
 scraper = "0.17.1"
+ureq = "2.8.0"

backend/src/online_classroom/session.rs

@@ -1,10 +1,12 @@
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use once_cell::sync::OnceCell;
-use reqwest::{Client, cookie::Jar, Url};
+use reqwest::{cookie::Jar, Client, Url};
 
-/// Stores the ch_sid cookie value
+use std::io::prelude::*;
-static SESSION_COOKIE: OnceCell<String> = OnceCell::new();
+
+/// Stores a client with a persistent cookie store
+static SESSION_COOKIE: OnceCell<Client> = OnceCell::new();
 /// Stores the last time a request was made, in seconds since UNIX epoch
 static SESSION_TIME: OnceCell<u64> = OnceCell::new();
 
@@ -14,37 +16,34 @@ pub async fn request(url: String) -> Result<String, String> {
 
     ensure_session().await?;
 
-    // Create a client & set cookies
+    // Get the stored client
-    let cookie = SESSION_COOKIE.get().expect("SESSION_COOKIE was not set, even after calling ensure_session");
+    let client = SESSION_COOKIE
-    let cookie = format!("ch_sid={};", cookie);
+        .get()
-    let cookie_url = format!("{}", classroom_url).parse::<Url>().expect("Error parsing CLASSROOM_URL into a url");
+        .expect("SESSION_COOKIE was not set, even after calling ensure_session");
-
-    let jar = Jar::default();
-    jar.add_cookie_str(cookie.as_str(), &cookie_url);
-
-    let client = reqwest::Client::builder()
-        .cookie_provider(jar.into())
-        .build();
-
-    let client = match client {
-        Ok(c) => c,
-        Err(error) => return Err(format!("Error creating client: {:?}", error))
-    };
 
     // Do the request
-    let response = client
+    let req_builder = client
         .get(format!("{}{}", classroom_url, url))
-        .send()
+        .build()
-        .await;
+        .unwrap();
+
+    println!("{:?}", req_builder);
+
+    let response = client.execute(req_builder).await;
 
     let response = match response {
         Ok(r) => r,
-        Err(err) => return Err(format!("Error sending request: {:?}", err))
+        Err(err) => return Err(format!("Error sending request: {:?}", err)),
     };
 
+    // Check if there's a new cookie
+    if let Some(session_cookie) = response.cookies().find(|c| c.name() == "ch_sid") {
+        println!("new cookie: {}", session_cookie.value());
+    }
+
     match response.text().await {
         Ok(t) => Ok(t),
-        Err(err) => Err(format!("Error getting text from response: {:?}", err))
+        Err(err) => Err(format!("Error getting text from response: {:?}", err)),
     }
 }
 
@@ -77,33 +76,60 @@ pub async fn ensure_session() -> Result<(), String> {
 /// Logins to the online classroom, and sets the session cookie
 async fn login() -> Result<(), String> {
     let classroom_url = std::env::var("CLASSROOM_URL").expect("CLASSROOM_URL env var is not set!");
-    let clasroom_user =
+    let classroom_user =
         std::env::var("CLASSROOM_USER").expect("CLASSROOM_USER env var is not set!");
-    let clasroom_password =
+    let classroom_password =
         std::env::var("CLASSROOM_PASSWORD").expect("CLASSROOM_PASSWORD env var is not set!");
 
     let params = [
-        ("login", clasroom_user),
+        ("login", classroom_user),
-        ("password", clasroom_password),
+        ("password", classroom_password),
         ("submitAuth", "".into()),
         ("_qf__formLogin", "".into()),
     ];
-    let client = Client::new();
+    let jar = Jar::default();
-    let result = client
+
+    
+    let client = Client::builder()
+        .cookie_store(true)
+        .cookie_provider(jar.into())
+        .build()
+        .unwrap();
+    // let client = Client::new();
+
+
+    let req = client
         .post(format!("{}/index.php", classroom_url))
         .form(&params)
-        .send()
+        .build().unwrap();
+
+    println!("{:?}\n", req);
+    let body_bytes= req.body().unwrap().as_bytes().unwrap();
+    println!("{:?}\n", std::str::from_utf8(body_bytes));
+
+    let result = client.execute(req)
         .await;
 
     match result {
         Ok(response) => {
+            println!("{:?}\n\n", response);
+
             let Some(session_cookie) = response.cookies().find(|c| c.name() == "ch_sid") else {
-                return Err("Response succeeded, but no session cookie was foun".into());
+                return Err("Response succeeded, but no session cookie was found".into());
             };
 
-            match SESSION_COOKIE.set(session_cookie.value().into()) {
+            // Save the client with the session cookie
+            println!("Got a cookie:  {}", session_cookie.value());
+
+            let text = response.text().await.unwrap();
+
+            // save text to file
+            let mut f = std::fs::File::create("scraps/test.html").unwrap();
+            f.write_all(text.as_bytes()).unwrap();
+
+            match SESSION_COOKIE.set(client) {
                 Ok(_) => Ok(()),
-                Err(error) => Err(format!("Error setting session cookie: {:?}", error)),
+                Err(error) => Err(format!("Error saving client: {:?}", error)),
             }
         }
         Err(error) => Err(format!("Error connecting to online classroom: {:?}", error)),

frontend/src/OnlineClassroom/index.tsx

@@ -1,9 +1,10 @@
-import { Show, createSignal } from "solid-js";
+import { For, Show, createSignal, onMount } from "solid-js";
 import { Search } from "../certs/Search";
 import { Person } from "../types/Person";
 import { FilledCard } from "../components/FilledCard";
 import { LinkIcon } from "../icons/LinkIcon";
 import { ClassroomUserCreation } from "./ClassroomUserCreation";
+import { ClassroomRegistrationUser } from "../types/ClassroomRegistrationUser";
 
 type TabType = "Vinculate" | "Create";
 
@@ -15,7 +16,9 @@ export function OnlineClassroom() {
             <Search setPerson={setPerson} />
             <div>
 
-                <ClassroomUser person={person()!} />
+                <Show when={person() !== null && !person()!.person_classroom_id}>
+                    <ClassroomUser person={person()!} />
+                </Show>
 
             </div>
         </div>
@@ -37,7 +40,9 @@ function ClassroomUser(props: {person: Person}) {
 
                 <div class="bg-c-surface">
                     <Show when={active() === "Vinculate"}>
-                        <ClassroomVinculation />
+                        <ClassroomVinculation
+                            person_surname={`${props.person.person_paternal_surname} ${props.person.person_maternal_surname}`}
+                        />
                     </Show>
                     <Show when={active() === "Create"}>
                         <ClassroomUserCreation />
@@ -49,15 +54,31 @@ function ClassroomUser(props: {person: Person}) {
     );
 }
 
-function ClassroomVinculation() {
+function ClassroomVinculation(props: {person_surname: string}) {
+    const [classroomUsers, setClassroomUsers] = createSignal<ClassroomRegistrationUser[]>([]);
+
+    const loadUsers = async() => {
+        const response = await fetch(`${import.meta.env.VITE_BACKEND_URL}/api/classroom/users/${encodeURIComponent(props.person_surname)}`);
+        const json = await response.json();
+
+        if (response.ok) {
+            setClassroomUsers(json);
+        } else {
+            console.error("Error loading users", json);
+        }
+
+    };
+
+    onMount(loadUsers);
+
     return (
         <div>
             <p class="py-2 px-4">
                 Vincule un usuario existente:
             </p>
-            <ClassroomSingleUser />
+            <For each={classroomUsers()}>
-            <ClassroomSingleUser />
+                {(_) => <ClassroomSingleUser />}
-            <ClassroomSingleUser />
+            </For>
         </div>
     );
 }

frontend/src/types/ClassroomRegistrationUser.ts

@@ -0,0 +1,6 @@
+export type ClassroomRegistrationUser = {
+    name: string,
+    surname: string,
+    username: string,
+    user_id: string,
+}