[BE] Broken scrappin with reqwest
Araozu
parent
8c1883d5dd
commit
495c60c323
|
|
@ -125,6 +125,7 @@ dependencies = [
|
|||
"scraper",
|
||||
"serde",
|
||||
"sqlx",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -317,6 +318,15 @@ version = "2.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.8"
|
||||
|
|
@ -551,6 +561,16 @@ dependencies = [
|
|||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flume"
|
||||
version = "0.10.14"
|
||||
|
|
@ -1967,6 +1987,7 @@ version = "0.21.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36"
|
||||
dependencies = [
|
||||
"log",
|
||||
"ring",
|
||||
"rustls-webpki",
|
||||
"sct",
|
||||
|
|
@ -2335,7 +2356,7 @@ dependencies = [
|
|||
"tokio-stream",
|
||||
"tracing",
|
||||
"url",
|
||||
"webpki-roots",
|
||||
"webpki-roots 0.24.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2928,6 +2949,22 @@ version = "0.7.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
|
||||
|
||||
[[package]]
|
||||
name = "ureq"
|
||||
version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f5ccd538d4a604753ebc2f17cd9946e89b77bf87f6a8e2309667c6f2e87855e3"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"flate2",
|
||||
"log",
|
||||
"once_cell",
|
||||
"rustls",
|
||||
"rustls-webpki",
|
||||
"url",
|
||||
"webpki-roots 0.25.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.4.0"
|
||||
|
|
@ -3078,6 +3115,12 @@ dependencies = [
|
|||
"rustls-webpki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.25.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
|
||||
|
||||
[[package]]
|
||||
name = "whoami"
|
||||
version = "1.4.1"
|
||||
|
|
|
|||
|
|
@ -14,3 +14,4 @@ dotenvy = "0.15.7"
|
|||
serde = "1.0.188"
|
||||
chrono = "0.4.27"
|
||||
scraper = "0.17.1"
|
||||
ureq = "2.8.0"
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use once_cell::sync::OnceCell;
|
||||
use reqwest::{Client, cookie::Jar, Url};
|
||||
use reqwest::{cookie::Jar, Client, Url};
|
||||
|
||||
/// Stores the ch_sid cookie value
|
||||
static SESSION_COOKIE: OnceCell<String> = OnceCell::new();
|
||||
use std::io::prelude::*;
|
||||
|
||||
/// Stores a client with a persistent cookie store
|
||||
static SESSION_COOKIE: OnceCell<Client> = OnceCell::new();
|
||||
/// Stores the last time a request was made, in seconds since UNIX epoch
|
||||
static SESSION_TIME: OnceCell<u64> = OnceCell::new();
|
||||
|
||||
|
|
@ -14,37 +16,34 @@ pub async fn request(url: String) -> Result<String, String> {
|
|||
|
||||
ensure_session().await?;
|
||||
|
||||
// Create a client & set cookies
|
||||
let cookie = SESSION_COOKIE.get().expect("SESSION_COOKIE was not set, even after calling ensure_session");
|
||||
let cookie = format!("ch_sid={};", cookie);
|
||||
let cookie_url = format!("{}", classroom_url).parse::<Url>().expect("Error parsing CLASSROOM_URL into a url");
|
||||
|
||||
let jar = Jar::default();
|
||||
jar.add_cookie_str(cookie.as_str(), &cookie_url);
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.cookie_provider(jar.into())
|
||||
.build();
|
||||
|
||||
let client = match client {
|
||||
Ok(c) => c,
|
||||
Err(error) => return Err(format!("Error creating client: {:?}", error))
|
||||
};
|
||||
// Get the stored client
|
||||
let client = SESSION_COOKIE
|
||||
.get()
|
||||
.expect("SESSION_COOKIE was not set, even after calling ensure_session");
|
||||
|
||||
// Do the request
|
||||
let response = client
|
||||
let req_builder = client
|
||||
.get(format!("{}{}", classroom_url, url))
|
||||
.send()
|
||||
.await;
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
println!("{:?}", req_builder);
|
||||
|
||||
let response = client.execute(req_builder).await;
|
||||
|
||||
let response = match response {
|
||||
Ok(r) => r,
|
||||
Err(err) => return Err(format!("Error sending request: {:?}", err))
|
||||
Err(err) => return Err(format!("Error sending request: {:?}", err)),
|
||||
};
|
||||
|
||||
// Check if there's a new cookie
|
||||
if let Some(session_cookie) = response.cookies().find(|c| c.name() == "ch_sid") {
|
||||
println!("new cookie: {}", session_cookie.value());
|
||||
}
|
||||
|
||||
match response.text().await {
|
||||
Ok(t) => Ok(t),
|
||||
Err(err) => Err(format!("Error getting text from response: {:?}", err))
|
||||
Err(err) => Err(format!("Error getting text from response: {:?}", err)),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -77,33 +76,60 @@ pub async fn ensure_session() -> Result<(), String> {
|
|||
/// Logins to the online classroom, and sets the session cookie
|
||||
async fn login() -> Result<(), String> {
|
||||
let classroom_url = std::env::var("CLASSROOM_URL").expect("CLASSROOM_URL env var is not set!");
|
||||
let clasroom_user =
|
||||
let classroom_user =
|
||||
std::env::var("CLASSROOM_USER").expect("CLASSROOM_USER env var is not set!");
|
||||
let clasroom_password =
|
||||
let classroom_password =
|
||||
std::env::var("CLASSROOM_PASSWORD").expect("CLASSROOM_PASSWORD env var is not set!");
|
||||
|
||||
let params = [
|
||||
("login", clasroom_user),
|
||||
("password", clasroom_password),
|
||||
("login", classroom_user),
|
||||
("password", classroom_password),
|
||||
("submitAuth", "".into()),
|
||||
("_qf__formLogin", "".into()),
|
||||
];
|
||||
let client = Client::new();
|
||||
let result = client
|
||||
let jar = Jar::default();
|
||||
|
||||
|
||||
let client = Client::builder()
|
||||
.cookie_store(true)
|
||||
.cookie_provider(jar.into())
|
||||
.build()
|
||||
.unwrap();
|
||||
// let client = Client::new();
|
||||
|
||||
|
||||
let req = client
|
||||
.post(format!("{}/index.php", classroom_url))
|
||||
.form(¶ms)
|
||||
.send()
|
||||
.build().unwrap();
|
||||
|
||||
println!("{:?}\n", req);
|
||||
let body_bytes= req.body().unwrap().as_bytes().unwrap();
|
||||
println!("{:?}\n", std::str::from_utf8(body_bytes));
|
||||
|
||||
let result = client.execute(req)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(response) => {
|
||||
println!("{:?}\n\n", response);
|
||||
|
||||
let Some(session_cookie) = response.cookies().find(|c| c.name() == "ch_sid") else {
|
||||
return Err("Response succeeded, but no session cookie was foun".into());
|
||||
return Err("Response succeeded, but no session cookie was found".into());
|
||||
};
|
||||
|
||||
match SESSION_COOKIE.set(session_cookie.value().into()) {
|
||||
// Save the client with the session cookie
|
||||
println!("Got a cookie: {}", session_cookie.value());
|
||||
|
||||
let text = response.text().await.unwrap();
|
||||
|
||||
// save text to file
|
||||
let mut f = std::fs::File::create("scraps/test.html").unwrap();
|
||||
f.write_all(text.as_bytes()).unwrap();
|
||||
|
||||
match SESSION_COOKIE.set(client) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(error) => Err(format!("Error setting session cookie: {:?}", error)),
|
||||
Err(error) => Err(format!("Error saving client: {:?}", error)),
|
||||
}
|
||||
}
|
||||
Err(error) => Err(format!("Error connecting to online classroom: {:?}", error)),
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import { Show, createSignal } from "solid-js";
|
||||
import { For, Show, createSignal, onMount } from "solid-js";
|
||||
import { Search } from "../certs/Search";
|
||||
import { Person } from "../types/Person";
|
||||
import { FilledCard } from "../components/FilledCard";
|
||||
import { LinkIcon } from "../icons/LinkIcon";
|
||||
import { ClassroomUserCreation } from "./ClassroomUserCreation";
|
||||
import { ClassroomRegistrationUser } from "../types/ClassroomRegistrationUser";
|
||||
|
||||
type TabType = "Vinculate" | "Create";
|
||||
|
||||
|
|
@ -15,7 +16,9 @@ export function OnlineClassroom() {
|
|||
<Search setPerson={setPerson} />
|
||||
<div>
|
||||
|
||||
<ClassroomUser person={person()!} />
|
||||
<Show when={person() !== null && !person()!.person_classroom_id}>
|
||||
<ClassroomUser person={person()!} />
|
||||
</Show>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -37,7 +40,9 @@ function ClassroomUser(props: {person: Person}) {
|
|||
|
||||
<div class="bg-c-surface">
|
||||
<Show when={active() === "Vinculate"}>
|
||||
<ClassroomVinculation />
|
||||
<ClassroomVinculation
|
||||
person_surname={`${props.person.person_paternal_surname} ${props.person.person_maternal_surname}`}
|
||||
/>
|
||||
</Show>
|
||||
<Show when={active() === "Create"}>
|
||||
<ClassroomUserCreation />
|
||||
|
|
@ -49,15 +54,31 @@ function ClassroomUser(props: {person: Person}) {
|
|||
);
|
||||
}
|
||||
|
||||
function ClassroomVinculation() {
|
||||
function ClassroomVinculation(props: {person_surname: string}) {
|
||||
const [classroomUsers, setClassroomUsers] = createSignal<ClassroomRegistrationUser[]>([]);
|
||||
|
||||
const loadUsers = async() => {
|
||||
const response = await fetch(`${import.meta.env.VITE_BACKEND_URL}/api/classroom/users/${encodeURIComponent(props.person_surname)}`);
|
||||
const json = await response.json();
|
||||
|
||||
if (response.ok) {
|
||||
setClassroomUsers(json);
|
||||
} else {
|
||||
console.error("Error loading users", json);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
onMount(loadUsers);
|
||||
|
||||
return (
|
||||
<div>
|
||||
<p class="py-2 px-4">
|
||||
Vincule un usuario existente:
|
||||
</p>
|
||||
<ClassroomSingleUser />
|
||||
<ClassroomSingleUser />
|
||||
<ClassroomSingleUser />
|
||||
<For each={classroomUsers()}>
|
||||
{(_) => <ClassroomSingleUser />}
|
||||
</For>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
export type ClassroomRegistrationUser = {
|
||||
name: string,
|
||||
surname: string,
|
||||
username: string,
|
||||
user_id: string,
|
||||
}
|
||||
Loading…
Reference in New Issue