mirror of
https://github.com/talwat/lowfi
synced 2025-08-16 06:32:14 +00:00
feat: start work on chillhop scraper
only the basic get request with caching has actually been implemented, but the rest shouldn't be too complicated.
This commit is contained in:
parent
620b568926
commit
3f55768754
1
.gitignore
vendored
1
.gitignore
vendored
@ -1 +1,2 @@
|
||||
/target
|
||||
/cache
|
@ -5,6 +5,8 @@
|
||||
use clap::{Parser, Subcommand};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::scrapers::Sources;
|
||||
|
||||
mod messages;
|
||||
mod play;
|
||||
mod player;
|
||||
@ -97,10 +99,13 @@ async fn main() -> eyre::Result<()> {
|
||||
match command {
|
||||
// TODO: Actually distinguish between sources.
|
||||
Commands::Scrape {
|
||||
source: _,
|
||||
source,
|
||||
extension,
|
||||
include_full,
|
||||
} => scrapers::lofigirl::scrape(extension, include_full).await?,
|
||||
} => match source {
|
||||
Sources::Lofigirl => scrapers::lofigirl::scrape(extension, include_full).await?,
|
||||
Sources::Chillhop => scrapers::chillhop::scrape().await,
|
||||
},
|
||||
}
|
||||
} else {
|
||||
play::play(cli).await?;
|
||||
|
@ -1,5 +1,6 @@
|
||||
use clap::ValueEnum;
|
||||
|
||||
pub mod chillhop;
|
||||
pub mod lofigirl;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug, ValueEnum)]
|
||||
|
53
src/scrapers/chillhop.rs
Normal file
53
src/scrapers/chillhop.rs
Normal file
@ -0,0 +1,53 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use reqwest::Client;
|
||||
use tokio::{
|
||||
fs::{self, File},
|
||||
io::AsyncWriteExt,
|
||||
};
|
||||
|
||||
struct Release {
|
||||
pub tracks: Vec<String>,
|
||||
pub author: String,
|
||||
}
|
||||
|
||||
struct Data {
|
||||
pub releases: Vec<Release>,
|
||||
}
|
||||
|
||||
/// Sends a get request, with caching.
|
||||
async fn get(client: &Client, path: &str) -> String {
|
||||
let cache = PathBuf::from(format!("./cache/chillhop/{path}.html"));
|
||||
if let Ok(x) = fs::read_to_string(&cache).await {
|
||||
x
|
||||
} else {
|
||||
let resp = client
|
||||
.get(format!("https://chillhop.com/{path}"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let text = resp.text().await.unwrap();
|
||||
|
||||
let parent = cache.parent();
|
||||
if let Some(x) = parent {
|
||||
if x != Path::new("") {
|
||||
fs::create_dir_all(x).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
let mut file = File::create(&cache).await.unwrap();
|
||||
file.write_all(text.as_bytes()).await.unwrap();
|
||||
|
||||
text
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn scrape() {
|
||||
const PAGE_COUNT: usize = 40;
|
||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36";
|
||||
|
||||
fs::create_dir_all("./cache/chillhop").await.unwrap();
|
||||
let client = Client::builder().user_agent(USER_AGENT).build().unwrap();
|
||||
|
||||
get(&client, "releases/?page=30").await;
|
||||
}
|
Loading…
x
Reference in New Issue
Block a user