feat: start work on chillhop scraper

only the basic get request with caching has actually been implemented,
but the rest shouldn't be too complicated.
This commit is contained in:
Tal 2025-08-07 15:18:04 +02:00
parent 620b568926
commit 3f55768754
4 changed files with 62 additions and 2 deletions

1
.gitignore vendored
View File

@ -1 +1,2 @@
/target
/cache

View File

@ -5,6 +5,8 @@
use clap::{Parser, Subcommand};
use std::path::PathBuf;
use crate::scrapers::Sources;
mod messages;
mod play;
mod player;
@ -97,10 +99,13 @@ async fn main() -> eyre::Result<()> {
match command {
// TODO: Actually distinguish between sources.
Commands::Scrape {
source: _,
source,
extension,
include_full,
} => scrapers::lofigirl::scrape(extension, include_full).await?,
} => match source {
Sources::Lofigirl => scrapers::lofigirl::scrape(extension, include_full).await?,
Sources::Chillhop => scrapers::chillhop::scrape().await,
},
}
} else {
play::play(cli).await?;

View File

@ -1,5 +1,6 @@
use clap::ValueEnum;
pub mod chillhop;
pub mod lofigirl;
#[derive(Clone, Copy, PartialEq, Eq, Debug, ValueEnum)]

53
src/scrapers/chillhop.rs Normal file
View File

@ -0,0 +1,53 @@
use std::path::{Path, PathBuf};
use reqwest::Client;
use tokio::{
fs::{self, File},
io::AsyncWriteExt,
};
struct Release {
pub tracks: Vec<String>,
pub author: String,
}
struct Data {
pub releases: Vec<Release>,
}
/// Sends a get request, with caching.
async fn get(client: &Client, path: &str) -> String {
let cache = PathBuf::from(format!("./cache/chillhop/{path}.html"));
if let Ok(x) = fs::read_to_string(&cache).await {
x
} else {
let resp = client
.get(format!("https://chillhop.com/{path}"))
.send()
.await
.unwrap();
let text = resp.text().await.unwrap();
let parent = cache.parent();
if let Some(x) = parent {
if x != Path::new("") {
fs::create_dir_all(x).await.unwrap();
}
}
let mut file = File::create(&cache).await.unwrap();
file.write_all(text.as_bytes()).await.unwrap();
text
}
}
pub async fn scrape() {
const PAGE_COUNT: usize = 40;
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36";
fs::create_dir_all("./cache/chillhop").await.unwrap();
let client = Client::builder().user_agent(USER_AGENT).build().unwrap();
get(&client, "releases/?page=30").await;
}