mirror of
https://github.com/talwat/lowfi
synced 2025-08-17 15:12:37 +00:00
feat: start work on chillhop scraper
only the basic get request with caching has actually been implemented, but the rest shouldn't be too complicated.
This commit is contained in:
parent
620b568926
commit
3f55768754
1
.gitignore
vendored
1
.gitignore
vendored
@ -1 +1,2 @@
|
|||||||
/target
|
/target
|
||||||
|
/cache
|
@ -5,6 +5,8 @@
|
|||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use crate::scrapers::Sources;
|
||||||
|
|
||||||
mod messages;
|
mod messages;
|
||||||
mod play;
|
mod play;
|
||||||
mod player;
|
mod player;
|
||||||
@ -97,10 +99,13 @@ async fn main() -> eyre::Result<()> {
|
|||||||
match command {
|
match command {
|
||||||
// TODO: Actually distinguish between sources.
|
// TODO: Actually distinguish between sources.
|
||||||
Commands::Scrape {
|
Commands::Scrape {
|
||||||
source: _,
|
source,
|
||||||
extension,
|
extension,
|
||||||
include_full,
|
include_full,
|
||||||
} => scrapers::lofigirl::scrape(extension, include_full).await?,
|
} => match source {
|
||||||
|
Sources::Lofigirl => scrapers::lofigirl::scrape(extension, include_full).await?,
|
||||||
|
Sources::Chillhop => scrapers::chillhop::scrape().await,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
play::play(cli).await?;
|
play::play(cli).await?;
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
use clap::ValueEnum;
|
use clap::ValueEnum;
|
||||||
|
|
||||||
|
pub mod chillhop;
|
||||||
pub mod lofigirl;
|
pub mod lofigirl;
|
||||||
|
|
||||||
#[derive(Clone, Copy, PartialEq, Eq, Debug, ValueEnum)]
|
#[derive(Clone, Copy, PartialEq, Eq, Debug, ValueEnum)]
|
||||||
|
53
src/scrapers/chillhop.rs
Normal file
53
src/scrapers/chillhop.rs
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use reqwest::Client;
|
||||||
|
use tokio::{
|
||||||
|
fs::{self, File},
|
||||||
|
io::AsyncWriteExt,
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Release {
|
||||||
|
pub tracks: Vec<String>,
|
||||||
|
pub author: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Data {
|
||||||
|
pub releases: Vec<Release>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sends a get request, with caching.
|
||||||
|
async fn get(client: &Client, path: &str) -> String {
|
||||||
|
let cache = PathBuf::from(format!("./cache/chillhop/{path}.html"));
|
||||||
|
if let Ok(x) = fs::read_to_string(&cache).await {
|
||||||
|
x
|
||||||
|
} else {
|
||||||
|
let resp = client
|
||||||
|
.get(format!("https://chillhop.com/{path}"))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let text = resp.text().await.unwrap();
|
||||||
|
|
||||||
|
let parent = cache.parent();
|
||||||
|
if let Some(x) = parent {
|
||||||
|
if x != Path::new("") {
|
||||||
|
fs::create_dir_all(x).await.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut file = File::create(&cache).await.unwrap();
|
||||||
|
file.write_all(text.as_bytes()).await.unwrap();
|
||||||
|
|
||||||
|
text
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn scrape() {
|
||||||
|
const PAGE_COUNT: usize = 40;
|
||||||
|
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36";
|
||||||
|
|
||||||
|
fs::create_dir_all("./cache/chillhop").await.unwrap();
|
||||||
|
let client = Client::builder().user_agent(USER_AGENT).build().unwrap();
|
||||||
|
|
||||||
|
get(&client, "releases/?page=30").await;
|
||||||
|
}
|
Loading…
x
Reference in New Issue
Block a user