From 980cbbce3f9f75a423153f2db7c7dadbb4282758 Mon Sep 17 00:00:00 2001 From: talwat <83217276+talwat@users.noreply.github.com> Date: Mon, 13 Apr 2026 13:15:10 +0200 Subject: [PATCH] fix: remove archive scraper --- src/main.rs | 1 - src/scrapers.rs | 4 --- src/scrapers/archive.rs | 74 --------------------------------------- src/ui/interface/clock.rs | 2 +- 4 files changed, 1 insertion(+), 80 deletions(-) delete mode 100644 src/scrapers/archive.rs diff --git a/src/main.rs b/src/main.rs index 4de1758..971391c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -121,7 +121,6 @@ async fn main() -> eyre::Result<()> { if let Some(command) = &args.command { return match command { Commands::Scrape { source } => match source { - Source::Archive => scrapers::archive::scrape().await, Source::Lofigirl => scrapers::lofigirl::scrape().await, Source::Chillhop => scrapers::chillhop::scrape().await, }, diff --git a/src/scrapers.rs b/src/scrapers.rs index fe6575a..0e28f5e 100644 --- a/src/scrapers.rs +++ b/src/scrapers.rs @@ -11,7 +11,6 @@ use tokio::{ io::AsyncWriteExt, }; -pub mod archive; pub mod chillhop; pub mod lofigirl; @@ -19,7 +18,6 @@ pub mod lofigirl; #[derive(Clone, Copy, PartialEq, Eq, Debug, ValueEnum)] pub enum Source { Lofigirl, - Archive, Chillhop, } @@ -28,7 +26,6 @@ impl Source { pub fn cache_dir(&self) -> &'static str { match self { Source::Lofigirl => "lofigirl", - Source::Archive => "archive", Source::Chillhop => "chillhop", } } @@ -37,7 +34,6 @@ impl Source { pub fn url(&self) -> &'static str { match self { Source::Chillhop => "https://chillhop.com", - Source::Archive => "https://ia601004.us.archive.org/31/items/lofigirl", Source::Lofigirl => "https://lofigirl.com/wp-content/uploads", } } diff --git a/src/scrapers/archive.rs b/src/scrapers/archive.rs deleted file mode 100644 index a4edc36..0000000 --- a/src/scrapers/archive.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Has all of the functions for the `scrape` command. -//! -//! This command is completely optional, and as such isn't subject to the same -//! quality standards as the rest of the codebase. - -use std::sync::LazyLock; - -use futures_util::{stream::FuturesOrdered, StreamExt}; -use reqwest::Client; -use scraper::{Html, Selector}; - -use crate::scrapers::{get, Source}; - -static SELECTOR: LazyLock = - LazyLock::new(|| Selector::parse("html > body > pre > a").unwrap()); - -async fn parse(client: &Client, path: &str) -> eyre::Result> { - let document = get(client, path, super::Source::Lofigirl).await?; - let html = Html::parse_document(&document); - - Ok(html - .select(&SELECTOR) - .skip(1) - .map(|x| String::from(x.attr("href").unwrap())) - .collect()) -} - -/// This function basically just scans the entire file server, and returns a list of paths to mp3 files. -/// -/// It's a bit hacky, and basically works by checking all of the years, then months, and then all of the files. -/// This is done as a way to avoid recursion, since async rust really hates recursive functions. -async fn scan() -> eyre::Result> { - let client = Client::new(); - - let mut releases = parse(&client, "/").await?; - releases.truncate(releases.len() - 4); - - // A little bit of async to run all of the months concurrently. - let mut futures = FuturesOrdered::new(); - - for release in releases { - let client = client.clone(); - futures.push_back(async move { - let items = parse(&client, &release).await.unwrap(); - items - .into_iter() - .filter_map(|x| { - if x.ends_with(".mp3") { - Some(format!("{release}{x}")) - } else { - None - } - }) - .collect::>() - }); - } - - let mut files = Vec::new(); - while let Some(mut result) = futures.next().await { - files.append(&mut result); - } - - eyre::Result::Ok(files) -} - -pub async fn scrape() -> eyre::Result<()> { - println!("{}/", Source::Lofigirl.url()); - let files = scan().await?; - for file in files { - println!("{file}"); - } - - Ok(()) -} diff --git a/src/ui/interface/clock.rs b/src/ui/interface/clock.rs index 0a3eb60..0196867 100644 --- a/src/ui/interface/clock.rs +++ b/src/ui/interface/clock.rs @@ -9,7 +9,7 @@ impl Clock { /// Small shorthand for getting the local time now, and formatting it. #[inline] fn now() -> chrono::format::DelayedFormat> { - chrono::Local::now().format("%H:%M:%S") + chrono::Local::now().format("%H:%M") } /// Checks if the last update was long enough ago, and if so,