mirror of
https://github.com/talwat/lowfi
synced 2025-08-13 13:04:16 +00:00
feat: chillhop scraper (#94)
* feat: start work on chillhop scraper only the basic get request with caching has actually been implemented, but the rest shouldn't be too complicated. * feat: finish chillhop scraper * chore: remove vscode dir * chore: upload new chillhop list for comparison * fix: improve scraper and remove duplicates * style: reorganize release scan function slightly * fix: make lowfi compile on non-linux * feat: make scraper fully reproducable (hopefully) * chore: remove useless mut * feat: add scrape feature flag * chore: update deps * chore: i hate macos * chore: add .DS_Store to gitignore * fix: ignore two tracks with lyrics * fix: seperate get function from chillhop scraper * fix: linux audio output stream * chore: replace chillhop old list
This commit is contained in:
Tal
GitHub
parent
19ca315509
commit
bdd508bfbb
2
.gitignore
vendored
2
.gitignore
vendored
@ -1 +1,3 @@
|
||||
/target
|
||||
/cache
|
||||
.DS_Store
|
||||
346
Cargo.lock
generated
346
Cargo.lock
generated
@ -29,15 +29,15 @@ checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.11"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"getrandom",
|
||||
"getrandom 0.3.3",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
"zerocopy 0.8.26",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -471,6 +471,19 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e09ced7ebbccb63b4c65413d821f2e00ce54c5ca4514ddc6b3c892fdbcbc69d"
|
||||
dependencies = [
|
||||
"encode_unicode",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"unicode-width",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.9.4"
|
||||
@ -609,9 +622,9 @@ checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "0.99.18"
|
||||
version = "0.99.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"
|
||||
checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@ -672,9 +685,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dtoa"
|
||||
version = "1.0.9"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"
|
||||
checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04"
|
||||
|
||||
[[package]]
|
||||
name = "dtoa-short"
|
||||
@ -691,6 +704,12 @@ version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c6ba7d4eec39eaa9ab24d44a0e73a7949a1095a8b3f3abb11eddf27dbb56a53"
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
@ -949,9 +968,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "getopts"
|
||||
version = "0.2.21"
|
||||
version = "0.2.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
|
||||
checksum = "cba6ae63eb948698e300f645f87c70f76630d505f23b8907cf1e193ee85048c1"
|
||||
dependencies = [
|
||||
"unicode-width",
|
||||
]
|
||||
@ -964,7 +983,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
"wasi 0.14.2+wasi-0.2.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1017,17 +1048,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.29.0"
|
||||
name = "html-escape"
|
||||
version = "0.2.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e15626aaf9c351bc696217cbe29cb9b5e86c43f8a46b5e2f5c6c5cf7cb904ce"
|
||||
checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
|
||||
dependencies = [
|
||||
"utf8-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c"
|
||||
dependencies = [
|
||||
"log",
|
||||
"mac",
|
||||
"markup5ever",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"match_token",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1297,6 +1335,19 @@ dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indicatif"
|
||||
version = "0.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd"
|
||||
dependencies = [
|
||||
"console",
|
||||
"portable-atomic",
|
||||
"unicode-width",
|
||||
"unit-prefix",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.10.1"
|
||||
@ -1410,6 +1461,8 @@ dependencies = [
|
||||
"dirs",
|
||||
"eyre",
|
||||
"futures",
|
||||
"html-escape",
|
||||
"indicatif",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"mpris-server",
|
||||
@ -1417,6 +1470,8 @@ dependencies = [
|
||||
"reqwest",
|
||||
"rodio",
|
||||
"scraper",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.12",
|
||||
"tokio",
|
||||
"unicode-segmentation",
|
||||
@ -1440,9 +1495,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.14.0"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "82c88c6129bd24319e62a0359cb6b958fa7e8be6e19bb1663bc396b90883aca5"
|
||||
checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18"
|
||||
dependencies = [
|
||||
"log",
|
||||
"phf",
|
||||
@ -1452,6 +1507,17 @@ dependencies = [
|
||||
"tendril",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "match_token"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
@ -1490,7 +1556,7 @@ checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"wasi",
|
||||
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
@ -1839,7 +1905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.11.3",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1848,18 +1914,8 @@ version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
|
||||
dependencies = [
|
||||
"phf_generator 0.11.3",
|
||||
"phf_shared 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
|
||||
dependencies = [
|
||||
"phf_shared 0.10.0",
|
||||
"rand",
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1868,7 +1924,7 @@ version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
|
||||
dependencies = [
|
||||
"phf_shared 0.11.3",
|
||||
"phf_shared",
|
||||
"rand",
|
||||
]
|
||||
|
||||
@ -1878,29 +1934,20 @@ version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
|
||||
dependencies = [
|
||||
"phf_generator 0.11.3",
|
||||
"phf_shared 0.11.3",
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
|
||||
dependencies = [
|
||||
"siphasher 0.3.11",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
|
||||
dependencies = [
|
||||
"siphasher 1.0.1",
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1947,13 +1994,19 @@ dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
|
||||
dependencies = [
|
||||
"zerocopy",
|
||||
"zerocopy 0.7.35",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1989,6 +2042,12 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "5.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
@ -2016,7 +2075,7 @@ version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"getrandom 0.2.15",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2034,7 +2093,7 @@ version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"getrandom 0.2.15",
|
||||
"libredox",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
@ -2120,7 +2179,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"getrandom",
|
||||
"getrandom 0.2.15",
|
||||
"libc",
|
||||
"spin",
|
||||
"untrusted",
|
||||
@ -2293,18 +2352,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.217"
|
||||
version = "1.0.219"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
|
||||
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.217"
|
||||
version = "1.0.219"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
|
||||
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@ -2313,9 +2372,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.135"
|
||||
version = "1.0.142"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9"
|
||||
checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
@ -2348,9 +2407,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "servo_arc"
|
||||
version = "0.4.0"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae65c4249478a2647db249fb43e23cec56a2c8974a427e7bd8cb5a1d0964921a"
|
||||
checksum = "204ea332803bd95a0b60388590d59cf6468ec9becf626e2451f1d26a1d972de4"
|
||||
dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
@ -2402,12 +2461,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.1"
|
||||
@ -2459,26 +2512,25 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.8.7"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
|
||||
checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"once_cell",
|
||||
"parking_lot",
|
||||
"phf_shared 0.10.0",
|
||||
"phf_shared",
|
||||
"precomputed-hash",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "string_cache_codegen"
|
||||
version = "0.5.2"
|
||||
version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"
|
||||
checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0"
|
||||
dependencies = [
|
||||
"phf_generator 0.10.0",
|
||||
"phf_shared 0.10.0",
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
@ -2702,7 +2754,7 @@ checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"fastrand",
|
||||
"getrandom",
|
||||
"getrandom 0.2.15",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.59.0",
|
||||
@ -2952,9 +3004,15 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.14"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
|
||||
checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c"
|
||||
|
||||
[[package]]
|
||||
name = "unit-prefix"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
@ -2985,6 +3043,12 @@ version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
|
||||
|
||||
[[package]]
|
||||
name = "utf8-width"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
@ -3034,6 +3098,15 @@ version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.14.2+wasi-0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
|
||||
dependencies = [
|
||||
"wit-bindgen-rt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.99"
|
||||
@ -3111,6 +3184,16 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-time"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
@ -3162,6 +3245,12 @@ dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
|
||||
|
||||
[[package]]
|
||||
name = "windows-registry"
|
||||
version = "0.2.0"
|
||||
@ -3237,6 +3326,15 @@ dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
|
||||
dependencies = [
|
||||
"windows-targets 0.53.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.42.2"
|
||||
@ -3276,13 +3374,30 @@ dependencies = [
|
||||
"windows_aarch64_gnullvm 0.52.6",
|
||||
"windows_aarch64_msvc 0.52.6",
|
||||
"windows_i686_gnu 0.52.6",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_gnullvm 0.52.6",
|
||||
"windows_i686_msvc 0.52.6",
|
||||
"windows_x86_64_gnu 0.52.6",
|
||||
"windows_x86_64_gnullvm 0.52.6",
|
||||
"windows_x86_64_msvc 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.53.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
"windows_aarch64_gnullvm 0.53.0",
|
||||
"windows_aarch64_msvc 0.53.0",
|
||||
"windows_i686_gnu 0.53.0",
|
||||
"windows_i686_gnullvm 0.53.0",
|
||||
"windows_i686_msvc 0.53.0",
|
||||
"windows_x86_64_gnu 0.53.0",
|
||||
"windows_x86_64_gnullvm 0.53.0",
|
||||
"windows_x86_64_msvc 0.53.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.42.2"
|
||||
@ -3301,6 +3416,12 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.42.2"
|
||||
@ -3319,6 +3440,12 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.42.2"
|
||||
@ -3337,12 +3464,24 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.42.2"
|
||||
@ -3361,6 +3500,12 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.42.2"
|
||||
@ -3379,6 +3524,12 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.42.2"
|
||||
@ -3397,6 +3548,12 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.42.2"
|
||||
@ -3415,6 +3572,12 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.53.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.6.22"
|
||||
@ -3424,6 +3587,15 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rt"
|
||||
version = "0.39.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "write16"
|
||||
version = "1.0.0"
|
||||
@ -3539,7 +3711,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"zerocopy-derive",
|
||||
"zerocopy-derive 0.7.35",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
|
||||
dependencies = [
|
||||
"zerocopy-derive 0.8.26",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3553,6 +3734,17 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.5"
|
||||
|
||||
22
Cargo.toml
22
Cargo.toml
@ -19,21 +19,18 @@ repository = "https://github.com/talwat/lowfi"
|
||||
[features]
|
||||
mpris = ["dep:mpris-server"]
|
||||
extra-audio-formats = ["rodio/default"]
|
||||
scrape = ["dep:serde", "dep:serde_json", "dep:html-escape", "dep:scraper", "dep:indicatif"]
|
||||
|
||||
[dependencies]
|
||||
# Basics
|
||||
clap = { version = "4.5.21", features = ["derive", "cargo"] }
|
||||
eyre = { version = "0.6.12" }
|
||||
eyre = "0.6.12"
|
||||
rand = "0.8.5"
|
||||
thiserror = "2.0.12"
|
||||
color-eyre = { version = "0.6.5", default-features = false }
|
||||
|
||||
# Async
|
||||
tokio = { version = "1.41.1", features = [
|
||||
"macros",
|
||||
"rt-multi-thread",
|
||||
"fs"
|
||||
], default-features = false }
|
||||
tokio = { version = "1.41.1", features = ["macros", "rt-multi-thread", "fs"], default-features = false }
|
||||
futures = "0.3.31"
|
||||
arc-swap = "1.7.1"
|
||||
|
||||
@ -48,9 +45,18 @@ mpris-server = { version = "0.8.1", optional = true }
|
||||
dirs = "5.0.1"
|
||||
|
||||
# Misc
|
||||
scraper = "0.21.0"
|
||||
Inflector = "0.11.4"
|
||||
lazy_static = "1.5.0"
|
||||
libc = "0.2.167"
|
||||
url = "2.5.4"
|
||||
unicode-segmentation = "1.12.0"
|
||||
|
||||
# Scraper
|
||||
serde = { version = "1.0.219", features = ["derive"], optional = true }
|
||||
serde_json = { version = "1.0.142", optional = true }
|
||||
scraper = { version = "0.21.0", optional = true }
|
||||
html-escape = { version = "0.2.13", optional = true }
|
||||
indicatif = { version = "0.18.0", optional = true }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libc = "0.2.167"
|
||||
|
||||
|
||||
2505
data/chillhop.txt
2505
data/chillhop.txt
File diff suppressed because it is too large
Load Diff
4
scripts/fix_cache.sh
Normal file
4
scripts/fix_cache.sh
Normal file
@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
|
||||
grep -rlZ "429 Too Many Requests" . | xargs -0 rm -f
|
||||
find . -type f -empty -delete
|
||||
14
src/main.rs
14
src/main.rs
@ -11,8 +11,11 @@ mod player;
|
||||
mod tracks;
|
||||
|
||||
#[allow(clippy::all, clippy::pedantic, clippy::nursery, clippy::restriction)]
|
||||
#[cfg(feature = "scrape")]
|
||||
mod scrapers;
|
||||
|
||||
#[cfg(feature = "scrape")]
|
||||
use crate::scrapers::Source;
|
||||
/// An extremely simple lofi player.
|
||||
#[derive(Parser, Clone)]
|
||||
#[command(about, version)]
|
||||
@ -64,9 +67,10 @@ struct Args {
|
||||
#[derive(Subcommand, Clone)]
|
||||
enum Commands {
|
||||
/// Scrapes a music source for files.
|
||||
#[cfg(feature = "scrape")]
|
||||
Scrape {
|
||||
// The source to scrape from.
|
||||
source: scrapers::Sources,
|
||||
source: scrapers::Source,
|
||||
|
||||
/// The file extension to search for, defaults to mp3.
|
||||
#[clap(long, short, default_value = "mp3")]
|
||||
@ -96,11 +100,15 @@ async fn main() -> eyre::Result<()> {
|
||||
if let Some(command) = cli.command {
|
||||
match command {
|
||||
// TODO: Actually distinguish between sources.
|
||||
#[cfg(feature = "scrape")]
|
||||
Commands::Scrape {
|
||||
source: _,
|
||||
source,
|
||||
extension,
|
||||
include_full,
|
||||
} => scrapers::lofigirl::scrape(extension, include_full).await?,
|
||||
} => match source {
|
||||
Source::Lofigirl => scrapers::lofigirl::scrape(extension, include_full).await?,
|
||||
Source::Chillhop => scrapers::chillhop::scrape().await?,
|
||||
},
|
||||
}
|
||||
} else {
|
||||
play::play(cli).await?;
|
||||
|
||||
@ -1,13 +1,7 @@
|
||||
#[cfg(target_os = "linux")]
|
||||
use rodio::OutputStream;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use crate::player;
|
||||
|
||||
/// This gets the output stream while also shutting up alsa with [libc].
|
||||
/// Uses raw libc calls, and therefore is functional only on Linux.
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn silent_get_output_stream() -> eyre::Result<OutputStream, player::Error> {
|
||||
pub fn silent_get_output_stream() -> eyre::Result<rodio::OutputStream, crate::player::Error> {
|
||||
use libc::freopen;
|
||||
use rodio::OutputStreamBuilder;
|
||||
use std::ffi::CString;
|
||||
|
||||
@ -1,9 +1,81 @@
|
||||
use clap::ValueEnum;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use clap::ValueEnum;
|
||||
use eyre::bail;
|
||||
use reqwest::Client;
|
||||
use tokio::{
|
||||
fs::{self, File},
|
||||
io::AsyncWriteExt,
|
||||
};
|
||||
|
||||
pub mod chillhop;
|
||||
pub mod lofigirl;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug, ValueEnum)]
|
||||
pub enum Sources {
|
||||
pub enum Source {
|
||||
Lofigirl,
|
||||
Chillhop,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
pub fn cache_dir(&self) -> &'static str {
|
||||
match self {
|
||||
Source::Lofigirl => "lofigirl",
|
||||
Source::Chillhop => "chillhop",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn url(&self) -> &'static str {
|
||||
match self {
|
||||
Source::Chillhop => "https://chillhop.com",
|
||||
Source::Lofigirl => "https://lofigirl.com/wp-content/uploads",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends a get request, with caching.
|
||||
async fn get(client: &Client, path: &str, source: Source) -> eyre::Result<String> {
|
||||
let trimmed = path.trim_matches('/');
|
||||
let cache = PathBuf::from(format!("./cache/{}/{trimmed}.html", source.cache_dir()));
|
||||
|
||||
if let Ok(x) = fs::read_to_string(&cache).await {
|
||||
Ok(x)
|
||||
} else {
|
||||
let resp = client
|
||||
.get(format!("{}/{trimmed}", source.url()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = resp.status();
|
||||
|
||||
if status == 429 {
|
||||
bail!("rate limit reached: {path}");
|
||||
}
|
||||
|
||||
if status != 404 && !status.is_success() && !status.is_redirection() {
|
||||
bail!("non success code {}: {path}", resp.status().as_u16());
|
||||
}
|
||||
|
||||
let text = resp.text().await?;
|
||||
|
||||
let parent = cache.parent();
|
||||
if let Some(x) = parent {
|
||||
if x != Path::new("") {
|
||||
fs::create_dir_all(x).await?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut file = File::create(&cache).await?;
|
||||
file.write_all(text.as_bytes()).await?;
|
||||
|
||||
if status.is_redirection() {
|
||||
bail!("redirect: {path}")
|
||||
}
|
||||
|
||||
if status == 404 {
|
||||
bail!("not found: {path}")
|
||||
}
|
||||
|
||||
Ok(text)
|
||||
}
|
||||
}
|
||||
|
||||
221
src/scrapers/chillhop.rs
Normal file
221
src/scrapers/chillhop.rs
Normal file
@ -0,0 +1,221 @@
|
||||
use eyre::eyre;
|
||||
use futures::stream::FuturesUnordered;
|
||||
use futures::StreamExt;
|
||||
use indicatif::ProgressBar;
|
||||
use lazy_static::lazy_static;
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
use reqwest::Client;
|
||||
use scraper::{Html, Selector};
|
||||
use serde::{
|
||||
de::{self, Visitor},
|
||||
Deserialize, Deserializer,
|
||||
};
|
||||
use tokio::fs;
|
||||
|
||||
use crate::scrapers::{get, Source};
|
||||
|
||||
lazy_static! {
|
||||
static ref RELEASES: Selector = Selector::parse(".table-body > a").unwrap();
|
||||
static ref RELEASE_LABEL: Selector = Selector::parse("label").unwrap();
|
||||
// static ref RELEASE_DATE: Selector = Selector::parse(".release-feat-props > .text-xs").unwrap();
|
||||
// static ref RELEASE_NAME: Selector = Selector::parse(".release-feat-props > h2").unwrap();
|
||||
static ref RELEASE_AUTHOR: Selector = Selector::parse(".release-feat-props .artist-link").unwrap();
|
||||
static ref RELEASE_TEXTAREA: Selector = Selector::parse("textarea").unwrap();
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Track {
|
||||
title: String,
|
||||
#[serde(deserialize_with = "deserialize_u32_from_string")]
|
||||
file_id: u32,
|
||||
artists: String,
|
||||
}
|
||||
|
||||
impl Track {
|
||||
pub fn clean(&mut self) {
|
||||
self.artists = html_escape::decode_html_entities(&self.artists).to_string();
|
||||
|
||||
self.title = html_escape::decode_html_entities(&self.title).to_string();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
struct Release {
|
||||
#[serde(skip)]
|
||||
pub path: String,
|
||||
|
||||
#[serde(skip)]
|
||||
pub index: usize,
|
||||
|
||||
pub tracks: Vec<Track>,
|
||||
}
|
||||
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
enum ReleaseError {
|
||||
#[error("invalid release: {0}")]
|
||||
Invalid(#[from] eyre::Error),
|
||||
}
|
||||
|
||||
impl Release {
|
||||
pub async fn scan(
|
||||
path: String,
|
||||
index: usize,
|
||||
client: Client,
|
||||
bar: ProgressBar,
|
||||
) -> Result<Self, ReleaseError> {
|
||||
let content = get(&client, &path, Source::Chillhop).await?;
|
||||
let html = Html::parse_document(&content);
|
||||
|
||||
let textarea = html
|
||||
.select(&RELEASE_TEXTAREA)
|
||||
.next()
|
||||
.ok_or(eyre!("unable to find textarea: {path}"))?;
|
||||
|
||||
let mut release: Self = serde_json::from_str(&textarea.inner_html()).unwrap();
|
||||
release.path = path;
|
||||
release.index = index;
|
||||
release.tracks.reverse();
|
||||
|
||||
bar.inc(release.tracks.len() as u64);
|
||||
|
||||
Ok(release)
|
||||
}
|
||||
}
|
||||
|
||||
async fn scan_page(
|
||||
number: usize,
|
||||
client: &Client,
|
||||
bar: ProgressBar,
|
||||
) -> eyre::Result<Vec<impl futures::Future<Output = Result<Release, ReleaseError>>>> {
|
||||
let path = format!("releases/?page={number}");
|
||||
let content = get(client, &path, Source::Chillhop).await?;
|
||||
let html = Html::parse_document(&content);
|
||||
|
||||
let elements = html.select(&RELEASES);
|
||||
Ok(elements
|
||||
.enumerate()
|
||||
.filter_map(|(i, x)| {
|
||||
let label = x.select(&RELEASE_LABEL).next()?.inner_html();
|
||||
if label == "Compilation" {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(Release::scan(
|
||||
x.attr("href")?.to_string(),
|
||||
(number * 12) + i,
|
||||
client.clone(),
|
||||
bar.clone(),
|
||||
))
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn scrape() -> eyre::Result<()> {
|
||||
const PAGE_COUNT: usize = 40;
|
||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36";
|
||||
const TRACK_COUNT: u64 = 1625;
|
||||
|
||||
const IGNORED_TRACKS: [u32; 3] = [
|
||||
74707, // 404
|
||||
21655, // Lyrics
|
||||
21773, // Lyrics
|
||||
];
|
||||
|
||||
const IGNORED_ARTISTS: [&str; 1] = [
|
||||
"Kenji", // Lyrics
|
||||
];
|
||||
|
||||
fs::create_dir_all("./cache/chillhop").await.unwrap();
|
||||
let client = Client::builder().user_agent(USER_AGENT).build().unwrap();
|
||||
|
||||
let futures = FuturesUnordered::new();
|
||||
let bar = ProgressBar::new(TRACK_COUNT + (12 * (PAGE_COUNT as u64)));
|
||||
|
||||
let mut errors = Vec::new();
|
||||
|
||||
// This is slightly less memory efficient than I'd hope, but it is what it is.
|
||||
for page in 0..=PAGE_COUNT {
|
||||
bar.inc(12);
|
||||
for x in scan_page(page, &client, bar.clone()).await? {
|
||||
futures.push(x);
|
||||
}
|
||||
}
|
||||
|
||||
let mut results: Vec<Result<Release, ReleaseError>> = futures.collect().await;
|
||||
bar.finish_and_clear();
|
||||
|
||||
// I mean, is it... optimal? Absolutely not. Does it work? Yes.
|
||||
eprintln!("sorting...");
|
||||
results.sort_by_key(|x| if let Ok(x) = x { x.index } else { 0 });
|
||||
results.reverse();
|
||||
|
||||
eprintln!("printing...");
|
||||
let mut printed = Vec::with_capacity(TRACK_COUNT as usize); // Lazy way to get rid of dupes.
|
||||
for result in results {
|
||||
let release = match result {
|
||||
Ok(release) => release,
|
||||
Err(error) => {
|
||||
errors.push(error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
for mut track in release.tracks {
|
||||
if IGNORED_TRACKS.contains(&track.file_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if IGNORED_ARTISTS.contains(&track.artists.as_ref()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if printed.contains(&track.file_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
printed.push(track.file_id);
|
||||
|
||||
track.clean();
|
||||
println!("{}!{}", track.file_id, track.title);
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!("-- ERROR REPORT --");
|
||||
for error in errors {
|
||||
eprintln!("{error}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn deserialize_u32_from_string<'de, D>(deserializer: D) -> Result<u32, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
struct U32FromStringVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for U32FromStringVisitor {
|
||||
type Value = u32;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
|
||||
formatter.write_str("a string containing an unsigned 32-bit integer")
|
||||
}
|
||||
|
||||
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
|
||||
where
|
||||
E: de::Error,
|
||||
{
|
||||
u32::from_str(value).map_err(|_| {
|
||||
de::Error::invalid_value(
|
||||
de::Unexpected::Str(value),
|
||||
&"a valid unsigned 32-bit integer",
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_str(U32FromStringVisitor)
|
||||
}
|
||||
@ -5,19 +5,19 @@
|
||||
|
||||
use futures::{stream::FuturesOrdered, StreamExt};
|
||||
use lazy_static::lazy_static;
|
||||
use reqwest::Client;
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
const BASE_URL: &str = "https://lofigirl.com/wp-content/uploads/";
|
||||
use crate::scrapers::{get, Source};
|
||||
|
||||
lazy_static! {
|
||||
static ref SELECTOR: Selector = Selector::parse("html > body > pre > a").unwrap();
|
||||
}
|
||||
|
||||
async fn parse(path: &str) -> eyre::Result<Vec<String>> {
|
||||
let response = reqwest::get(format!("{}{}", BASE_URL, path)).await?;
|
||||
let document = response.text().await?;
|
||||
|
||||
async fn parse(client: &Client, path: &str) -> eyre::Result<Vec<String>> {
|
||||
let document = get(client, path, super::Source::Lofigirl).await?;
|
||||
let html = Html::parse_document(&document);
|
||||
|
||||
Ok(html
|
||||
.select(&SELECTOR)
|
||||
.skip(5)
|
||||
@ -30,9 +30,10 @@ async fn parse(path: &str) -> eyre::Result<Vec<String>> {
|
||||
/// It's a bit hacky, and basically works by checking all of the years, then months, and then all of the files.
|
||||
/// This is done as a way to avoid recursion, since async rust really hates recursive functions.
|
||||
async fn scan(extension: &str, include_full: bool) -> eyre::Result<Vec<String>> {
|
||||
let client = Client::new();
|
||||
let extension = &format!(".{}", extension);
|
||||
|
||||
let items = parse("").await?;
|
||||
let items = parse(&client, "/").await?;
|
||||
|
||||
let mut years: Vec<u32> = items
|
||||
.iter()
|
||||
@ -48,19 +49,20 @@ async fn scan(extension: &str, include_full: bool) -> eyre::Result<Vec<String>>
|
||||
let mut futures = FuturesOrdered::new();
|
||||
|
||||
for year in years {
|
||||
let months = parse(&year.to_string()).await?;
|
||||
let months = parse(&client, &year.to_string()).await?;
|
||||
|
||||
for month in months {
|
||||
let client = client.clone();
|
||||
futures.push_back(async move {
|
||||
let path = format!("{}/{}", year, month);
|
||||
|
||||
let items = parse(&path).await.unwrap();
|
||||
let items = parse(&client, &path).await.unwrap();
|
||||
items
|
||||
.into_iter()
|
||||
.filter_map(|x| {
|
||||
if x.ends_with(extension) {
|
||||
if include_full {
|
||||
Some(format!("{BASE_URL}{path}{x}"))
|
||||
Some(format!("{}/{path}{x}", Source::Lofigirl.url()))
|
||||
} else {
|
||||
Some(format!("{path}{x}"))
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user