commit - 08360f954b312a5566c5a39e6d82256257b77644
commit + e039dc9be309339563524e1606a744336453c5ce
blob - 5e5ded631b7135c166fb13031e2f6335bdc4e3c8
blob + 06368397a1ad7a4dfe9c33b7086b93885ff67823
--- Cargo.lock
+++ Cargo.lock
[[package]]
name = "futures-channel"
-version = "0.3.23"
+version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bfc52cbddcfd745bf1740338492bb0bd83d76c67b445f91c5fb29fae29ecaa1"
+checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
dependencies = [
"futures-core",
]
[[package]]
name = "futures-core"
-version = "0.3.23"
+version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2acedae88d38235936c3922476b10fced7b2b68136f5e3c03c2d5be348a1115"
+checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
[[package]]
-name = "futures-io"
-version = "0.3.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93a66fc6d035a26a3ae255a6d2bca35eda63ae4c5512bef54449113f7a1228e5"
-
-[[package]]
name = "futures-sink"
-version = "0.3.23"
+version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca0bae1fe9752cf7fd9b0064c674ae63f97b37bc714d745cbde0afb7ec4e6765"
+checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
[[package]]
name = "futures-task"
-version = "0.3.23"
+version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "842fc63b931f4056a24d59de13fb1272134ce261816e063e634ad0c15cdc5306"
+checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
[[package]]
name = "futures-util"
-version = "0.3.23"
+version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0828a5471e340229c11c77ca80017937ce3c58cb788a17e5f1c2d5c485a9577"
+checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
dependencies = [
"futures-core",
- "futures-io",
"futures-task",
- "memchr",
"pin-project-lite",
"pin-utils",
- "slab",
]
[[package]]
[[package]]
name = "num_cpus"
-version = "1.13.1"
+version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
+checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
dependencies = [
"hermit-abi",
"libc",
"reqwest",
"rss",
"serde",
+ "tokio",
"toml",
]
[[package]]
name = "tokio"
-version = "1.20.1"
+version = "1.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581"
+checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099"
dependencies = [
"autocfg",
"bytes",
"memchr",
"mio",
"num_cpus",
- "once_cell",
"pin-project-lite",
"socket2",
+ "tokio-macros",
"winapi",
]
[[package]]
+name = "tokio-macros"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
name = "tokio-rustls"
version = "0.23.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
blob - 93f7b32ac4c60607d4a91913294e9d22c1c78654
blob + dacdd829277c0d5384d2a0a86e198c7bd7533a05
--- Cargo.toml
+++ Cargo.toml
diesel = { version = "1.4", features = ["sqlite"] }
rss = "2.0"
anyhow = "1.0"
-reqwest = {version = "0.11", default-features = false, features = ["blocking", "rustls-tls"]}
+reqwest = {version = "0.11", default-features = false, features = ["rustls-tls"]}
clap = { version = "3", features = ["derive"] }
chrono = "0.4"
toml = "0.5.8"
directories = "4.0.1"
log = "0.4.17"
env_logger = "0.9.0"
+tokio = { version = "1.21.2", default-features = false, features = ["rt-multi-thread", "macros"] }
blob - 156cc104f6b1d2ca9ecf492f4f9893c292758ee2
blob + b181138712b3a96686ba3774132fc7fa124cfbc2
--- src/main.rs
+++ src/main.rs
fs::File,
io::{BufRead, BufReader},
};
+use tokio::task::JoinSet;
-fn main() -> anyhow::Result<()> {
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
diesel_migrations::embed_migrations!("migrations/");
env_logger::init();
let conn = SqliteConnection::establish(args.database_path.unwrap().to_str().unwrap())?;
embedded_migrations::run(&conn)?;
+ let mut set = JoinSet::new();
for u in urls {
- debug!("Fetching feed from {u:?}");
- let new = rss::fetch_new(u)?;
- for i in new.items() {
+ set.spawn(async move { rss::fetch_new(u).await });
+ }
+
+ while let Some(new) = set.join_next().await {
+ let new = new??;
+ let items = new.items();
+
+ debug!("Found {} new items", items.len());
+
+ for i in items {
let _ = db::insert_item(&conn, i)?;
}
}
blob - 52903660f78d83ec81015281338c89ae1ba63af2
blob + af377d139f71bc562e75be4281aecfad59e9702f
--- src/rss.rs
+++ src/rss.rs
use reqwest;
use rss;
-pub fn fetch_new<S: AsRef<str>>(url: S) -> anyhow::Result<rss::Channel> {
- let content = reqwest::blocking::get(url.as_ref())?.bytes()?;
+pub async fn fetch_new<S: AsRef<str>>(url: S) -> anyhow::Result<rss::Channel> {
+ debug!("Fetching feed for {}", url.as_ref());
+
+ let content = reqwest::get(url.as_ref()).await?.bytes().await?;
let channel = rss::Channel::read_from(&content[..])?;
Ok(channel)