From ae7a2408c444721ef02257be255ef486fc19e917 Mon Sep 17 00:00:00 2001 From: SpikeHD <25207995+SpikeHD@users.noreply.github.com> Date: Fri, 28 Feb 2025 12:24:44 -0800 Subject: [PATCH 1/7] feat: begin ureq backend --- Cargo.lock | 44 ++++++++++++++++++++++ Cargo.toml | 1 + examples/screenshot.rs | 10 ++--- examples/url.rs | 4 +- packages/blitz-net/Cargo.toml | 10 ++++- packages/blitz-net/src/backend/mod.rs | 32 ++++++++++++++++ packages/blitz-net/src/backend/reqwest.rs | 45 +++++++++++++++++++++++ packages/blitz-net/src/backend/ureq.rs | 39 ++++++++++++++++++++ packages/blitz-net/src/lib.rs | 44 +++++++++++----------- packages/blitz-traits/src/net.rs | 17 +++++++++ 10 files changed, 213 insertions(+), 33 deletions(-) create mode 100644 packages/blitz-net/src/backend/mod.rs create mode 100644 packages/blitz-net/src/backend/reqwest.rs create mode 100644 packages/blitz-net/src/backend/ureq.rs diff --git a/Cargo.lock b/Cargo.lock index 2cc092760..c66d6aad1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -705,9 +705,12 @@ version = "0.1.0" dependencies = [ "blitz-traits", "data-url", + "http", "reqwest", "thiserror 1.0.69", "tokio", + "ureq", + "url", ] [[package]] @@ -4772,7 +4775,9 @@ version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -6122,6 +6127,36 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06f78313c985f2fba11100dd06d60dd402d0cabb458af4d94791b8e09c025323" +dependencies = [ + "base64", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "ureq-proto", + "utf-8", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64adb55464bad1ab1aa9229133d0d59d2f679180f4d15f0d9debe616f541f25e" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.4" @@ -6546,6 +6581,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "weezl" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index 62fe165a6..86f7a1c64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ http = "1.1.0" data-url = "0.3.1" tokio = "1.42" reqwest = "0.12" +ureq = "3.0" # Media & Decoding image = { version = "0.25", default-features = false } diff --git a/examples/screenshot.rs b/examples/screenshot.rs index 020cc3c3c..6990d2087 100644 --- a/examples/screenshot.rs +++ b/examples/screenshot.rs @@ -22,9 +22,7 @@ const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/2010010 async fn main() { let mut timer = Timer::init(); - let url_string = std::env::args() - .skip(1) - .next() + let url_string = std::env::args().nth(1) .unwrap_or_else(|| "https://www.google.com".into()); println!("{}", url_string); @@ -55,9 +53,7 @@ async fn main() { // Setup viewport. TODO: make configurable. let scale = 2; let height = 800; - let width: u32 = std::env::args() - .skip(2) - .next() + let width: u32 = std::env::args().nth(2) .and_then(|arg| arg.parse().ok()) .unwrap_or(1200); @@ -151,7 +147,7 @@ fn write_png(writer: W, buffer: &[u8], width: u32, height: u32) { // Write PNG data to writer let mut writer = encoder.write_header().unwrap(); - writer.write_image_data(&buffer).unwrap(); + writer.write_image_data(buffer).unwrap(); writer.finish().unwrap(); } diff --git a/examples/url.rs b/examples/url.rs index 99a2acffc..6339e21de 100644 --- a/examples/url.rs +++ b/examples/url.rs @@ -1,9 +1,7 @@ //! Load first CLI argument as a url. Fallback to google.com if no CLI argument is provided. fn main() { - let url = std::env::args() - .skip(1) - .next() + let url = std::env::args().nth(1) .unwrap_or_else(|| "https://www.google.com".into()); blitz::launch_url(&url); } diff --git a/packages/blitz-net/Cargo.toml b/packages/blitz-net/Cargo.toml index 0102c0146..a98647999 100644 --- a/packages/blitz-net/Cargo.toml +++ b/packages/blitz-net/Cargo.toml @@ -4,9 +4,17 @@ version = "0.1.0" license.workspace = true edition = "2021" +[features] +default = ["reqwest"] +reqwest = ["dep:reqwest"] +ureq = ["dep:ureq"] + [dependencies] blitz-traits = { path = "../blitz-traits" } tokio = { workspace = true } -reqwest = { workspace = true } +reqwest = { workspace = true, optional = true } +ureq = { workspace = true, optional = true } data-url = { workspace = true } thiserror = { workspace = true } +http = { workspace = true } +url = { workspace = true } \ No newline at end of file diff --git a/packages/blitz-net/src/backend/mod.rs b/packages/blitz-net/src/backend/mod.rs new file mode 100644 index 000000000..27dd05225 --- /dev/null +++ b/packages/blitz-net/src/backend/mod.rs @@ -0,0 +1,32 @@ +use std::fmt::{Display, Formatter}; + +use blitz_traits::net::{Request, Response}; +use thiserror::Error; + +#[cfg(feature = "reqwest")] +mod reqwest; +#[cfg(feature = "ureq")] +mod ureq; + +#[cfg(feature = "reqwest")] +pub use reqwest::Backend; +#[cfg(feature = "ureq")] +pub use ureq::Backend; + +pub trait RequestBackend { + fn new() -> Self + where + Self: Sized; + async fn request(&mut self, request: Request) -> Result; +} + +#[derive(Debug, Error)] +pub struct BackendError { + pub message: String, +} + +impl Display for BackendError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} diff --git a/packages/blitz-net/src/backend/reqwest.rs b/packages/blitz-net/src/backend/reqwest.rs new file mode 100644 index 000000000..c333a9b23 --- /dev/null +++ b/packages/blitz-net/src/backend/reqwest.rs @@ -0,0 +1,45 @@ +use super::{BackendError, RequestBackend, Response}; +use blitz_traits::net::Request; + +// Compat with reqwest +impl From for BackendError { + fn from(e: reqwest::Error) -> Self { + BackendError { + message: e.to_string(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Backend { + client: reqwest::Client, +} + +impl RequestBackend for Backend { + fn new() -> Self + where + Self: Sized, + { + Self { + client: reqwest::Client::new(), + } + } + + async fn request(&mut self, request: Request) -> Result { + let request = self + .client + .request(request.method, request.url.clone()) + .headers(request.headers); + + let response = request.send().await?; + let status = response.status(); + let headers = response.headers().clone(); + let body = response.bytes().await?; + + Ok(Response { + status: status.as_u16(), + headers, + body, + }) + } +} diff --git a/packages/blitz-net/src/backend/ureq.rs b/packages/blitz-net/src/backend/ureq.rs new file mode 100644 index 000000000..9cfdfc3a0 --- /dev/null +++ b/packages/blitz-net/src/backend/ureq.rs @@ -0,0 +1,39 @@ +use blitz_traits::net::{Request, Response}; + +use super::{BackendError, RequestBackend}; + +impl From for BackendError { + fn from(e: ureq::Error) -> Self { + BackendError { + message: e.to_string(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Backend { + client: ureq::Agent, +} + +impl RequestBackend for Backend { + fn new() -> Self + where + Self: Sized, + { + Self { + client: ureq::agent(), + } + } + + async fn request(&mut self, request: Request) -> Result { + let mut response = self.client.run(request.into())?; + + let status = response.status().as_u16(); + + Ok(Response { + status, + headers: response.headers().clone(), + body: response.body_mut().read_to_vec()?.into(), + }) + } +} diff --git a/packages/blitz-net/src/lib.rs b/packages/blitz-net/src/lib.rs index 6262ef166..1f85b4bfb 100644 --- a/packages/blitz-net/src/lib.rs +++ b/packages/blitz-net/src/lib.rs @@ -1,37 +1,39 @@ +use backend::{Backend, RequestBackend}; use blitz_traits::net::{BoxedHandler, Bytes, NetCallback, NetProvider, Request, SharedCallback}; use data_url::DataUrl; -use reqwest::Client; +use http::HeaderValue; use std::sync::Arc; use thiserror::Error; use tokio::{ runtime::Handle, sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, }; +use url::Url; + +#[cfg(all(feature = "reqwest", feature = "ureq"))] +compile_error!("multiple request backends cannot be enabled at the same time"); + +mod backend; const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"; pub async fn get_text(url: &str) -> String { - Client::new() - .get(url) - .header("User-Agent", USER_AGENT) - .send() - .await - .unwrap() - .text() - .await - .unwrap() + let mut backend = Backend::new(); + let request = Request::get(Url::parse(url).unwrap()); + let response = backend.request(request).await.unwrap(); + String::from_utf8_lossy(&response.body).to_string() } pub struct Provider { rt: Handle, - client: Client, + client: Backend, resource_callback: SharedCallback, } impl Provider { pub fn new(res_callback: SharedCallback) -> Self { Self { rt: Handle::current(), - client: Client::new(), + client: Backend::new(), resource_callback: res_callback, } } @@ -44,7 +46,7 @@ impl Provider { } impl Provider { async fn fetch_inner( - client: Client, + mut client: Backend, doc_id: usize, request: Request, handler: BoxedHandler, @@ -61,15 +63,13 @@ impl Provider { handler.bytes(doc_id, Bytes::from(file_content), res_callback); } _ => { - let response = client - .request(request.method, request.url) - .headers(request.headers) - .header("User-Agent", USER_AGENT) - .body(request.body) - .send() - .await?; + let mut request = Request::get(request.url); + request + .headers + .insert("User-Agent", HeaderValue::from_static(USER_AGENT)); + let response = client.request(request).await?; - handler.bytes(doc_id, response.bytes().await?, res_callback); + handler.bytes(doc_id, response.body, res_callback); } } Ok(()) @@ -103,7 +103,7 @@ enum ProviderError { #[error("{0}")] DataUrlBas64(#[from] data_url::forgiving_base64::InvalidBase64), #[error("{0}")] - ReqwestError(#[from] reqwest::Error), + BackendError(#[from] backend::BackendError), } pub struct MpscCallback(UnboundedSender<(usize, T)>); diff --git a/packages/blitz-traits/src/net.rs b/packages/blitz-traits/src/net.rs index a6bf13fc2..e585c9645 100644 --- a/packages/blitz-traits/src/net.rs +++ b/packages/blitz-traits/src/net.rs @@ -51,6 +51,23 @@ impl Request { } } +impl Into>> for Request { + fn into(self) -> http::Request> { + let mut request = http::Request::new(self.body.into()); + request.headers_mut().extend(self.headers); + *request.method_mut() = self.method; + request + } +} + +#[derive(Debug)] +/// An HTTP response +pub struct Response { + pub status: u16, + pub headers: HeaderMap, + pub body: Bytes, +} + /// A default noop NetProvider pub struct DummyNetProvider(PhantomData); impl Default for DummyNetProvider { From d5bf0a4692d994177650106e600df80d53b5a161 Mon Sep 17 00:00:00 2001 From: SpikeHD <25207995+SpikeHD@users.noreply.github.com> Date: Fri, 28 Feb 2025 13:06:49 -0800 Subject: [PATCH 2/7] feat: move provider around, feature-gated callback helpers --- examples/screenshot.rs | 2 +- packages/blitz-net/Cargo.toml | 4 +- packages/blitz-net/src/backend/mod.rs | 3 + packages/blitz-net/src/backend/reqwest.rs | 52 ++++++++- packages/blitz-net/src/callback/mod.rs | 9 ++ packages/blitz-net/src/callback/reqwest.rs | 16 +++ packages/blitz-net/src/lib.rs | 118 ++++++--------------- 7 files changed, 115 insertions(+), 89 deletions(-) create mode 100644 packages/blitz-net/src/callback/mod.rs create mode 100644 packages/blitz-net/src/callback/reqwest.rs diff --git a/examples/screenshot.rs b/examples/screenshot.rs index 6990d2087..02b4b7416 100644 --- a/examples/screenshot.rs +++ b/examples/screenshot.rs @@ -2,7 +2,7 @@ use blitz_dom::net::Resource; use blitz_html::HtmlDocument; -use blitz_net::{MpscCallback, Provider}; +use blitz_net::{callback::MpscCallback, Provider}; use blitz_renderer_vello::render_to_buffer; use blitz_traits::navigation::DummyNavigationProvider; use blitz_traits::net::SharedProvider; diff --git a/packages/blitz-net/Cargo.toml b/packages/blitz-net/Cargo.toml index a98647999..0258b75fe 100644 --- a/packages/blitz-net/Cargo.toml +++ b/packages/blitz-net/Cargo.toml @@ -6,12 +6,12 @@ edition = "2021" [features] default = ["reqwest"] -reqwest = ["dep:reqwest"] +reqwest = ["dep:reqwest", "dep:tokio"] ureq = ["dep:ureq"] [dependencies] blitz-traits = { path = "../blitz-traits" } -tokio = { workspace = true } +tokio = { workspace = true, optional = true } reqwest = { workspace = true, optional = true } ureq = { workspace = true, optional = true } data-url = { workspace = true } diff --git a/packages/blitz-net/src/backend/mod.rs b/packages/blitz-net/src/backend/mod.rs index 27dd05225..7cef0e797 100644 --- a/packages/blitz-net/src/backend/mod.rs +++ b/packages/blitz-net/src/backend/mod.rs @@ -10,6 +10,9 @@ mod ureq; #[cfg(feature = "reqwest")] pub use reqwest::Backend; +#[cfg(feature = "reqwest")] +pub use reqwest::Provider; + #[cfg(feature = "ureq")] pub use ureq::Backend; diff --git a/packages/blitz-net/src/backend/reqwest.rs b/packages/blitz-net/src/backend/reqwest.rs index c333a9b23..5d4cceb7d 100644 --- a/packages/blitz-net/src/backend/reqwest.rs +++ b/packages/blitz-net/src/backend/reqwest.rs @@ -1,5 +1,12 @@ +use std::sync::Arc; + +use crate::{ProviderError, USER_AGENT}; + use super::{BackendError, RequestBackend, Response}; -use blitz_traits::net::Request; +use blitz_traits::net::{BoxedHandler, Bytes, NetCallback, NetProvider, Request, SharedCallback}; +use data_url::DataUrl; +use http::HeaderValue; +use tokio::{runtime::Handle, sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}}; // Compat with reqwest impl From for BackendError { @@ -43,3 +50,46 @@ impl RequestBackend for Backend { }) } } + +pub struct Provider { + rt: Handle, + client: Backend, + resource_callback: SharedCallback, +} + +impl Provider { + pub fn new(res_callback: SharedCallback) -> Self { + Self { + rt: Handle::current(), + client: Backend::new(), + resource_callback: res_callback, + } + } + + pub fn shared(res_callback: SharedCallback) -> Arc> { + Arc::new(Self::new(res_callback)) + } + + pub fn is_empty(&self) -> bool { + Arc::strong_count(&self.resource_callback) == 1 + } +} + +impl NetProvider for Provider { + type Data = D; + + fn fetch(&self, doc_id: usize, request: Request, handler: BoxedHandler) { + let client = self.client.clone(); + let callback = Arc::clone(&self.resource_callback); + println!("Fetching {}", &request.url); + drop(self.rt.spawn(async move { + let url = request.url.to_string(); + let res = Self::fetch_inner(client, doc_id, request, handler, callback).await; + if let Err(e) = res { + eprintln!("Error fetching {}: {e}", url); + } else { + println!("Success {}", url); + } + })); + } +} diff --git a/packages/blitz-net/src/callback/mod.rs b/packages/blitz-net/src/callback/mod.rs new file mode 100644 index 000000000..9a5328e0a --- /dev/null +++ b/packages/blitz-net/src/callback/mod.rs @@ -0,0 +1,9 @@ +#[cfg(feature = "reqwest")] +pub mod reqwest; +#[cfg(feature = "ureq")] +pub mod ureq; + +#[cfg(feature = "reqwest")] +pub use reqwest::*; +#[cfg(feature = "ureq")] +pub use ureq::*; \ No newline at end of file diff --git a/packages/blitz-net/src/callback/reqwest.rs b/packages/blitz-net/src/callback/reqwest.rs new file mode 100644 index 000000000..8cbda6943 --- /dev/null +++ b/packages/blitz-net/src/callback/reqwest.rs @@ -0,0 +1,16 @@ +use blitz_traits::net::NetCallback; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; + +pub struct MpscCallback(UnboundedSender<(usize, T)>); +impl MpscCallback { + pub fn new() -> (UnboundedReceiver<(usize, T)>, Self) { + let (send, recv) = unbounded_channel(); + (recv, Self(send)) + } +} +impl NetCallback for MpscCallback { + type Data = T; + fn call(&self, doc_id: usize, data: Self::Data) { + let _ = self.0.send((doc_id, data)); + } +} diff --git a/packages/blitz-net/src/lib.rs b/packages/blitz-net/src/lib.rs index 1f85b4bfb..12fc0cc47 100644 --- a/packages/blitz-net/src/lib.rs +++ b/packages/blitz-net/src/lib.rs @@ -1,18 +1,15 @@ use backend::{Backend, RequestBackend}; -use blitz_traits::net::{BoxedHandler, Bytes, NetCallback, NetProvider, Request, SharedCallback}; +use blitz_traits::net::{BoxedHandler, Bytes, Request, SharedCallback}; use data_url::DataUrl; use http::HeaderValue; -use std::sync::Arc; use thiserror::Error; -use tokio::{ - runtime::Handle, - sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, -}; + use url::Url; #[cfg(all(feature = "reqwest", feature = "ureq"))] compile_error!("multiple request backends cannot be enabled at the same time"); +pub mod callback; mod backend; const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"; @@ -24,74 +21,38 @@ pub async fn get_text(url: &str) -> String { String::from_utf8_lossy(&response.body).to_string() } -pub struct Provider { - rt: Handle, - client: Backend, - resource_callback: SharedCallback, -} -impl Provider { - pub fn new(res_callback: SharedCallback) -> Self { - Self { - rt: Handle::current(), - client: Backend::new(), - resource_callback: res_callback, - } - } - pub fn shared(res_callback: SharedCallback) -> Arc> { - Arc::new(Self::new(res_callback)) - } - pub fn is_empty(&self) -> bool { - Arc::strong_count(&self.resource_callback) == 1 - } -} -impl Provider { - async fn fetch_inner( - mut client: Backend, - doc_id: usize, - request: Request, - handler: BoxedHandler, - res_callback: SharedCallback, - ) -> Result<(), ProviderError> { - match request.url.scheme() { - "data" => { - let data_url = DataUrl::process(request.url.as_str())?; - let decoded = data_url.decode_to_vec()?; - handler.bytes(doc_id, Bytes::from(decoded.0), res_callback); - } - "file" => { - let file_content = std::fs::read(request.url.path())?; - handler.bytes(doc_id, Bytes::from(file_content), res_callback); - } - _ => { - let mut request = Request::get(request.url); - request - .headers - .insert("User-Agent", HeaderValue::from_static(USER_AGENT)); - let response = client.request(request).await?; +pub use backend::Provider; - handler.bytes(doc_id, response.body, res_callback); - } - } - Ok(()) - } -} +impl Provider { + async fn fetch_inner( + mut client: Backend, + doc_id: usize, + request: Request, + handler: BoxedHandler, + res_callback: SharedCallback, + ) -> Result<(), ProviderError> { + match request.url.scheme() { + "data" => { + let data_url = DataUrl::process(request.url.as_str())?; + let decoded = data_url.decode_to_vec()?; + handler.bytes(doc_id, Bytes::from(decoded.0), res_callback); + } + "file" => { + let file_content = std::fs::read(request.url.path())?; + handler.bytes(doc_id, Bytes::from(file_content), res_callback); + } + _ => { + let mut request = Request::get(request.url); + request + .headers + .insert("User-Agent", HeaderValue::from_static(USER_AGENT)); + let response = client.request(request).await?; -impl NetProvider for Provider { - type Data = D; - fn fetch(&self, doc_id: usize, request: Request, handler: BoxedHandler) { - let client = self.client.clone(); - let callback = Arc::clone(&self.resource_callback); - println!("Fetching {}", &request.url); - drop(self.rt.spawn(async move { - let url = request.url.to_string(); - let res = Self::fetch_inner(client, doc_id, request, handler, callback).await; - if let Err(e) = res { - eprintln!("Error fetching {}: {e}", url); - } else { - println!("Success {}", url); - } - })); - } + handler.bytes(doc_id, response.body, res_callback); + } + } + Ok(()) + } } #[derive(Error, Debug)] @@ -106,16 +67,3 @@ enum ProviderError { BackendError(#[from] backend::BackendError), } -pub struct MpscCallback(UnboundedSender<(usize, T)>); -impl MpscCallback { - pub fn new() -> (UnboundedReceiver<(usize, T)>, Self) { - let (send, recv) = unbounded_channel(); - (recv, Self(send)) - } -} -impl NetCallback for MpscCallback { - type Data = T; - fn call(&self, doc_id: usize, data: Self::Data) { - let _ = self.0.send((doc_id, data)); - } -} From 1f8ddd0b6fbbb8048c57ed15cfd7a0905f09e5bb Mon Sep 17 00:00:00 2001 From: SpikeHD <25207995+SpikeHD@users.noreply.github.com> Date: Fri, 28 Feb 2025 13:07:46 -0800 Subject: [PATCH 3/7] fix: clippy + fmt --- examples/screenshot.rs | 6 ++- examples/url.rs | 3 +- packages/blitz-net/src/backend/reqwest.rs | 43 +++++++++-------- packages/blitz-net/src/callback/mod.rs | 2 +- packages/blitz-net/src/callback/ureq.rs | 1 + packages/blitz-net/src/lib.rs | 59 +++++++++++------------ 6 files changed, 60 insertions(+), 54 deletions(-) create mode 100644 packages/blitz-net/src/callback/ureq.rs diff --git a/examples/screenshot.rs b/examples/screenshot.rs index 02b4b7416..91684c148 100644 --- a/examples/screenshot.rs +++ b/examples/screenshot.rs @@ -22,7 +22,8 @@ const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/2010010 async fn main() { let mut timer = Timer::init(); - let url_string = std::env::args().nth(1) + let url_string = std::env::args() + .nth(1) .unwrap_or_else(|| "https://www.google.com".into()); println!("{}", url_string); @@ -53,7 +54,8 @@ async fn main() { // Setup viewport. TODO: make configurable. let scale = 2; let height = 800; - let width: u32 = std::env::args().nth(2) + let width: u32 = std::env::args() + .nth(2) .and_then(|arg| arg.parse().ok()) .unwrap_or(1200); diff --git a/examples/url.rs b/examples/url.rs index 6339e21de..606abf95b 100644 --- a/examples/url.rs +++ b/examples/url.rs @@ -1,7 +1,8 @@ //! Load first CLI argument as a url. Fallback to google.com if no CLI argument is provided. fn main() { - let url = std::env::args().nth(1) + let url = std::env::args() + .nth(1) .unwrap_or_else(|| "https://www.google.com".into()); blitz::launch_url(&url); } diff --git a/packages/blitz-net/src/backend/reqwest.rs b/packages/blitz-net/src/backend/reqwest.rs index 5d4cceb7d..05a588c5d 100644 --- a/packages/blitz-net/src/backend/reqwest.rs +++ b/packages/blitz-net/src/backend/reqwest.rs @@ -3,10 +3,13 @@ use std::sync::Arc; use crate::{ProviderError, USER_AGENT}; use super::{BackendError, RequestBackend, Response}; -use blitz_traits::net::{BoxedHandler, Bytes, NetCallback, NetProvider, Request, SharedCallback}; +use blitz_traits::net::{BoxedHandler, NetProvider, Request, SharedCallback}; use data_url::DataUrl; use http::HeaderValue; -use tokio::{runtime::Handle, sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}}; +use tokio::{ + runtime::Handle, + sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, +}; // Compat with reqwest impl From for BackendError { @@ -52,9 +55,9 @@ impl RequestBackend for Backend { } pub struct Provider { - rt: Handle, - client: Backend, - resource_callback: SharedCallback, + rt: Handle, + client: Backend, + resource_callback: SharedCallback, } impl Provider { @@ -76,20 +79,20 @@ impl Provider { } impl NetProvider for Provider { - type Data = D; + type Data = D; - fn fetch(&self, doc_id: usize, request: Request, handler: BoxedHandler) { - let client = self.client.clone(); - let callback = Arc::clone(&self.resource_callback); - println!("Fetching {}", &request.url); - drop(self.rt.spawn(async move { - let url = request.url.to_string(); - let res = Self::fetch_inner(client, doc_id, request, handler, callback).await; - if let Err(e) = res { - eprintln!("Error fetching {}: {e}", url); - } else { - println!("Success {}", url); - } - })); - } + fn fetch(&self, doc_id: usize, request: Request, handler: BoxedHandler) { + let client = self.client.clone(); + let callback = Arc::clone(&self.resource_callback); + println!("Fetching {}", &request.url); + drop(self.rt.spawn(async move { + let url = request.url.to_string(); + let res = Self::fetch_inner(client, doc_id, request, handler, callback).await; + if let Err(e) = res { + eprintln!("Error fetching {}: {e}", url); + } else { + println!("Success {}", url); + } + })); + } } diff --git a/packages/blitz-net/src/callback/mod.rs b/packages/blitz-net/src/callback/mod.rs index 9a5328e0a..581e4da0f 100644 --- a/packages/blitz-net/src/callback/mod.rs +++ b/packages/blitz-net/src/callback/mod.rs @@ -6,4 +6,4 @@ pub mod ureq; #[cfg(feature = "reqwest")] pub use reqwest::*; #[cfg(feature = "ureq")] -pub use ureq::*; \ No newline at end of file +pub use ureq::*; diff --git a/packages/blitz-net/src/callback/ureq.rs b/packages/blitz-net/src/callback/ureq.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/blitz-net/src/callback/ureq.rs @@ -0,0 +1 @@ + diff --git a/packages/blitz-net/src/lib.rs b/packages/blitz-net/src/lib.rs index 12fc0cc47..577520426 100644 --- a/packages/blitz-net/src/lib.rs +++ b/packages/blitz-net/src/lib.rs @@ -9,8 +9,8 @@ use url::Url; #[cfg(all(feature = "reqwest", feature = "ureq"))] compile_error!("multiple request backends cannot be enabled at the same time"); -pub mod callback; mod backend; +pub mod callback; const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"; @@ -24,35 +24,35 @@ pub async fn get_text(url: &str) -> String { pub use backend::Provider; impl Provider { - async fn fetch_inner( - mut client: Backend, - doc_id: usize, - request: Request, - handler: BoxedHandler, - res_callback: SharedCallback, - ) -> Result<(), ProviderError> { - match request.url.scheme() { - "data" => { - let data_url = DataUrl::process(request.url.as_str())?; - let decoded = data_url.decode_to_vec()?; - handler.bytes(doc_id, Bytes::from(decoded.0), res_callback); - } - "file" => { - let file_content = std::fs::read(request.url.path())?; - handler.bytes(doc_id, Bytes::from(file_content), res_callback); - } - _ => { - let mut request = Request::get(request.url); - request - .headers - .insert("User-Agent", HeaderValue::from_static(USER_AGENT)); - let response = client.request(request).await?; + async fn fetch_inner( + mut client: Backend, + doc_id: usize, + request: Request, + handler: BoxedHandler, + res_callback: SharedCallback, + ) -> Result<(), ProviderError> { + match request.url.scheme() { + "data" => { + let data_url = DataUrl::process(request.url.as_str())?; + let decoded = data_url.decode_to_vec()?; + handler.bytes(doc_id, Bytes::from(decoded.0), res_callback); + } + "file" => { + let file_content = std::fs::read(request.url.path())?; + handler.bytes(doc_id, Bytes::from(file_content), res_callback); + } + _ => { + let mut request = Request::get(request.url); + request + .headers + .insert("User-Agent", HeaderValue::from_static(USER_AGENT)); + let response = client.request(request).await?; - handler.bytes(doc_id, response.body, res_callback); - } - } - Ok(()) - } + handler.bytes(doc_id, response.body, res_callback); + } + } + Ok(()) + } } #[derive(Error, Debug)] @@ -66,4 +66,3 @@ enum ProviderError { #[error("{0}")] BackendError(#[from] backend::BackendError), } - From 60ea05e6812e8b1d4975d582c534b1672da5c234 Mon Sep 17 00:00:00 2001 From: SpikeHD <25207995+SpikeHD@users.noreply.github.com> Date: Fri, 28 Feb 2025 14:31:06 -0800 Subject: [PATCH 4/7] feat: separate providers, multithread ureq implementation --- Cargo.lock | 1 + Cargo.toml | 1 + packages/blitz-net/Cargo.toml | 5 +- packages/blitz-net/src/backend/mod.rs | 15 +-- packages/blitz-net/src/backend/reqwest.rs | 56 ++++++++--- packages/blitz-net/src/backend/ureq.rs | 113 ++++++++++++++++++++-- packages/blitz-net/src/callback/mod.rs | 2 - packages/blitz-net/src/lib.rs | 49 +--------- packages/blitz-traits/src/net.rs | 13 +++ 9 files changed, 170 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c66d6aad1..47d4e5e50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -706,6 +706,7 @@ dependencies = [ "blitz-traits", "data-url", "http", + "rayon", "reqwest", "thiserror 1.0.69", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 86f7a1c64..b119305e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ data-url = "0.3.1" tokio = "1.42" reqwest = "0.12" ureq = "3.0" +rayon = "1.10" # Media & Decoding image = { version = "0.25", default-features = false } diff --git a/packages/blitz-net/Cargo.toml b/packages/blitz-net/Cargo.toml index 0258b75fe..6fe347e13 100644 --- a/packages/blitz-net/Cargo.toml +++ b/packages/blitz-net/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [features] default = ["reqwest"] reqwest = ["dep:reqwest", "dep:tokio"] -ureq = ["dep:ureq"] +ureq = ["dep:ureq", "dep:rayon"] [dependencies] blitz-traits = { path = "../blitz-traits" } @@ -17,4 +17,5 @@ ureq = { workspace = true, optional = true } data-url = { workspace = true } thiserror = { workspace = true } http = { workspace = true } -url = { workspace = true } \ No newline at end of file +url = { workspace = true } +rayon = { workspace = true, optional = true } \ No newline at end of file diff --git a/packages/blitz-net/src/backend/mod.rs b/packages/blitz-net/src/backend/mod.rs index 7cef0e797..f035acdc9 100644 --- a/packages/blitz-net/src/backend/mod.rs +++ b/packages/blitz-net/src/backend/mod.rs @@ -1,6 +1,5 @@ use std::fmt::{Display, Formatter}; -use blitz_traits::net::{Request, Response}; use thiserror::Error; #[cfg(feature = "reqwest")] @@ -9,19 +8,9 @@ mod reqwest; mod ureq; #[cfg(feature = "reqwest")] -pub use reqwest::Backend; -#[cfg(feature = "reqwest")] -pub use reqwest::Provider; - +pub use reqwest::{get_text, Provider}; #[cfg(feature = "ureq")] -pub use ureq::Backend; - -pub trait RequestBackend { - fn new() -> Self - where - Self: Sized; - async fn request(&mut self, request: Request) -> Result; -} +pub use ureq::{get_text, Provider}; #[derive(Debug, Error)] pub struct BackendError { diff --git a/packages/blitz-net/src/backend/reqwest.rs b/packages/blitz-net/src/backend/reqwest.rs index 05a588c5d..c75285bc4 100644 --- a/packages/blitz-net/src/backend/reqwest.rs +++ b/packages/blitz-net/src/backend/reqwest.rs @@ -2,14 +2,12 @@ use std::sync::Arc; use crate::{ProviderError, USER_AGENT}; -use super::{BackendError, RequestBackend, Response}; -use blitz_traits::net::{BoxedHandler, NetProvider, Request, SharedCallback}; +use super::BackendError; +use blitz_traits::net::{BoxedHandler, Bytes, NetProvider, Request, Response, SharedCallback}; use data_url::DataUrl; use http::HeaderValue; -use tokio::{ - runtime::Handle, - sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, -}; +use tokio::runtime::Handle; +use url::Url; // Compat with reqwest impl From for BackendError { @@ -25,17 +23,14 @@ pub struct Backend { client: reqwest::Client, } -impl RequestBackend for Backend { - fn new() -> Self - where - Self: Sized, - { +impl Backend { + pub fn new() -> Self { Self { client: reqwest::Client::new(), } } - async fn request(&mut self, request: Request) -> Result { + pub async fn request(&mut self, request: Request) -> Result { let request = self .client .request(request.method, request.url.clone()) @@ -54,6 +49,13 @@ impl RequestBackend for Backend { } } +pub async fn get_text(url: &str) -> String { + let mut backend = Backend::new(); + let request = Request::get(Url::parse(url).unwrap()); + let response = backend.request(request).await.unwrap(); + String::from_utf8_lossy(&response.body).to_string() +} + pub struct Provider { rt: Handle, client: Backend, @@ -76,6 +78,36 @@ impl Provider { pub fn is_empty(&self) -> bool { Arc::strong_count(&self.resource_callback) == 1 } + + async fn fetch_inner( + mut client: Backend, + doc_id: usize, + request: Request, + handler: BoxedHandler, + res_callback: SharedCallback, + ) -> Result<(), ProviderError> { + match request.url.scheme() { + "data" => { + let data_url = DataUrl::process(request.url.as_str())?; + let decoded = data_url.decode_to_vec()?; + handler.bytes(doc_id, Bytes::from(decoded.0), res_callback); + } + "file" => { + let file_content = std::fs::read(request.url.path())?; + handler.bytes(doc_id, Bytes::from(file_content), res_callback); + } + _ => { + let mut request = Request::get(request.url); + request + .headers + .insert("User-Agent", HeaderValue::from_static(USER_AGENT)); + let response = client.request(request).await?; + + handler.bytes(doc_id, response.body, res_callback); + } + } + Ok(()) + } } impl NetProvider for Provider { diff --git a/packages/blitz-net/src/backend/ureq.rs b/packages/blitz-net/src/backend/ureq.rs index 9cfdfc3a0..3585b23d8 100644 --- a/packages/blitz-net/src/backend/ureq.rs +++ b/packages/blitz-net/src/backend/ureq.rs @@ -1,6 +1,13 @@ -use blitz_traits::net::{Request, Response}; +use blitz_traits::net::{BoxedHandler, Bytes, NetProvider, Request, Response, SharedCallback}; +use data_url::DataUrl; +use http::HeaderValue; +use rayon::{ThreadPool, ThreadPoolBuilder}; +use std::sync::Arc; +use url::Url; -use super::{BackendError, RequestBackend}; +use crate::{ProviderError, USER_AGENT}; + +use super::BackendError; impl From for BackendError { fn from(e: ureq::Error) -> Self { @@ -15,19 +22,26 @@ pub struct Backend { client: ureq::Agent, } -impl RequestBackend for Backend { - fn new() -> Self - where - Self: Sized, - { +impl Backend { + pub fn new() -> Self { Self { client: ureq::agent(), } } - async fn request(&mut self, request: Request) -> Result { - let mut response = self.client.run(request.into())?; + pub fn request(&mut self, mut request: Request) -> Result { + request + .headers + .insert("User-Agent", HeaderValue::from_static(&USER_AGENT)); + let mut response = if request.body.is_empty() { + self.client + .run(>>::into(request))? + } else { + self.client.run(>, + >>::into(request))? + }; let status = response.status().as_u16(); Ok(Response { @@ -37,3 +51,84 @@ impl RequestBackend for Backend { }) } } + +pub fn get_text(url: &str) -> String { + let mut backend = Backend::new(); + let request = Request::get(Url::parse(url).unwrap()); + let response = backend.request(request).unwrap(); + String::from_utf8_lossy(&response.body).to_string() +} + +pub struct Provider { + thread_pool: ThreadPool, + client: Backend, + resource_callback: SharedCallback, +} + +impl Provider { + pub fn new(res_callback: SharedCallback) -> Self { + let thread_pool = ThreadPoolBuilder::new().num_threads(0).build().unwrap(); + + Self { + thread_pool, + client: Backend::new(), + resource_callback: res_callback, + } + } + + pub fn shared(res_callback: SharedCallback) -> Arc> { + Arc::new(Self::new(res_callback)) + } + + pub fn is_empty(&self) -> bool { + Arc::strong_count(&self.resource_callback) == 1 + } + + fn fetch_inner( + mut client: Backend, + doc_id: usize, + request: Request, + handler: BoxedHandler, + callback: SharedCallback, + ) -> Result<(), ProviderError> { + match request.url.scheme() { + "data" => { + let data_url = DataUrl::process(request.url.as_str())?; + let decoded = data_url.decode_to_vec()?; + handler.bytes(doc_id, Bytes::from(decoded.0), callback); + } + "file" => { + let file_content = std::fs::read(request.url.path())?; + handler.bytes(doc_id, Bytes::from(file_content), callback); + } + _ => { + let mut request = Request::get(request.url); + request + .headers + .insert("User-Agent", HeaderValue::from_static(USER_AGENT)); + let response = client.request(request)?; + + handler.bytes(doc_id, response.body, callback); + } + } + Ok(()) + } +} + +impl NetProvider for Provider { + type Data = D; + + fn fetch(&self, doc_id: usize, request: Request, handler: BoxedHandler) { + let client = self.client.clone(); + let callback = Arc::clone(&self.resource_callback); + self.thread_pool.spawn(move || { + let url = request.url.to_string(); + let res = Self::fetch_inner(client, doc_id, request, handler, callback); + if let Err(e) = res { + eprintln!("Error fetching {}: {e}", url); + } else { + println!("Success {}", url); + } + }); + } +} diff --git a/packages/blitz-net/src/callback/mod.rs b/packages/blitz-net/src/callback/mod.rs index 581e4da0f..69b1bce3c 100644 --- a/packages/blitz-net/src/callback/mod.rs +++ b/packages/blitz-net/src/callback/mod.rs @@ -5,5 +5,3 @@ pub mod ureq; #[cfg(feature = "reqwest")] pub use reqwest::*; -#[cfg(feature = "ureq")] -pub use ureq::*; diff --git a/packages/blitz-net/src/lib.rs b/packages/blitz-net/src/lib.rs index 577520426..2d5b76278 100644 --- a/packages/blitz-net/src/lib.rs +++ b/packages/blitz-net/src/lib.rs @@ -1,59 +1,14 @@ -use backend::{Backend, RequestBackend}; -use blitz_traits::net::{BoxedHandler, Bytes, Request, SharedCallback}; -use data_url::DataUrl; -use http::HeaderValue; use thiserror::Error; -use url::Url; - #[cfg(all(feature = "reqwest", feature = "ureq"))] -compile_error!("multiple request backends cannot be enabled at the same time"); +compile_error!("multiple request backends cannot be enabled at the same time. either use reqwest or ureq, but not both"); mod backend; pub mod callback; const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"; -pub async fn get_text(url: &str) -> String { - let mut backend = Backend::new(); - let request = Request::get(Url::parse(url).unwrap()); - let response = backend.request(request).await.unwrap(); - String::from_utf8_lossy(&response.body).to_string() -} - -pub use backend::Provider; - -impl Provider { - async fn fetch_inner( - mut client: Backend, - doc_id: usize, - request: Request, - handler: BoxedHandler, - res_callback: SharedCallback, - ) -> Result<(), ProviderError> { - match request.url.scheme() { - "data" => { - let data_url = DataUrl::process(request.url.as_str())?; - let decoded = data_url.decode_to_vec()?; - handler.bytes(doc_id, Bytes::from(decoded.0), res_callback); - } - "file" => { - let file_content = std::fs::read(request.url.path())?; - handler.bytes(doc_id, Bytes::from(file_content), res_callback); - } - _ => { - let mut request = Request::get(request.url); - request - .headers - .insert("User-Agent", HeaderValue::from_static(USER_AGENT)); - let response = client.request(request).await?; - - handler.bytes(doc_id, response.body, res_callback); - } - } - Ok(()) - } -} +pub use backend::{get_text, Provider}; #[derive(Error, Debug)] enum ProviderError { diff --git a/packages/blitz-traits/src/net.rs b/packages/blitz-traits/src/net.rs index e585c9645..d3b1cacb7 100644 --- a/packages/blitz-traits/src/net.rs +++ b/packages/blitz-traits/src/net.rs @@ -1,6 +1,8 @@ pub use bytes::Bytes; +use http::Uri; pub use http::{self, HeaderMap, Method}; use std::marker::PhantomData; +use std::str::FromStr; use std::sync::Arc; pub use url::Url; @@ -51,10 +53,21 @@ impl Request { } } +impl Into> for Request { + fn into(self) -> http::Request<()> { + let mut request = http::Request::new(()); + request.headers_mut().extend(self.headers); + *request.uri_mut() = Uri::from_str(&self.url.to_string()).unwrap(); + *request.method_mut() = self.method; + request + } +} + impl Into>> for Request { fn into(self) -> http::Request> { let mut request = http::Request::new(self.body.into()); request.headers_mut().extend(self.headers); + *request.uri_mut() = Uri::from_str(&self.url.to_string()).unwrap(); *request.method_mut() = self.method; request } From 1a4d5c268773d1d8bb308f49fac027427fc12115 Mon Sep 17 00:00:00 2001 From: SpikeHD <25207995+SpikeHD@users.noreply.github.com> Date: Wed, 5 Mar 2025 07:42:13 -0800 Subject: [PATCH 5/7] fix: impl From<> --- packages/blitz-traits/src/net.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/blitz-traits/src/net.rs b/packages/blitz-traits/src/net.rs index d3b1cacb7..e496c488b 100644 --- a/packages/blitz-traits/src/net.rs +++ b/packages/blitz-traits/src/net.rs @@ -53,22 +53,22 @@ impl Request { } } -impl Into> for Request { - fn into(self) -> http::Request<()> { +impl From for http::Request<()> { + fn from(req: Request) -> http::Request<()> { let mut request = http::Request::new(()); - request.headers_mut().extend(self.headers); - *request.uri_mut() = Uri::from_str(&self.url.to_string()).unwrap(); - *request.method_mut() = self.method; + request.headers_mut().extend(req.headers); + *request.uri_mut() = Uri::from_str(req.url.as_ref()).unwrap(); + *request.method_mut() = req.method; request } } -impl Into>> for Request { - fn into(self) -> http::Request> { - let mut request = http::Request::new(self.body.into()); - request.headers_mut().extend(self.headers); - *request.uri_mut() = Uri::from_str(&self.url.to_string()).unwrap(); - *request.method_mut() = self.method; +impl From for http::Request> { + fn from(req: Request) -> http::Request> { + let mut request = http::Request::new(req.body.into()); + request.headers_mut().extend(req.headers); + *request.uri_mut() = Uri::from_str(req.url.as_ref()).unwrap(); + *request.method_mut() = req.method; request } } From d11fe9e90600a4abc42013e2b9f1d4d4b0928106 Mon Sep 17 00:00:00 2001 From: SpikeHD <25207995+SpikeHD@users.noreply.github.com> Date: Wed, 5 Mar 2025 07:52:03 -0800 Subject: [PATCH 6/7] feat: clippy, mpsc channel for ureq --- packages/blitz-net/src/callback/ureq.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/blitz-net/src/callback/ureq.rs b/packages/blitz-net/src/callback/ureq.rs index 8b1378917..253399b52 100644 --- a/packages/blitz-net/src/callback/ureq.rs +++ b/packages/blitz-net/src/callback/ureq.rs @@ -1 +1,16 @@ +use blitz_traits::net::NetCallback; +use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; +pub struct MpscCallback(SyncSender<(usize, T)>); +impl MpscCallback { + pub fn new() -> (Receiver<(usize, T)>, Self) { + let (send, recv) = sync_channel(0); + (recv, Self(send)) + } +} +impl NetCallback for MpscCallback { + type Data = T; + fn call(&self, doc_id: usize, data: Self::Data) { + let _ = self.0.send((doc_id, data)); + } +} From 2d47bb32572ed44db866e76e00b87427844da1de Mon Sep 17 00:00:00 2001 From: SpikeHD <25207995+SpikeHD@users.noreply.github.com> Date: Wed, 5 Mar 2025 07:58:28 -0800 Subject: [PATCH 7/7] fix: remove log --- packages/blitz-net/src/backend/reqwest.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/blitz-net/src/backend/reqwest.rs b/packages/blitz-net/src/backend/reqwest.rs index c75285bc4..3ba82396e 100644 --- a/packages/blitz-net/src/backend/reqwest.rs +++ b/packages/blitz-net/src/backend/reqwest.rs @@ -116,7 +116,6 @@ impl NetProvider for Provider { fn fetch(&self, doc_id: usize, request: Request, handler: BoxedHandler) { let client = self.client.clone(); let callback = Arc::clone(&self.resource_callback); - println!("Fetching {}", &request.url); drop(self.rt.spawn(async move { let url = request.url.to_string(); let res = Self::fetch_inner(client, doc_id, request, handler, callback).await;