diff --git a/Cargo.lock b/Cargo.lock index c6dd21128..93d0948fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -705,9 +705,13 @@ version = "0.1.0" dependencies = [ "blitz-traits", "data-url", + "http", + "rayon", "reqwest", "thiserror 1.0.69", "tokio", + "ureq", + "url", ] [[package]] @@ -4804,7 +4808,9 @@ version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -6188,6 +6194,36 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06f78313c985f2fba11100dd06d60dd402d0cabb458af4d94791b8e09c025323" +dependencies = [ + "base64", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "ureq-proto", + "utf-8", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64adb55464bad1ab1aa9229133d0d59d2f679180f4d15f0d9debe616f541f25e" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.4" @@ -6612,6 +6648,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "weezl" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index 2d5f96f39..2aa7e8b23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,8 @@ http = "1.1.0" data-url = "0.3.1" tokio = "1.42" reqwest = "0.12" +ureq = "3.0" +rayon = "1.10" # Media & Decoding image = { version = "0.25", default-features = false } diff --git a/examples/screenshot.rs b/examples/screenshot.rs index 020cc3c3c..91684c148 100644 --- a/examples/screenshot.rs +++ b/examples/screenshot.rs @@ -2,7 +2,7 @@ use blitz_dom::net::Resource; use blitz_html::HtmlDocument; -use blitz_net::{MpscCallback, Provider}; +use blitz_net::{callback::MpscCallback, Provider}; use blitz_renderer_vello::render_to_buffer; use blitz_traits::navigation::DummyNavigationProvider; use blitz_traits::net::SharedProvider; @@ -23,8 +23,7 @@ async fn main() { let mut timer = Timer::init(); let url_string = std::env::args() - .skip(1) - .next() + .nth(1) .unwrap_or_else(|| "https://www.google.com".into()); println!("{}", url_string); @@ -56,8 +55,7 @@ async fn main() { let scale = 2; let height = 800; let width: u32 = std::env::args() - .skip(2) - .next() + .nth(2) .and_then(|arg| arg.parse().ok()) .unwrap_or(1200); @@ -151,7 +149,7 @@ fn write_png(writer: W, buffer: &[u8], width: u32, height: u32) { // Write PNG data to writer let mut writer = encoder.write_header().unwrap(); - writer.write_image_data(&buffer).unwrap(); + writer.write_image_data(buffer).unwrap(); writer.finish().unwrap(); } diff --git a/examples/url.rs b/examples/url.rs index 99a2acffc..606abf95b 100644 --- a/examples/url.rs +++ b/examples/url.rs @@ -2,8 +2,7 @@ fn main() { let url = std::env::args() - .skip(1) - .next() + .nth(1) .unwrap_or_else(|| "https://www.google.com".into()); blitz::launch_url(&url); } diff --git a/packages/blitz-net/Cargo.toml b/packages/blitz-net/Cargo.toml index ffc07a9c2..e19d449d4 100644 --- a/packages/blitz-net/Cargo.toml +++ b/packages/blitz-net/Cargo.toml @@ -4,9 +4,18 @@ version = "0.1.0" license.workspace = true edition = "2024" +[features] +default = ["reqwest"] +reqwest = ["dep:reqwest", "dep:tokio"] +ureq = ["dep:ureq", "dep:rayon"] + [dependencies] blitz-traits = { path = "../blitz-traits" } -tokio = { workspace = true } -reqwest = { workspace = true } +tokio = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } +ureq = { workspace = true, optional = true } data-url = { workspace = true } thiserror = { workspace = true } +http = { workspace = true } +url = { workspace = true } +rayon = { workspace = true, optional = true } \ No newline at end of file diff --git a/packages/blitz-net/src/backend/mod.rs b/packages/blitz-net/src/backend/mod.rs new file mode 100644 index 000000000..f035acdc9 --- /dev/null +++ b/packages/blitz-net/src/backend/mod.rs @@ -0,0 +1,24 @@ +use std::fmt::{Display, Formatter}; + +use thiserror::Error; + +#[cfg(feature = "reqwest")] +mod reqwest; +#[cfg(feature = "ureq")] +mod ureq; + +#[cfg(feature = "reqwest")] +pub use reqwest::{get_text, Provider}; +#[cfg(feature = "ureq")] +pub use ureq::{get_text, Provider}; + +#[derive(Debug, Error)] +pub struct BackendError { + pub message: String, +} + +impl Display for BackendError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} diff --git a/packages/blitz-net/src/backend/reqwest.rs b/packages/blitz-net/src/backend/reqwest.rs new file mode 100644 index 000000000..3ba82396e --- /dev/null +++ b/packages/blitz-net/src/backend/reqwest.rs @@ -0,0 +1,129 @@ +use std::sync::Arc; + +use crate::{ProviderError, USER_AGENT}; + +use super::BackendError; +use blitz_traits::net::{BoxedHandler, Bytes, NetProvider, Request, Response, SharedCallback}; +use data_url::DataUrl; +use http::HeaderValue; +use tokio::runtime::Handle; +use url::Url; + +// Compat with reqwest +impl From for BackendError { + fn from(e: reqwest::Error) -> Self { + BackendError { + message: e.to_string(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Backend { + client: reqwest::Client, +} + +impl Backend { + pub fn new() -> Self { + Self { + client: reqwest::Client::new(), + } + } + + pub async fn request(&mut self, request: Request) -> Result { + let request = self + .client + .request(request.method, request.url.clone()) + .headers(request.headers); + + let response = request.send().await?; + let status = response.status(); + let headers = response.headers().clone(); + let body = response.bytes().await?; + + Ok(Response { + status: status.as_u16(), + headers, + body, + }) + } +} + +pub async fn get_text(url: &str) -> String { + let mut backend = Backend::new(); + let request = Request::get(Url::parse(url).unwrap()); + let response = backend.request(request).await.unwrap(); + String::from_utf8_lossy(&response.body).to_string() +} + +pub struct Provider { + rt: Handle, + client: Backend, + resource_callback: SharedCallback, +} + +impl Provider { + pub fn new(res_callback: SharedCallback) -> Self { + Self { + rt: Handle::current(), + client: Backend::new(), + resource_callback: res_callback, + } + } + + pub fn shared(res_callback: SharedCallback) -> Arc> { + Arc::new(Self::new(res_callback)) + } + + pub fn is_empty(&self) -> bool { + Arc::strong_count(&self.resource_callback) == 1 + } + + async fn fetch_inner( + mut client: Backend, + doc_id: usize, + request: Request, + handler: BoxedHandler, + res_callback: SharedCallback, + ) -> Result<(), ProviderError> { + match request.url.scheme() { + "data" => { + let data_url = DataUrl::process(request.url.as_str())?; + let decoded = data_url.decode_to_vec()?; + handler.bytes(doc_id, Bytes::from(decoded.0), res_callback); + } + "file" => { + let file_content = std::fs::read(request.url.path())?; + handler.bytes(doc_id, Bytes::from(file_content), res_callback); + } + _ => { + let mut request = Request::get(request.url); + request + .headers + .insert("User-Agent", HeaderValue::from_static(USER_AGENT)); + let response = client.request(request).await?; + + handler.bytes(doc_id, response.body, res_callback); + } + } + Ok(()) + } +} + +impl NetProvider for Provider { + type Data = D; + + fn fetch(&self, doc_id: usize, request: Request, handler: BoxedHandler) { + let client = self.client.clone(); + let callback = Arc::clone(&self.resource_callback); + drop(self.rt.spawn(async move { + let url = request.url.to_string(); + let res = Self::fetch_inner(client, doc_id, request, handler, callback).await; + if let Err(e) = res { + eprintln!("Error fetching {}: {e}", url); + } else { + println!("Success {}", url); + } + })); + } +} diff --git a/packages/blitz-net/src/backend/ureq.rs b/packages/blitz-net/src/backend/ureq.rs new file mode 100644 index 000000000..3585b23d8 --- /dev/null +++ b/packages/blitz-net/src/backend/ureq.rs @@ -0,0 +1,134 @@ +use blitz_traits::net::{BoxedHandler, Bytes, NetProvider, Request, Response, SharedCallback}; +use data_url::DataUrl; +use http::HeaderValue; +use rayon::{ThreadPool, ThreadPoolBuilder}; +use std::sync::Arc; +use url::Url; + +use crate::{ProviderError, USER_AGENT}; + +use super::BackendError; + +impl From for BackendError { + fn from(e: ureq::Error) -> Self { + BackendError { + message: e.to_string(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Backend { + client: ureq::Agent, +} + +impl Backend { + pub fn new() -> Self { + Self { + client: ureq::agent(), + } + } + + pub fn request(&mut self, mut request: Request) -> Result { + request + .headers + .insert("User-Agent", HeaderValue::from_static(&USER_AGENT)); + + let mut response = if request.body.is_empty() { + self.client + .run(>>::into(request))? + } else { + self.client.run(>, + >>::into(request))? + }; + let status = response.status().as_u16(); + + Ok(Response { + status, + headers: response.headers().clone(), + body: response.body_mut().read_to_vec()?.into(), + }) + } +} + +pub fn get_text(url: &str) -> String { + let mut backend = Backend::new(); + let request = Request::get(Url::parse(url).unwrap()); + let response = backend.request(request).unwrap(); + String::from_utf8_lossy(&response.body).to_string() +} + +pub struct Provider { + thread_pool: ThreadPool, + client: Backend, + resource_callback: SharedCallback, +} + +impl Provider { + pub fn new(res_callback: SharedCallback) -> Self { + let thread_pool = ThreadPoolBuilder::new().num_threads(0).build().unwrap(); + + Self { + thread_pool, + client: Backend::new(), + resource_callback: res_callback, + } + } + + pub fn shared(res_callback: SharedCallback) -> Arc> { + Arc::new(Self::new(res_callback)) + } + + pub fn is_empty(&self) -> bool { + Arc::strong_count(&self.resource_callback) == 1 + } + + fn fetch_inner( + mut client: Backend, + doc_id: usize, + request: Request, + handler: BoxedHandler, + callback: SharedCallback, + ) -> Result<(), ProviderError> { + match request.url.scheme() { + "data" => { + let data_url = DataUrl::process(request.url.as_str())?; + let decoded = data_url.decode_to_vec()?; + handler.bytes(doc_id, Bytes::from(decoded.0), callback); + } + "file" => { + let file_content = std::fs::read(request.url.path())?; + handler.bytes(doc_id, Bytes::from(file_content), callback); + } + _ => { + let mut request = Request::get(request.url); + request + .headers + .insert("User-Agent", HeaderValue::from_static(USER_AGENT)); + let response = client.request(request)?; + + handler.bytes(doc_id, response.body, callback); + } + } + Ok(()) + } +} + +impl NetProvider for Provider { + type Data = D; + + fn fetch(&self, doc_id: usize, request: Request, handler: BoxedHandler) { + let client = self.client.clone(); + let callback = Arc::clone(&self.resource_callback); + self.thread_pool.spawn(move || { + let url = request.url.to_string(); + let res = Self::fetch_inner(client, doc_id, request, handler, callback); + if let Err(e) = res { + eprintln!("Error fetching {}: {e}", url); + } else { + println!("Success {}", url); + } + }); + } +} diff --git a/packages/blitz-net/src/callback/mod.rs b/packages/blitz-net/src/callback/mod.rs new file mode 100644 index 000000000..69b1bce3c --- /dev/null +++ b/packages/blitz-net/src/callback/mod.rs @@ -0,0 +1,7 @@ +#[cfg(feature = "reqwest")] +pub mod reqwest; +#[cfg(feature = "ureq")] +pub mod ureq; + +#[cfg(feature = "reqwest")] +pub use reqwest::*; diff --git a/packages/blitz-net/src/callback/reqwest.rs b/packages/blitz-net/src/callback/reqwest.rs new file mode 100644 index 000000000..8cbda6943 --- /dev/null +++ b/packages/blitz-net/src/callback/reqwest.rs @@ -0,0 +1,16 @@ +use blitz_traits::net::NetCallback; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; + +pub struct MpscCallback(UnboundedSender<(usize, T)>); +impl MpscCallback { + pub fn new() -> (UnboundedReceiver<(usize, T)>, Self) { + let (send, recv) = unbounded_channel(); + (recv, Self(send)) + } +} +impl NetCallback for MpscCallback { + type Data = T; + fn call(&self, doc_id: usize, data: Self::Data) { + let _ = self.0.send((doc_id, data)); + } +} diff --git a/packages/blitz-net/src/callback/ureq.rs b/packages/blitz-net/src/callback/ureq.rs new file mode 100644 index 000000000..253399b52 --- /dev/null +++ b/packages/blitz-net/src/callback/ureq.rs @@ -0,0 +1,16 @@ +use blitz_traits::net::NetCallback; +use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; + +pub struct MpscCallback(SyncSender<(usize, T)>); +impl MpscCallback { + pub fn new() -> (Receiver<(usize, T)>, Self) { + let (send, recv) = sync_channel(0); + (recv, Self(send)) + } +} +impl NetCallback for MpscCallback { + type Data = T; + fn call(&self, doc_id: usize, data: Self::Data) { + let _ = self.0.send((doc_id, data)); + } +} diff --git a/packages/blitz-net/src/lib.rs b/packages/blitz-net/src/lib.rs index 2157e22b8..2d5b76278 100644 --- a/packages/blitz-net/src/lib.rs +++ b/packages/blitz-net/src/lib.rs @@ -1,98 +1,14 @@ -use blitz_traits::net::{BoxedHandler, Bytes, NetCallback, NetProvider, Request, SharedCallback}; -use data_url::DataUrl; -use reqwest::Client; -use std::sync::Arc; use thiserror::Error; -use tokio::{ - runtime::Handle, - sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}, -}; -const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"; - -pub async fn get_text(url: &str) -> String { - Client::new() - .get(url) - .header("User-Agent", USER_AGENT) - .send() - .await - .unwrap() - .text() - .await - .unwrap() -} +#[cfg(all(feature = "reqwest", feature = "ureq"))] +compile_error!("multiple request backends cannot be enabled at the same time. either use reqwest or ureq, but not both"); -pub struct Provider { - rt: Handle, - client: Client, - resource_callback: SharedCallback, -} -impl Provider { - pub fn new(res_callback: SharedCallback) -> Self { - Self { - rt: Handle::current(), - client: Client::new(), - resource_callback: res_callback, - } - } - pub fn shared(res_callback: SharedCallback) -> Arc> { - Arc::new(Self::new(res_callback)) - } - pub fn is_empty(&self) -> bool { - Arc::strong_count(&self.resource_callback) == 1 - } -} -impl Provider { - async fn fetch_inner( - client: Client, - doc_id: usize, - request: Request, - handler: BoxedHandler, - res_callback: SharedCallback, - ) -> Result<(), ProviderError> { - match request.url.scheme() { - "data" => { - let data_url = DataUrl::process(request.url.as_str())?; - let decoded = data_url.decode_to_vec()?; - handler.bytes(doc_id, Bytes::from(decoded.0), res_callback); - } - "file" => { - let file_content = std::fs::read(request.url.path())?; - handler.bytes(doc_id, Bytes::from(file_content), res_callback); - } - _ => { - let response = client - .request(request.method, request.url) - .headers(request.headers) - .header("User-Agent", USER_AGENT) - .body(request.body) - .send() - .await?; +mod backend; +pub mod callback; - handler.bytes(doc_id, response.bytes().await?, res_callback); - } - } - Ok(()) - } -} +const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"; -impl NetProvider for Provider { - type Data = D; - fn fetch(&self, doc_id: usize, request: Request, handler: BoxedHandler) { - let client = self.client.clone(); - let callback = Arc::clone(&self.resource_callback); - println!("Fetching {}", &request.url); - self.rt.spawn(async move { - let url = request.url.to_string(); - let res = Self::fetch_inner(client, doc_id, request, handler, callback).await; - if let Err(e) = res { - eprintln!("Error fetching {}: {e}", url); - } else { - println!("Success {}", url); - } - }); - } -} +pub use backend::{get_text, Provider}; #[derive(Error, Debug)] enum ProviderError { @@ -103,19 +19,5 @@ enum ProviderError { #[error("{0}")] DataUrlBas64(#[from] data_url::forgiving_base64::InvalidBase64), #[error("{0}")] - ReqwestError(#[from] reqwest::Error), -} - -pub struct MpscCallback(UnboundedSender<(usize, T)>); -impl MpscCallback { - pub fn new() -> (UnboundedReceiver<(usize, T)>, Self) { - let (send, recv) = unbounded_channel(); - (recv, Self(send)) - } -} -impl NetCallback for MpscCallback { - type Data = T; - fn call(&self, doc_id: usize, data: Self::Data) { - let _ = self.0.send((doc_id, data)); - } + BackendError(#[from] backend::BackendError), } diff --git a/packages/blitz-traits/src/net.rs b/packages/blitz-traits/src/net.rs index a6bf13fc2..e496c488b 100644 --- a/packages/blitz-traits/src/net.rs +++ b/packages/blitz-traits/src/net.rs @@ -1,6 +1,8 @@ pub use bytes::Bytes; +use http::Uri; pub use http::{self, HeaderMap, Method}; use std::marker::PhantomData; +use std::str::FromStr; use std::sync::Arc; pub use url::Url; @@ -51,6 +53,34 @@ impl Request { } } +impl From for http::Request<()> { + fn from(req: Request) -> http::Request<()> { + let mut request = http::Request::new(()); + request.headers_mut().extend(req.headers); + *request.uri_mut() = Uri::from_str(req.url.as_ref()).unwrap(); + *request.method_mut() = req.method; + request + } +} + +impl From for http::Request> { + fn from(req: Request) -> http::Request> { + let mut request = http::Request::new(req.body.into()); + request.headers_mut().extend(req.headers); + *request.uri_mut() = Uri::from_str(req.url.as_ref()).unwrap(); + *request.method_mut() = req.method; + request + } +} + +#[derive(Debug)] +/// An HTTP response +pub struct Response { + pub status: u16, + pub headers: HeaderMap, + pub body: Bytes, +} + /// A default noop NetProvider pub struct DummyNetProvider(PhantomData); impl Default for DummyNetProvider {