add a partial commit implementation

supports:
- small files
- git lfs files
- git lfs multipart files
- model repos

does not yet support:
- dataset repos, spaces
- file copies
- file deletions
- folder uploads
- much of anything else

Author: arilotter

Cargo.toml

@@ -36,6 +36,13 @@ ureq = { version = "2.8.0", optional = true, features = [
   "json",
   "socks-proxy",
 ] }
+sha2 = "0.10"
+base64 = "0.22.1"
+hex = "0.4.3"
+regex = "1.11.1"
+lazy_static = "1.5.0"
+sha1 = "0.10.6"
+urlencoding = "2.1.3"
 
 [features]
 default = ["default-tls", "tokio", "ureq"]
@@ -68,6 +75,8 @@ ureq = [
 ]
 
 [dev-dependencies]
+env_logger = "0.11.5"
 hex-literal = "0.4.1"
+rand = "0.8.5"
 sha2 = "0.10"
 tokio-test = "0.4.2"

README.md

@@ -1,3 +1,9 @@
+## temporary todo
+
+to run the test, do `RUST_LOG=trace HF_TOKEN=token_here HF_REPO=repo_here cargo run --example upload --release`
+
+## real readme here
+
 This crates aims to emulate and be compatible with the
 [huggingface_hub](https://github.com/huggingface/huggingface_hub/) python package.
 
@@ -18,14 +24,14 @@ However allowing new features or creating new features might be denied by lack o
 time. We're focusing on what we currently internally need. Hopefully that subset is already interesting
 to more users.
 
-
-# How to use 
+# How to use
 
 Add the dependency
 
 ```bash
 cargo add hf-hub  # --features tokio
 ```
+
 `tokio` feature will enable an async (and potentially faster) API.
 
 Use the crate:

examples/upload.rs

@@ -0,0 +1,56 @@
+use std::time::Instant;
+
+use hf_hub::{api::tokio::ApiBuilder, Repo};
+use rand::Rng;
+
+const ONE_MB: usize = 1024 * 1024;
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    env_logger::init();
+    let token =
+        std::env::var("HF_TOKEN").map_err(|_| format!("HF_TOKEN environment variable not set"))?;
+    let hf_repo = std::env::var("HF_REPO")
+        .map_err(|_| format!("HF_REPO environment variable not set, e.g. apyh/gronk"))?;
+
+    let api = ApiBuilder::new().with_token(Some(token)).build()?;
+    let repo = Repo::model(hf_repo);
+    let api_repo = api.repo(repo);
+    for (filepath, contents) in [
+        (
+            "tiny_file.txt",
+            format!("im a tiny file {:?}", Instant::now())
+                .as_bytes()
+                .to_vec(),
+        ),
+        ("1m_file.txt", {
+            let mut data = vec![0u8; ONE_MB];
+            rand::thread_rng().fill(&mut data[..]);
+            data
+        }),
+        ("10m_file.txt", {
+            let mut data = vec![0u8; 10 * ONE_MB];
+            rand::thread_rng().fill(&mut data[..]);
+            data
+        }),
+        ("20m_file.txt", {
+            let mut data = vec![0u8; 20 * ONE_MB];
+            rand::thread_rng().fill(&mut data[..]);
+            data
+        }),
+    ] {
+        let res = api_repo
+            .upload_file(
+                contents,
+                filepath,
+                None,
+                format!("update {}", filepath).into(),
+                false,
+            )
+            .await?;
+        log::info!("Uploaded file {:?}", filepath);
+        log::info!("{:?}", res);
+        log::info!("Success!!");
+    }
+    Ok(())
+}