Remove Capnp as a Required Dependency

.github/workflows/ci.yaml

@@ -18,42 +18,43 @@ env:
 jobs:
   test_dev:
     runs-on: ubuntu-latest
+    # see capnp-runner.dockerfile
+    container: ghcr.io/jonoprest/capnp-runner:latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: Swatinem/rust-cache@v2
-    - name: Install Dependencies
-      run: |
-          export DEBIAN_FRONTEND=noninteractive
-          sudo apt-get install -y capnproto libcapnp-dev
-    - name: Build
-      run: cargo build
-    - name: Test
-      run: cargo test
+      - uses: actions/checkout@v3
+      # selecting a toolchain either by action or manual `rustup` calls should happen
+      # before the plugin, as the cache uses the current rustc version as its cache key
+      - run: rustup toolchain install stable
+      - uses: Swatinem/rust-cache@v2
+      - name: Build
+        run: cargo build
+      - name: Test
+        run: cargo test
+      - name: Check capnp schema should match the generated code
+        working-directory: ./hypersync-net-types
+        run: |
+          make capnp-build
+          git diff --exit-code
 
   test_release:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: Swatinem/rust-cache@v2
-    - name: Install Dependencies
-      run: |
-          export DEBIAN_FRONTEND=noninteractive
-          sudo apt-get install -y capnproto libcapnp-dev
-    - name: Build
-      run: cargo build --release
-    - name: Test
-      run: cargo test --release
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+      - name: Build
+        run: cargo build --release
+      - name: Test
+        run: cargo test --release
 
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: Swatinem/rust-cache@v2
-    - name: Install Dependencies
-      run: |
-          export DEBIAN_FRONTEND=noninteractive
-          sudo apt-get install -y capnproto libcapnp-dev
-    - name: Rustfmt
-      run: cargo fmt --check
-    - name: Clippy
-      run: cargo clippy -- -Dwarnings
+      - uses: actions/checkout@v3
+      # selecting a toolchain either by action or manual `rustup` calls should happen
+      # before the plugin, as the cache uses the current rustc version as its cache key
+      - run: rustup toolchain install stable
+      - uses: Swatinem/rust-cache@v2
+      - name: Rustfmt
+        run: cargo fmt --check
+      - name: Clippy
+        run: cargo clippy -- -Dwarnings

README.md

@@ -2,7 +2,7 @@
 
 [![CI](https://github.com/enviodev/hypersync-client-rust/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/enviodev/hypersync-client-rust/actions/workflows/ci.yaml)
 <a href="https://crates.io/crates/hypersync-client">
-    <img src="https://img.shields.io/crates/v/hypersync-client.svg?style=flat-square"
+<img src="https://img.shields.io/crates/v/hypersync-client.svg?style=flat-square"
     alt="Crates.io version" />
 </a>
 
@@ -12,7 +12,7 @@ Rust crate for [Envio's](https://envio.dev/) HyperSync client.
 
 ### Dependencies
 
-Need to install capnproto tool in order to build the library.
+In order to update the capnp schema on hypersync-net-types, you will need to install the `capnproto` tool.
 
 #### Linux
 
@@ -31,3 +31,14 @@ choco install capnproto
 ```bash
 brew install capnp
 ```
+
+## Building
+
+To build the schema, run the following command:
+
+```bash
+cd hypersync-net-types
+make capnp-build
+```
+
+This triggers the build script in hypersync-net-types, which will generate the `hypersync_net_types_capnp.rs` file in the `src` directory.

capnp-runner.dockerfile

@@ -0,0 +1,33 @@
+# This Dockerfile is used for ci to have the latest version of Cap'n Proto (1.1.0) 
+# built from source. Since ubuntu package manager is not up to date.
+
+# Start with a base image
+FROM ubuntu:22.04
+
+# Set non-interactive mode to prevent prompts during installation
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install dependencies
+RUN apt update && apt install -y \
+    build-essential \
+    cmake \
+    curl \
+    git \
+    sudo \
+    && rm -rf /var/lib/apt/lists/*
+
+# Download and install Cap'n Proto 1.1.0
+WORKDIR /opt
+RUN curl -O https://capnproto.org/capnproto-c++-1.1.0.tar.gz && \
+    tar zxf capnproto-c++-1.1.0.tar.gz && \
+    cd capnproto-c++-1.1.0 && \
+    ./configure && \
+    make -j$(nproc) check && \
+    sudo make install && \
+    cd .. && rm -rf capnproto-c++-1.1.0 capnproto-c++-1.1.0.tar.gz
+
+# Set environment variables
+ENV PATH="/usr/local/bin:$PATH"
+
+# Keep the container running
+CMD ["tail", "-f", "/dev/null"]

hypersync-client/src/stream.rs

@@ -169,7 +169,7 @@ pub async fn stream_arrow(
                         let x = x as u32;
 
                         let batch_size = cmp::max((x as f64 * ratio) as u64, min_batch_size);
-                        let step = batch_size | u64::from(next_generation) << 32;
+                        let step = batch_size | (u64::from(next_generation) << 32);
                         Some(step)
                     })
                     .ok();
@@ -180,7 +180,7 @@ pub async fn stream_arrow(
                         let x = x as u32;
 
                         let batch_size = cmp::min((x as f64 * ratio) as u64, max_batch_size);
-                        let step = batch_size | u64::from(next_generation) << 32;
+                        let step = batch_size | (u64::from(next_generation) << 32);
                         Some(step)
                     })
                     .ok();

hypersync-format/src/types/bloom_filter_wrapper.rs

@@ -72,7 +72,7 @@ impl<'de> Deserialize<'de> for FilterWrapper {
     {
         struct FilterWrapperVisitor;
 
-        impl<'de> Visitor<'de> for FilterWrapperVisitor {
+        impl Visitor<'_> for FilterWrapperVisitor {
             type Value = FilterWrapper;
 
             fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {

hypersync-format/src/types/data.rs

@@ -38,7 +38,7 @@ impl<const N: usize> From<[u8; N]> for Data {
 
 struct DataVisitor;
 
-impl<'de> Visitor<'de> for DataVisitor {
+impl Visitor<'_> for DataVisitor {
     type Value = Data;
 
     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {

hypersync-format/src/types/fixed_size_data.rs

@@ -135,7 +135,7 @@ impl<const N: usize> fmt::Display for FixedSizeData<N> {
 
 struct FixedSizeDataVisitor<const N: usize>;
 
-impl<'de, const N: usize> Visitor<'de> for FixedSizeDataVisitor<N> {
+impl<const N: usize> Visitor<'_> for FixedSizeDataVisitor<N> {
     type Value = FixedSizeData<N>;
 
     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
@@ -191,6 +191,8 @@ impl<const N: usize> fmt::Debug for FixedSizeData<N> {
 #[cfg(test)]
 mod tests {
     type FixedSizeData = super::FixedSizeData<4>;
+    type FSD4 = FixedSizeData;
+    use derive_more::FromStr;
     use hex_literal::hex;
     use serde_test::{assert_tokens, Token};
 

hypersync-format/src/types/quantity.rs

@@ -91,7 +91,7 @@ impl<const N: usize> From<[u8; N]> for Quantity {
 
 struct QuantityVisitor;
 
-impl<'de> Visitor<'de> for QuantityVisitor {
+impl Visitor<'_> for QuantityVisitor {
     type Value = Quantity;
 
     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {

hypersync-format/src/types/transaction_status.rs

@@ -53,7 +53,7 @@ impl TransactionStatus {
 
 struct TransactionStatusVisitor;
 
-impl<'de> Visitor<'de> for TransactionStatusVisitor {
+impl Visitor<'_> for TransactionStatusVisitor {
     type Value = TransactionStatus;
 
     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {

hypersync-format/src/types/transaction_type.rs

@@ -25,7 +25,7 @@ pub struct TransactionType(pub u8);
 
 struct TransactionTypeVisitor;
 
-impl<'de> Visitor<'de> for TransactionTypeVisitor {
+impl Visitor<'_> for TransactionTypeVisitor {
     type Value = TransactionType;
 
     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {

hypersync-format/src/types/uint.rs

@@ -59,7 +59,7 @@ impl From<UInt> for ethabi::ethereum_types::U256 {
 
 struct UIntVisitor;
 
-impl<'de> Visitor<'de> for UIntVisitor {
+impl Visitor<'_> for UIntVisitor {
     type Value = UInt;
 
     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {

hypersync-format/src/types/util.rs

@@ -1,5 +1,5 @@
 pub fn decode_hex(hex: &str) -> Result<Vec<u8>, faster_hex::Error> {
-    let len = hex.as_bytes().len();
+    let len = hex.len();
     let mut dst = vec![0; len / 2];
 
     faster_hex::hex_decode(hex.as_bytes(), &mut dst)?;

hypersync-net-types/Cargo.toml

@@ -11,6 +11,3 @@ serde = { version = "1", features = ["derive"] }
 arrayvec = { version = "0.7", features = ["serde"] }
 
 hypersync-format = { path = "../hypersync-format", version = "0.4" }
-
-[build-dependencies]
-capnpc = "0.19"

hypersync-net-types/README.md

@@ -0,0 +1,19 @@
+# hypersync-net-types
+
+This crate contains the capnp schema for the hypersync network protocol.
+
+## Building
+
+To build the schema, run the following command:
+
+```bash
+make capnp-build
+```
+
+This will generate the `hypersync_net_types_capnp.rs` file in the `src` directory.
+
+The build script will not run on every build, so you will need to run it manually if you make changes to the schema.
+
+The ci tests will fail if the schema is not up to date.
+
+This is to avoid the need to have a capnp compiler installed on the consumers of this crate.

hypersync-net-types/makefile

@@ -0,0 +1,8 @@
+.PHONY: capnp-build
+
+capnp-build:
+	# Ensure we are building with the same capnp (1.1.0) in all environments
+	capnp --version | grep -q "1.1.0"
+	cargo install capnpc
+	capnpc -orust:src hypersync_net_types.capnp
+	rustfmt -- src/hypersync_net_types_capnp.rs