Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/scripts/rustfmt.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail

readonly RUSTFMT_TOOLCHAIN="nightly-2025-08-07"

exec cargo +"${RUSTFMT_TOOLCHAIN}" fmt "$@"
3 changes: 2 additions & 1 deletion .github/workflows/ci-dotnet.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ jobs:
dotnet tool install -g dotnet-format
echo "$HOME/.dotnet/tools" >> $GITHUB_PATH

- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # nightly
with:
toolchain: nightly-2025-08-07
components: rustfmt

- name: Format all files
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/ci-go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,9 @@ jobs:
with:
go-version: '1.25'

- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # nightly
with:
toolchain: nightly-2025-08-07
components: rustfmt

- name: Format all files
Expand Down Expand Up @@ -222,4 +223,4 @@ jobs:

- name: Test Proto batch example builds
working-directory: go/examples/proto/batch
run: go build -v
run: go build -v
5 changes: 3 additions & 2 deletions .github/workflows/ci-python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,9 @@ jobs:
with:
python-version: '3.12'

- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # nightly
with:
toolchain: nightly-2025-08-07
components: rustfmt

- name: Copy root LICENSE for maturin
Expand Down Expand Up @@ -140,4 +141,4 @@ jobs:
run: cp ../LICENSE LICENSE

- name: Lint code
run: make dev lint
run: make dev lint
5 changes: 3 additions & 2 deletions .github/workflows/ci-rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ jobs:
- name: Configure Cargo registry
shell: bash
run: bash "$GITHUB_WORKSPACE/.github/scripts/configure-cargo.sh"
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # nightly
with:
toolchain: nightly-2025-08-07
components: rustfmt
- name: Format all files
run: make fmt
Expand Down Expand Up @@ -80,4 +81,4 @@ jobs:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- name: Run tests
run: cargo test --workspace
run: cargo test --workspace
13 changes: 13 additions & 0 deletions .rustfmt.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# cargo fmt reads each crate's edition from Cargo.toml; this fallback keeps
# direct rustfmt usable for the repository's Rust 2021 crates.
edition = "2021"
style_edition = "2021"
newline_style = "Unix"
group_imports = "StdExternalCrate"
imports_layout = "HorizontalVertical"
imports_granularity = "Module"

max_width = 100
fn_call_width = 100
attr_fn_like_width = 100
array_width = 100
2 changes: 1 addition & 1 deletion dotnet/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ fmt-dotnet:

fmt-rust:
@echo "Formatting Rust code..."
cd ../rust/ffi && cargo fmt --all
cd ../rust/ffi && ../../.github/scripts/rustfmt.sh --all

lint: lint-dotnet lint-rust

Expand Down
2 changes: 1 addition & 1 deletion go/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ fmt-go:

fmt-rust:
@echo "Formatting Rust code..."
cd ../rust/ffi && cargo fmt --all
cd ../rust/ffi && ../../.github/scripts/rustfmt.sh --all

lint: lint-go lint-rust

Expand Down
2 changes: 1 addition & 1 deletion python/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ fmt:
$(VENV) -m black zerobus examples tests
$(VENV) -m autoflake -ri --exclude '*_pb2*.py' zerobus examples tests
$(VENV) -m isort zerobus examples tests
cd rust && cargo fmt --all
cd rust && ../../.github/scripts/rustfmt.sh --all

lint:
$(VENV) -m pycodestyle --exclude='*_pb2*.py' --max-line-length=120 --ignore=E203,W503 zerobus
Expand Down
16 changes: 7 additions & 9 deletions python/rust/src/arrow.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,15 @@

use std::sync::Arc;

use pyo3::prelude::*;
use pyo3::types::PyBytes;
use tokio::sync::RwLock;

use databricks_zerobus_ingest_sdk::{
StreamBuilder, ZerobusArrowStream as RustZerobusArrowStream, ZerobusError as RustError,
StreamBuilder,
ZerobusArrowStream as RustZerobusArrowStream,
ZerobusError as RustError,
ZerobusSdk as RustSdk,
};
use pyo3::prelude::*;
use pyo3::types::PyBytes;
use tokio::sync::RwLock;

use crate::auth::HeadersProviderWrapper;
use crate::common::map_error;
Expand Down Expand Up @@ -249,10 +250,7 @@ impl ArrowStreamConfigurationOptions {
("recovery_timeout_ms", self.recovery_timeout_ms),
("recovery_backoff_ms", self.recovery_backoff_ms),
("recovery_retries", self.recovery_retries as i64),
(
"server_lack_of_ack_timeout_ms",
self.server_lack_of_ack_timeout_ms,
),
("server_lack_of_ack_timeout_ms", self.server_lack_of_ack_timeout_ms),
("flush_timeout_ms", self.flush_timeout_ms),
("connection_timeout_ms", self.connection_timeout_ms),
] {
Expand Down
19 changes: 13 additions & 6 deletions python/rust/src/async_wrapper.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,27 @@ use std::future::Future;
use std::pin::Pin;
use std::sync::{Arc, Mutex};

use databricks_zerobus_ingest_sdk::{
StreamBuilder,
ZerobusSdk as RustSdk,
ZerobusStream as RustStream,
};
use pyo3::prelude::*;
use pyo3_asyncio::tokio::future_into_py;
use tokio::sync::RwLock;

use databricks_zerobus_ingest_sdk::{
StreamBuilder, ZerobusSdk as RustSdk, ZerobusStream as RustStream,
};

use crate::arrow;
use crate::arrow::{ArrowStreamConfigurationOptions, AsyncZerobusArrowStream};
use crate::auth::HeadersProviderWrapper;
use crate::common::{
apply_grpc_options, encoded_record_to_pybytes, extract_record_payload, extract_record_payloads,
map_error, StreamConfigurationOptions, TableProperties, SDK_IDENTIFIER_PREFIX,
apply_grpc_options,
encoded_record_to_pybytes,
extract_record_payload,
extract_record_payloads,
map_error,
StreamConfigurationOptions,
TableProperties,
SDK_IDENTIFIER_PREFIX,
};

// =============================================================================
Expand Down
14 changes: 7 additions & 7 deletions python/rust/src/auth.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
use async_trait::async_trait;
use pyo3::exceptions::PyNotImplementedError;
use pyo3::prelude::*;
use std::collections::HashMap;

use async_trait::async_trait;
use databricks_zerobus_ingest_sdk::{
HeadersProvider as RustHeadersProvider, ZerobusError as RustError, ZerobusResult as RustResult,
HeadersProvider as RustHeadersProvider,
ZerobusError as RustError,
ZerobusResult as RustResult,
};
use pyo3::exceptions::PyNotImplementedError;
use pyo3::prelude::*;

use crate::common::intern_header_name;

Expand Down Expand Up @@ -41,9 +43,7 @@ impl HeadersProvider {
/// Returns:
/// List of (header_name, header_value) tuples
fn get_headers(&self, _py: Python) -> PyResult<PyObject> {
Err(PyNotImplementedError::new_err(
"Subclasses must implement get_headers()",
))
Err(PyNotImplementedError::new_err("Subclasses must implement get_headers()"))
}
}

Expand Down
34 changes: 12 additions & 22 deletions python/rust/src/common.rs
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex, OnceLock};

use databricks_zerobus_ingest_sdk::{
AckCallback as RustAckCallback,
EncodedRecord,
OffsetId,
StreamBuilder,
};
use prost::Message;
use pyo3::exceptions::{PyException, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyDict, PyList, PyTuple};

use databricks_zerobus_ingest_sdk::{
AckCallback as RustAckCallback, EncodedRecord, OffsetId, StreamBuilder,
};

/// User-agent prefix emitted by this wrapper SDK. Combined with the wrapper
/// crate version via `env!("CARGO_PKG_VERSION")` at the call site.
pub(crate) const SDK_IDENTIFIER_PREFIX: &str = "zerobus-sdk-py";
Expand Down Expand Up @@ -273,34 +275,24 @@ impl StreamConfigurationOptions {
/// Validate that all numeric fields are non-negative before casting to unsigned types.
pub fn validate(&self) -> PyResult<()> {
if self.max_inflight_records < 0 {
return Err(PyValueError::new_err(
"max_inflight_records must be non-negative",
));
return Err(PyValueError::new_err("max_inflight_records must be non-negative"));
}
if self.recovery_timeout_ms < 0 {
return Err(PyValueError::new_err(
"recovery_timeout_ms must be non-negative",
));
return Err(PyValueError::new_err("recovery_timeout_ms must be non-negative"));
}
if self.recovery_backoff_ms < 0 {
return Err(PyValueError::new_err(
"recovery_backoff_ms must be non-negative",
));
return Err(PyValueError::new_err("recovery_backoff_ms must be non-negative"));
}
if self.recovery_retries < 0 {
return Err(PyValueError::new_err(
"recovery_retries must be non-negative",
));
return Err(PyValueError::new_err("recovery_retries must be non-negative"));
}
if self.server_lack_of_ack_timeout_ms < 0 {
return Err(PyValueError::new_err(
"server_lack_of_ack_timeout_ms must be non-negative",
));
}
if self.flush_timeout_ms < 0 {
return Err(PyValueError::new_err(
"flush_timeout_ms must be non-negative",
));
return Err(PyValueError::new_err("flush_timeout_ms must be non-negative"));
}
if let Some(v) = self.stream_paused_max_wait_time_ms {
if v < 0 {
Expand Down Expand Up @@ -454,9 +446,7 @@ pub(crate) fn extract_record_payloads(payloads: &PyAny) -> PyResult<Vec<EncodedR
out.push(EncodedRecord::Json(json));
}
} else {
return Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
"Payloads must be a list",
));
return Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>("Payloads must be a list"));
}

Ok(out)
Expand Down
10 changes: 2 additions & 8 deletions python/rust/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,8 @@ fn _zerobus_core(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<common::AckCallback>()?;

// Add exception types
m.add(
"ZerobusException",
py.get_type::<common::ZerobusException>(),
)?;
m.add(
"NonRetriableException",
py.get_type::<common::NonRetriableException>(),
)?;
m.add("ZerobusException", py.get_type::<common::ZerobusException>())?;
m.add("NonRetriableException", py.get_type::<common::NonRetriableException>())?;

// Add authentication classes
m.add_class::<auth::HeadersProvider>()?;
Expand Down
19 changes: 13 additions & 6 deletions python/rust/src/sync_wrapper.rs
Original file line number Diff line number Diff line change
@@ -1,20 +1,27 @@
use std::sync::Arc;

use databricks_zerobus_ingest_sdk::{
StreamBuilder,
ZerobusSdk as RustSdk,
ZerobusStream as RustStream,
};
use pyo3::prelude::*;
use pyo3::types::PyBytes;
use tokio::runtime::Runtime;
use tokio::sync::RwLock;

use databricks_zerobus_ingest_sdk::{
StreamBuilder, ZerobusSdk as RustSdk, ZerobusStream as RustStream,
};

use crate::arrow;
use crate::arrow::{ArrowStreamConfigurationOptions, ZerobusArrowStream};
use crate::auth::HeadersProviderWrapper;
use crate::common::{
apply_grpc_options, encoded_record_to_pybytes, extract_record_payload, extract_record_payloads,
map_error, StreamConfigurationOptions, TableProperties, SDK_IDENTIFIER_PREFIX,
apply_grpc_options,
encoded_record_to_pybytes,
extract_record_payload,
extract_record_payloads,
map_error,
StreamConfigurationOptions,
TableProperties,
SDK_IDENTIFIER_PREFIX,
};

// =============================================================================
Expand Down
3 changes: 2 additions & 1 deletion rust/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ This document covers Rust-specific development setup and workflow.

- Git
- Rust 1.70+ (stable toolchain)
- Pinned formatter: `rustup toolchain install nightly-2025-08-07 --profile minimal --component rustfmt`
- Protocol Buffers compiler (`protoc`)

### Setting Up Your Development Environment
Expand Down Expand Up @@ -37,7 +38,7 @@ Code style is enforced by a formatter check in your pull request. We use `rustfm
make fmt
```

This runs `cargo fmt --all` to format all crates in the workspace.
This runs `cargo fmt --all` with the same pinned rustfmt used by CI. Compilation, linting, and tests continue to use stable Rust.

### Running Linters

Expand Down
2 changes: 1 addition & 1 deletion rust/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ clean:
cargo clean

fmt:
cargo fmt --all
../.github/scripts/rustfmt.sh --all

lint:
cargo clippy --all -- -D warnings
Expand Down
12 changes: 6 additions & 6 deletions rust/examples/arrow/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ use std::error::Error;
use std::sync::Arc;

use arrow_array::{
Float64Array, Int32Array, LargeStringArray, RecordBatch, TimestampMicrosecondArray,
Float64Array,
Int32Array,
LargeStringArray,
RecordBatch,
TimestampMicrosecondArray,
};
use arrow_ipc::CompressionType;
use databricks_zerobus_ingest_sdk::{ArrowSchema, DataType, Field, TimeUnit, ZerobusSdk};
Expand Down Expand Up @@ -128,11 +132,7 @@ async fn main() -> Result<(), Box<dyn Error>> {

if (i + 1) % WAIT_EVERY == 0 {
stream.wait_for_offset(offset_id).await?;
println!(
"Acknowledged through batch {} (offset ID {})",
i + 1,
offset_id
);
println!("Acknowledged through batch {} (offset ID {})", i + 1, offset_id);
}
}

Expand Down
Loading
Loading