From 2b2a7b4b0ea6527d9c6653dc990329178e5e38e4 Mon Sep 17 00:00:00 2001 From: ferris Date: Fri, 13 Mar 2026 13:51:26 +0100 Subject: [PATCH 1/5] Add minimal record reading support --- resources/example_xy_little_endian_record.npy | Bin 0 -> 144 bytes src/npy/elements/mod.rs | 2 + src/npy/elements/record.rs | 29 ++++++++++++ src/npy/mod.rs | 1 + tests/integration/examples.rs | 42 ++++++++++++++++++ 5 files changed, 74 insertions(+) create mode 100644 resources/example_xy_little_endian_record.npy create mode 100644 src/npy/elements/record.rs diff --git a/resources/example_xy_little_endian_record.npy b/resources/example_xy_little_endian_record.npy new file mode 100644 index 0000000000000000000000000000000000000000..0219a42c4ace5fdc1b3d60783f748d1bf2066e7f GIT binary patch literal 144 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zlw^dp3 literal 0 HcmV?d00001 diff --git a/src/npy/elements/mod.rs b/src/npy/elements/mod.rs index 4e4186d..53ed016 100644 --- a/src/npy/elements/mod.rs +++ b/src/npy/elements/mod.rs @@ -200,3 +200,5 @@ mod bool; #[cfg(feature = "num-complex-0_4")] mod complex; mod num; +mod record; +pub use record::RecordFromSlice; diff --git a/src/npy/elements/record.rs b/src/npy/elements/record.rs new file mode 100644 index 0000000..044319d --- /dev/null +++ b/src/npy/elements/record.rs @@ -0,0 +1,29 @@ +use crate::{ReadDataError, ReadableElement}; +use py_literal::Value as PyValue; + +pub trait RecordFromSlice: Sized { + const SIZE: usize; + + // TODO(perf): validate type_descr once? + + fn from_raw_slice( + type_descr: &PyValue, + reader: &mut R, + ) -> Result; +} + +impl ReadableElement for T { + fn read_to_end_exact_vec( + mut reader: R, + type_desc: &PyValue, + len: usize, + ) -> Result, ReadDataError> { + let mut out = Vec::new(); // NOTE(perf): in theory could be MaybeUinitVec? + + for _ in 0..len { + out.push(RecordFromSlice::from_raw_slice(type_desc, &mut reader)?); + } + + Ok(out) + } +} diff --git a/src/npy/mod.rs b/src/npy/mod.rs index b14a8a3..e6b0297 100644 --- a/src/npy/mod.rs +++ b/src/npy/mod.rs @@ -4,6 +4,7 @@ mod elements; pub mod header; +pub use elements::RecordFromSlice; use self::header::{ FormatHeaderError, Header, Layout, ParseHeaderError, ReadHeaderError, WriteHeaderError, diff --git a/tests/integration/examples.rs b/tests/integration/examples.rs index 6c02fb7..5beb6d0 100644 --- a/tests/integration/examples.rs +++ b/tests/integration/examples.rs @@ -440,3 +440,45 @@ fn zeroed() { assert_eq!(arr, Array3::::zeros(SHAPE)); assert!(arr.is_standard_layout()); } + +#[cfg(target_endian = "little")] +#[test] +fn record_reading() { + use ndarray_npy::npy::RecordFromSlice; + + #[derive(PartialEq, Debug)] + struct Record { + x: i32, + y: i32, + } + + impl RecordFromSlice for Record { + const SIZE: usize = 8; + + fn from_raw_slice( + type_descr: &py_literal::Value, + reader: &mut R, + ) -> Result { + let mut buf = [0u8; 4]; + + reader + .read_exact(&mut buf) + .map_err(ndarray_npy::ReadDataError::Io)?; + let x = i32::from_le_bytes(buf); + reader + .read_exact(&mut buf) + .map_err(ndarray_npy::ReadDataError::Io)?; + let y = i32::from_le_bytes(buf); + Ok(Record { x, y }) + } + } + + // np.save("example_xy_little_endian_record", np.rec.array([(42, 42), (35, 35)], dtype=[('x', np.int32), ('y', np.int32)])) + let record_array: Array1 = + ndarray_npy::read_npy("resources/example_xy_little_endian_record.npy") + .expect("Failed to load npy"); + + assert_eq!(record_array[0], Record { x: 42, y: 42 }); + assert_eq!(record_array[1], Record { x: 35, y: 35 }); + assert!(record_array.len() == 2); +} From 1b114673f0439ff6954ea40a2e7b68d67674e73a Mon Sep 17 00:00:00 2001 From: ferris Date: Fri, 13 Mar 2026 14:30:31 +0100 Subject: [PATCH 2/5] Validation & perf --- src/npy/elements/record.rs | 16 +++++++--------- tests/integration/examples.rs | 7 ++++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/npy/elements/record.rs b/src/npy/elements/record.rs index 044319d..4ce9b12 100644 --- a/src/npy/elements/record.rs +++ b/src/npy/elements/record.rs @@ -2,14 +2,9 @@ use crate::{ReadDataError, ReadableElement}; use py_literal::Value as PyValue; pub trait RecordFromSlice: Sized { - const SIZE: usize; + fn compatible_schema(type_descr: &PyValue) -> bool; - // TODO(perf): validate type_descr once? - - fn from_raw_slice( - type_descr: &PyValue, - reader: &mut R, - ) -> Result; + fn from_raw_slice(reader: &mut R) -> Result; } impl ReadableElement for T { @@ -18,10 +13,13 @@ impl ReadableElement for T { type_desc: &PyValue, len: usize, ) -> Result, ReadDataError> { - let mut out = Vec::new(); // NOTE(perf): in theory could be MaybeUinitVec? + if !T::compatible_schema(type_desc) { + return Err(ReadDataError::WrongDescriptor(type_desc.clone())); + } + let mut out = Vec::with_capacity(len); for _ in 0..len { - out.push(RecordFromSlice::from_raw_slice(type_desc, &mut reader)?); + out.push(RecordFromSlice::from_raw_slice(&mut reader)?); } Ok(out) diff --git a/tests/integration/examples.rs b/tests/integration/examples.rs index 5beb6d0..8063f8f 100644 --- a/tests/integration/examples.rs +++ b/tests/integration/examples.rs @@ -453,10 +453,7 @@ fn record_reading() { } impl RecordFromSlice for Record { - const SIZE: usize = 8; - fn from_raw_slice( - type_descr: &py_literal::Value, reader: &mut R, ) -> Result { let mut buf = [0u8; 4]; @@ -471,6 +468,10 @@ fn record_reading() { let y = i32::from_le_bytes(buf); Ok(Record { x, y }) } + + fn compatible_schema(type_descr: &py_literal::Value) -> bool { + true + } } // np.save("example_xy_little_endian_record", np.rec.array([(42, 42), (35, 35)], dtype=[('x', np.int32), ('y', np.int32)])) From 2e3645808cf5f9902685e4d9ed37ad0db492a2cc Mon Sep 17 00:00:00 2001 From: ferris Date: Fri, 13 Mar 2026 14:46:16 +0100 Subject: [PATCH 3/5] Add partial validation example & 2D array test --- .../example_xy_little_endian_record_2d.npy | Bin 0 -> 144 bytes tests/integration/examples.rs | 22 ++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 resources/example_xy_little_endian_record_2d.npy diff --git a/resources/example_xy_little_endian_record_2d.npy b/resources/example_xy_little_endian_record_2d.npy new file mode 100644 index 0000000000000000000000000000000000000000..29ba73987a009a6830ff3fc4877a96a9c267452e GIT binary patch literal 144 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zlw^dpp3;;uZ9vJ`t literal 0 HcmV?d00001 diff --git a/tests/integration/examples.rs b/tests/integration/examples.rs index 8063f8f..802a4b0 100644 --- a/tests/integration/examples.rs +++ b/tests/integration/examples.rs @@ -469,8 +469,16 @@ fn record_reading() { Ok(Record { x, y }) } + // NOTE: partial validation fn compatible_schema(type_descr: &py_literal::Value) -> bool { - true + match type_descr { + py_literal::Value::List(values) => matches!( + &values[..], + // 2 Values per record + [py_literal::Value::Tuple(..), py_literal::Value::Tuple(..)] + ), + _ => false, + } } } @@ -481,5 +489,15 @@ fn record_reading() { assert_eq!(record_array[0], Record { x: 42, y: 42 }); assert_eq!(record_array[1], Record { x: 35, y: 35 }); - assert!(record_array.len() == 2); + assert_eq!(record_array.len(), 2); + + // np.save("example_xy_little_endian_record_2d.npy", np.rec.array([[(42, 42), (35, 35)]], dtype=[('x', np.int32), ('y', np.int32)])) + let record_array: Array2 = + ndarray_npy::read_npy("resources/example_xy_little_endian_record_2d.npy") + .expect("Failed to load npy"); + + assert_eq!(record_array[[0, 0]], Record { x: 42, y: 42 }); + assert_eq!(record_array[[0, 1]], Record { x: 35, y: 35 }); + assert_eq!(record_array.len_of(Axis(0)), 1); + assert_eq!(record_array.len_of(Axis(1)), 2); } From 9ca2157d01074488cdce7fdc0954172b78774781 Mon Sep 17 00:00:00 2001 From: ferris Date: Mon, 16 Mar 2026 10:52:05 +0100 Subject: [PATCH 4/5] Expose pyliteral --- src/npy/elements/record.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/npy/elements/record.rs b/src/npy/elements/record.rs index 4ce9b12..c053424 100644 --- a/src/npy/elements/record.rs +++ b/src/npy/elements/record.rs @@ -1,8 +1,11 @@ +//! Currently only little-endian is supported + use crate::{ReadDataError, ReadableElement}; -use py_literal::Value as PyValue; +pub use ndarray_derive::RecordFromSlice; +pub use py_literal; pub trait RecordFromSlice: Sized { - fn compatible_schema(type_descr: &PyValue) -> bool; + fn compatible_schema(type_descr: &py_literal::Value) -> bool; fn from_raw_slice(reader: &mut R) -> Result; } @@ -10,7 +13,7 @@ pub trait RecordFromSlice: Sized { impl ReadableElement for T { fn read_to_end_exact_vec( mut reader: R, - type_desc: &PyValue, + type_desc: &py_literal::Value, len: usize, ) -> Result, ReadDataError> { if !T::compatible_schema(type_desc) { From ba08687e9aff271201de7e20ed153461e2517652 Mon Sep 17 00:00:00 2001 From: ferris Date: Mon, 16 Mar 2026 10:57:16 +0100 Subject: [PATCH 5/5] Re-expose py_literal --- src/lib.rs | 1 + src/npy/elements/record.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d3ca5bb..a005a97 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,3 +61,4 @@ pub use crate::npy::{ }; #[cfg(feature = "npz")] pub use crate::npz::{NpzReader, NpzWriter, ReadNpzError, WriteNpzError}; +pub use py_literal; diff --git a/src/npy/elements/record.rs b/src/npy/elements/record.rs index c053424..0890912 100644 --- a/src/npy/elements/record.rs +++ b/src/npy/elements/record.rs @@ -1,8 +1,8 @@ //! Currently only little-endian is supported use crate::{ReadDataError, ReadableElement}; -pub use ndarray_derive::RecordFromSlice; -pub use py_literal; +// pub use ndarray_derive::RecordFromSlice; +use py_literal; pub trait RecordFromSlice: Sized { fn compatible_schema(type_descr: &py_literal::Value) -> bool;