diff --git a/.github/workflows/prgate.yaml b/.github/workflows/prgate.yaml index 9b24ede..404c381 100644 --- a/.github/workflows/prgate.yaml +++ b/.github/workflows/prgate.yaml @@ -59,6 +59,58 @@ jobs: working-directory: ${{github.workspace}}/build run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + linux-examples-images: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Git config for fetching pull requests + run: | + git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/* + + - name: CMake config + working-directory: ${{github.workspace}}/examples/images + run: cmake -B ${{github.workspace}}/examples/images/build + env: + LIBNPY_REPO: https://github.com/${{github.repository}} + LIBNPY_TAG: ${{github.sha}} + + - name: CMake build + working-directory: ${{github.workspace}}/examples/images/build + run: make + + - name: CMake test + working-directory: ${{github.workspace}}/examples/images/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + + linux-examples-custom_tensors: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Git config for fetching pull requests + run: | + git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/* + + - name: CMake config + working-directory: ${{github.workspace}}/examples/custom_tensors + run: cmake -B ${{github.workspace}}/examples/custom_tensors/build + env: + LIBNPY_REPO: https://github.com/${{github.repository}} + LIBNPY_TAG: ${{github.sha}} + + - name: CMake build + working-directory: ${{github.workspace}}/examples/custom_tensors/build + run: make + + - name: CMake test + working-directory: ${{github.workspace}}/examples/custom_tensors/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test -L npy + linux-asan: runs-on: ubuntu-latest @@ -122,6 +174,58 @@ jobs: working-directory: ${{github.workspace}}/build run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + windows-examples-images: + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Git config for fetching pull requests + run: | + git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/* + + - name: CMake config + working-directory: ${{github.workspace}}/examples/images + run: cmake -B ${{github.workspace}}/examples/images/build + env: + LIBNPY_REPO: https://github.com/${{github.repository}} + LIBNPY_TAG: ${{github.sha}} + + - name: CMake build + working-directory: ${{github.workspace}}/examples/images/build + run: cmake --build . --config Release + + - name: CMake test + working-directory: ${{github.workspace}}/examples/images/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + + windows-examples-custom_tensors: + runs-on: windows-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Git config for fetching pull requests + run: | + git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/* + + - name: CMake config + working-directory: ${{github.workspace}}/examples/custom_tensors + run: cmake -B ${{github.workspace}}/examples/custom_tensors/build + env: + LIBNPY_REPO: https://github.com/${{github.repository}} + LIBNPY_TAG: ${{github.sha}} + + - name: CMake build + working-directory: ${{github.workspace}}/examples/custom_tensors/build + run: cmake --build . --config Release + + - name: CMake test + working-directory: ${{github.workspace}}/examples/custom_tensors/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test -L npy + macos: runs-on: macos-latest @@ -154,3 +258,140 @@ jobs: - name: CMake test working-directory: ${{github.workspace}}/build run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + + macos-examples-images: + runs-on: macos-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Git config for fetching pull requests + run: | + git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/* + + - name: CMake config + working-directory: ${{github.workspace}}/examples/images + run: cmake -B ${{github.workspace}}/examples/images/build + env: + LIBNPY_REPO: https://github.com/${{github.repository}} + LIBNPY_TAG: ${{github.sha}} + + - name: CMake build + working-directory: ${{github.workspace}}/examples/images/build + run: make + + - name: CMake test + working-directory: ${{github.workspace}}/examples/images/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + + macos-examples-custom_tensors: + runs-on: macos-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Git config for fetching pull requests + run: | + git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/* + + - name: CMake config + working-directory: ${{github.workspace}}/examples/custom_tensors + run: cmake -B ${{github.workspace}}/examples/custom_tensors/build + env: + LIBNPY_REPO: https://github.com/${{github.repository}} + LIBNPY_TAG: ${{github.sha}} + + - name: CMake build + working-directory: ${{github.workspace}}/examples/custom_tensors/build + run: make + + - name: CMake test + working-directory: ${{github.workspace}}/examples/custom_tensors/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test -L npy + + macos-x86_64: + runs-on: macos-15-intel + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get dependencies + run: | + brew update && brew install ninja + + - name: CMake config + run: cmake -B ${{github.workspace}}/build --preset release-clang + + - name: CMake build + working-directory: ${{github.workspace}}/build + run: ninja + + - name: Use Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Generate large NPZ files + working-directory: ${{github.workspace}}/ + run: | + pip install numpy + python ${{github.workspace}}/test/generate_large_test.py + + - name: CMake test + working-directory: ${{github.workspace}}/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + + macos-x86_64-examples-images: + runs-on: macos-15-intel + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Git config for fetching pull requests + run: | + git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/* + + - name: CMake config + working-directory: ${{github.workspace}}/examples/images + run: cmake -B ${{github.workspace}}/examples/images/build + env: + LIBNPY_REPO: https://github.com/${{github.repository}} + LIBNPY_TAG: ${{github.sha}} + + - name: CMake build + working-directory: ${{github.workspace}}/examples/images/build + run: make + + - name: CMake test + working-directory: ${{github.workspace}}/examples/images/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test + + macos-x86_64-examples-custom_tensors: + runs-on: macos-15-intel + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Git config for fetching pull requests + run: | + git config --global --add remote.origin.fetch +refs/pull/*/merge:refs/remotes/pull/* + + - name: CMake config + working-directory: ${{github.workspace}}/examples/custom_tensors + run: cmake -B ${{github.workspace}}/examples/custom_tensors/build + env: + LIBNPY_REPO: https://github.com/${{github.repository}} + LIBNPY_TAG: ${{github.sha}} + + - name: CMake build + working-directory: ${{github.workspace}}/examples/custom_tensors/build + run: make + + - name: CMake test + working-directory: ${{github.workspace}}/examples/custom_tensors/build + run: ctest -V --build-config Release --timeout 120 --output-on-failure -T Test -L npy \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 91b6bac..b78c6db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [2026-01-14 - Version 2.0.0](https://github.com/matajoh/libnpy/releases/tag/v2.0.0) + +Major version with breaking changes. + +**Breaking Changes**: +- `onpzstream` has been split into `npzfilewriter` and `npzstringwriter` +- `inpzstream` has been split into `npzfilereader` and `npzstringreader` +- The library interface has been merged into a single `npy.h` header + +Improvements: +- Support for complex numbers has been added +- The library design has been greatly simplified +- The example projects have been broken out into self-contained projects (which fetch + the main project using `FetchContent`) and a `custom_tensors` example has been added + showing a non-trivial custom tensor implementation +- Custom tensor support is greatly improved, and is now fully featured + +Bugfixes: +- Fixed an issue with broken CRC checks on Windows + ## [2024-11-01 - Version 1.5.3](https://github.com/matajoh/libnpy/releases/tag/v1.5.3) Improvements: diff --git a/CMakeLists.txt b/CMakeLists.txt index 41ce88d..b7fd3aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required( VERSION 3.13...3.16 FATAL_ERROR ) +cmake_minimum_required( VERSION 3.15 ) # -------------------- Version -------------------------------- @@ -16,14 +16,12 @@ set( LIBNPY_VERSION ${LIBNPY_VERSION_MAJOR}.${LIBNPY_VERSION_MINOR}.${LIBNPY_VER message("Configure LIBNPY_VERSION at ${LIBNPY_VERSION}") -project( libnpy VERSION ${LIBNPY_VERSION} LANGUAGES CXX) +project( npy VERSION ${LIBNPY_VERSION} LANGUAGES CXX) # -------------------- Options -------------------------------- option( LIBNPY_BUILD_TESTS "Specifies whether to build the tests" OFF ) -option( LIBNPY_BUILD_SAMPLES "Specifies whether to build the samples" OFF ) option( LIBNPY_BUILD_DOCUMENTATION "Specifies whether to build the documentation for the API and XML" OFF ) -option( LIBNPY_INCLUDE_CSHARP "Specifies whether to build libnpy with C# bindings" OFF ) set( LIBNPY_SANITIZE "" CACHE STRING "Argument to pass to sanitize (disabled by default)") set(CMAKE_CXX_STANDARD 17) @@ -31,30 +29,8 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # -------------------- Find packages -------------------------- -if ( WIN32 AND INCLUDE_CSHARP ) - enable_language( CSharp ) -else() - set( INCLUDE_CSHARP OFF ) -endif() - set( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake ) -if( INCLUDE_CSHARP ) - find_package( SWIG REQUIRED ) - - # Select the .NET architecture - set( CSHARP_PLATFORM_DESC "C# target platform: x86, x64, anycpu, or itanium" ) - if( CMAKE_SIZEOF_VOID_P EQUAL 8 ) - set( CSHARP_PLATFORM "x64" CACHE STRING ${CSHARP_PLATFORM_DESC}) - else() - set( CSHARP_PLATFORM "x86" CACHE STRING ${CSHARP_PLATFORM_DESC}) - endif() -endif() - -if( BUILD_DOCUMENTATION ) - find_package( Doxygen REQUIRED ) -endif() - find_program(CLANG_FORMAT NAMES clang-format-10 clang-format-14 clang-format-18 ) string(COMPARE EQUAL ${CLANG_FORMAT} "CLANG_FORMAT-NOTFOUND" CLANG_FORMAT_NOT_FOUND) @@ -67,7 +43,7 @@ else() include/libnpy/*.h test/*.cpp test/*.h - examples/*.cpp + examples/**/*.cpp ) add_custom_target(libnpy_format @@ -86,13 +62,10 @@ if( LIBNPY_BUILD_SAMPLES ) endif() if( LIBNPY_BUILD_DOCUMENTATION ) + find_package( Doxygen REQUIRED ) add_subdirectory( doc ) endif() -if( LIBNPY_INCLUDE_CSHARP ) - add_subdirectory( CSharpWrapper ) -endif() - target_include_directories(npy PUBLIC $ @@ -104,65 +77,58 @@ target_include_directories(npy # -------------------- Testing ------------------------------------ if( LIBNPY_BUILD_TESTS ) - if( MSVC ) - set( LIBNPY_CSHARP_DIR ${CMAKE_BINARY_DIR}/CSharpWrapper/$ ) - endif() - include( CTest ) add_subdirectory( test ) endif() -# -------------------- Build settings ----------------------------- - -# use C++11 -target_compile_features(npy PRIVATE cxx_std_11) - # -------------------- INSTALL ------------------------------------ -set(INSTALL_CONFIGDIR "cmake") - -install(TARGETS npy - EXPORT npy-targets - ARCHIVE DESTINATION "build/native/lib" - LIBRARY DESTINATION "build/native/lib" -) - -install(DIRECTORY include/ DESTINATION "build/native/include") +set(INSTALL_CONFIGDIR cmake) +set(INSTALL_LIBDIR lib) +set(INSTALL_INCLUDEDIR include) +set(LIBNPY_INSTALL_TARGETS npy) -install(EXPORT npy-targets - FILE - npyTargets.cmake - NAMESPACE - npy:: - DESTINATION - ${INSTALL_CONFIGDIR} +install(TARGETS ${LIBNPY_INSTALL_TARGETS} + EXPORT ${PROJECT_NAME}_Targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) +# Create a ConfigVersion.cmake file include(CMakePackageConfigHelpers) write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/npyConfigVersion.cmake + ${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake VERSION ${PROJECT_VERSION} COMPATIBILITY AnyNewerVersion ) -configure_package_config_file(${CMAKE_CURRENT_LIST_DIR}/cmake/npyConfig.cmake.in - ${CMAKE_CURRENT_BINARY_DIR}/npyConfig.cmake - INSTALL_DESTINATION ${INSTALL_CONFIGDIR} +configure_package_config_file(${PROJECT_SOURCE_DIR}/cmake/${PROJECT_NAME}Config.cmake.in + ${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake + INSTALL_DESTINATION + ${CMAKE_INSTALL_PREFIX}/${PROJECT_NAME}/cmake ) -install(FILES - ${CMAKE_CURRENT_BINARY_DIR}/npyConfig.cmake - ${CMAKE_CURRENT_BINARY_DIR}/npyConfigVersion.cmake - DESTINATION ${INSTALL_CONFIGDIR} -) +install(EXPORT ${PROJECT_NAME}_Targets + FILE ${PROJECT_NAME}Targets.cmake + NAMESPACE ${PROJECT_NAME}:: + DESTINATION ${CMAKE_INSTALL_PREFIX}/${PROJECT_NAME}/cmake) -export(EXPORT npy-targets - FILE ${CMAKE_CURRENT_BINARY_DIR}/npyTargets.cmake - NAMESPACE npy:: -) +install(FILES ${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake + ${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_PREFIX}/${PROJECT_NAME}/cmake) + +export(EXPORT ${PROJECT_NAME}_Targets + FILE ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Targets.cmake + NAMESPACE ${PROJECT_NAME}::) -export(PACKAGE npy) +export(PACKAGE ${PROJECT_NAME}) + +install(DIRECTORY include/ DESTINATION ${INSTALL_INCLUDEDIR}) +install(FILES + DESTINATION ${INSTALL_INCLUDEDIR}/npy +) # -------------------- Package ------------------------------------ @@ -171,22 +137,9 @@ set( PROJECT_FILES CHANGELOG.md ) -# copy these files into the root of the distribution zip -install( FILES ${PROJECT_FILES} DESTINATION "." ) - -if( MSVC ) - # NuGet files - set( LIBNPY_NUGET_NAME "npy-${SYSTEM_TOOLKIT}-${SYSTEM_BITS}-${CMAKE_BUILD_TYPE}" CACHE STRING "npy NuGet Name" FORCE ) - file( READ RELEASE_NOTES LIBNPY_RELEASE_NOTES ) - - configure_file("${CMAKE_CURRENT_SOURCE_DIR}/nuget/template.nuspec.in" "${CMAKE_CURRENT_BINARY_DIR}/nuget/${LIBNPY_NUGET_NAME}.nuspec" @ONLY ) - configure_file("${CMAKE_CURRENT_SOURCE_DIR}/nuget/template.targets.in" - "${CMAKE_CURRENT_BINARY_DIR}/nuget/build/native/${LIBNPY_NUGET_NAME}.targets" @ONLY ) -else() - set( CPACK_SYSTEM_NAME ${SYSTEM_NAME} ) - set( CPACK_PACKAGE_VERSION "${LIBNPY_VERSION}" ) - set( CPACK_GENERATOR "ZIP" ) - set( CPACK_SOURCE_GENERATOR "ZIP" ) - set( CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0 ) - include( CPack ) -endif() +set( CPACK_SYSTEM_NAME ${SYSTEM_NAME} ) +set( CPACK_PACKAGE_VERSION "${LIBNPY_VERSION}" ) +set( CPACK_GENERATOR "ZIP" ) +set( CPACK_SOURCE_GENERATOR "ZIP" ) +set( CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0 ) +include( CPack ) \ No newline at end of file diff --git a/CSharpWrapper/AssemblyInfo.cs.in b/CSharpWrapper/AssemblyInfo.cs.in deleted file mode 100644 index 714e573..0000000 --- a/CSharpWrapper/AssemblyInfo.cs.in +++ /dev/null @@ -1,17 +0,0 @@ -using System.Reflection; -using System.Runtime.CompilerServices; - -[assembly: AssemblyVersion("@LIBNPY_VERSION@")] -[assembly: AssemblyTitle("@PROJECT_NAME@ C# @CSHARP_PLATFORM@ wrapper @LIBNPY_VERSION@")] -[assembly: AssemblyCopyright("Copyright Matthew Johnson (c) 2021")] -[assembly: AssemblyDescription("NumpyIO C# @CSHARP_PLATFORM@ wrapper @LIBNPY_VERSION_CSHARP_AssemblyVersion@, compiled with @CSHARP_TYPE@ @CSHARP_VERSION@")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyFileVersion("@LIBNPY_VERSION@")] -[assembly: AssemblyCulture("")] -[assembly: AssemblyTrademark("NumpyIO")] -[assembly: AssemblyProduct("NumpyIO")] -[assembly: AssemblyInformationalVersion("@LIBNPY_VERSION_CSHARP_AssemblyVersion@")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyDelaySign(false)] -[assembly: AssemblyKeyName("")] -[assembly: AssemblyKeyFile("")] diff --git a/CSharpWrapper/CMakeLists.txt b/CSharpWrapper/CMakeLists.txt deleted file mode 100644 index 1537734..0000000 --- a/CSharpWrapper/CMakeLists.txt +++ /dev/null @@ -1,84 +0,0 @@ -include( ${SWIG_USE_FILE} ) - -execute_process( - COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - OUTPUT_VARIABLE GIT_BRANCH - OUTPUT_STRIP_TRAILING_WHITESPACE -) - -# Get the latest abbreviated commit hash of the working branch -execute_process( - COMMAND ${GIT_EXECUTABLE} log -1 --format=%h - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - OUTPUT_VARIABLE GIT_COMMIT_HASH - OUTPUT_STRIP_TRAILING_WHITESPACE -) - -set_source_files_properties ( NumpyIONative.i PROPERTIES CPLUSPLUS ON ) - -# CSharp version requirements: http://msdn.microsoft.com/en-us/library/system.reflection.assemblyversionattribute.aspx -# major.minor[.build[.revision]] where all components are 16-bit unsigned integers - -set(NUMPYIO_VERSION_CSHARP_AssemblyVersion "${LIBNPY_VERSION_MAJOR}.${LIBNPY_VERSION_MINOR}.${GIT_BRANCH}.${GIT_COMMIT_HASH}") - - # Make sure the nested directory structure exists -set(CSHARP_SOURCE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/swig CACHE INTERNAL "") -file(MAKE_DIRECTORY ${CSHARP_SOURCE_DIRECTORY}) - - # Create swig target -set(CMAKE_SWIG_OUTDIR ${CSHARP_SOURCE_DIRECTORY}) - -set(CMAKE_SWIG_FLAGS -I${CMAKE_CURRENT_SOURCE_DIR} -namespace \"NumpyIO\" ${CMAKE_SWIG_GLOBAL_FLAGS} ${CMAKE_SWIG_FLAGS}) - -SET_SOURCE_FILES_PROPERTIES(NumpyIONative.i PROPERTIES SWIG_FLAGS "-includeall") - -SWIG_ADD_LIBRARY( NumpyIONative - LANGUAGE csharp - TYPE SHARED - SOURCES NumpyIONative.i -) - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) - -SWIG_LINK_LIBRARIES( ${SWIG_MODULE_NumpyIONative_REAL_NAME} npy ) - -if( UNIX ) - set_target_properties(${SWIG_MODULE_NumpyIONative_REAL_NAME} PROPERTIES PREFIX "lib" SUFFIX ".so") - set( NUMPYIO_NATIVE libNumpyIONative.so CACHE INTERNAL "The NumpyIO built library" ) -else() - set_target_properties(${SWIG_MODULE_NumpyIONative_REAL_NAME} PROPERTIES SUFFIX ".dll") - set( NUMPYIO_NATIVE NumpyIONative.dll CACHE INTERNAL "The NumpyIONative built library" ) -endif() - -# Configure AssemblyInfo.cs -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/AssemblyInfo.cs.in - ${CSHARP_SOURCE_DIRECTORY}/AssemblyInfo.cs - @ONLY -) - -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/Tensor.cs - ${CSHARP_SOURCE_DIRECTORY}/Tensor.cs - COPYONLY -) - -# build the SWIG CSharp files into a wrapper library -FILE(GLOB SWIG_CSHARP_SOURCES ${CSHARP_SOURCE_DIRECTORY}/*.cs) - -foreach( source ${SWIG_CSHARP_SOURCES} ) - set_source_files_properties(${source} PROPERTIES GENERATED TRUE) -endforeach(source) - -# Add managed wrapper -add_library( - NumpyIO - SHARED - ${SWIG_CSHARP_SOURCES} -) - -target_compile_options( NumpyIO PUBLIC "/unsafe" ) -target_link_libraries( NumpyIO ${SWIG_MODULE_NumpyIONative_REAL_NAME} ) - -install( TARGETS NumpyIO ${SWIG_MODULE_NumpyIONative_REAL_NAME} RUNTIME DESTINATION "lib/net472") \ No newline at end of file diff --git a/CSharpWrapper/NumpyIONative.i b/CSharpWrapper/NumpyIONative.i deleted file mode 100644 index e6a2fa3..0000000 --- a/CSharpWrapper/NumpyIONative.i +++ /dev/null @@ -1,342 +0,0 @@ -%module NumpyIO -%{ - #include "npy/tensor.h" - #include "npy/npy.h" - #include "npy/npz.h" - using namespace npy; -%} - -%include "std_vector.i" -%include "std_string.i" -%include "std_wstring.i" -%include "stdint.i" -%include "arrays_csharp.i" -%include "typemaps.i" -%include "attribute.i" - -%rename(DataType) data_type_t; -%typemap(csbase) data_type_t "byte"; -enum class data_type_t : char { - INT8, - UINT8, - INT16, - UINT16, - INT32, - UINT32, - INT64, - UINT64, - FLOAT32, - FLOAT64, - UNICODE_STRING -}; - -%rename(Endian) endian_t; -%typemap(csbase) endian_t "byte"; -enum class endian_t : char { - NATIVE, - BIG, - LITTLE -}; - -%rename(CompressionMethod) compression_method_t; -%typemap(csbase) compression_method_t "ushort"; -enum class compression_method_t : std::uint16_t { - STORED = 0, - DEFLATED = 8 -}; - -%typemap(ctype, out="void *") const wstring * "wchar_t *" -%typemap(imtype, - inattributes="[global::System.Runtime.InteropServices.MarshalAs(UnmanagedType.LPArray, ArraySubType=UnmanagedType.LPStr)]", - outattributes="[return: global::System.Runtime.InteropServices.MarshalAs(UnmanagedType.LPArray, ArraySubType=UnmanagedType.LPStr)]" - ) const wstring * "string[]" -%typemap(cstype) const wstring * "string[]" - -%template(UInt8Buffer) std::vector; -%template(Int8Buffer) std::vector; -%template(UInt16Buffer) std::vector; -%template(Int16Buffer) std::vector; -%template(UInt32Buffer) std::vector; -%template(Int32Buffer) std::vector; -%template(UInt64Buffer) std::vector; -%template(Int64Buffer) std::vector; -%template(Float32Buffer) std::vector; -%template(Float64Buffer) std::vector; -%apply const std::wstring & {std::wstring &}; -%template(UnicodeStringBuffer) std::vector; -%template(StringList) std::vector; - -%template(Shape) std::vector; - -%rename(HeaderInfo) header_info; -struct header_info -{ - header_info(data_type_t dtype, - endian_t endianness, - bool fortran_order, - const std::vector &shape); - - - %rename(DataType) dtype; - data_type_t dtype; - - %rename(Endianness) endianness; - endian_t endianness; - - %rename(FortranOrder) fortran_order; - bool fortran_order; - - %rename(Shape) shape; - std::vector shape; -}; - -%exception peek(const std::string& path) %{ - try{ - $action - } catch( std::invalid_argument& e) { - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Invalid path location", e.what()); - return $null; - } catch( std::logic_error& e) { - SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); - return $null; - } -%} - -%rename(Peek) peek; -header_info peek(const std::string& path); - -template -class tensor { -public: - %exception tensor(const std::string& path) %{ - try{ - $action - } catch( std::invalid_argument& e) { - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Invalid path location", e.what()); - return $null; - } catch( std::logic_error& e) { - SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); - return $null; - } - %} - - explicit tensor(const std::string& path); - - tensor(const std::vector& shape); - - tensor(const std::vector& shape, bool fortran_order); - - %exception save(const std::string& path, endian_t endian = endian_t::NATIVE) %{ - try{ - $action - } catch (std::invalid_argument& e) { - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Invalid path location", e.what()); - return $null; - } catch (std::logic_error& e) { - SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); - return $null; - } - %} - - %csmethodmodifiers save "public override"; - %rename(Save) save; - void save(const std::string& path, endian_t endian = endian_t::NATIVE); - - %exception copy_from(const std::vector& source) %{ - try{ - $action - } catch (std::invalid_argument& e){ - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Incorrect number of items", e.what()); - return $null; - } - %} - - %csmethodmodifiers copy_from "public unsafe override"; - %rename(CopyFrom) copy_from; - void copy_from(const std::vector& source); - - %csmethodmodifiers values "protected override" - %rename(getValues) values; - const std::vector& values() const; - - %csmethodmodifiers shape "protected override" - %rename(getShape) shape; - const std::vector shape() const; - - %csmethodmodifiers fortran_order "protected override" - %rename(getFortranOrder) fortran_order; - bool fortran_order() const; - - %csmethodmodifiers dtype "protected override" - %rename(getDataType) dtype; - data_type_t dtype() const; - - %csmethodmodifiers size "protected override" - %rename(getSize) size; - size_t size() const; - - %exception get(const std::vector& index) const %{ - try{ - $action - }catch(std::invalid_argument& e){ - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "incorrect index size", e.what()); - return $null; - }catch(std::out_of_range& e){ - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentOutOfRangeException, "index out of range", e.what()); - return $null; - } - %} - - %csmethodmodifiers get "protected override" - const T& get(const std::vector& index) const; - - %exception set(const std::vector& index, const T& value) const %{ - try{ - $action - }catch(std::invalid_argument& e){ - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "incorrect index size", e.what()); - return $null; - }catch(std::out_of_range& e){ - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentOutOfRangeException, "index out of range", e.what()); - return $null; - } - %} - - %csmethodmodifiers set "protected override" - void set(const std::vector& index, const T& value); -}; - -%typemap(csbase) SWIGTYPE "Tensor"; -%template(UInt8Tensor) tensor; -%typemap(csbase) SWIGTYPE "Tensor"; -%template(Int8Tensor) tensor; -%typemap(csbase) SWIGTYPE "Tensor"; -%template(UInt16Tensor) tensor; -%typemap(csbase) SWIGTYPE "Tensor"; -%template(Int16Tensor) tensor; -%typemap(csbase) SWIGTYPE "Tensor"; -%template(UInt32Tensor) tensor; -%typemap(csbase) SWIGTYPE "Tensor"; -%template(Int32Tensor) tensor; -%typemap(csbase) SWIGTYPE "Tensor"; -%template(UInt64Tensor) tensor; -%typemap(csbase) SWIGTYPE "Tensor"; -%template(Int64Tensor) tensor; -%typemap(csbase) SWIGTYPE "Tensor"; -%template(Float32Tensor) tensor; -%typemap(csbase) SWIGTYPE "Tensor"; -%template(Float64Tensor) tensor; -%typemap(csbase) SWIGTYPE "Tensor"; -%template(UnicodeStringTensor) tensor; - -%typemap(csbase) SWIGTYPE "" - -%rename(NPZOutputStream) onpzstream; -class onpzstream { -public: - onpzstream(const std::string& path, compression_method_t compression=compression_method_t::STORED, endian_t endian=endian_t::NATIVE); - - %rename(Close) close; - void close(); - - %exception write(const std::string& filename, const tensor& tensor) %{ - try{ - $action - }catch(std::invalid_argument& e){ - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Unsupported compression method", e.what()); - return $null; - }catch(std::logic_error& e){ - SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); - return $null; - } - %} - - template - void write(const std::string& filename, const tensor& tensor); -}; - -%extend onpzstream { - %template(Write) write; - %template(Write) write; - %template(Write) write; - %template(Write) write; - %template(Write) write; - %template(Write) write; - %template(Write) write; - %template(Write) write; - %template(Write) write; - %template(Write) write; - %template(Write) write; -}; - -%rename(NPZInputStream) inpzstream; -class inpzstream { -public: - %exception inpzstream(const std::string& path) %{ - try{ - $action - }catch(std::invalid_argument& e){ - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "File does not exist", e.what()); - return $null; - }catch(std::logic_error& e){ - SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); - return $null; - } - %} - - inpzstream(const std::string& path); - - %rename(Keys) keys; - const std::vector& keys() const; - - %rename(Contains) contains; - bool contains(const std::string& filename); - - %exception peek(const std::string& filename) %{ - try{ - $action - }catch(std::invalid_argument& e){ - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Filename does not exist in archive", e.what()); - return $null; - }catch(std::logic_error& e){ - SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); - return $null; - } - %} - - %rename(Peek) peek; - header_info peek(const std::string& filename); - - - %exception read(const std::string& filename) %{ - try{ - $action - }catch(std::invalid_argument& e){ - SWIG_CSharpSetPendingExceptionArgument(SWIG_CSharpArgumentException, "Filename does not exist in archive", e.what()); - return $null; - }catch(std::logic_error& e){ - SWIG_CSharpSetPendingException(SWIG_CSharpIOException, e.what()); - return $null; - } - %} - - template - tensor read(const std::string& filename); - - %rename(Close) close; - void close(); -}; - -%extend inpzstream { - %template(ReadUInt8) read; - %template(ReadInt8) read; - %template(ReadUInt16) read; - %template(ReadInt16) read; - %template(ReadUInt32) read; - %template(ReadInt32) read; - %template(ReadUInt64) read; - %template(ReadInt64) read; - %template(ReadFloat32) read; - %template(ReadFloat64) read; - %template(ReadUnicodeString) read; -}; diff --git a/CSharpWrapper/Tensor.cs b/CSharpWrapper/Tensor.cs deleted file mode 100644 index 6ef27cc..0000000 --- a/CSharpWrapper/Tensor.cs +++ /dev/null @@ -1,173 +0,0 @@ -// ---------------------------------------------------------------------------- -// -// Tensor.cs -- abstract base class for .NET Tensor objects -// -// Copyright (C) 2021 Matthew Johnson -// -// For conditions of distribution and use, see copyright notice in LICENSE -// -// ---------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; - -namespace NumpyIO -{ - /// - /// This class is the abstract base class for all .NET tensor objects. - /// The autogenerated classes will all be subclasses of this class. - /// - /// The data type - /// The buffer type - public abstract class Tensor where B : IList - { - /// - /// Copy the data from the provided buffer. These values will - /// be copied into the underlying C++ type. - /// - /// The source buffer - public abstract void CopyFrom(B source); - - /// - /// Save the tensor to the provided location on the disk. - /// - /// a valid location on the disk - public abstract void Save(string path); - - /// - /// Save the tensor to the provided location on the disk. - /// - /// a valid location on the disk - /// the endianness with which to write the tensor - public abstract void Save(string path, Endian endian); - - /// - /// A readonly list of values. Changing values in this object - /// will NOT affect the underlying Tensor. - /// - public IList Values - { - get - { - return this.getValues(); - } - } - - /// - /// The shape of the tensor. - /// - public Shape Shape - { - get - { - return this.getShape(); - } - } - - /// - /// Whether the data is stored in FORTRAN, or column-major, order. - /// - public bool FortranOrder - { - get - { - return this.getFortranOrder(); - } - } - - /// - /// The data type of the tensor. - /// - public DataType DataType - { - get - { - return this.getDataType(); - } - } - - /// - /// The number of elements in the tensor. - /// - public uint Size - { - get - { - return this.getSize(); - } - } - - /// - /// Index operator. - /// - /// The index into the tensor - /// The value at the provided index - public T this[params int[] multiIndex] - { - get - { - if (multiIndex.Length != this.Shape.Count) - { - throw new ArgumentException("Incorrect number of indices"); - } - - return this.get(new Int32Buffer(multiIndex)); - } - set - { - if (multiIndex.Length != this.Shape.Count) - { - throw new ArgumentException("Incorrect number of indices"); - } - - this.set(new Int32Buffer(multiIndex), value); - } - } - - /// - /// Autogenerated subclasses will implement this method. Maps to - /// \link npy::tensor::values \endlink. - /// - /// A read-only buffer object - protected abstract B getValues(); - - /// - /// Autogenerated subclasses will implement this method. Maps to - /// \link npy::tensor::shape \endlink. - /// - /// A Shape object - protected abstract Shape getShape(); - - /// - /// Autogenerated subclasses will implement this method. Maps to - /// \link npy::tensor::fortran_order \endlink. - /// - /// Whether the data is stored in FORTRAN, or column-major, order - protected abstract bool getFortranOrder(); - - /// - /// Autogenerated subclasses will implement this method. Maps to - /// \link npy::tensor::dtype \endlink. - /// - /// The data type of the tensor - protected abstract DataType getDataType(); - - /// - /// Autogenerated subclasses will implement this method. Maps to - /// \link npy::tensor::size \endlink. - /// - /// The number of elements in the tensor - protected abstract uint getSize(); - - /// - /// Autogenerated subclasses will implement this method. Maps to - /// \link npy::tensor::get \endlink - protected abstract T get(Int32Buffer multiIndex); - - /// - /// Autogenereted subclasses will implement this method. Maps to - /// \link npy::tensor::set \endlink - /// - protected abstract void set(Int32Buffer multiIndex, T value); - } -} \ No newline at end of file diff --git a/README.md b/README.md index 06f8b7a..87402bc 100644 --- a/README.md +++ b/README.md @@ -49,21 +49,20 @@ You can then build and run the tests using: Once the library has been built and installed, you can begin to use it in your code. We have provided some -[sample programs](https://github.com/matajoh/libnpy/tree/main/samples) -(and naturally the [tests](https://github.com/matajoh/libnpy/tree/main/test) -as well) which show how to use the library, but the basic concepts are as follows. +[example programs](examples) +(and naturally the [tests](test) as well) which show how to use the library, +but the basic concepts are as follows. For the purpose of this sample code we will use the built-in [tensor](src/tensor.h) -class, but you should use your own tensor class as appropriate. +class, but you should use your own tensor class as appropriate (see +the [custom tensor example](examples/custom_tensors/) for details.) ```C++ -#include "tensor.h" -#include "npy.h" -#include "npz.h" +#include "npy/npy.h" -... +int main() +{ // create a tensor object - std::vector shape({32, 32, 3}); - npy::tensor color(shape); + npy::tensor color({32, 32, 3}); // fill it with some data for (int row = 0; row < color.shape(0); ++row) @@ -92,8 +91,7 @@ class, but you should use your own tensor class as appropriate. color = npy::load("color.npy"); // let's create a second tensor as well - shape = {32, 32}; - npy::tensor gray(shape); + npy::tensor gray({32, 32}); for (int row = 0; row < gray.shape(0); ++row) { @@ -107,14 +105,14 @@ class, but you should use your own tensor class as appropriate. // we can write them to an NPZ file { - npy::onpzstream output("test.npz"); + npy::npzfilewriter output("test.npz"); output.write("color.npy", color); output.write("gray.npy", gray); } // and we can read them back out again { - npy::inpzstream input("test.npz"); + npy::npzfilereader input("test.npz"); // we can test to see if the archive contains a file if (input.contains("color.npy")) @@ -123,9 +121,12 @@ class, but you should use your own tensor class as appropriate. header = input.peek("color.npy"); } - color = input.read("color.npy"); - gray = input.read("gray.npy"); + color = input.read>("color.npy"); + gray = input.read>("gray.npy"); } + + return 0; +} ``` The generated documentation contains more details on all of the functionality. diff --git a/VERSION b/VERSION index 1d5e9e0..359a5b9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.3 \ No newline at end of file +2.0.0 \ No newline at end of file diff --git a/assets/test/complex128.npy b/assets/test/complex128.npy new file mode 100644 index 0000000..9945cd9 Binary files /dev/null and b/assets/test/complex128.npy differ diff --git a/assets/test/complex64.npy b/assets/test/complex64.npy new file mode 100644 index 0000000..1fa626f Binary files /dev/null and b/assets/test/complex64.npy differ diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 11e5e87..4adb6bc 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -13,7 +13,7 @@ configure_file( ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY ) -add_custom_target( npy_doc +add_custom_target( libnpy_doc ALL ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 6f06061..a43c37e 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -1,4 +1,4 @@ -# Doxyfile 1.8.15 +# Doxyfile 1.9.8 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -12,6 +12,16 @@ # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). +# +# Note: +# +# Use doxygen to compare the used configuration file with the template +# configuration file: +# doxygen -x [configFile] +# Use doxygen to compare the used configuration file with the template +# configuration file without replacing the environment variables or CMake type +# replacement variables: +# doxygen -x_noenv [configFile] #--------------------------------------------------------------------------- # Project related configuration options @@ -60,16 +70,28 @@ PROJECT_LOGO = OUTPUT_DIRECTORY = doc -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 +# sub-directories (in 2 levels) under the output directory of each output format +# and will distribute the generated files over these directories. Enabling this # option can be useful when feeding doxygen a huge amount of source files, where # putting all generated files in the same directory would otherwise causes -# performance problems for the file system. +# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to +# control the number of sub-directories. # The default value is: NO. CREATE_SUBDIRS = NO +# Controls the number of sub-directories that will be created when +# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every +# level increment doubles the number of directories, resulting in 4096 +# directories at level 8 which is the default and also the maximum value. The +# sub-directories are organized in 2 levels, the first level always has a fixed +# number of 16 directories. +# Minimum value: 0, maximum value: 8, default value: 8. +# This tag requires that the tag CREATE_SUBDIRS is set to YES. + +CREATE_SUBDIRS_LEVEL = 8 + # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode @@ -81,26 +103,18 @@ ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, +# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English +# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, +# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with +# English messages), Korean, Korean-en (Korean with English messages), Latvian, +# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, +# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, +# Swedish, Turkish, Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. @@ -195,7 +209,17 @@ SHORT_NAMES = NO # description.) # The default value is: NO. -JAVADOC_AUTOBRIEF = YES +JAVADOC_AUTOBRIEF = NO + +# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line +# such as +# /*************** +# as being the beginning of a Javadoc-style comment "banner". If set to NO, the +# Javadoc-style will behave just like regular comments and it will not be +# interpreted by doxygen. +# The default value is: NO. + +JAVADOC_BANNER = NO # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first # line (until the first dot) of a Qt-style comment as the brief description. If @@ -217,6 +241,14 @@ QT_AUTOBRIEF = NO MULTILINE_CPP_IS_BRIEF = NO +# By default Python docstrings are displayed as preformatted text and doxygen's +# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the +# doxygen's special commands can be used and the contents of the docstring +# documentation blocks is shown as doxygen documentation. +# The default value is: YES. + +PYTHON_DOCSTRING = YES + # If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the # documentation from any documented member that it re-implements. # The default value is: YES. @@ -240,16 +272,16 @@ TAB_SIZE = 4 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) ALIASES = @@ -293,19 +325,22 @@ OPTIMIZE_OUTPUT_SLICE = NO # parses. With this tag you can assign which parser to use for a given # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, Javascript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the -# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat -# .inc files as Fortran files (default is PHP), and .f files as C (default is -# Fortran), use: inc=Fortran f=C. +# default for Fortran type files). For instance to make doxygen treat .inc files +# as Fortran files (default is PHP), and .f files as C (default is Fortran), +# use: inc=Fortran f=C. # # Note: For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. +# the files are not read by doxygen. When specifying no_extension you should add +# * to the FILE_PATTERNS. +# +# Note see also the list of default file extension mappings. EXTENSION_MAPPING = @@ -323,10 +358,21 @@ MARKDOWN_SUPPORT = YES # to that level are automatically included in the table of contents, even if # they do not have an id attribute. # Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 0. +# Minimum value: 0, maximum value: 99, default value: 5. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 5 + +# The MARKDOWN_ID_STYLE tag can be used to specify the algorithm used to +# generate identifiers for the Markdown headings. Note: Every identifier is +# unique. +# Possible values are: DOXYGEN use a fixed 'autotoc_md' string followed by a +# sequence number starting at 0 and GITHUB use the lower case version of title +# with any whitespace replaced by '-' and punctuation characters removed. +# The default value is: DOXYGEN. # This tag requires that the tag MARKDOWN_SUPPORT is set to YES. -TOC_INCLUDE_HEADINGS = 0 +MARKDOWN_ID_STYLE = DOXYGEN # When enabled doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can @@ -439,6 +485,27 @@ TYPEDEF_HIDES_STRUCT = NO LOOKUP_CACHE_SIZE = 0 +# The NUM_PROC_THREADS specifies the number of threads doxygen is allowed to use +# during processing. When set to 0 doxygen will based this on the number of +# cores available in the system. You can set it explicitly to a value larger +# than 0 to get more control over the balance between CPU load and processing +# speed. At this moment only the input processing can be done using multiple +# threads. Since this is still an experimental feature the default is set to 1, +# which effectively disables parallel processing. Please report any issues you +# encounter. Generating dot graphs in parallel is controlled by the +# DOT_NUM_THREADS setting. +# Minimum value: 0, maximum value: 32, default value: 1. + +NUM_PROC_THREADS = 1 + +# If the TIMESTAMP tag is set different from NO then each generated page will +# contain the date or date and time when the page was generated. Setting this to +# NO can help when comparing the output of multiple runs. +# Possible values are: YES, NO, DATETIME and DATE. +# The default value is: NO. + +TIMESTAMP = NO + #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- @@ -459,6 +526,12 @@ EXTRACT_ALL = YES EXTRACT_PRIVATE = NO +# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual +# methods of a class will be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIV_VIRTUAL = NO + # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal # scope will be included in the documentation. # The default value is: NO. @@ -496,6 +569,13 @@ EXTRACT_LOCAL_METHODS = NO EXTRACT_ANON_NSPACES = NO +# If this flag is set to YES, the name of an unnamed parameter in a declaration +# will be determined by the corresponding definition. By default unnamed +# parameters remain unnamed in the output. +# The default value is: YES. + +RESOLVE_UNNAMED_PARAMS = YES + # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation @@ -507,14 +587,15 @@ HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. +# will also hide undocumented C++ concepts if enabled. This option has no effect +# if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# (class|struct|union) declarations. If set to NO, these declarations will be -# included in the documentation. +# declarations. If set to NO, these declarations will be included in the +# documentation. # The default value is: NO. HIDE_FRIEND_COMPOUNDS = NO @@ -533,12 +614,20 @@ HIDE_IN_BODY_DOCS = NO INTERNAL_DOCS = NO -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. -# The default value is: system dependent. +# With the correct setting of option CASE_SENSE_NAMES doxygen will better be +# able to match the capabilities of the underlying filesystem. In case the +# filesystem is case sensitive (i.e. it supports files in the same directory +# whose names only differ in casing), the option must be set to YES to properly +# deal with such files in case they appear in the input. For filesystems that +# are not case sensitive the option should be set to NO to properly deal with +# output files written for symbols that only differ in casing, such as for two +# classes, one named CLASS and the other named Class, and to also support +# references to files without having to specify the exact matching casing. On +# Windows (including Cygwin) and MacOS, users should typically set this option +# to NO, whereas on Linux or other Unix flavors it should typically be set to +# YES. +# Possible values are: SYSTEM, NO and YES. +# The default value is: SYSTEM. CASE_SENSE_NAMES = NO @@ -556,6 +645,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -713,7 +808,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -759,24 +855,50 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = YES +# If WARN_IF_UNDOC_ENUM_VAL option is set to YES, doxygen will warn about +# undocumented enumeration values. If set to NO, doxygen will accept +# undocumented enumeration values. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: NO. + +WARN_IF_UNDOC_ENUM_VAL = NO + # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. +# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS +# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but +# at the end of the doxygen process doxygen will return with a non-zero status. +# If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS_PRINT then doxygen behaves +# like FAIL_ON_WARNINGS but in case no WARN_LOGFILE is defined doxygen will not +# write the warning messages in between other messages but write them at the end +# of a run, in case a WARN_LOGFILE is defined the warning messages will be +# besides being in the defined file also be shown at the end of a run, unless +# the WARN_LOGFILE is defined as - i.e. standard output (stdout) in that case +# the behavior will remain as with the setting FAIL_ON_WARNINGS. +# Possible values are: NO, YES, FAIL_ON_WARNINGS and FAIL_ON_WARNINGS_PRINT. # The default value is: NO. WARN_AS_ERROR = NO @@ -787,13 +909,27 @@ WARN_AS_ERROR = NO # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) +# See also: WARN_LINE_FORMAT # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" +# In the $text part of the WARN_FORMAT command it is possible that a reference +# to a more specific place is given. To make it easier to jump to this place +# (outside of doxygen) the user can define a custom "cut" / "paste" string. +# Example: +# WARN_LINE_FORMAT = "'vi $file +$line'" +# See also: WARN_FORMAT +# The default value is: at line $line of file $file. + +WARN_LINE_FORMAT = "at line $line of file $file" + # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard -# error (stderr). +# error (stderr). In case the file specified cannot be opened for writing the +# warning and error messages are written to standard error. When as file - is +# specified the warning and error messages are written to standard output +# (stdout). WARN_LOGFILE = @@ -814,12 +950,23 @@ INPUT += @CSHARP_SOURCE_DIRECTORY@ # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: https://www.gnu.org/software/libiconv/) for the list of -# possible encodings. +# documentation (see: +# https://www.gnu.org/software/libiconv/) for the list of possible encodings. +# See also: INPUT_FILE_ENCODING # The default value is: UTF-8. INPUT_ENCODING = UTF-8 +# This tag can be used to specify the character encoding of the source files +# that doxygen parses The INPUT_FILE_ENCODING tag can be used to specify +# character encoding on a per file pattern basis. Doxygen will compare the file +# name with each pattern and apply the encoding instead of the default +# INPUT_ENCODING) if there is a match. The character encodings are a list of the +# form: pattern=encoding (like *.php=ISO-8859-1). See cfg_input_encoding +# "INPUT_ENCODING" for further information on supported encodings. + +INPUT_FILE_ENCODING = + # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. @@ -828,17 +975,24 @@ INPUT_ENCODING = UTF-8 # need to set EXTENSION_MAPPING for the extension otherwise the files are not # read by doxygen. # -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, -# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. +# Note the list of default checked file patterns might differ from the list of +# default file extension mappings. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cxxm, +# *.cpp, *.cppm, *.c++, *.c++m, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, +# *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp, *.h++, *.ixx, *.l, *.cs, *.d, *.php, +# *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be +# provided as doxygen C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f18, *.f, *.for, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.c \ *.cc \ *.cxx \ + *.cxxm \ *.cpp \ + *.cppm \ *.c++ \ + *.c++m \ *.java \ *.ii \ *.ixx \ @@ -853,6 +1007,8 @@ FILE_PATTERNS = *.c \ *.hxx \ *.hpp \ *.h++ \ + *.ixx \ + *.l \ *.cs \ *.d \ *.php \ @@ -871,9 +1027,9 @@ FILE_PATTERNS = *.c \ *.f95 \ *.f03 \ *.f08 \ + *.f18 \ *.f \ *.for \ - *.tcl \ *.vhd \ *.vhdl \ *.ucf \ @@ -915,10 +1071,7 @@ EXCLUDE_PATTERNS = # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* +# ANamespace::AClass, ANamespace::*Test EXCLUDE_SYMBOLS = @@ -963,6 +1116,11 @@ IMAGE_PATH = # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. # +# Note that doxygen will use the data processed and written to standard output +# for further processing, therefore nothing else, like debug statements or used +# commands (so in case of a Windows batch file always use @echo OFF), should be +# written to standard output. +# # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not # properly processed by doxygen. @@ -1002,7 +1160,16 @@ FILTER_SOURCE_PATTERNS = # (index.html). This can be useful if you have a project on for instance GitHub # and want to reuse the introduction page also for the doxygen output. -USE_MDFILE_AS_MAINPAGE = @CMAKE_SOURCE_DIR@/README.md +USE_MDFILE_AS_MAINPAGE = + +# The Fortran standard specifies that for fixed formatted Fortran code all +# characters from position 72 are to be considered as comment. A common +# extension is to allow longer lines before the automatic comment starts. The +# setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can +# be processed before the automatic comment starts. +# Minimum value: 7, maximum value: 10000, default value: 72. + +FORTRAN_COMMENT_AFTER = 72 #--------------------------------------------------------------------------- # Configuration options related to source browsing @@ -1091,16 +1258,24 @@ USE_HTAGS = NO VERBATIM_HEADERS = YES # If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the -# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the -# cost of reduced performance. This can be particularly helpful with template -# rich C++ code for which doxygen's built-in parser lacks the necessary type -# information. +# clang parser (see: +# http://clang.llvm.org/) for more accurate parsing at the cost of reduced +# performance. This can be particularly helpful with template rich C++ code for +# which doxygen's built-in parser lacks the necessary type information. # Note: The availability of this option depends on whether or not doxygen was # generated with the -Duse_libclang=ON option for CMake. # The default value is: NO. CLANG_ASSISTED_PARSING = NO +# If the CLANG_ASSISTED_PARSING tag is set to YES and the CLANG_ADD_INC_PATHS +# tag is set to YES then doxygen will add the directory of each input to the +# include path. +# The default value is: YES. +# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. + +CLANG_ADD_INC_PATHS = YES + # If clang assisted parsing is enabled you can provide the compiler with command # line options that you would normally use when invoking the compiler. Note that # the include paths will already be set by doxygen for the files and directories @@ -1110,10 +1285,13 @@ CLANG_ASSISTED_PARSING = NO CLANG_OPTIONS = # If clang assisted parsing is enabled you can provide the clang parser with the -# path to the compilation database (see: -# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) used when the files -# were built. This is equivalent to specifying the "-p" option to a clang tool, -# such as clang-check. These options will then be passed to the parser. +# path to the directory containing a file called compile_commands.json. This +# file is the compilation database (see: +# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) containing the +# options used when the source files were built. This is equivalent to +# specifying the -p option to a clang tool, such as clang-check. These options +# will then be passed to the parser. Any options specified with CLANG_OPTIONS +# will be added as well. # Note: The availability of this option depends on whether or not doxygen was # generated with the -Duse_libclang=ON option for CMake. @@ -1130,10 +1308,11 @@ CLANG_DATABASE_PATH = ALPHABETICAL_INDEX = YES -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. +# The IGNORE_PREFIX tag can be used to specify a prefix (or a list of prefixes) +# that should be ignored while generating the index headers. The IGNORE_PREFIX +# tag works for classes, function and member names. The entity will be placed in +# the alphabetical list under the first letter of the entity name that remains +# after removing the prefix. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = @@ -1212,7 +1391,12 @@ HTML_STYLESHEET = # Doxygen will copy the style sheet files to the output directory. # Note: The order of the extra style sheet files is of importance (e.g. the last # style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. +# list). +# Note: Since the styling of scrollbars can currently not be overruled in +# Webkit/Chromium, the styling will be left out of the default doxygen.css if +# one or more extra stylesheets have been specified. So if scrollbar +# customization is desired it has to be added explicitly. For an example see the +# documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_STYLESHEET = @@ -1227,9 +1411,22 @@ HTML_EXTRA_STYLESHEET = HTML_EXTRA_FILES = +# The HTML_COLORSTYLE tag can be used to specify if the generated HTML output +# should be rendered with a dark or light theme. +# Possible values are: LIGHT always generate light mode output, DARK always +# generate dark mode output, AUTO_LIGHT automatically set the mode according to +# the user preference, use light mode if no preference is set (the default), +# AUTO_DARK automatically set the mode according to the user preference, use +# dark mode if no preference is set and TOGGLE allow to user to switch between +# light and dark mode via a button. +# The default value is: AUTO_LIGHT. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE = AUTO_LIGHT + # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see +# this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. @@ -1239,7 +1436,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1257,20 +1454,11 @@ HTML_COLORSTYLE_SAT = 100 HTML_COLORSTYLE_GAMMA = 80 -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = YES - # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML # documentation will contain a main index with vertical navigation menus that -# are dynamically created via Javascript. If disabled, the navigation index will +# are dynamically created via JavaScript. If disabled, the navigation index will # consists of multiple levels of tabs that are statically embedded in every HTML -# page. Disable this option to support browsers that do not have Javascript, +# page. Disable this option to support browsers that do not have JavaScript, # like the Qt help browser. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1285,6 +1473,13 @@ HTML_DYNAMIC_MENUS = YES HTML_DYNAMIC_SECTIONS = NO +# If the HTML_CODE_FOLDING tag is set to YES then classes and functions can be +# dynamically folded and expanded in the generated HTML source code. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_CODE_FOLDING = YES + # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand # and collapse entries dynamically later on. Doxygen will expand the tree to @@ -1300,10 +1495,11 @@ HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: https://developer.apple.com/xcode/), introduced with OSX -# 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in +# environment (see: +# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To +# create a documentation set, doxygen will generate a Makefile in the HTML +# output directory. Running make will produce the docset in that directory and +# running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at # startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy # genXcode/_index.html for more information. @@ -1320,6 +1516,13 @@ GENERATE_DOCSET = NO DOCSET_FEEDNAME = "Doxygen generated docs" +# This tag determines the URL of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDURL = + # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. @@ -1345,8 +1548,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1376,7 +1583,7 @@ CHM_FILE = HHC_LOCATION = # The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the master .chm file (NO). +# (YES) or that it should be included in the main .chm file (NO). # The default value is: NO. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. @@ -1403,6 +1610,16 @@ BINARY_TOC = NO TOC_EXPAND = NO +# The SITEMAP_URL tag is used to specify the full URL of the place where the +# generated documentation will be placed on the server by the user during the +# deployment of the documentation. The generated sitemap is called sitemap.xml +# and placed on the directory specified by HTML_OUTPUT. In case no SITEMAP_URL +# is specified no sitemap is generated. For information about the sitemap +# protocol see https://www.sitemaps.org +# This tag requires that the tag GENERATE_HTML is set to YES. + +SITEMAP_URL = + # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help @@ -1421,7 +1638,8 @@ QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace -# (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). +# (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1429,8 +1647,8 @@ QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- -# folders). +# Folders (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1438,30 +1656,30 @@ QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's filter section matches. Qt Help Project / Filter Attributes (see: -# http://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_SECT_FILTER_ATTRS = -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. +# The QHG_LOCATION tag can be used to specify the location (absolute path +# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to +# run qhelpgenerator on the generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = @@ -1504,16 +1722,28 @@ DISABLE_INDEX = NO # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NO +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATE_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1538,6 +1768,24 @@ TREEVIEW_WIDTH = 250 EXT_LINKS_IN_WINDOW = NO +# If the OBFUSCATE_EMAILS tag is set to YES, doxygen will obfuscate email +# addresses. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +OBFUSCATE_EMAILS = YES + +# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg +# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see +# https://inkscape.org) to generate formulas as SVG images instead of PNGs for +# the HTML output. These images will generally look nicer at scaled resolutions. +# Possible values are: png (the default) and svg (looks nicer but requires the +# pdf2svg or inkscape tool). +# The default value is: png. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FORMULA_FORMAT = png + # Use this tag to change the font size of LaTeX formulas included as images in # the HTML documentation. When you change the font size after a successful # doxygen run you need to manually remove any form_*.png images from the HTML @@ -1547,19 +1795,14 @@ EXT_LINKS_IN_WINDOW = NO FORMULA_FONTSIZE = 10 -# Use the FORMULA_TRANSPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. +# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands +# to create new LaTeX commands to be used in formulas as building blocks. See +# the section "Including formulas" for details. -FORMULA_TRANSPARENT = YES +FORMULA_MACROFILE = # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# https://www.mathjax.org) which uses client side Javascript for the rendering +# https://www.mathjax.org) which uses client side JavaScript for the rendering # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX # installed or if you want to formulas look prettier in the HTML output. When # enabled you may also need to install MathJax separately and configure the path @@ -1569,11 +1812,29 @@ FORMULA_TRANSPARENT = YES USE_MATHJAX = NO +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1586,22 +1847,29 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. -MATHJAX_RELPATH = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/ +MATHJAX_RELPATH = # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see +# https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1629,7 +1897,7 @@ MATHJAX_CODEFILE = SEARCHENGINE = YES # When the SERVER_BASED_SEARCH tag is enabled the search engine will be -# implemented using a web server instead of a web client using Javascript. There +# implemented using a web server instead of a web client using JavaScript. There # are two flavors of web server based searching depending on the EXTERNAL_SEARCH # setting. When disabled, doxygen will generate a PHP script for searching and # an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing @@ -1648,7 +1916,8 @@ SERVER_BASED_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: https://xapian.org/). +# Xapian (see: +# https://xapian.org/). # # See the section "External Indexing and Searching" for details. # The default value is: NO. @@ -1661,8 +1930,9 @@ EXTERNAL_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: https://xapian.org/). See the section "External Indexing and -# Searching" for details. +# Xapian (see: +# https://xapian.org/). See the section "External Indexing and Searching" for +# details. # This tag requires that the tag SEARCHENGINE is set to YES. SEARCHENGINE_URL = @@ -1733,13 +2003,14 @@ LATEX_CMD_NAME = MAKEINDEX_CMD_NAME = makeindex # The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to -# generate index for LaTeX. +# generate index for LaTeX. In case there is no backslash (\) as first character +# it will be automatically added in the LaTeX code. # Note: This tag is used in the generated output file (.tex). # See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat. -# The default value is: \makeindex. +# The default value is: makeindex. # This tag requires that the tag GENERATE_LATEX is set to YES. -LATEX_MAKEINDEX_CMD = \makeindex +LATEX_MAKEINDEX_CMD = makeindex # If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX # documents. This may be useful for small projects and may help to save some @@ -1770,29 +2041,31 @@ PAPER_TYPE = a4 EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1825,18 +2098,26 @@ LATEX_EXTRA_FILES = PDF_HYPERLINKS = YES -# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate -# the PDF file directly from the LaTeX files. Set this option to YES, to get a -# higher quality PDF documentation. +# If the USE_PDFLATEX tag is set to YES, doxygen will use the engine as +# specified with LATEX_CMD_NAME to generate the PDF file directly from the LaTeX +# files. Set this option to YES, to get a higher quality PDF documentation. +# +# See also section LATEX_CMD_NAME for selecting the engine. # The default value is: YES. # This tag requires that the tag GENERATE_LATEX is set to YES. USE_PDFLATEX = YES -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode -# command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# The LATEX_BATCHMODE tag signals the behavior of LaTeX in case of an error. +# Possible values are: NO same as ERROR_STOP, YES same as BATCH, BATCH In batch +# mode nothing is printed on the terminal, errors are scrolled as if is +# hit at every error; missing files that TeX tries to input or request from +# keyboard input (\read on a not open input stream) cause the job to abort, +# NON_STOP In nonstop mode the diagnostic message will appear on the terminal, +# but there is no possibility of user interaction just like in batch mode, +# SCROLL In scroll mode, TeX will stop only for missing files to input or if +# keyboard input is necessary and ERROR_STOP In errorstop mode, TeX will stop at +# each error, asking for user intervention. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1849,16 +2130,6 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # https://en.wikipedia.org/wiki/BibTeX and \cite for more info. @@ -1867,14 +2138,6 @@ LATEX_SOURCE_CODE = NO LATEX_BIB_STYLE = plain -# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated -# page will contain the date and time when the page was generated. Setting this -# to NO can help when comparing the output of multiple runs. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_TIMESTAMP = NO - # The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute) # path from which the emoji images will be read. If a relative path is entered, # it will be relative to the LATEX_OUTPUT directory. If left blank the @@ -1939,16 +2202,6 @@ RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_RTF is set to YES. - -RTF_SOURCE_CODE = NO - #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- @@ -2045,27 +2298,44 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an -# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures +# AutoGen Definitions (see https://autogen.sourceforge.net/) file that captures # the structure of the code including all documentation. Note that this feature # is still experimental and incomplete at the moment. # The default value is: NO. GENERATE_AUTOGEN_DEF = NO +#--------------------------------------------------------------------------- +# Configuration options related to Sqlite3 output +#--------------------------------------------------------------------------- + +# If the GENERATE_SQLITE3 tag is set to YES doxygen will generate a Sqlite3 +# database with symbols found by doxygen stored in tables. +# The default value is: NO. + +GENERATE_SQLITE3 = NO + +# The SQLITE3_OUTPUT tag is used to specify where the Sqlite3 database will be +# put. If a relative path is entered the value of OUTPUT_DIRECTORY will be put +# in front of it. +# The default directory is: sqlite3. +# This tag requires that the tag GENERATE_SQLITE3 is set to YES. + +SQLITE3_OUTPUT = sqlite3 + +# The SQLITE3_OVERWRITE_DB tag is set to YES, the existing doxygen_sqlite3.db +# database file will be recreated with each doxygen run. If set to NO, doxygen +# will warn if an a database file is already found and not modify it. +# The default value is: YES. +# This tag requires that the tag GENERATE_SQLITE3 is set to YES. + +SQLITE3_RECREATE_DB = YES + #--------------------------------------------------------------------------- # Configuration options related to the Perl module output #--------------------------------------------------------------------------- @@ -2140,7 +2410,8 @@ SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the -# preprocessor. +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @@ -2207,15 +2478,15 @@ TAGFILES = GENERATE_TAGFILE = -# If the ALLEXTERNALS tag is set to YES, all external class will be listed in -# the class index. If set to NO, only the inherited external classes will be -# listed. +# If the ALLEXTERNALS tag is set to YES, all external classes and namespaces +# will be listed in the class and namespace index. If set to NO, only the +# inherited external classes will be listed. # The default value is: NO. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will be +# in the topic index. If set to NO, only the current project's groups will be # listed. # The default value is: YES. @@ -2229,25 +2500,9 @@ EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES #--------------------------------------------------------------------------- -# Configuration options related to the dot tool +# Configuration options related to diagram generator tools #--------------------------------------------------------------------------- -# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram -# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to -# NO turns the diagrams off. Note that this option also works with HAVE_DOT -# disabled, but it is recommended to install and use dot, since it yields more -# powerful graphs. -# The default value is: YES. - -CLASS_DIAGRAMS = YES - -# You can include diagrams made with dia in doxygen documentation. Doxygen will -# then run dia to produce the diagram and insert it in the documentation. The -# DIA_PATH tag allows you to specify the directory where the dia binary resides. -# If left empty dia is assumed to be found in the default search path. - -DIA_PATH = - # If set to YES the inheritance and collaboration graphs will hide inheritance # and usage relations if the target is undocumented or is not a class. # The default value is: YES. @@ -2256,12 +2511,12 @@ HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz (see: -# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# https://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent # Bell Labs. The other options in this section have no effect if this option is # set to NO -# The default value is: NO. +# The default value is: YES. -HAVE_DOT = NO +HAVE_DOT = YES # The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed # to run in parallel. When set to 0 doxygen will base this on the number of @@ -2273,49 +2528,73 @@ HAVE_DOT = NO DOT_NUM_THREADS = 0 -# When you want a differently looking font in the dot files that doxygen -# generates you can specify the font name using DOT_FONTNAME. You need to make -# sure dot is able to find the font, which can be done by putting it in a -# standard location or by setting the DOTFONTPATH environment variable or by -# setting DOT_FONTPATH to the directory containing the font. -# The default value is: Helvetica. +# DOT_COMMON_ATTR is common attributes for nodes, edges and labels of +# subgraphs. When you want a differently looking font in the dot files that +# doxygen generates you can specify fontname, fontcolor and fontsize attributes. +# For details please see Node, +# Edge and Graph Attributes specification You need to make sure dot is able +# to find the font, which can be done by putting it in a standard location or by +# setting the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the +# directory containing the font. Default graphviz fontsize is 14. +# The default value is: fontname=Helvetica,fontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTNAME = Helvetica +DOT_COMMON_ATTR = "fontname=Helvetica,fontsize=10" -# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of -# dot graphs. -# Minimum value: 4, maximum value: 24, default value: 10. +# DOT_EDGE_ATTR is concatenated with DOT_COMMON_ATTR. For elegant style you can +# add 'arrowhead=open, arrowtail=open, arrowsize=0.5'. Complete documentation about +# arrows shapes. +# The default value is: labelfontname=Helvetica,labelfontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTSIZE = 10 +DOT_EDGE_ATTR = "labelfontname=Helvetica,labelfontsize=10" -# By default doxygen will tell dot to use the default font as specified with -# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set -# the path where dot can find it using this tag. +# DOT_NODE_ATTR is concatenated with DOT_COMMON_ATTR. For view without boxes +# around nodes set 'shape=plain' or 'shape=plaintext' Shapes specification +# The default value is: shape=box,height=0.2,width=0.4. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" + +# You can set the path where dot can find font specified with fontname in +# DOT_COMMON_ATTR and others dot attributes. # This tag requires that the tag HAVE_DOT is set to YES. DOT_FONTPATH = -# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for -# each documented class showing the direct and indirect inheritance relations. -# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# If the CLASS_GRAPH tag is set to YES or GRAPH or BUILTIN then doxygen will +# generate a graph for each documented class showing the direct and indirect +# inheritance relations. In case the CLASS_GRAPH tag is set to YES or GRAPH and +# HAVE_DOT is enabled as well, then dot will be used to draw the graph. In case +# the CLASS_GRAPH tag is set to YES and HAVE_DOT is disabled or if the +# CLASS_GRAPH tag is set to BUILTIN, then the built-in generator will be used. +# If the CLASS_GRAPH tag is set to TEXT the direct and indirect inheritance +# relations will be shown as texts / links. +# Possible values are: NO, YES, TEXT, GRAPH and BUILTIN. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. CLASS_GRAPH = YES # If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a # graph for each documented class showing the direct and indirect implementation # dependencies (inheritance, containment, and class references variables) of the -# class with other documented classes. +# class with other documented classes. Explicit enabling a collaboration graph, +# when COLLABORATION_GRAPH is set to NO, can be accomplished by means of the +# command \collaborationgraph. Disabling a collaboration graph can be +# accomplished by means of the command \hidecollaborationgraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for -# groups, showing the direct groups dependencies. +# groups, showing the direct groups dependencies. Explicit enabling a group +# dependency graph, when GROUP_GRAPHS is set to NO, can be accomplished by means +# of the command \groupgraph. Disabling a directory graph can be accomplished by +# means of the command \hidegroupgraph. See also the chapter Grouping in the +# manual. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2338,10 +2617,32 @@ UML_LOOK = NO # but if the number exceeds 15, the total amount of fields shown is limited to # 10. # Minimum value: 0, maximum value: 100, default value: 10. -# This tag requires that the tag HAVE_DOT is set to YES. +# This tag requires that the tag UML_LOOK is set to YES. UML_LIMIT_NUM_FIELDS = 10 +# If the DOT_UML_DETAILS tag is set to NO, doxygen will show attributes and +# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS +# tag is set to YES, doxygen will add type and arguments for attributes and +# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, doxygen +# will not generate fields with class member information in the UML graphs. The +# class diagrams will look similar to the default class diagrams but using UML +# notation for the relationships. +# Possible values are: NO, YES and NONE. +# The default value is: NO. +# This tag requires that the tag UML_LOOK is set to YES. + +DOT_UML_DETAILS = NO + +# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters +# to display on a single line. If the actual line length exceeds this threshold +# significantly it will wrapped across multiple lines. Some heuristics are apply +# to avoid ugly line breaks. +# Minimum value: 0, maximum value: 1000, default value: 17. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_WRAP_THRESHOLD = 17 + # If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and # collaboration graphs will show the relations between templates and their # instances. @@ -2353,7 +2654,9 @@ TEMPLATE_RELATIONS = NO # If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to # YES then doxygen will generate a graph for each documented file showing the # direct and indirect include dependencies of the file with other documented -# files. +# files. Explicit enabling an include graph, when INCLUDE_GRAPH is is set to NO, +# can be accomplished by means of the command \includegraph. Disabling an +# include graph can be accomplished by means of the command \hideincludegraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2362,7 +2665,10 @@ INCLUDE_GRAPH = YES # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are # set to YES then doxygen will generate a graph for each documented file showing # the direct and indirect include dependencies of the file with other documented -# files. +# files. Explicit enabling an included by graph, when INCLUDED_BY_GRAPH is set +# to NO, can be accomplished by means of the command \includedbygraph. Disabling +# an included by graph can be accomplished by means of the command +# \hideincludedbygraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2402,21 +2708,32 @@ GRAPHICAL_HIERARCHY = YES # If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the # dependencies a directory has on other directories in a graphical way. The # dependency relations are determined by the #include relations between the -# files in the directories. +# files in the directories. Explicit enabling a directory graph, when +# DIRECTORY_GRAPH is set to NO, can be accomplished by means of the command +# \directorygraph. Disabling a directory graph can be accomplished by means of +# the command \hidedirectorygraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. DIRECTORY_GRAPH = YES +# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels +# of child directories generated in directory dependency graphs by dot. +# Minimum value: 1, maximum value: 25, default value: 1. +# This tag requires that the tag DIRECTORY_GRAPH is set to YES. + +DIR_GRAPH_MAX_DEPTH = 1 + # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. For an explanation of the image formats see the section # output formats in the documentation of the dot tool (Graphviz (see: -# http://www.graphviz.org/)). +# https://www.graphviz.org/)). # Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order # to make the SVG files visible in IE 9+ (other browsers do not have this # requirement). -# Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo, -# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and +# Possible values are: png, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, +# gif, gif:cairo, gif:cairo:gd, gif:gd, gif:gd:gd, svg, png:gd, png:gd:gd, +# png:cairo, png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and # png:gdiplus:gdiplus. # The default value is: png. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2448,11 +2765,12 @@ DOT_PATH = DOTFILE_DIRS = -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the \mscfile -# command). +# You can include diagrams made with dia in doxygen documentation. Doxygen will +# then run dia to produce the diagram and insert it in the documentation. The +# DIA_PATH tag allows you to specify the directory where the dia binary resides. +# If left empty dia is assumed to be found in the default search path. -MSCFILE_DIRS = +DIA_PATH = # The DIAFILE_DIRS tag can be used to specify one or more directories that # contain dia files that are included in the documentation (see the \diafile @@ -2461,10 +2779,10 @@ MSCFILE_DIRS = DIAFILE_DIRS = # When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the -# path where java can find the plantuml.jar file. If left blank, it is assumed -# PlantUML is not used or called during a preprocessing step. Doxygen will -# generate a warning when it encounters a \startuml command in this case and -# will not generate output for the diagram. +# path where java can find the plantuml.jar file or to the filename of jar file +# to be used. If left blank, it is assumed PlantUML is not used or called during +# a preprocessing step. Doxygen will generate a warning when it encounters a +# \startuml command in this case and will not generate output for the diagram. PLANTUML_JAR_PATH = @@ -2502,18 +2820,6 @@ DOT_GRAPH_MAX_NODES = 50 MAX_DOT_GRAPH_DEPTH = 0 -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, because dot on Windows does not seem -# to support this out of the box. -# -# Warning: Depending on the platform used, enabling this option may lead to -# badly anti-aliased labels on the edges of a graph (i.e. they become hard to -# read). -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_TRANSPARENT = NO - # Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) support @@ -2526,14 +2832,34 @@ DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page # explaining the meaning of the various boxes and arrows in the dot generated # graphs. +# Note: This tag requires that UML_LOOK isn't set, i.e. the doxygen internal +# graphical representation for inheritance and collaboration diagrams is used. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GENERATE_LEGEND = YES -# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot +# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. +# +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. DOT_CLEANUP = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. If the MSCGEN_TOOL tag is left empty (the default), then doxygen will +# use a built-in version of mscgen tool to produce the charts. Alternatively, +# the MSCGEN_TOOL tag can also specify the name an external tool. For instance, +# specifying prog as the value, doxygen will call the tool as prog -T +# -o . The external tool should support +# output file formats "png", "eps", "svg", and "ismap". + +MSCGEN_TOOL = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the \mscfile +# command). + +MSCFILE_DIRS = diff --git a/examples/custom_tensors/CMakeLists.txt b/examples/custom_tensors/CMakeLists.txt new file mode 100644 index 0000000..8c49c1e --- /dev/null +++ b/examples/custom_tensors/CMakeLists.txt @@ -0,0 +1,51 @@ +cmake_minimum_required(VERSION 3.15) + +include(FetchContent) + +project( npy_custom VERSION 0.1.0 LANGUAGES CXX ) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +if (DEFINED ENV{LIBNPY_REPO}) + set(LIBNPY_REPO $ENV{LIBNPY_REPO}) +else () + set(LIBNPY_REPO "https://github.com/matajoh/libnpy/") +endif () + +if (DEFINED ENV{LIBNPY_TAG}) + set(LIBNPY_TAG $ENV{LIBNPY_TAG}) +else () + set(LIBNPY_TAG "main") +endif() + +FetchContent_Declare( + libnpy + GIT_REPOSITORY ${LIBNPY_REPO} + GIT_TAG ${LIBNPY_TAG} +) + +FetchContent_Declare( + Eigen + GIT_REPOSITORY https://gitlab.com/libeigen/eigen.git + GIT_TAG 3.4.0 +) + +FetchContent_MakeAvailable(libnpy) + +FetchContent_GetProperties(Eigen) +if(NOT eigen_POPULATED) + FetchContent_Populate(Eigen) + + add_subdirectory(${eigen_SOURCE_DIR} ${eigen_BINARY_DIR} EXCLUDE_FROM_ALL) +endif() + +add_executable( npy_custom custom.cpp ) +target_link_libraries( npy_custom npy::npy Eigen3::Eigen) + +enable_testing() +add_test( + NAME npy_custom + COMMAND npy_custom +) +set_property(TEST npy_custom PROPERTY LABELS TESTLABEL npy) diff --git a/examples/custom_tensors/custom.cpp b/examples/custom_tensors/custom.cpp new file mode 100644 index 0000000..6d761d6 --- /dev/null +++ b/examples/custom_tensors/custom.cpp @@ -0,0 +1,203 @@ +#include "npy/npy.h" +#include +#include +#include + +/// @brief A custom tensor class which uses Eigen matrices as the backing +/// storage. +/// @details This serves as a more complex example of how to implement a custom +/// tensor type which is compatible with the npy::load and npy::save functions. +template class EigenTensor { +public: + using MatrixT = + Eigen::Matrix; + + /// @brief Constructor. + /// @param rows the number of rows + /// @param columns the number of columns + EigenTensor(size_t rows, size_t columns) : m_matrix(rows, columns) {} + + /// @brief Constructor. + /// @param matrix an Eigen matrix to use as the backing storage + EigenTensor(MatrixT &&matrix) : m_matrix(std::move(matrix)) {} + + /// @brief Constructor. + /// @param matrix an Eigen matrix to use as the backing storage + EigenTensor(const MatrixT &matrix) : m_matrix(matrix) {} + + /// @brief Load a tensor from the provided stream. + /// @details This is one of the methods required by the library to + /// read NPY files. Note how we handle both row-major and column-major + /// storage here by using the npy::read_values function appropriately. + /// @param input the input stream + /// @param info the header information + /// @return an instance of the tensor read from the stream + static EigenTensor load(std::basic_istream &input, + const npy::header_info &info) { + if (info.shape.size() > 2) { + throw std::invalid_argument( + "Matrices of dimensionality > 2 not supported"); + } + + size_t rows = info.shape[0]; + size_t columns = 1; + if (info.shape.size() == 2) { + columns = info.shape[1]; + } + + MatrixT result(rows, columns); + + if (info.fortran_order) { + if (StorageOrder == Eigen::ColMajor || columns == 1) { + npy::read_values(input, result.data(), rows * columns, info); + } else { + /// Copy into a temporary column-major matrix and then assign to the + /// row-major result. + Eigen::Matrix cm( + rows, columns); + npy::read_values(input, cm.data(), rows * columns, info); + result = cm; + } + } else { + if (StorageOrder == Eigen::RowMajor || columns == 1) { + npy::read_values(input, result.data(), rows * columns, info); + } else { + /// Copy into a temporary row-major matrix and then assign to the + /// column-major result. + Eigen::Matrix rm( + rows, columns); + npy::read_values(input, rm.data(), rows * columns, info); + result = rm; + } + } + + return result; + } + + /// @brief Save the tensor to the provided stream. + /// @details This is one of the methods required by the library to + /// write NPY files. + void save(std::basic_ostream &output, npy::endian_t endianness) const { + npy::write_values(output, data(), size(), endianness); + } + + /// @brief Return the number of dimensions of the tensor. + /// @details This is one of the methods required by the library to + /// read and write NPY files. + /// @return the number of dimensions (2) + size_t ndim() const { return 2; } + + /// @brief Returns the dimensionality of the tensor at the specified index. + /// @details This is one of the methods required by the library to + /// read and write NPY files. + /// @param index index into the shape + /// @return the dimensionality at the index + size_t shape(int index) const { + switch (index) { + case 0: + return m_matrix.rows(); + + case 1: + return m_matrix.cols(); + + default: + throw std::invalid_argument( + "Matrix only has two dimensions (rows, columns)"); + } + } + + /// @brief The number of elements in the tensor. + /// @return the number of elements + size_t size() const { return m_matrix.rows() * m_matrix.cols(); } + + /// @brief Whether the tensor data is stored in FORTRAN, or column-major, + /// order. + /// @details This is one of the methods required by the library to + /// read and write NPY files. + /// @return whether the tensor is stored in FORTRAN order + bool fortran_order() const { return StorageOrder == Eigen::ColMajor; } + + /// @brief A pointer to the start of the underlying values buffer. + /// @return a pointer to the data + T *data() { return m_matrix.data(); } + + /// @brief A pointer to the start of the underlying values buffer. + /// @return a pointer to the data + const T *data() const { return m_matrix.data(); } + + /// @brief The data type of the tensor. + /// @details This is one of the methods required by the library to + /// read and write NPY files. + /// @return the data type + npy::data_type_t dtype() const { + if constexpr (std::is_same()) { + return npy::data_type_t::FLOAT32; + } + + if constexpr (std::is_same()) { + return npy::data_type_t::FLOAT64; + } + + throw std::invalid_argument("Unsupported matrix type"); + } + + /// @brief The data type of the tensor as a dtype string. + /// @details This is one of the methods required by the library to + /// read and write NPY files. Note that you can use the npy::to_dtype + /// function to convert a data_type_t into a dtype string with the desired + /// endianness. + /// @param endianness the endianness to use in the dtype string + /// @return the data type string + std::string dtype(npy::endian_t endianness) const { + return npy::to_dtype(dtype(), endianness); + } + + /// @brief Access the underlying Eigen matrix. + /// @return a reference to the Eigen matrix + MatrixT &matrix() { return m_matrix; } + + /// @brief Access the underlying Eigen matrix. + /// @return a const reference to the Eigen matrix + const MatrixT &matrix() const { return m_matrix; } + +private: + MatrixT m_matrix; +}; + +int main() { + EigenTensor a(Eigen::MatrixXf::Random(256, 256)); + EigenTensor b(Eigen::MatrixXf::Random(256, 512)); + auto ab = a.matrix() * b.matrix(); + // Create a double-precision version of the result in row-major order + EigenTensor c(ab.cast()); + + { + npy::npzfilewriter npz("custom.npz"); + npz.write("a", a); + npz.write("b", b); + npz.write("c", c); + } + + npy::npzfilereader npz("custom.npz"); + EigenTensor a_out = npz.read>("a"); + EigenTensor b_out = npz.read>("b"); + // read the result back out, but in column-major order this time + EigenTensor c_out = npz.read>("c"); + + if (a.matrix() != a_out.matrix()) { + std::cout << "a and a_out differ!" << std::endl; + return 1; + } + + if (b.matrix() != b_out.matrix()) { + std::cout << "b and b_out differ!" << std::endl; + return 1; + } + + if (c.matrix() != c_out.matrix()) { + std::cout << "c and c_out differ!" << std::endl; + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/examples/images/CMakeLists.txt b/examples/images/CMakeLists.txt new file mode 100644 index 0000000..be133cb --- /dev/null +++ b/examples/images/CMakeLists.txt @@ -0,0 +1,37 @@ +cmake_minimum_required(VERSION 3.15) + +include(FetchContent) + +project( npy_images VERSION 0.1.0 LANGUAGES CXX ) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +if (DEFINED ENV{LIBNPY_REPO}) + set(LIBNPY_REPO $ENV{LIBNPY_REPO}) +else () + set(LIBNPY_REPO "https://github.com/matajoh/libnpy/") +endif () + +if (DEFINED ENV{LIBNPY_TAG}) + set(LIBNPY_TAG $ENV{LIBNPY_TAG}) +else () + set(LIBNPY_TAG "main") +endif() + +FetchContent_Declare( + libnpy + GIT_REPOSITORY ${LIBNPY_REPO} + GIT_TAG ${LIBNPY_TAG} +) + +FetchContent_MakeAvailable(libnpy) + +add_executable( npy_images images.cpp ) +target_link_libraries( npy_images npy::npy ) + +enable_testing() +add_test( + NAME npy_images + COMMAND npy_images +) \ No newline at end of file diff --git a/samples/README.md b/examples/images/README.md similarity index 100% rename from samples/README.md rename to examples/images/README.md diff --git a/samples/display.png b/examples/images/display.png similarity index 100% rename from samples/display.png rename to examples/images/display.png diff --git a/samples/display.py b/examples/images/display.py similarity index 100% rename from samples/display.py rename to examples/images/display.py diff --git a/examples/images/images.cpp b/examples/images/images.cpp new file mode 100644 index 0000000..e6c694d --- /dev/null +++ b/examples/images/images.cpp @@ -0,0 +1,63 @@ +#include "npy/npy.h" + +int main() { + // create a tensor object + npy::tensor color({32, 32, 3}); + + // fill it with some data + for (int row = 0; row < color.shape(0); ++row) { + for (int col = 0; col < color.shape(1); ++col) { + color(row, col, 0) = static_cast(row << 3); + color(row, col, 1) = static_cast(col << 3); + color(row, col, 2) = 128; + } + } + + // save it to disk as an NPY file + npy::save("color.npy", color); + + // we can manually set the endianness to use + npy::save("color.npy", color, npy::endian_t::BIG); + + // the built-in tensor class also has a save method + color.save("color.npy"); + + // we can peek at the header of the file + npy::header_info header = npy::peek("color.npy"); + + // we can load it back the same way + color = npy::load("color.npy"); + + // let's create a second tensor as well + npy::tensor gray({32, 32}); + + for (int row = 0; row < gray.shape(0); ++row) { + for (int col = 0; col < gray.shape(1); ++col) { + gray(row, col) = 0.21f * color(row, col, 0) + 0.72f * color(row, col, 1) + + 0.07f * color(row, col, 2); + } + } + + // we can write them to an NPZ file + { + npy::npzfilewriter output("test.npz"); + output.write("color.npy", color); + output.write("gray.npy", gray); + } + + // and we can read them back out again + { + npy::npzfilereader input("test.npz"); + + // we can test to see if the archive contains a file + if (input.contains("color.npy")) { + // and peek at its header + header = input.peek("color.npy"); + } + + color = input.read>("color.npy"); + gray = input.read>("gray.npy"); + } + + return 0; +} \ No newline at end of file diff --git a/samples/requirements.txt b/examples/images/requirements.txt similarity index 100% rename from samples/requirements.txt rename to examples/images/requirements.txt diff --git a/include/npy/core.h b/include/npy/core.h deleted file mode 100644 index ede7a6e..0000000 --- a/include/npy/core.h +++ /dev/null @@ -1,90 +0,0 @@ -// ---------------------------------------------------------------------------- -// -// core.h -- core types, enums and functions used by the library -// -// Copyright (C) 2021 Matthew Johnson -// -// For conditions of distribution and use, see copyright notice in LICENSE -// -// ---------------------------------------------------------------------------- - -#ifndef _CORE_H_ -#define _CORE_H_ - -#include -#include -#include - -namespace npy { -/** Enumeration which represents a type of endianness */ -enum class endian_t : char { - /** Indicates that the native endianness should be used. Native in this case - * means that of the hardware the program is currently running on. - */ - NATIVE, - /** Indicates the use of big-endian encoding */ - BIG, - /** Indicates the use of little-endian encoding */ - LITTLE -}; - -/** This function will return the endianness of the current hardware. */ -inline endian_t native_endian() { - union { - std::uint32_t i; - char c[4]; - } endian_test = {0x01020304}; - - return endian_test.c[0] == 1 ? endian_t::BIG : endian_t::LITTLE; -}; - -/** This enum represents the different types of tensor data that can be stored. - */ -enum class data_type_t : char { - /** 8 bit signed integer */ - INT8, - /** 8 bit unsigned integer */ - UINT8, - /** 16-bit signed integer (short) */ - INT16, - /** 16-bit unsigned integer (ushort) */ - UINT16, - /** 32-bit signed integer (int) */ - INT32, - /** 32-bit unsigned integer (uint) */ - UINT32, - /** 64-bit integer (long) */ - INT64, - /** 64-bit unsigned integer (long) */ - UINT64, - /** 32-bit floating point value (float) */ - FLOAT32, - /** 64-bit floating point value (double) */ - FLOAT64, - /** Unicode string (std::wstring) */ - UNICODE_STRING -}; - -/** Convert a data type and endianness to a NPY dtype string. - * \param dtype the data type - * \param endian the endianness. Defaults to the current endianness of the - * caller. \return the NPY dtype string - */ -const std::string &to_dtype(data_type_t dtype, - endian_t endian = endian_t::NATIVE); - -/** Converts from an NPY dtype string to a data type and endianness. - * \param dtype the NPY dtype string - * \return a pair of data type and endianness corresponding to the input - */ -const std::pair &from_dtype(const std::string &dtype); - -typedef std::basic_istringstream imemstream; -typedef std::basic_ostringstream omemstream; - -std::ostream &operator<<(std::ostream &os, const endian_t &obj); -std::ostream &operator<<(std::ostream &os, const data_type_t &obj); - -} // namespace npy - -#endif \ No newline at end of file diff --git a/include/npy/npy.h b/include/npy/npy.h index cec5f4e..f8c529e 100644 --- a/include/npy/npy.h +++ b/include/npy/npy.h @@ -1,14 +1,19 @@ -// ---------------------------------------------------------------------------- -// -// npy.h -- methods for reading and writing the numpy lib (NPY) format. The -// implementation is based upon the description available at: -// https://docs.scipy.org/doc/numpy/reference/generated/numpy.lib.format.html -// -// Copyright (C) 2021 Matthew Johnson -// -// For conditions of distribution and use, see copyright notice in LICENSE -// -// ---------------------------------------------------------------------------- +/// ---------------------------------------------------------------------------- +/// +/// @file npy.h +/// @brief Definitions for reading and writing NPY and NPZ files +/// The libnpy library provides a means to read and write NPY and NPY +/// files from C++. methods for reading and writing the numpy lib (NPY) format. +/// The implementation is based upon the description available at: +/// https://docs.scipy.org/doc/numpy/reference/generated/numpy.lib.format.html +/// The NPZ implementation draws heavily from the PKZIP Application note: +/// https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. +/// +/// Copyright (C) 2021 Matthew Johnson +/// +/// For conditions of distribution and use, see copyright notice in LICENSE +/// +/// ---------------------------------------------------------------------------- #ifndef _NPY_H_ #define _NPY_H_ @@ -16,55 +21,128 @@ #include #include #include +#include #include +#include #include #include #include #include -#include "core.h" +#define NPY_VERSION_MAJOR 2 +#define NPY_VERSION_MINOR 0 +#define NPY_VERSION_PATCH 0 +#define NPY_VERSION_STRING "2.0.0" const int STATIC_HEADER_LENGTH = 10; namespace npy { -/** Class representing the header info for an NPY file */ +/// @brief Enumeration which represents a type of endianness +enum class endian_t : char { + /// Indicates that the native endianness should be used. Native in this case + /// means that of the hardware the program is currently running on. + NATIVE, + /// Indicates the use of big-endian encoding + BIG, + /// Indicates the use of little-endian encoding + LITTLE +}; + +/// @brief This function will return the endianness of the current hardware. +inline endian_t native_endian() { + union { + std::uint32_t i; + char c[4]; + } endian_test = {0x01020304}; + + return endian_test.c[0] == 1 ? endian_t::BIG : endian_t::LITTLE; +}; + +/// @brief This enum represents the different types of tensor data that can be +/// stored. +enum class data_type_t : char { + /// 8 bit signed integer + INT8, + /// 8 bit unsigned integer + UINT8, + /// 16-bit signed integer (short) + INT16, + /// 16-bit unsigned integer (ushort) + UINT16, + /// 32-bit signed integer (int) + INT32, + /// 32-bit unsigned integer (uint) + UINT32, + /// 64-bit integer (long) + INT64, + /// 64-bit unsigned integer (long) + UINT64, + /// 32-bit floating-point value (float) + FLOAT32, + /// 64-bit floating-point value (double) + FLOAT64, + /// 64-bit complex number (std::complex) + COMPLEX64, + /// 128-bit complex number (std::complex) + COMPLEX128, + /// Unicode string (std::wstring) + UNICODE_STRING +}; + +/// @brief Convert a data type and endianness to a NPY dtype string. +/// @param dtype the data type +/// @param endian the endianness. Defaults to the current endianness of the +/// caller. +/// @return the NPY dtype string +const std::string &to_dtype(data_type_t dtype, + endian_t endian = endian_t::NATIVE); + +/// Converts from an NPY dtype string to a data type and endianness. +/// @param dtype the NPY dtype string +/// @return a pair of data type and endianness corresponding to the input +const std::pair &from_dtype(const std::string &dtype); + +std::ostream &operator<<(std::ostream &os, const endian_t &obj); +std::ostream &operator<<(std::ostream &os, const data_type_t &obj); + +/// @brief Class representing the header info for an NPY file struct header_info { - /** Constructor. - * \param dictionary a Python-encoded dictionary containing the header - * information - */ + /// Constructor. + /// @param dictionary a Python-encoded dictionary containing the header + /// information explicit header_info(const std::string &dictionary); - /** Constructor */ + /// Constructor header_info(data_type_t dtype, npy::endian_t endianness, bool fortran_order, const std::vector &shape); - /** The data type of the NPY file */ + /// The data type of the NPY file data_type_t dtype; - /** The endianness of the data in the NPY file */ + /// The endianness of the data in the NPY file npy::endian_t endianness; - /** Whether the values in the tensor are stored in FORTRAN, or column major, - * order */ + /// Whether the values in the tensor are stored in FORTRAN, or column major, + /// order bool fortran_order; - /** A vector of values indicating the shape of each dimension of the tensor. - */ + /// A vector of values indicating the shape of each dimension of the tensor. std::vector shape; - /** Value used to indicate the maximum length of an element (used by Unicode - * strings) */ + /// Value used to indicate the maximum length of an element (used by Unicode + /// strings) std::size_t max_element_length; }; -/** Writes an NPY header to the provided stream. - * \param output the output stream - * \param dtype the NPY-encoded dtype string (includes data type and - * endianness) \param fortran_order whether the data is encoded in FORTRAN (i.e. - * column major) order \param shape a sequence of values indicating the shape of - * each dimension of the tensor \sa npy::to_dtype - */ +/// @brief Writes an NPY header to the provided stream. +/// @param output the output stream +/// @param dtype the NPY-encoded dtype string (includes data type and +/// endianness) +/// @param fortran_order whether the data is encoded in FORTRAN (i.e. column +/// major) order +/// @param shape a sequence of values indicating the shape of each dimension of +/// the tensor +/// @sa npy::to_dtype template void write_npy_header(std::basic_ostream &output, const std::string &dtype, bool fortran_order, @@ -97,116 +175,88 @@ void write_npy_header(std::basic_ostream &output, } const char header[STATIC_HEADER_LENGTH] = { - static_cast(0x93), 'N', 'U', - 'M', 'P', 'Y', - 0x01, 0x00, static_cast(dict_length), - 0x00}; + static_cast(0x93), 'N', 'U', 'M', 'P', 'Y', 0x01, 0x00, + static_cast(dict_length), 0x00}; output.write(header, STATIC_HEADER_LENGTH); output.write(reinterpret_cast(dictionary.data()), dictionary.length()); output.write(reinterpret_cast(end.data()), end.length()); } +/// @brief Write values to the provided stream. +/// @tparam T the data type +/// @tparam CHAR the character type of the output stream +/// @param output the output stream +/// @param data_ptr pointer to the start of the data buffer +/// @param num_elements the number of elements to write +/// @param endianness the endianness to use in writing the data template -void copy_to(const T *data_ptr, std::size_t num_elements, - std::basic_ostream &output, npy::endian_t endianness) { - if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { - output.write(reinterpret_cast(data_ptr), - num_elements * sizeof(T)); - } else { - CHAR buffer[sizeof(T)]; - for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { - const CHAR *start = reinterpret_cast(curr); - std::reverse_copy(start, start + sizeof(T), buffer); - output.write(buffer, sizeof(T)); - } +void write_values(std::basic_ostream &output, const T *data_ptr, + size_t num_elements, endian_t endianness); + +/// @brief Saves a tensor to the provided stream. +/// @tparam T the tensor type +/// @tparam CHAR the character type of the output stream +/// @param output the output stream +/// @param tensor the tensor +/// @param endianness the endianness to use in saving the tensor +template +void save(std::basic_ostream &output, const T &tensor, + endian_t endianness = npy::endian_t::NATIVE) { + std::vector shape; + for (size_t d = 0; d < tensor.ndim(); ++d) { + shape.push_back(tensor.shape(d)); } -} -/** Saves a tensor to the provided stream. - * \tparam T the data type - * \tparam TENSOR the tensor type. - * \param output the output stream - * \param tensor the tensor - * \param endianness the endianness to use in saving the tensor - * \sa npy::tensor - */ -template class TENSOR, typename CHAR, - std::enable_if_t::value, int> = 42> -void save(std::basic_ostream &output, const TENSOR &tensor, - endian_t endianness = npy::endian_t::NATIVE) { - auto dtype = to_dtype(tensor.dtype(), endianness); - write_npy_header(output, dtype, tensor.fortran_order(), tensor.shape()); - copy_to(tensor.data(), tensor.size(), output, endianness); + write_npy_header(output, tensor.dtype(endianness), tensor.fortran_order(), + shape); + tensor.save(output, endianness); }; -/** Saves a unicode string tensor to the provided stream. - * \tparam TENSOR the tensor type. - * \param output the output stream - * \param tensor the tensor - * \param endianness the endianness to use in saving the tensor - * \sa npy::tensor - */ -template class TENSOR, typename CHAR, - std::enable_if_t::value, int> = 42> -void save(std::basic_ostream &output, const TENSOR &tensor, +/// @brief Saves a tensor to the provided stream. +/// @tparam T the data type +/// @tparam TENSOR the tensor type +/// @tparam CHAR the character type of the output stream +/// @param output the output stream +/// @param tensor the tensor +/// @param endianness the endianness to use in saving the tensor +template class TENSOR, typename CHAR> +void save(std::basic_ostream &output, const TENSOR &tensor, endian_t endianness = npy::endian_t::NATIVE) { - std::size_t max_length = 0; - for (const auto &element : tensor) { - if (element.size() > max_length) { - max_length = element.size(); - } - } - - if (endianness == npy::endian_t::NATIVE) { - endianness = native_endian(); - } - - std::string dtype = ">U" + std::to_string(max_length); - if (endianness == npy::endian_t::LITTLE) { - dtype = " unicode(tensor.size() * max_length, 0); - auto word_start = unicode.begin(); - for (const auto &element : tensor) { - auto char_it = word_start; - for (const auto &wchar : element) { - *char_it = static_cast(wchar); - char_it += 1; - } - - word_start += max_length; - } - - copy_to(unicode.data(), unicode.size(), output, endianness); -}; + save, CHAR>(output, tensor, endianness); +} -/** Saves a tensor to the provided location on disk. - * \tparam T the data type - * \tparam TENSOR the tensor type. - * \param path a path to a valid location on disk - * \param tensor the tensor - * \param endianness the endianness to use in saving the tensor - * \sa npy::tensor - */ -template class TENSOR> -void save(const std::string &path, const TENSOR &tensor, +/// @brief Saves a tensor to the provided location on disk. +/// @tparam T the tensor type +/// @param path a path to a valid location on disk +/// @param tensor the tensor +/// @param endianness the endianness to use in saving the tensor +template +void save(const std::string &path, T &tensor, endian_t endianness = npy::endian_t::NATIVE) { std::ofstream output(path, std::ios::out | std::ios::binary); if (!output.is_open()) { throw std::invalid_argument("path"); } - save(output, tensor, endianness); + save(output, tensor, endianness); }; -/** Read an NPY header from the provided stream. - * \param input the input stream - * \return the header information - */ +/// @brief Saves a tensor to the provided location on disk. +/// @tparam T the data type +/// @tparam TENSOR the tensor type +/// @param path a path to a valid location on disk +/// @param tensor the tensor +/// @param endianness the endianness to use in saving the tensor +template class TENSOR> +void save(const std::string &path, T &tensor, + endian_t endianness = npy::endian_t::NATIVE) { + save>(path, tensor, endianness); +}; + +/// @brief Read an NPY header from the provided stream. +/// @param input the input stream +/// @return the header information template header_info read_npy_header(std::basic_istream &input) { std::uint8_t header[STATIC_HEADER_LENGTH]; @@ -233,103 +283,734 @@ header_info read_npy_header(std::basic_istream &input) { return header_info(dictionary); } +/// @brief Read values from the provided stream. +/// @tparam T the data type +/// @tparam CHAR the character type of the input stream +/// @param input the input stream +/// @param data_ptr pointer to the start of the data buffer +/// @param num_elements the number of elements to read +/// @param info the header information template -void copy_to(std::basic_istream &input, T *data_ptr, - std::size_t num_elements, npy::endian_t endianness) { - if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { - CHAR *start = reinterpret_cast(data_ptr); - input.read(start, num_elements * sizeof(T)); - } else { - CHAR buffer[sizeof(T)]; - for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { - input.read(buffer, sizeof(T)); - CHAR *start = reinterpret_cast(curr); - std::reverse_copy(buffer, buffer + sizeof(T), start); - } - } -} +void read_values(std::basic_istream &input, T *data_ptr, + size_t num_elements, const header_info &info); -/** Loads a tensor in NPY format from the provided stream. The type of the - * tensor must match the data to be read. \tparam T the data type \tparam TENSOR - * the tensor type \param input the input stream \return an object of type - * TENSOR read from the stream \sa npy::tensor - */ -template class TENSOR, typename CHAR, - std::enable_if_t::value, int> = 42> -TENSOR load(std::basic_istream &input) { +template T load(std::basic_istream &input) { header_info info = read_npy_header(input); - TENSOR tensor(info.shape, info.fortran_order); - if (info.dtype != tensor.dtype()) { - throw std::logic_error("requested dtype does not match stream's dtype"); + return T::load(input, info); +} + +/// @brief Loads a tensor in NPY format from the specified location on the disk. +/// The type of the tensor must match the data to be read. +/// @tparam T the data type +/// @tparam TENSOR the tensor type +/// @param path a valid location on the disk +/// @return an object of type TENSOR read from the stream +template T load(const std::string &path) { + std::ifstream input(path, std::ios::in | std::ios::binary); + if (!input.is_open()) { + throw std::invalid_argument("path"); } - copy_to(input, tensor.data(), tensor.size(), info.endianness); - return tensor; + return load(input); } -/** Loads a unicode string tensor in NPY format from the provided stream. The - * type of the tensor must match the data to be read. \tparam T the data type - * \tparam TENSOR the tensor type - * \param input the input stream - * \return an object of type TENSOR read from the stream - * \sa npy::tensor - */ -template class TENSOR, typename CHAR, - std::enable_if_t::value, int> = 42> -TENSOR load(std::basic_istream &input) { - header_info info = read_npy_header(input); - TENSOR tensor(info.shape, info.fortran_order); - if (info.dtype != tensor.dtype()) { - throw std::logic_error("requested dtype does not match stream's dtype"); +/// @brief Loads a tensor in NPY format from the specified location on the disk. +/// The type of the tensor must match the data to be read. +/// @tparam T the data type +/// @tparam TENSOR the tensor type +/// @param path a valid location on the disk +/// @return an object of type TENSOR read from the stream +template class TENSOR> +TENSOR load(const std::string &path) { + return load>(path); +} + +/// @brief Return the header information for an NPY file. +/// @param input the input stream containing the NPY-encoded bytes +/// @return the NPY header information +template header_info peek(std::basic_istream &input) { + return read_npy_header(input); +} + +/// @brief Return the header information for an NPY file. +/// @param path the path to the NPY file on disk +/// @return the NPY header information +header_info peek(const std::string &path); + +/// @brief Enumeration indicating the compression method to use for data in the +/// NPZ archive. +enum class compression_method_t : std::uint16_t { + /// Store the data with no compression + STORED = 0, + /// Use the DEFLATE algorithm to compress the data + DEFLATED = 8 +}; + +/// @brief Struct representing a file in the NPZ archive. +struct file_entry { + /// The name of the file + std::string filename; + /// The CRC32 checksum of the uncompressed data + std::uint32_t crc32; + /// The size of the compressed data + std::uint64_t compressed_size; + /// The size of the uncompressed data + std::uint64_t uncompressed_size; + /// The method used to compress the data + std::uint16_t compression_method; + /// The offset of the file in the archive + std::uint64_t offset; + + /// Check if this entry matches another entry + /// @param other the other entry + /// @return if these entries match + bool check(const file_entry &other) const; +}; + +/// @brief Class which handles writing of an NPZ to an in-memory string stream. +class npzstringwriter { +public: + /// @brief Constructor. + /// @param compression how the entries should be compressed + /// @param endianness the endianness to use in writing the entries + npzstringwriter( + compression_method_t compression = compression_method_t::STORED, + endian_t endianness = npy::endian_t::NATIVE); + + /// @brief Destructor. This will call @ref npy::npzstringwriter::close, if it has + /// not been called already. + ~npzstringwriter(); + + /// @brief Returns the contents of the string stream as a string. + /// @return the state of the in-memory stream + std::string str() const; + + /// @brief Writes the directory and end-matter of the NPZ file. Further writes + /// will fail. + void close(); + + /// @brief Write a tensor to the NPZ archive. + /// @tparam T the tensor type + /// @param filename the name of the file in the archive + /// @param tensor the tensor to write + template + void write(const std::string &filename, const T &tensor) { + if (m_closed) { + throw std::runtime_error("Stream is closed"); + } + + std::ostringstream output; + save(output, tensor, m_endianness); + + std::string suffix = ".npy"; + std::string name = filename; + if (name.size() < 4 || + !std::equal(suffix.rbegin(), suffix.rend(), name.rbegin())) { + name += ".npy"; + } + + write_file(name, output.str()); } - std::vector unicode(tensor.size() * info.max_element_length, 0); - copy_to(input, unicode.data(), unicode.size(), info.endianness); +private: + /// Write a file to the stream. + /// @param filename the name of the file + /// @param bytes the file data + void write_file(const std::string &filename, std::string &&bytes); + + bool m_closed; + std::ostringstream m_output; + compression_method_t m_compression_method; + endian_t m_endianness; + std::vector m_entries; +}; + +/// @brief Class which handles writing of an NPZ archive to disk. +class npzfilewriter { +public: + /// @brief Constructor. + /// @param path path to the output NPZ file + /// @param compression how the entries should be compressed + /// @param endianness the endianness to use in writing the entries + npzfilewriter(const std::string &path, + compression_method_t compression = compression_method_t::STORED, + endian_t endianness = npy::endian_t::NATIVE); + + /// @brief Constructor. + /// @param path path to the output NPZ file + /// @param compression how the entries should be compressed + /// @param endianness the endianness to use in writing the entries + npzfilewriter(const char *path, + compression_method_t compression = compression_method_t::STORED, + endian_t endianness = npy::endian_t::NATIVE); + + /// @brief Constructor. + /// @param path path to the output NPZ file + /// @param compression how the entries should be compressed + /// @param endianness the endianness to use in writing the entries + npzfilewriter(const std::filesystem::path &path, + compression_method_t compression = compression_method_t::STORED, + endian_t endianness = npy::endian_t::NATIVE); + + /// @brief Destructor. This will call @ref npy::npzfilewriter::close, if it has + /// not been called already. + ~npzfilewriter(); + + /// @brief Returns whether the NPZ file is open. + bool is_open() const; + + /// @brief Writes the directory and end-matter of the NPZ file, and closes the + /// file. Further writes will fail. + void close(); + + /// @brief Write a tensor to the NPZ archive. + /// @tparam T the tensor type + /// @param filename the name of the file in the archive + /// @param tensor the tensor to write + template + void write(const std::string &filename, const T &tensor) { + if (m_closed) { + throw std::runtime_error("Stream is closed"); + } + + std::ostringstream output; + save(output, tensor, m_endianness); - auto word_start = unicode.begin(); - for (auto &element : tensor) { - auto char_it = word_start; - for (std::size_t i = 0; i < info.max_element_length && *char_it > 0; - ++i, ++char_it) { - element.push_back(static_cast(*char_it)); + std::string suffix = ".npy"; + std::string name = filename; + if (name.size() < 4 || + !std::equal(suffix.rbegin(), suffix.rend(), name.rbegin())) { + name += ".npy"; } - word_start += info.max_element_length; + write_file(name, output.str()); } - return tensor; -} +private: + /// @brief Write a file to the stream. + /// @param filename the name of the file + /// @param bytes the file data + void write_file(const std::string &filename, std::string &&bytes); + + bool m_closed; + std::ofstream m_output; + compression_method_t m_compression_method; + endian_t m_endianness; + std::vector m_entries; +}; -/** Loads a tensor in NPY format from the specified location on the disk. The - * type of the tensor must match the data to be read. \tparam T the data type - * \tparam TENSOR the tensor type - * \param path a valid location on the disk - * \return an object of type TENSOR read from the stream - * \sa npy::tensor - */ -template class TENSOR> -TENSOR load(const std::string &path) { - std::ifstream input(path, std::ios::in | std::ios::binary); - if (!input.is_open()) { - throw std::invalid_argument("path"); +/// @brief Class handling reading of an NPZ from an in-memory string stream. +class npzstringreader { +public: + /// @brief Constructor. + /// @param bytes the contents of the stream + npzstringreader(const std::string &bytes); + + /// @brief Constructor. + /// @param bytes the contents of the stream + npzstringreader(std::string &&bytes); + + /// @brief The keys of the tensors in the NPZ + const std::vector &keys() const; + + /// @brief Returns whether this NPZ contains the specified tensor + /// @param filename the name of the tensor in the archive + /// @return whether the tensor is in the archive + bool contains(const std::string &filename); + + /// @brief Returns the header for a specified tensor. + /// @param filename the name of the tensor in the archive + /// @return the header for the tensor + header_info peek(const std::string &filename); + + /// @brief Read a tensor from the archive. + /// @details This method will throw an exception if + /// the tensor does not exist, or if the data type of the tensor does not + /// match the template type. + /// @tparam T the tensor type + /// @param filename the name of the tensor in the archive + /// @return an instance of T read from the archive + /// @sa npy::tensor + template T read(const std::string &filename) { + std::istringstream stream(read_file(filename)); + return load(stream); } - return load(input); -} + template class TENSOR> + TENSOR read(const std::string &filename) { + return read>(filename); + } -/** Return the header information for an NPY file. - * \param input the input stream containing the NPY-encoded bytes - * \return the NPY header information - */ -template header_info peek(std::basic_istream &input) { - return read_npy_header(input); -} +private: + /// @brief Reads the bytes for a file from the archive. + /// @param filename the name of the file + /// @return the raw file bytes + std::string read_file(const std::string &filename); -/** Return the header information for an NPY file. - * \param path the path to the NPY file on disk - * \return the NPY header information - */ -header_info peek(const std::string &path); + /// @brief Read all entries from the directory. + void read_entries(); + + std::istringstream m_input; + std::map m_entries; + std::vector m_keys; +}; + +/// @brief Class handling reading of an NPZ from a file on disk. +class npzfilereader { +public: + /// @brief Constructor. + /// @param path path to the input NPZ file + npzfilereader(const std::string &path); + + /// @brief Constructor. + /// @param path path to the input NPZ file + npzfilereader(const char *path); + + /// @brief Constructor. + /// @param path path to the input NPZ file + npzfilereader(const std::filesystem::path &path); + + /// @brief Whether the NPZ file is open. + bool is_open() const; + + /// @brief Closes the NPZ file. + void close(); + + /// @brief The keys of the tensors in the NPZ + const std::vector &keys() const; + + /// @brief Returns whether this NPZ contains the specified tensor + /// @param filename the name of the tensor in the archive + /// @return whether the tensor is in the archive + bool contains(const std::string &filename); + + /// @brief Returns the header for a specified tensor. + /// @param filename the name of the tensor in the archive + /// @return the header for the tensor + header_info peek(const std::string &filename); + + /// @brief Read a tensor from the archive. + /// @details This method will throw an exception if + /// the tensor does not exist, or if the data type of the tensor does not + /// match the template type. + /// @tparam T the tensor type + /// @param filename the name of the tensor in the archive + /// @return an instance of T read from the archive + template T read(const std::string &filename) { + std::istringstream stream(read_file(filename)); + return load(stream); + } + + /// @brief Read a tensor from the archive. + /// @details This method will throw an exception if + /// the tensor does not exist, or if the data type of the tensor does not + /// match the template type. + /// @tparam T the data type + /// @tparam TENSOR the tensor type + /// @param filename the name of the tensor in the archive + /// @return an instance of TENSOR read from the archive + template class TENSOR> + TENSOR read(const std::string &filename) { + read>(filename); + } + +private: + /// @brief Reads the bytes for a file from the archive. + /// @param filename the name of the file + /// @return the raw file bytes + std::string read_file(const std::string &filename); + + /// @brief Read all entries from the directory. + void read_entries(); + + std::ifstream m_input; + std::map m_entries; + std::vector m_keys; +}; + +/// @brief The default tensor class. +/// @details This class can be used as a data exchange format +/// for the library, but the methods and classes will also work with your own +/// tensor implementation. The library methods require the following methods to +/// be present in a tensor type: +/// - @ref load +/// - @ref save +/// - @ref shape +/// - @ref ndim +/// - @ref dtype +/// - @ref fortran_order +/// +/// As long as these are present and have the same semantics, the library +/// should handle them in the same was as this implementation. Only certain type +/// of tensor objects are natively supported (see @ref npy::data_type_t). +/// @note This class is not optimized for access speed. It is intended as a +/// simple data exchange format. Once the raw data has been extracted from the +/// NPY or NPZ, it is recommended to convert it to a more efficient format for +/// processing using the data() method. +template class tensor { +public: + /// The value type of the tensor. + typedef T value_type; + /// The reference type of the tensor. + typedef value_type &reference; + /// The const reference type of the tensor. + typedef const value_type &const_reference; + /// The pointer type of the tensor. + typedef value_type *pointer; + /// The const pointer type of the tensor. + typedef const value_type *const_pointer; + + /// @brief Constructor. + /// @details This will allocate a data buffer of the appropriate size in + /// row-major order. + /// @param shape the shape of the tensor + tensor(const std::vector &shape) : tensor(shape, false) {} + + /// @brief Constructor. + /// @details This will allocate a data buffer of the appropriate size. + /// @param shape the shape of the tensor + /// @param fortran_order whether the data is stored in FORTRAN, or column + /// major, order + tensor(const std::vector &shape, bool fortran_order) + : m_shape(shape), + m_ravel_strides(tensor::get_ravel_strides(shape, fortran_order)), + m_fortran_order(fortran_order), m_dtype(tensor::get_dtype()), + m_values(tensor::get_size(shape)) {} + + /// @brief Copy constructor. + tensor(const tensor &other) + : m_shape(other.m_shape), m_ravel_strides(other.m_ravel_strides), + m_fortran_order(other.m_fortran_order), m_dtype(other.m_dtype), + m_values(other.m_values) {} + + /// @brief Move constructor. + tensor(tensor &&other) + : m_shape(std::move(other.m_shape)), + m_ravel_strides(std::move(other.m_ravel_strides)), + m_fortran_order(other.m_fortran_order), m_dtype(other.m_dtype), + m_values(std::move(other.m_values)) {} + + /// @brief Load a tensor from the specified location on disk. + static tensor from_file(const std::string &path) { + return npy::load>(path); + } + + /// @brief Load a tensor from the provided stream. + /// @details This is one of the methods required by the library to + /// read NPY files. If you implement this in a custom tensor, you will + /// need to populate your internal data structure using the provided + /// stream and header information. The @ref npy::read_values method can be + /// used to read raw data from the stream. + /// @param input the input stream + /// @param info the header information + /// @return an instance of the tensor read from the stream + /// @sa npy::read_values + static tensor load(std::basic_istream &input, + const header_info &info) { + tensor result(info.shape, info.fortran_order); + if (info.dtype != result.dtype()) { + throw std::runtime_error("requested dtype does not match stream's dtype"); + } + + read_values(input, result.m_values.data(), result.m_values.size(), info); + + return result; + } + + /// @brief Save the tensor to the provided stream. + /// @details This is one of the methods required by the library to + /// write NPY files. If you implement this in a custom tensor, you will + /// need to write your internal data structure to the provided stream. The + /// @ref npy::write_values method can be used to write raw data to the stream. + /// @param output the output stream + /// @param endianness the endianness to use in writing the data + /// @sa npy::write_values + void save(std::basic_ostream &output, endian_t endianness) const { + write_values(output, m_values.data(), m_values.size(), endianness); + } + + /// @brief Variable parameter index function. + /// @param index an index into the tensor. Can be negative (in which case it + /// will work as in numpy) + /// @return the value at the provided index + template const T &operator()(Indices... index) const { + return m_values[ravel(std::vector({index...}))]; + } + + /// @brief Index function. + /// @param multi_index the index into the tensor + /// @return the value at the provided index + const T &operator()(const std::vector &multi_index) const { + return m_values[ravel(multi_index)]; + } + + /// @brief Variable parameter index function. + /// @param index an index into the tensor. Can be negative (in which case it + /// will work as in numpy) + /// @return the value at the provided index + template T &operator()(Indices... index) { + return m_values[ravel(std::vector({index...}))]; + } + + /// @brief Index function. + /// @param multi_index the index into the tensor + /// @return the value at the provided index + T &operator()(const std::vector &multi_index) { + return m_values[ravel(multi_index)]; + } + + /// @brief Iterator pointing at the beginning of the tensor in memory. + typename std::vector::iterator begin() { return m_values.begin(); } + + /// @brief Iterator pointing at the beginning of the tensor in memory. + typename std::vector::const_iterator begin() const { + return m_values.begin(); + } + + /// @brief Iterator pointing at the end of the tensor in memory. + typename std::vector::iterator end() { return m_values.end(); } + + /// @brief Iterator pointing at the end of the tensor in memory. + typename std::vector::const_iterator end() const { return m_values.end(); } + + /// @brief Sets the value at the provided index. + /// @param multi_index an index into the tensor + /// @param value the value to set + void set(const std::vector &multi_index, const T &value) { + m_values[ravel(multi_index)] = value; + } + + /// @brief Gets the value at the provided index. + /// @param multi_index the index into the tensor + /// @return the value at the provided index + const T &get(const std::vector &multi_index) const { + return m_values[ravel(multi_index)]; + } + + /// @brief The data type of the tensor. + std::string dtype(endian_t endianness) const { + return to_dtype(m_dtype, endianness); + } + + /// @brief The data type of the tensor. + /// @details This is one of the methods required by the library to + /// read and write NPY files. + /// @return the data type of the tensor + data_type_t dtype() const { return m_dtype; }; + + /// @brief The underlying values buffer. + const std::vector &values() const { return m_values; } + + /// @brief Copy values from the source to this tensor. + /// @param source pointer to the start of the source buffer + /// @param nitems the number of items to copy. Should be equal to @ref size. + void copy_from(const T *source, size_t nitems) { + if (nitems != size()) { + throw std::invalid_argument("nitems"); + } + + std::copy(source, source + nitems, m_values.begin()); + } + + /// @brief Copy values from the provided vector. + /// @param source the source vector. Should have the same size as @ref values. + void copy_from(const std::vector &source) { + if (source.size() != size()) { + throw std::invalid_argument("source.size"); + } + + std::copy(source.begin(), source.end(), m_values.begin()); + } + + /// @brief Move values from the provided vector. + /// @param source the source vector. Should have the same size as @ref values. + void move_from(std::vector &&source) { + if (source.size() != size()) { + throw std::invalid_argument("source.size"); + } + + m_values = std::move(source); + } + + /// @brief A pointer to the start of the underlying values buffer. + T *data() { return m_values.data(); } + + /// @brief A pointer to the start of the underlying values buffer. + const T *data() const { return m_values.data(); } + + /// @brief The number of elements in the tensor. + size_t size() const { return m_values.size(); } + + /// @brief The shape of the vector. Each element is the size of the + /// corresponding dimension. + const std::vector &shape() const { return m_shape; } + + /// @brief Returns the dimensionality of the tensor at the specified index. + /// @details This is one of the methods required by the library to + /// read and write NPY files. + /// @param index index into the shape + /// @return the dimensionality at the index + size_t shape(int index) const { return m_shape[index]; } + + /// @brief The number of dimensions of the tensor. + /// @details This is one of the methods required by the library to + /// read and write NPY files. + /// @return the number of dimensions + size_t ndim() const { return m_shape.size(); } + + /// @brief Whether the tensor data is stored in FORTRAN, or column-major, + /// order. + /// @details This is one of the methods required by the library to + /// read and write NPY files. + /// @return whether the tensor is stored in FORTRAN order + bool fortran_order() const { return m_fortran_order; } + + /// @brief Copy assignment operator. + tensor &operator=(const tensor &other) { + m_shape = other.m_shape; + m_ravel_strides = other.m_ravel_strides; + m_fortran_order = other.m_fortran_order; + m_dtype = other.m_dtype; + m_values = other.m_values; + return *this; + } + + /// @brief Move assignment operator. + tensor &operator=(tensor &&other) { + m_shape = std::move(other.m_shape); + m_ravel_strides = std::move(other.m_ravel_strides); + m_fortran_order = other.m_fortran_order; + m_dtype = other.m_dtype; + m_values = std::move(other.m_values); + return *this; + } + + /// @brief Save this tensor to the provided location on disk. + /// @param path a valid location on disk + /// @param endianness the endianness to use in writing the tensor + void save(const std::string &path, + endian_t endianness = npy::endian_t::NATIVE) { + npy::save(path, *this, endianness); + } + + /// @brief Ravels a multi-index into a single value indexing the buffer. + /// @tparam INDEX_IT the index iterator class + /// @tparam SHAPE_IT the shape iterator class + /// @param index the multi-index iterator + /// @param shape the shape iterator + /// @return the single value in the buffer corresponding to the multi-index + template + size_t ravel(INDEX_IT index, SHAPE_IT shape) const { + std::size_t ravel = 0; + for (auto stride = m_ravel_strides.begin(); stride < m_ravel_strides.end(); + ++index, ++shape, ++stride) { + if (*index >= *shape) { + throw std::invalid_argument("multi_index"); + } + + ravel += *index * *stride; + } + + return ravel; + } + + /// @brief Ravels a multi-index into a single value indexing the buffer. + /// @param multi_index the multi-index value + /// @return the single value in the buffer corresponding to the multi-index + size_t ravel(const std::vector &multi_index) const { + if (multi_index.size() != m_shape.size()) { + throw std::invalid_argument("multi_index"); + } + + std::vector abs_multi_index(multi_index.size()); + std::transform(multi_index.begin(), multi_index.end(), m_shape.begin(), + abs_multi_index.begin(), + [](std::int32_t index, std::size_t shape) -> std::size_t { + if (index < 0) { + return static_cast(shape + index); + } + + return static_cast(index); + }); + + return ravel(abs_multi_index); + } + + /// @brief Ravels a multi-index into a single value indexing the buffer. + /// @param abs_multi_index the multi-index value + /// @return the single value in the buffer corresponding to the multi-index + size_t ravel(const std::vector &abs_multi_index) const { + if (m_fortran_order) { + return ravel(abs_multi_index.rbegin(), m_shape.rbegin()); + } + + return ravel(abs_multi_index.begin(), m_shape.begin()); + } + +private: + std::vector m_shape; + std::vector m_ravel_strides; + bool m_fortran_order; + data_type_t m_dtype; + std::vector m_values; + + /// @brief Returns the data type for this tensor. + static data_type_t get_dtype(); + + /// @brief Gets the size of a tensor given its shape + static size_t get_size(const std::vector &shape) { + size_t size = 1; + for (auto &dim : shape) { + size *= dim; + } + + return size; + } + + /// @brief Gets the strides for ravelling + static std::vector get_ravel_strides(const std::vector &shape, + bool fortran_order) { + std::vector ravel_strides(shape.size()); + size_t stride = 1; + auto ravel = ravel_strides.rbegin(); + if (fortran_order) { + for (auto max_index = shape.begin(); max_index < shape.end(); + ++max_index, ++ravel) { + *ravel = stride; + stride *= *max_index; + } + } else { + for (auto max_index = shape.rbegin(); max_index < shape.rend(); + ++max_index, ++ravel) { + *ravel = stride; + stride *= *max_index; + } + } + + return ravel_strides; + } +}; + +/// @brief Specialization of dtype for std::wstring tensors. +template <> +inline std::string tensor::dtype(endian_t endianness) const { + std::size_t max_length = 0; + for (const auto &element : m_values) { + if (element.size() > max_length) { + max_length = element.size(); + } + } + + if (endianness == npy::endian_t::NATIVE) { + endianness = native_endian(); + } + + if (endianness == npy::endian_t::LITTLE) { + return "U" + std::to_string(max_length); +} } // namespace npy diff --git a/include/npy/npz.h b/include/npy/npz.h deleted file mode 100644 index bedd9eb..0000000 --- a/include/npy/npz.h +++ /dev/null @@ -1,213 +0,0 @@ -// ---------------------------------------------------------------------------- -// -// npz.h -- methods for reading and writing the numpy archive (NPZ) file -// format. The implementation here is based upon the PKZIP Application -// note: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. -// -// Copyright (C) 2021 Matthew Johnson -// -// For conditions of distribution and use, see copyright notice in LICENSE -// -// ---------------------------------------------------------------------------- - -#ifndef _NPZ_H_ -#define _NPZ_H_ - -#include -#include -#include -#include -#include -#include -#include - -#include "core.h" -#include "npy.h" -#include "tensor.h" - -namespace npy { -/** Enumeration indicating the compression method to use for data in the NPZ - * archive. */ -enum class compression_method_t : std::uint16_t { - /** Store the data with no compression */ - STORED = 0, - /** Use the DEFLATE algorithm to compress the data */ - DEFLATED = 8 -}; - -/** Struct representing a file in the NPZ archive. */ -struct file_entry { - /** The name of the file */ - std::string filename; - /** The CRC32 checksum of the uncompressed data */ - std::uint32_t crc32; - /** The size of the compressed data */ - std::uint64_t compressed_size; - /** The size of the uncompressed data */ - std::uint64_t uncompressed_size; - /** The method used to compress the data */ - std::uint16_t compression_method; - /** The offset of the file in the archive */ - std::uint64_t offset; - - /** Check if this entry matches another entry - * \param other the other entry - * \return if these entries match - */ - bool check(const file_entry &other) const; -}; - -/** Class representing an output stream for an NPZ archive file */ -class onpzstream { -public: - /** Constructor - * \param output the output stream to write to - * \param compression how the entries should be compressed - * \param endianness the endianness to use in writing the entries - */ - onpzstream(const std::shared_ptr &output, - compression_method_t compression = compression_method_t::STORED, - endian_t endianness = npy::endian_t::NATIVE); - - /** Constructor. - * \param path the path to the file on disk - * \param compression how the entries should be compressed - * \param endianness the endianness to use in writing the entries - */ - onpzstream(const std::string &path, - compression_method_t compression = compression_method_t::STORED, - endian_t endianness = npy::endian_t::NATIVE); - - /** Whether the underlying stream has successfully been opened. */ - bool is_open() const; - - /** Closes this stream. This will write the directory and close - * the underlying stream as well. */ - void close(); - - /** Write a tensor to the NPZ archive. - * \tparam T the data type - * \tparam TENSOR the tensor type - * \param filename the name of the file in the archive - * \param tensor the tensor to write - */ - template class TENSOR> - void write(const std::string &filename, const TENSOR &tensor) { - if (m_closed) { - throw std::logic_error("Stream is closed"); - } - - omemstream output; - save(output, tensor, m_endianness); - - std::string suffix = ".npy"; - std::string name = filename; - if (name.size() < 4 || - !std::equal(suffix.rbegin(), suffix.rend(), name.rbegin())) { - name += ".npy"; - } - - write_file(name, output.str()); - } - - /** Write a tensor to the NPZ archive. - * \tparam T the data type - * \param filename the name of the file in the archive - * \param tensor the tensor to write - */ - template - void write(const std::string &filename, const tensor &tensor) { - write(filename, tensor); - } - - /** Destructor. This will call - * \link npy::onpzstream::close \endlink, if it has not been called already. - */ - ~onpzstream(); - -private: - /** Write a file to the stream. - * \param filename the name of the file - * \param bytes the file data - */ - void write_file(const std::string &filename, std::string &&bytes); - - bool m_closed; - std::shared_ptr m_output; - compression_method_t m_compression_method; - endian_t m_endianness; - std::vector m_entries; -}; - -/** Class representing an input stream from an NPZ archive file */ -class inpzstream { -public: - /** Constructor. - * \param stream the input stream to read from - */ - inpzstream(const std::shared_ptr &stream); - - /** Constructor. - * \param path the path to the NPZ file on the disk - */ - inpzstream(const std::string &path); - - /** Whether the underlying stream has successfully been opened. */ - bool is_open() const; - - /** Closes the underlying stream. */ - void close(); - - /** The keys of the tensors in the NPZ */ - const std::vector &keys() const; - - /** Returns whether this NPZ contains the specified tensor - * \param filename the name of the tensor in the archive - * \return whether the tensor is in the archive - */ - bool contains(const std::string &filename); - - /** Returns the header for a specified tensor. - * \param filename the name of the tensor in the archive - * \return the header for the tensor - */ - header_info peek(const std::string &filename); - - /** Read a tensor from the archive. This method will thrown an exception if - * the tensor does not exist, or if the data type of the tensor does not - * match the template type. \tparam T the data type \tparam TENSOR the tensor - * type \param filename the name of the tensor in the archive. \return a - * instance of TENSOR read from the archive. \sa npy::tensor - */ - template class TENSOR> - TENSOR read(const std::string &filename) { - imemstream stream(read_file(filename)); - return load(stream); - } - - /** Read a tensor from the archive. This method will thrown an exception if - * the tensor does not exist, or if the data type of the tensor does not - * match. \tparam T the data type \param filename the name of the tensor in - * the archive. \return a instance of tensor read from the archive. - */ - template tensor read(const std::string &filename) { - return read(filename); - } - -private: - /** Reads the bytes for a file from the archive. - * \param filename the name of the file - * \return the raw file bytes - */ - std::string read_file(const std::string &filename); - - /** Read all entries from the directory. */ - void read_entries(); - - std::shared_ptr m_input; - std::map m_entries; - std::vector m_keys; -}; -} // namespace npy - -#endif \ No newline at end of file diff --git a/include/npy/tensor.h b/include/npy/tensor.h deleted file mode 100644 index d2763fe..0000000 --- a/include/npy/tensor.h +++ /dev/null @@ -1,334 +0,0 @@ -// ---------------------------------------------------------------------------- -// -// tensor.h -- default tensor class for use with the library. -// -// Copyright (C) 2021 Matthew Johnson -// -// For conditions of distribution and use, see copyright notice in LICENSE -// -// ---------------------------------------------------------------------------- - -#ifndef _TENSOR_H_ -#define _TENSOR_H_ - -#include -#include -#include -#include -#include -#include - -#include "core.h" -#include "npy.h" - -namespace npy { -/** The default tensor class. This class can be used as a data exchange format - * for the library, but the methods and classes will also work with your own - * tensor implementation. The library methods require the following methods to - * be present in a tensor type: - * - \link data \endlink - * - \link shape \endlink - * - \link size \endlink - * - \link dtype \endlink - * - \link fortran_order \endlink - * - * As long as these are present and have the same semantics, the library should - * handle them in the same was as this implementation. Only certain type of - * tensor objects are natively supported (see \link npy::data_type_t \endlink). - */ -template class tensor { -public: - /** Constructor. - * \param path the path to an NPY file on the disk - */ - explicit tensor(const std::string &path) - : tensor(npy::load(path)) {} - - /** Constructor. This will allocate a data buffer of the appropriate size in - * row-major order. \param shape the shape of the tensor - */ - tensor(const std::vector &shape) : tensor(shape, false) {} - - /** Constructor. This will allocate a data buffer of the appropriate size. - * \param shape the shape of the tensor - * \param fortran_order whether the data is stored in FORTRAN, or column - * major, order - */ - tensor(const std::vector &shape, bool fortran_order) - : m_shape(shape), - m_ravel_strides(tensor::get_ravel_strides(shape, fortran_order)), - m_fortran_order(fortran_order), m_dtype(tensor::get_dtype()), - m_values(tensor::get_size(shape)) {} - - /** Copy constructor. */ - tensor(const tensor &other) - : m_shape(other.m_shape), m_ravel_strides(other.m_ravel_strides), - m_fortran_order(other.m_fortran_order), m_dtype(other.m_dtype), - m_values(other.m_values) {} - - /** Move constructor. */ - tensor(tensor &&other) - : m_shape(std::move(other.m_shape)), - m_ravel_strides(std::move(other.m_ravel_strides)), - m_fortran_order(other.m_fortran_order), m_dtype(other.m_dtype), - m_values(std::move(other.m_values)) {} - - /** Variable parameter index function. - * \param index an index into the tensor. Can be negative (in which case it - * will work as in numpy) \return the value at the provided index - */ - template const T &operator()(Indices... index) const { - return m_values[ravel(std::vector({index...}))]; - } - - /** Index function. - * \param multi_index the index into the tensor. - * \return the value at the provided index - */ - const T &operator()(const std::vector &multi_index) const { - return m_values[ravel(multi_index)]; - } - - /** Variable parameter index function. - * \param index an index into the tensor. Can be negative (in which case it - * will work as in numpy) \return the value at the provided index - */ - template T &operator()(Indices... index) { - return m_values[ravel(std::vector({index...}))]; - } - - /** Index function. - * \param multi_index the index into the tensor. - * \return the value at the provided index - */ - T &operator()(const std::vector &multi_index) { - return m_values[ravel(multi_index)]; - } - - /** Iterator pointing at the beginning of the tensor in memory. */ - typename std::vector::iterator begin() { return m_values.begin(); } - - /** Iterator pointing at the beginning of the tensor in memory. */ - typename std::vector::const_iterator begin() const { - return m_values.begin(); - } - - /** Iterator pointing at the end of the tensor in memory. */ - typename std::vector::iterator end() { return m_values.end(); } - - /** Iterator pointing at the end of the tensor in memory. */ - typename std::vector::const_iterator end() const { return m_values.end(); } - - /** Sets the value at the provided index. - * \param multi_index an index into the tensor - * \param value the value to set - */ - void set(const std::vector &multi_index, const T &value) { - m_values[ravel(multi_index)] = value; - } - - /** Gets the value at the provided index. - * \param multi_index the index into the tensor - * \return the value at the provided index - */ - const T &get(const std::vector &multi_index) const { - return m_values[ravel(multi_index)]; - } - - /** The data type of the tensor. */ - const data_type_t dtype() const { return m_dtype; } - - /** The underlying values buffer. */ - const std::vector &values() const { return m_values; } - - /** Copy values from the source to this tensor. - * \param source pointer to the start of the source buffer - * \param nitems the number of items to copy. Should be equal to \link size - * \endlink. - */ - void copy_from(const T *source, size_t nitems) { - if (nitems != size()) { - throw std::invalid_argument("nitems"); - } - - std::copy(source, source + nitems, m_values.begin()); - } - - /** Copy values from the provided vector. - * \param source the source vector. Should have the same size as \link values - * \endlink. - */ - void copy_from(const std::vector &source) { - if (source.size() != size()) { - throw std::invalid_argument("source.size"); - } - - std::copy(source.begin(), source.end(), m_values.begin()); - } - - /** Move values from the provided vector. - * \param source the source vector. Should have the same size as \link values - * \endlink. - */ - void move_from(std::vector &&source) { - if (source.size() != size()) { - throw std::invalid_argument("source.size"); - } - - m_values = std::move(source); - } - - /** A pointer to the start of the underlying values buffer. */ - T *data() { return m_values.data(); } - - /** A pointer to the start of the underlying values buffer. */ - const T *data() const { return m_values.data(); } - - /** The number of elements in the tensor. */ - size_t size() const { return m_values.size(); } - - /** The shape of the vector. Each element is the size of the - * corresponding dimension. */ - const std::vector &shape() const { return m_shape; } - - /** Returns the dimensionality of the tensor at the specified index. - * \param index index into the shape - * \return the dimensionality at the index - */ - const size_t shape(int index) const { return m_shape[index]; } - - /** Whether the tensor data is stored in FORTRAN, or column-major, order. */ - bool fortran_order() const { return m_fortran_order; } - - /** Copy assignment operator. */ - tensor &operator=(const tensor &other) { - m_shape = other.m_shape; - m_ravel_strides = other.m_ravel_strides; - m_fortran_order = other.m_fortran_order; - m_dtype = other.m_dtype; - m_values = other.m_values; - return *this; - } - - /** Move assignment operator. */ - tensor &operator=(tensor &&other) { - m_shape = std::move(other.m_shape); - m_ravel_strides = std::move(other.m_ravel_strides); - m_fortran_order = other.m_fortran_order; - m_dtype = other.m_dtype; - m_values = std::move(other.m_values); - return *this; - } - - /** Save this tensor to the provided location on disk. - * \param path a valid location on disk - * \param endianness the endianness to use in writing the tensor - */ - void save(const std::string &path, - endian_t endianness = npy::endian_t::NATIVE) { - npy::save(path, *this, endianness); - } - - /** Ravels a multi-index into a single value indexing the buffer. - * \tparam INDEX_IT the index iterator class - * \tparam SHAPE_IT the shape iterator class - * \param index the multi-index iterator - * \param shape the shape iterator - * \return the single value in the buffer corresponding to the multi-index - */ - template - size_t ravel(INDEX_IT index, SHAPE_IT shape) const { - std::size_t ravel = 0; - for (auto stride = m_ravel_strides.begin(); stride < m_ravel_strides.end(); - ++index, ++shape, ++stride) { - if (*index >= *shape) { - throw std::out_of_range("multi_index"); - } - - ravel += *index * *stride; - } - - return ravel; - } - - /** Ravels a multi-index into a single value indexing the buffer. - * \param multi_index the multi-index value - * \return the single value in the buffer corresponding to the multi-index - */ - size_t ravel(const std::vector &multi_index) const { - if (multi_index.size() != m_shape.size()) { - throw std::invalid_argument("multi_index"); - } - - std::vector abs_multi_index(multi_index.size()); - std::transform(multi_index.begin(), multi_index.end(), m_shape.begin(), - abs_multi_index.begin(), - [](std::int32_t index, std::size_t shape) -> std::size_t { - if (index < 0) { - return static_cast(shape + index); - } - - return static_cast(index); - }); - - return ravel(abs_multi_index); - } - - /** Ravels a multi-index into a single value indexing the buffer. - * \param abs_multi_index the multi-index value - * \return the single value in the buffer corresponding to the multi-index - */ - size_t ravel(const std::vector &abs_multi_index) const { - if (m_fortran_order) { - return ravel(abs_multi_index.rbegin(), m_shape.rbegin()); - } - - return ravel(abs_multi_index.begin(), m_shape.begin()); - } - -private: - std::vector m_shape; - std::vector m_ravel_strides; - bool m_fortran_order; - data_type_t m_dtype; - std::vector m_values; - - /** Returns the data type for this tensor. */ - static data_type_t get_dtype(); - - /** Gets the size of a tensor given its shape */ - static size_t get_size(const std::vector &shape) { - size_t size = 1; - for (auto &dim : shape) { - size *= dim; - } - - return size; - } - - /** Gets the strides for ravelling */ - static std::vector get_ravel_strides(const std::vector &shape, - bool fortran_order) { - std::vector ravel_strides(shape.size()); - size_t stride = 1; - auto ravel = ravel_strides.rbegin(); - if (fortran_order) { - for (auto max_index = shape.begin(); max_index < shape.end(); - ++max_index, ++ravel) { - *ravel = stride; - stride *= *max_index; - } - } else { - for (auto max_index = shape.rbegin(); max_index < shape.rend(); - ++max_index, ++ravel) { - *ravel = stride; - stride *= *max_index; - } - } - - return ravel_strides; - } -}; -} // namespace npy - -#endif \ No newline at end of file diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt deleted file mode 100644 index 487e6d7..0000000 --- a/samples/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -add_executable( npy_images images.cpp ) -target_link_libraries( npy_images npy::npy ) - -if( INCLUDE_CSHARP ) - add_executable( npy_images_net images_net.cs ) - target_link_libraries( npy_images_net NumpyIO ${SWIG_MODULE_NumpyIONative_REAL_NAME} ) -endif() \ No newline at end of file diff --git a/samples/images.cpp b/samples/images.cpp deleted file mode 100644 index 05b5a57..0000000 --- a/samples/images.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include - -#include "npy/tensor.h" -#include "npy/npy.h" -#include "npy/npz.h" - -int main() -{ - // create a tensor object - std::vector shape({32, 32, 3}); - npy::tensor color(shape); - - // fill it with some data - for (int row = 0; row < color.shape(0); ++row) - { - for (int col = 0; col < color.shape(1); ++col) - { - color(row, col, 0) = static_cast(row << 3); - color(row, col, 1) = static_cast(col << 3); - color(row, col, 2) = 128; - } - } - - // save it to disk as an NPY file - npy::save("color.npy", color); - - // we can manually set the endianness to use - npy::save("color.npy", color, npy::endian_t::BIG); - - // the built-in tensor class also has a save method - color.save("color.npy"); - - // we can peek at the header of the file - npy::header_info header = npy::peek("color.npy"); - - // we can load it back the same way - color = npy::load("color.npy"); - - // let's create a second tensor as well - shape = {32, 32}; - npy::tensor gray(shape); - - for (int row = 0; row < gray.shape(0); ++row) - { - for (int col = 0; col < gray.shape(1); ++col) - { - gray(row, col) = 0.21f * color(row, col, 0) + - 0.72f * color(row, col, 1) + - 0.07f * color(row, col, 2); - } - } - - // we can write them to an NPZ file - { - npy::onpzstream output("test.npz"); - output.write("color.npy", color); - output.write("gray.npy", gray); - } - - // and we can read them back out again - { - npy::inpzstream input("test.npz"); - - // we can test to see if the archive contains a file - if (input.contains("color.npy")) - { - // and peek at its header - header = input.peek("color.npy"); - } - - color = input.read("color.npy"); - gray = input.read("gray.npy"); - } - - return 0; -} \ No newline at end of file diff --git a/samples/images_net.cs b/samples/images_net.cs deleted file mode 100644 index 297532d..0000000 --- a/samples/images_net.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System; -using NumpyIO; - -public unsafe class Sample -{ - public static void Main(string[] args) - { - // create a tensor object - Shape shape = new Shape(new uint[] { 32, 32, 3 }); - UInt8Tensor color = new UInt8Tensor(shape); - - // fill it with some data. - for (int row = 0; row < color.Shape[0]; ++row) - { - for (int col = 0; col < color.Shape[1]; ++col) - { - color[row, col, 0] = (byte)(row << 3); - color[row, col, 1] = (byte)(col << 3); - color[row, col, 2] = 128; - } - } - - // save it to disk as an NPY file - color.Save("color.npy"); - - // we can manually set the endianness to use - color.Save("color.npy", Endian.BIG); - - // we can peek at the header of a file - HeaderInfo header = NumpyIO.NumpyIO.Peek("color.npy"); - - // we can load it using the path constructor - color = new UInt8Tensor("color.npy"); - - // let's create a second tensor as well - shape = new Shape(new uint[] { 32, 32 }); - Float32Tensor gray = new Float32Tensor(shape); - for (int row = 0; row < gray.Shape[0]; ++row) - { - for (int col = 0; col < gray.Shape[1]; ++col) - { - gray[row, col] = 0.21f * color[row, col, 0] + - 0.72f * color[row, col, 1] + - 0.07f * color[row, col, 2]; - } - } - - // we can write them to an NPZ file - NPZOutputStream output = new NPZOutputStream("test.npz"); - output.Write("color.npy", color); - output.Write("gray.npy", gray); - output.Close(); - - // and we can read them back out again - NPZInputStream input = new NPZInputStream("test.npz"); - - // we can check of an archive contains a file - if (input.Contains("color.npy")) - { - // and peek at its header - header = input.Peek("color.npy"); - } - - color = input.ReadUInt8("color.npy"); - gray = input.ReadFloat32("gray.npy"); - input.Close(); - } -} \ No newline at end of file diff --git a/src/dtype.cpp b/src/dtype.cpp index 3968cb6..8f81735 100644 --- a/src/dtype.cpp +++ b/src/dtype.cpp @@ -1,15 +1,21 @@ #include +#include #include +#include #include -#include "npy/core.h" +#include "npy/npy.h" + +#define GETC(x) static_cast((x).get()) namespace { -std::array BIG_ENDIAN_DTYPES = { - "|i1", "|u1", ">i2", ">u2", ">i4", ">u4", ">i8", ">u8", ">f4", ">f8"}; +std::array BIG_ENDIAN_DTYPES = {"|i1", "|u1", ">i2", ">u2", + ">i4", ">u4", ">i8", ">u8", + ">f4", ">f8", ">c8", ">c16"}; -std::array LITTLE_ENDIAN_DTYPES = { - "|i1", "|u1", " LITTLE_ENDIAN_DTYPES = { + "|i1", "|u1", "> DTYPE_MAP = { {"|u1", {npy::data_type_t::UINT8, npy::endian_t::NATIVE}}, @@ -30,11 +36,18 @@ std::map> DTYPE_MAP = { {">f4", {npy::data_type_t::FLOAT32, npy::endian_t::BIG}}, {"f8", {npy::data_type_t::FLOAT64, npy::endian_t::BIG}}, -}; + {"c8", {npy::data_type_t::COMPLEX64, npy::endian_t::BIG}}, + {"c16", {npy::data_type_t::COMPLEX128, npy::endian_t::BIG}}}; } // namespace namespace npy { const std::string &to_dtype(data_type_t dtype, endian_t endianness) { + if (dtype == data_type_t::UNICODE_STRING) { + throw std::invalid_argument("U dtype must be computed dynamically"); + } + if (endianness == npy::endian_t::NATIVE) { endianness = native_endian(); } @@ -60,4 +73,496 @@ std::ostream &operator<<(std::ostream &os, const endian_t &value) { return os; } +template <> +void write_values<>(std::basic_ostream &output, const uint8_t *data_ptr, + size_t num_elements, endian_t) { + output.write(reinterpret_cast(data_ptr), num_elements); +} + +template <> +void read_values<>(std::basic_istream &input, uint8_t *data_ptr, + size_t num_elements, const header_info &) { + char *start = reinterpret_cast(data_ptr); + input.read(start, num_elements); +} + +template <> +void write_values<>(std::basic_ostream &output, const int8_t *data_ptr, + size_t num_elements, endian_t) { + output.write(reinterpret_cast(data_ptr), num_elements); +} + +template <> +void read_values<>(std::basic_istream &input, int8_t *data_ptr, + size_t num_elements, const header_info &) { + char *start = reinterpret_cast(data_ptr); + input.read(start, num_elements); +} + +template <> +void write_values<>(std::basic_ostream &output, + const uint_least16_t *data_ptr, size_t num_elements, + endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), num_elements * 2); + } else { + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const char *start = reinterpret_cast(curr); + output.put(start[1]); + output.put(start[0]); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, uint_least16_t *data_ptr, + size_t num_elements, const header_info &info) { + char *start = reinterpret_cast(data_ptr); + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + input.read(start, num_elements * 2); + } else { + char *ptr = start; + for (size_t i = 0; i < num_elements; ++i, ptr += 2) { + ptr[1] = GETC(input); + ptr[0] = GETC(input); + } + } +} + +template <> +void write_values<>(std::basic_ostream &output, + const int_least16_t *data_ptr, size_t num_elements, + endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), num_elements * 2); + } else { + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const char *start = reinterpret_cast(curr); + output.put(start[1]); + output.put(start[0]); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, int_least16_t *data_ptr, + size_t num_elements, const header_info &info) { + char *start = reinterpret_cast(data_ptr); + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + input.read(start, num_elements * 2); + } else { + char *ptr = start; + for (size_t i = 0; i < num_elements; ++i, ptr += 2) { + ptr[1] = GETC(input); + ptr[0] = GETC(input); + } + } +} + +template <> +void write_values<>(std::basic_ostream &output, + const uint_least32_t *data_ptr, size_t num_elements, + endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), num_elements * 4); + } else { + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const char *start = reinterpret_cast(curr); + output.put(start[3]); + output.put(start[2]); + output.put(start[1]); + output.put(start[0]); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, uint_least32_t *data_ptr, + size_t num_elements, const header_info &info) { + char *start = reinterpret_cast(data_ptr); + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + input.read(start, num_elements * 4); + } else { + char *ptr = start; + for (size_t i = 0; i < num_elements; ++i, ptr += 4) { + ptr[3] = GETC(input); + ptr[2] = GETC(input); + ptr[1] = GETC(input); + ptr[0] = GETC(input); + } + } +} + +template <> +void write_values<>(std::basic_ostream &output, + const int_least32_t *data_ptr, size_t num_elements, + endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), num_elements * 4); + } else { + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const char *start = reinterpret_cast(curr); + output.put(start[3]); + output.put(start[2]); + output.put(start[1]); + output.put(start[0]); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, int_least32_t *data_ptr, + size_t num_elements, const header_info &info) { + char *start = reinterpret_cast(data_ptr); + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + input.read(start, num_elements * 4); + } else { + char *ptr = start; + for (size_t i = 0; i < num_elements; ++i, ptr += 4) { + ptr[3] = GETC(input); + ptr[2] = GETC(input); + ptr[1] = GETC(input); + ptr[0] = GETC(input); + } + } +} + +template <> +void write_values<>(std::basic_ostream &output, const float *data_ptr, + size_t num_elements, endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), num_elements * 4); + } else { + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const char *start = reinterpret_cast(curr); + output.put(start[3]); + output.put(start[2]); + output.put(start[1]); + output.put(start[0]); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, float *data_ptr, + size_t num_elements, const header_info &info) { + char *start = reinterpret_cast(data_ptr); + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + input.read(start, num_elements * 4); + } else { + char *ptr = start; + for (size_t i = 0; i < num_elements; ++i, ptr += 4) { + ptr[3] = GETC(input); + ptr[2] = GETC(input); + ptr[1] = GETC(input); + ptr[0] = GETC(input); + } + } +} + +template <> +void write_values<>(std::basic_ostream &output, + const uint_least64_t *data_ptr, size_t num_elements, + endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), num_elements * 8); + } else { + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const char *start = reinterpret_cast(curr); + output.put(start[7]); + output.put(start[6]); + output.put(start[5]); + output.put(start[4]); + output.put(start[3]); + output.put(start[2]); + output.put(start[1]); + output.put(start[0]); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, uint_least64_t *data_ptr, + size_t num_elements, const header_info &info) { + char *start = reinterpret_cast(data_ptr); + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + input.read(start, num_elements * 8); + } else { + char *ptr = start; + for (size_t i = 0; i < num_elements; ++i, ptr += 8) { + ptr[7] = GETC(input); + ptr[6] = GETC(input); + ptr[5] = GETC(input); + ptr[4] = GETC(input); + ptr[3] = GETC(input); + ptr[2] = GETC(input); + ptr[1] = GETC(input); + ptr[0] = GETC(input); + } + } +} + +template <> +void write_values<>(std::basic_ostream &output, + const int_least64_t *data_ptr, size_t num_elements, + endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), num_elements * 8); + } else { + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const char *start = reinterpret_cast(curr); + output.put(start[7]); + output.put(start[6]); + output.put(start[5]); + output.put(start[4]); + output.put(start[3]); + output.put(start[2]); + output.put(start[1]); + output.put(start[0]); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, int_least64_t *data_ptr, + size_t num_elements, const header_info &info) { + char *start = reinterpret_cast(data_ptr); + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + input.read(start, num_elements * 8); + } else { + char *ptr = start; + for (size_t i = 0; i < num_elements; ++i, ptr += 8) { + ptr[7] = GETC(input); + ptr[6] = GETC(input); + ptr[5] = GETC(input); + ptr[4] = GETC(input); + ptr[3] = GETC(input); + ptr[2] = GETC(input); + ptr[1] = GETC(input); + ptr[0] = GETC(input); + } + } +} + +template <> +void write_values<>(std::basic_ostream &output, const double *data_ptr, + size_t num_elements, endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), num_elements * 8); + } else { + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const char *start = reinterpret_cast(curr); + output.put(start[7]); + output.put(start[6]); + output.put(start[5]); + output.put(start[4]); + output.put(start[3]); + output.put(start[2]); + output.put(start[1]); + output.put(start[0]); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, double *data_ptr, + size_t num_elements, const header_info &info) { + char *start = reinterpret_cast(data_ptr); + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + input.read(start, num_elements * 8); + } else { + char *ptr = start; + for (size_t i = 0; i < num_elements; ++i, ptr += 8) { + ptr[7] = GETC(input); + ptr[6] = GETC(input); + ptr[5] = GETC(input); + ptr[4] = GETC(input); + ptr[3] = GETC(input); + ptr[2] = GETC(input); + ptr[1] = GETC(input); + ptr[0] = GETC(input); + } + } +} + +template <> +void write_values<>(std::basic_ostream &output, + const std::complex *data_ptr, size_t num_elements, + endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), num_elements * 8); + } else { + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const char *start = reinterpret_cast(curr); + output.put(start[7]); + output.put(start[6]); + output.put(start[5]); + output.put(start[4]); + output.put(start[3]); + output.put(start[2]); + output.put(start[1]); + output.put(start[0]); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, + std::complex *data_ptr, size_t num_elements, + const header_info &info) { + char *start = reinterpret_cast(data_ptr); + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + input.read(start, num_elements * 8); + } else { + char *ptr = start; + for (size_t i = 0; i < num_elements; ++i, ptr += 8) { + ptr[7] = GETC(input); + ptr[6] = GETC(input); + ptr[5] = GETC(input); + ptr[4] = GETC(input); + ptr[3] = GETC(input); + ptr[2] = GETC(input); + ptr[1] = GETC(input); + ptr[0] = GETC(input); + } + } +} + +template <> +void write_values<>(std::basic_ostream &output, + const std::complex *data_ptr, size_t num_elements, + endian_t endianness) { + if (endianness == npy::endian_t::NATIVE || endianness == native_endian()) { + output.write(reinterpret_cast(data_ptr), num_elements * 16); + } else { + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + const char *start = reinterpret_cast(curr); + output.put(start[15]); + output.put(start[14]); + output.put(start[13]); + output.put(start[12]); + output.put(start[11]); + output.put(start[10]); + output.put(start[9]); + output.put(start[8]); + output.put(start[7]); + output.put(start[6]); + output.put(start[5]); + output.put(start[4]); + output.put(start[3]); + output.put(start[2]); + output.put(start[1]); + output.put(start[0]); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, + std::complex *data_ptr, size_t num_elements, + const header_info &info) { + char *start = reinterpret_cast(data_ptr); + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + input.read(start, num_elements * 16); + } else { + char *ptr = start; + for (size_t i = 0; i < num_elements; ++i, ptr += 16) { + ptr[15] = GETC(input); + ptr[14] = GETC(input); + ptr[13] = GETC(input); + ptr[12] = GETC(input); + ptr[11] = GETC(input); + ptr[10] = GETC(input); + ptr[9] = GETC(input); + ptr[8] = GETC(input); + ptr[7] = GETC(input); + ptr[6] = GETC(input); + ptr[5] = GETC(input); + ptr[4] = GETC(input); + ptr[3] = GETC(input); + ptr[2] = GETC(input); + ptr[1] = GETC(input); + ptr[0] = GETC(input); + } + } +} + +template <> +void write_values<>(std::basic_ostream &output, + const std::wstring *data_ptr, size_t num_elements, + endian_t endianness) { + size_t max_element_length = 0; + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + if (curr->size() > max_element_length) { + max_element_length = curr->size(); + } + } + + for (auto curr = data_ptr; curr < data_ptr + num_elements; ++curr) { + for (size_t i = 0; i < curr->size(); ++i) { + std::int_least32_t value = static_cast(curr->at(i)); + + const char *start = reinterpret_cast(&value); + if (endianness == npy::endian_t::NATIVE || + endianness == native_endian()) { + output.put(start[0]); + output.put(start[1]); + output.put(start[2]); + output.put(start[3]); + } else { + output.put(start[3]); + output.put(start[2]); + output.put(start[1]); + output.put(start[0]); + } + } + + for (size_t i = curr->size(); i < max_element_length; ++i) { + output.put(0); + output.put(0); + output.put(0); + output.put(0); + } + } +} + +template <> +void read_values<>(std::basic_istream &input, std::wstring *data_ptr, + size_t num_elements, const header_info &info) { + std::wstring *ptr = data_ptr; + for (size_t i = 0; i < num_elements; ++i, ++ptr) { + std::int_least32_t value = 0; + char *bytes = reinterpret_cast(&value); + for (size_t j = 0; j < info.max_element_length; ++j) { + if (info.endianness == npy::endian_t::NATIVE || + info.endianness == native_endian()) { + bytes[0] = GETC(input); + bytes[1] = GETC(input); + bytes[2] = GETC(input); + bytes[3] = GETC(input); + } else { + bytes[3] = GETC(input); + bytes[2] = GETC(input); + bytes[1] = GETC(input); + bytes[0] = GETC(input); + } + if (value == 0) { + continue; + } + + ptr->push_back(static_cast(value)); + } + } +} + } // namespace npy \ No newline at end of file diff --git a/src/npy.cpp b/src/npy.cpp index 8966983..db19808 100644 --- a/src/npy.cpp +++ b/src/npy.cpp @@ -57,12 +57,12 @@ bool read_bool(std::istream &input) { return false; } - throw std::logic_error("Dictionary value is not a boolean"); + throw std::runtime_error("Dictionary value is not a boolean"); } std::vector read_shape(std::istream &input) { read(input, '('); - std::stringstream tuple(read_to(input, ')')); + std::istringstream tuple(read_to(input, ')')); read(input, ')'); std::vector shape; @@ -107,7 +107,7 @@ header_info::header_info(const std::string &dictionary) { } else if (key == "shape") { shape = read_shape(input); } else { - throw std::logic_error("Unsupported key: " + key); + throw std::runtime_error("Unsupported key: " + key); } read(input, ','); diff --git a/src/npz.cpp b/src/npz.cpp index a192d14..51a25e0 100644 --- a/src/npz.cpp +++ b/src/npz.cpp @@ -5,10 +5,12 @@ #include #include -#include "npy/npz.h" +#include "npy/npy.h" #include "zip.h" namespace { +using namespace npy; + const std::array LOCAL_HEADER_SIG = {0x50, 0x4B, 0x03, 0x04}; const std::array CD_HEADER_SIG = {0x50, 0x4B, 0x01, 0x02}; const std::array CD_END_SIG = {0x50, 0x4B, 0x05, 0x06}; @@ -80,11 +82,17 @@ std::uint64_t read64(std::istream &stream) { } void assert_sig(std::istream &stream, - const std::array &expected) { + const std::array &expected, + const char *entity) { std::array actual; stream.read(reinterpret_cast(actual.data()), actual.size()); if (actual != expected) { - throw std::logic_error("Invalid signature (Not a valid NPZ file)"); + printf("Invalid signature when reading %s:\n", entity); + printf("actual: [%d, %d, %d, %d]\n", actual[0], actual[1], actual[2], + actual[3]); + printf("expected: [%d, %d, %d, %d]\n", expected[0], expected[1], + expected[2], expected[3]); + throw std::runtime_error("Invalid signature (Not a valid NPZ file)"); } } @@ -132,7 +140,7 @@ void read_zip64_extra(std::istream &stream, npy::file_entry &header, bool include_offset) { std::uint16_t tag = read16(stream); if (tag != ZIP64_TAG) { - throw std::logic_error("Invalid tag (expected ZIP64)"); + throw std::runtime_error("Invalid tag (expected ZIP64)"); } std::uint16_t actual_size = read16(stream); @@ -154,7 +162,7 @@ void read_zip64_extra(std::istream &stream, npy::file_entry &header, } if (actual_size < expected_size) { - throw std::logic_error("ZIP64 extra info missing"); + throw std::runtime_error("ZIP64 extra info missing"); } if (actual_size > expected_size) { @@ -200,10 +208,10 @@ void write_local_header(std::ostream &stream, const npy::file_entry &header, } npy::file_entry read_local_header(std::istream &stream) { - assert_sig(stream, LOCAL_HEADER_SIG); + assert_sig(stream, LOCAL_HEADER_SIG, "local_header"); std::uint16_t version = read16(stream); if (version > ZIP64_VERSION) { - throw std::logic_error("Unsupported NPZ version"); + throw std::runtime_error("Unsupported NPZ version"); } npy::file_entry entry; @@ -246,11 +254,11 @@ void write_central_directory_header(std::ostream &stream, } npy::file_entry read_central_directory_header(std::istream &stream) { - assert_sig(stream, CD_HEADER_SIG); + assert_sig(stream, CD_HEADER_SIG, "central_directory"); read16(stream); // version made by std::uint16_t version = read16(stream); if (version > ZIP64_VERSION) { - throw std::logic_error("Unsupported NPZ version"); + throw std::runtime_error("Unsupported NPZ version"); } npy::file_entry entry; @@ -295,7 +303,7 @@ void write_end_of_central_directory(std::ostream &stream, } CentralDirectory read_end_of_central_directory(std::istream &stream) { - assert_sig(stream, CD_END_SIG); + assert_sig(stream, CD_END_SIG, "end_of_central_directory"); CentralDirectory result; read16(stream); // number of this disk @@ -307,180 +315,266 @@ CentralDirectory read_end_of_central_directory(std::istream &stream) { return result; } -} // namespace +void read_entries(std::istream &input, + std::map &entries, + std::vector &keys) { + input.seekg(-CD_END_SIZE, std::ios::end); + CentralDirectory dir = read_end_of_central_directory(input); -namespace npy { -bool file_entry::check(const file_entry &other) const { - return !(other.filename != this->filename || other.crc32 != this->crc32 || - other.compression_method != this->compression_method || - other.compressed_size != this->compressed_size || - other.uncompressed_size != this->uncompressed_size); -} + input.seekg(dir.offset, std::ios::beg); -onpzstream::onpzstream(const std::shared_ptr &output, - compression_method_t compression, endian_t endianness) - : m_closed(false), m_output(output), m_compression_method(compression), - m_endianness(endianness) {} + for (size_t i = 0; i < dir.num_entries; ++i) { + file_entry entry = read_central_directory_header(input); + entries[entry.filename] = entry; + keys.push_back(entry.filename); + } -onpzstream::onpzstream(const std::string &path, compression_method_t method, - endian_t endianness) - : m_closed(false), m_output(std::make_shared( - path, std::ios::out | std::ios::binary)), - m_compression_method(method), m_endianness(endianness) {} + std::sort(keys.begin(), keys.end()); +} -onpzstream::~onpzstream() { - if (!m_closed) { - close(); +void close(std::ostream &output, const std::vector &entries) { + CentralDirectory dir; + dir.offset = static_cast(output.tellp()); + for (auto &header : entries) { + write_central_directory_header(output, header); } + + dir.size = static_cast(output.tellp()) - dir.offset; + dir.num_entries = static_cast(entries.size()); + write_end_of_central_directory(output, dir); } -void onpzstream::write_file(const std::string &filename, std::string &&bytes) { +void write_file(std::ostream &output, std::vector &entries, + const std::string &filename, + compression_method_t compression_method, std::string &&bytes) { std::uint32_t uncompressed_size = static_cast(bytes.size()); std::uint32_t compressed_size = 0; std::string compressed_bytes; std::uint32_t checksum = npy_crc32(bytes); - if (m_compression_method == compression_method_t::STORED) { + if (compression_method == compression_method_t::STORED) { compressed_bytes = bytes; compressed_size = uncompressed_size; - } else if (m_compression_method == compression_method_t::DEFLATED) { + } else if (compression_method == compression_method_t::DEFLATED) { compressed_bytes = npy_deflate(std::move(bytes)); compressed_size = static_cast(compressed_bytes.size()); } else { - throw std::invalid_argument("m_compression_method"); + throw std::invalid_argument("Unsupported compression method"); } file_entry entry = {filename, checksum, compressed_size, uncompressed_size, - static_cast(m_compression_method), - static_cast(m_output->tellp())}; + static_cast(compression_method), + static_cast(output.tellp())}; bool zip64 = uncompressed_size > ZIP64_LIMIT || compressed_size > ZIP64_LIMIT; - write_local_header(*m_output, entry, zip64); - m_output->write(compressed_bytes.data(), compressed_size); - m_entries.push_back(std::move(entry)); + write_local_header(output, entry, zip64); + output.write(compressed_bytes.data(), compressed_size); + entries.push_back(std::move(entry)); } -bool onpzstream::is_open() const { - std::shared_ptr output = - std::dynamic_pointer_cast(m_output); - if (output) { - return output->is_open(); +typedef union crcbytes_u { + std::uint32_t value; + std::uint8_t bytes[4]; +} CRCBytes; + +std::string read_file(std::istream &input, + const std::map &entries, + const std::string &temp_filename) { + std::string filename = temp_filename; + if (entries.count(filename) == 0) { + filename += ".npy"; + if (entries.count(filename) == 0) { + throw std::invalid_argument("filename"); + } + } + + const file_entry &entry = entries.at(filename); + input.seekg(entry.offset, std::ios::beg); + + file_entry local = read_local_header(input); + if (!entry.check(local)) { + throw std::runtime_error("Central directory and local headers disagree"); } - return true; + std::string uncompressed_bytes; + uncompressed_bytes.resize(entry.compressed_size); + input.read(reinterpret_cast(uncompressed_bytes.data()), + uncompressed_bytes.size()); + compression_method_t cmethod = + static_cast(entry.compression_method); + if (cmethod == compression_method_t::DEFLATED) { + uncompressed_bytes = npy_inflate(std::move(uncompressed_bytes)); + } + + std::uint32_t actual_crc32 = npy_crc32(uncompressed_bytes); + if (actual_crc32 != entry.crc32) { + CRCBytes actual_bytes{actual_crc32}; + CRCBytes expected_bytes{entry.crc32}; + printf("CRC mismatch when reading %s:\n", filename.c_str()); + printf("actual: [0x%x, 0x%x, 0x%x, 0x%x]\n", actual_bytes.bytes[0], + actual_bytes.bytes[1], actual_bytes.bytes[2], actual_bytes.bytes[3]); + printf("expected: [0x%x, 0x%x, 0x%x, 0x%x]\n", expected_bytes.bytes[0], + expected_bytes.bytes[1], expected_bytes.bytes[2], + expected_bytes.bytes[3]); + throw std::runtime_error("CRC mismatch"); + } + + return uncompressed_bytes; } -void onpzstream::close() { - if (!m_closed) { - CentralDirectory dir; - dir.offset = static_cast(m_output->tellp()); - for (auto &header : m_entries) { - write_central_directory_header(*m_output, header); - } +} // namespace - dir.size = static_cast(m_output->tellp()) - dir.offset; - dir.num_entries = static_cast(m_entries.size()); - write_end_of_central_directory(*m_output, dir); +namespace npy { +bool file_entry::check(const file_entry &other) const { + return !(other.filename != this->filename || other.crc32 != this->crc32 || + other.compression_method != this->compression_method || + other.compressed_size != this->compressed_size || + other.uncompressed_size != this->uncompressed_size); +} - std::shared_ptr output = - std::dynamic_pointer_cast(m_output); - if (output) { - output->close(); - } +npzstringwriter::npzstringwriter(compression_method_t compression, + endian_t endianness) + : m_closed(false), m_compression_method(compression), + m_endianness(endianness) {} - m_closed = true; +npzstringwriter::~npzstringwriter() { + if (!m_closed) { + close(); } } -inpzstream::inpzstream(const std::shared_ptr &stream) - : m_input(stream) { - read_entries(); +std::string npzstringwriter::str() const { return m_output.str(); } + +void npzstringwriter::write_file(const std::string &filename, + std::string &&bytes) { + if (m_closed) { + throw std::runtime_error("NPZ file has been closed"); + } + + ::write_file(m_output, m_entries, filename, m_compression_method, + std::move(bytes)); } -inpzstream::inpzstream(const std::string &path) { - m_input = - std::make_shared(path, std::ios::in | std::ios::binary); - read_entries(); +void npzstringwriter::close() { + if (!m_closed) { + ::close(m_output, m_entries); + m_closed = true; + } } -void inpzstream::read_entries() { - m_input->seekg(-CD_END_SIZE, std::ios::end); - CentralDirectory dir = read_end_of_central_directory(*m_input); +npzfilewriter::npzfilewriter(const std::string &path, + compression_method_t compression, + endian_t endianness) + : m_closed(false), m_output(path, std::ios::binary), + m_compression_method(compression), m_endianness(endianness) {} - m_input->seekg(dir.offset, std::ios::beg); +npzfilewriter::npzfilewriter(const std::filesystem::path &path, + compression_method_t compression, + endian_t endianness) + : m_closed(false), m_output(path, std::ios::binary), + m_compression_method(compression), m_endianness(endianness) {} - for (size_t i = 0; i < dir.num_entries; ++i) { - file_entry entry = read_central_directory_header(*m_input); - m_entries[entry.filename] = entry; - m_keys.push_back(entry.filename); - } +npzfilewriter::npzfilewriter(const char *path, compression_method_t compression, + endian_t endianness) + : m_closed(false), m_output(path, std::ios::binary), + m_compression_method(compression), m_endianness(endianness) {} - std::sort(m_keys.begin(), m_keys.end()); +npzfilewriter::~npzfilewriter() { + if (!m_closed) { + close(); + } } -const std::vector &inpzstream::keys() const { return m_keys; } +bool npzfilewriter::is_open() const { return m_output.is_open(); } -std::string inpzstream::read_file(const std::string &temp_filename) { - std::string filename = temp_filename; - if (m_entries.count(filename) == 0) { - filename += ".npy"; - if (m_entries.count(filename) == 0) { - throw std::invalid_argument("filename"); - } +void npzfilewriter::write_file(const std::string &filename, + std::string &&bytes) { + if (m_closed) { + throw std::runtime_error("NPZ file has been closed"); } - const file_entry &entry = m_entries[filename]; - m_input->seekg(entry.offset, std::ios::beg); + ::write_file(m_output, m_entries, filename, m_compression_method, + std::move(bytes)); +} - file_entry local = read_local_header(*m_input); - if (!entry.check(local)) { - throw std::logic_error("Central directory and local headers disagree"); +void npzfilewriter::close() { + if (!m_closed) { + ::close(m_output, m_entries); + m_closed = true; + m_output.close(); } +} - std::string uncompressed_bytes; - uncompressed_bytes.resize(entry.compressed_size); - m_input->read(reinterpret_cast(uncompressed_bytes.data()), - uncompressed_bytes.size()); - compression_method_t cmethod = - static_cast(entry.compression_method); - if (cmethod == compression_method_t::DEFLATED) { - uncompressed_bytes = npy_inflate(std::move(uncompressed_bytes)); - } +npzstringreader::npzstringreader(const std::string &bytes) : m_input(bytes) { + read_entries(); +} - std::uint32_t actual_crc32 = npy_crc32(uncompressed_bytes); - if (actual_crc32 != entry.crc32) { - throw std::logic_error("CRC mismatch"); - } +npzstringreader::npzstringreader(std::string &&bytes) + : m_input(std::move(bytes)) { + read_entries(); +} - return uncompressed_bytes; +void npzstringreader::read_entries() { + ::read_entries(m_input, m_entries, m_keys); } -bool inpzstream::is_open() const { - std::shared_ptr input = - std::dynamic_pointer_cast(m_input); - if (input) { - return input->is_open(); - } +const std::vector &npzstringreader::keys() const { return m_keys; } + +std::string npzstringreader::read_file(const std::string &filename) { + return ::read_file(m_input, m_entries, filename); +} + +bool npzstringreader::contains(const std::string &filename) { + return m_entries.count(filename); +} + +header_info npzstringreader::peek(const std::string &filename) { + std::istringstream stream(read_file(filename)); + return npy::peek(stream); +} - return true; +npzfilereader::npzfilereader(const std::string &path) + : m_input(path, std::ios::binary) { + read_entries(); +} + +npzfilereader::npzfilereader(const char *path) + : m_input(path, std::ios::binary) { + read_entries(); +} + +npzfilereader::npzfilereader(const std::filesystem::path &path) + : m_input(path, std::ios::binary) { + read_entries(); } -void inpzstream::close() { - std::shared_ptr input = - std::dynamic_pointer_cast(m_input); - if (input) { - input->close(); +void npzfilereader::read_entries() { + if (!m_input.is_open()) { + throw std::invalid_argument("File not found"); } + + ::read_entries(m_input, m_entries, m_keys); +} + +const std::vector &npzfilereader::keys() const { return m_keys; } + +std::string npzfilereader::read_file(const std::string &filename) { + return ::read_file(m_input, m_entries, filename); } -bool inpzstream::contains(const std::string &filename) { +bool npzfilereader::contains(const std::string &filename) { return m_entries.count(filename); } -header_info inpzstream::peek(const std::string &filename) { - imemstream stream(read_file(filename)); +header_info npzfilereader::peek(const std::string &filename) { + std::istringstream stream(read_file(filename)); return npy::peek(stream); } + +bool npzfilereader::is_open() const { return m_input.is_open(); } + +void npzfilereader::close() { m_input.close(); } + } // namespace npy \ No newline at end of file diff --git a/src/tensor.cpp b/src/tensor.cpp index 04497f2..3493dcf 100644 --- a/src/tensor.cpp +++ b/src/tensor.cpp @@ -1,4 +1,5 @@ -#include "npy/tensor.h" +#include "npy/npy.h" +#include namespace npy { template <> data_type_t tensor::get_dtype() { @@ -41,6 +42,14 @@ template <> data_type_t tensor::get_dtype() { return data_type_t::FLOAT64; }; +template <> data_type_t tensor>::get_dtype() { + return data_type_t::COMPLEX64; +} + +template <> data_type_t tensor>::get_dtype() { + return data_type_t::COMPLEX128; +} + template <> data_type_t tensor::get_dtype() { return data_type_t::UNICODE_STRING; } diff --git a/src/zip.cpp b/src/zip.cpp index 1269704..b4d1046 100644 --- a/src/zip.cpp +++ b/src/zip.cpp @@ -1,10 +1,9 @@ +#include "zip.h" #include "miniz/miniz.h" #include +#include #include -#include "npy/core.h" -#include "zip.h" - namespace { const size_t CHUNK = 1024 * 1024; const int WINDOW_BITS = -15; @@ -34,12 +33,12 @@ std::string npy_deflate(std::string &&bytes) { ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, WINDOW_BITS, MEM_LEVEL, Z_DEFAULT_STRATEGY); if (ret != Z_OK) { - throw std::logic_error("Unable to initialize deflate algorithm"); + throw std::runtime_error("Unable to initialize deflate algorithm"); } std::string str(bytes.begin(), bytes.end()); - imemstream input(std::move(str)); - omemstream output; + std::istringstream input(std::move(str)); + std::ostringstream output; /* compress until end of file */ do { @@ -50,7 +49,7 @@ std::string npy_deflate(std::string &&bytes) { } else { if (input.fail() || input.bad()) { (void)deflateEnd(&strm); - throw std::logic_error("Error reading from input stream"); + throw std::runtime_error("Error reading from input stream"); } flush = Z_NO_FLUSH; @@ -69,7 +68,7 @@ std::string npy_deflate(std::string &&bytes) { output.write(reinterpret_cast(out.data()), have); if (output.fail() || output.bad()) { (void)deflateEnd(&strm); - throw std::logic_error("Error writing to output stream"); + throw std::runtime_error("Error writing to output stream"); } } while (strm.avail_out == 0); assert(strm.avail_in == 0); /* all input will be used */ @@ -97,12 +96,12 @@ std::string npy_inflate(std::string &&bytes) { strm.next_in = Z_NULL; ret = inflateInit2(&strm, WINDOW_BITS); if (ret != Z_OK) { - throw std::logic_error("Unable to initialize inflate algorithm"); + throw std::runtime_error("Unable to initialize inflate algorithm"); } std::string str(bytes.begin(), bytes.end()); - imemstream input(std::move(str)); - omemstream output; + std::istringstream input(std::move(str)); + std::ostringstream output; /* decompress until deflate stream ends or end of file */ do { @@ -110,7 +109,7 @@ std::string npy_inflate(std::string &&bytes) { strm.avail_in = static_cast(input.gcount()); if ((input.fail() && !input.eof()) || input.bad()) { (void)inflateEnd(&strm); - throw std::logic_error("Error reading from input stream"); + throw std::runtime_error("Error reading from input stream"); } if (strm.avail_in == 0) { @@ -131,14 +130,14 @@ std::string npy_inflate(std::string &&bytes) { case Z_DATA_ERROR: case Z_MEM_ERROR: (void)inflateEnd(&strm); - throw std::logic_error("Error inflating stream"); + throw std::runtime_error("Error inflating stream"); } have = CHUNK - strm.avail_out; output.write(reinterpret_cast(out.data()), have); if (output.fail() || output.bad()) { (void)inflateEnd(&strm); - throw std::logic_error("Error writing to output stream"); + throw std::runtime_error("Error writing to output stream"); } } while (strm.avail_out == 0); @@ -152,7 +151,7 @@ std::string npy_inflate(std::string &&bytes) { return output.str(); } - throw std::logic_error("Error inflating stream"); + throw std::runtime_error("Error inflating stream"); } } // namespace npy \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 43c06b9..53716c7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -6,7 +6,6 @@ set( CPP_TEST_SOURCES set( TESTS crc32 exceptions - memstream npy_peek npy_read npy_write @@ -14,6 +13,7 @@ set( TESTS npz_read npz_write tensor + custom_tensor ) foreach( test ${TESTS} ) @@ -30,8 +30,3 @@ add_test( NAME ${test} COMMAND ${TEST_DRIVER} ${test} WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY} ) endforeach() - - -if( INCLUDE_CSHARP ) - add_subdirectory( CSharpTests ) -endif() diff --git a/test/CSharpTests/CMakeLists.txt b/test/CSharpTests/CMakeLists.txt deleted file mode 100644 index 5c428cf..0000000 --- a/test/CSharpTests/CMakeLists.txt +++ /dev/null @@ -1,38 +0,0 @@ -set( CSHARP_TEST_SOURCES - test_exceptions.cs - test_npy_peek.cs - test_npy_read.cs - test_npy_write.cs - test_npz_peek.cs - test_npz_read.cs - test_npz_write.cs -) - -if( WIN32 ) - set( TEST_FOLDER ${CMAKE_CURRENT_BINARY_DIR}/$ ) -else() - set( TEST_FOLDER ${CMAKE_CURRENT_BINARY_DIR} ) -endif() - -foreach( file ${CSHARP_TEST_SOURCES} ) - get_filename_component( test_name "${file}" NAME_WE ) - set( test_name "CSharpTests_${test_name}" ) - - add_executable( ${test_name} ${file} Test.cs ) - target_link_libraries( ${test_name} NumpyIO ${SWIG_MODULE_NumpyIONative_REAL_NAME} ) - - # copy the NumpyIO dynamic libraries to the test build folder - add_custom_command( TARGET ${test_name} POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different ${LIBNPY_CSHARP_DIR}/NumpyIO.dll ${TEST_FOLDER}/NumpyIO.dll ) - add_custom_command( TARGET ${test_name} POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different ${LIBNPY_CSHARP_DIR}/${NUMPYIO_NATIVE} ${TEST_FOLDER}/${NUMPYIO_NATIVE} ) - - if( NOT WIN32 ) - add_test( NAME ${test_name} - COMMAND ${CSHARP_INTERPRETER} ${CMAKE_CURRENT_BINARY_DIR}/${test_name}.exe - WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY} ) - else() - add_test( NAME ${test_name} - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_name}.exe - WORKING_DIRECTORY ${CMAKE_HOME_DIRECTORY} ) - endif() - -endforeach() \ No newline at end of file diff --git a/test/CSharpTests/test.cs b/test/CSharpTests/test.cs deleted file mode 100644 index d83c8a2..0000000 --- a/test/CSharpTests/test.cs +++ /dev/null @@ -1,92 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Runtime.InteropServices; -using System.Linq; -using NumpyIO; - -namespace Testing -{ - static class Test - { - public const int EXIT_SUCCESS = 0; - public const int EXIT_FAILURE = 1; - - public static void AssertEqual(T expected, T actual, ref int result, string tag) - { - if (!actual.Equals(expected)) - { - Console.WriteLine("{0} is incorrect: {1} != {2}", tag, actual, expected); - result = EXIT_FAILURE; - } - } - - public static void AssertEqual(B expected, B actual, ref int result, string tag) where B : IList - { - AssertEqual(expected.Count, actual.Count, ref result, tag + " Count"); - if (result == EXIT_SUCCESS) - { - for (int i = 0; i < actual.Count; ++i) - { - AssertEqual(expected[i], actual[i], ref result, tag + "[" + i + "]"); - if (result == EXIT_FAILURE) - { - break; - } - } - } - } - - public static void AssertEqual(Tensor expected, Tensor actual, ref int result, string tag) where B : IList - { - AssertEqual(expected.DataType, actual.DataType, ref result, tag + " DataType"); - AssertEqual(expected.FortranOrder, actual.FortranOrder, ref result, tag + " FortranOrder"); - AssertEqual>(expected.Shape.ToList(), actual.Shape.ToList(), ref result, tag + " Shape"); - AssertEqual>(expected.Values, actual.Values, ref result, tag); - } - - public static void AssertEqual(HeaderInfo expected, HeaderInfo actual, ref int result, string tag) - { - AssertEqual(expected.DataType, actual.DataType, ref result, tag + " DataType"); - AssertEqual(expected.Endianness, actual.Endianness, ref result, tag + " Endianness"); - AssertEqual(expected.FortranOrder, actual.FortranOrder, ref result, tag + " FortranOrder"); - AssertEqual>(expected.Shape.ToList(), actual.Shape.ToList(), ref result, tag + " Shape"); - } - - public static void AssertThrows(Action action, ref int result, string tag) where E:Exception - { - try - { - action(); - result = EXIT_FAILURE; - Console.WriteLine("{0} did not throw an exception", tag); - } - catch (E) - { - } - catch(Exception e) - { - result = EXIT_FAILURE; - Console.WriteLine("{0} threw an unexpected exception: {1}", tag, e); - } - } - - public static T Tensor(Shape shape) - where B : IList - where T : Tensor - { - T tensor = (T)Activator.CreateInstance(typeof(T), new object[] { shape }); - for (int i = 0; i < tensor.Size; ++i) - { - tensor.Values[i] = (D)Convert.ChangeType(i, typeof(D)); - } - - return tensor; - } - - public static string AssetPath(string filename) - { - return Path.Combine("assets", "test", filename); - } - } -} \ No newline at end of file diff --git a/test/CSharpTests/test_exceptions.cs b/test/CSharpTests/test_exceptions.cs deleted file mode 100644 index 7d130ca..0000000 --- a/test/CSharpTests/test_exceptions.cs +++ /dev/null @@ -1,111 +0,0 @@ -using System; -using System.IO; -using NumpyIO; - -namespace Testing -{ - class TestExceptions - { - static UInt8Tensor TENSOR = new UInt8Tensor(new Shape(new uint[] { 5, 2, 5 })); - static string TEMP_NPZ = "temp.npz"; - - static void PeekInvalidPath() - { - NumpyIO.NumpyIO.Peek(Path.Combine("does_not_exist", "bad.npy")); - } - - static void SaveInvalidPath() - { - TENSOR.Save(Path.Combine("does_not_exist", "bad.npy")); - } - - static void LoadInvalidPath() - { - var tensor = new UInt8Tensor(Path.Combine("does_not_exist", "bad.npy")); - } - - static void NPZInputStreamInvalidPath() - { - var stream = new NPZInputStream(Path.Combine("does_not_exist", "bad.npz")); - } - - static void NPZOutputStreamCompression() - { - CompressionMethod method = (CompressionMethod)99; - using(var stream = new NPZOutputStream(TEMP_NPZ, method)) - { - stream.Write("error.npy", TENSOR); - } - } - - static void NPZInputStreamReadInvalidFilename() - { - using(var stream = new NPZInputStream(Test.AssetPath("test.npz"))) - { - var tensor = stream.ReadUInt8("not_there.npy"); - } - } - - static void NPZInputStreamPeekInvalidFilename() - { - using(var stream = new NPZInputStream(Test.AssetPath("test.npz"))) - { - var header = stream.Peek("not_there.npy"); - } - } - - static void TensorCopyFrom() - { - UInt8Buffer buffer = new UInt8Buffer(new byte[10]); - TENSOR.CopyFrom(buffer); - } - - static void TensorIndexSize() - { - byte value = TENSOR[0, 0]; - } - - static void TensorIndexRange() - { - byte value = TENSOR[2, 3, 3]; - } - - static void NPZOutputStreamClosed() - { - using(var stream = new NPZOutputStream(TEMP_NPZ)) - { - stream.Close(); - stream.Write("error.npy", TENSOR); - } - } - - static void NPZInputStreamInvalidFile() - { - var stream = new NPZInputStream(Test.AssetPath("uint8.npy")); - } - - public static int Main(string[] args) - { - int result = Test.EXIT_SUCCESS; - - Test.AssertThrows(PeekInvalidPath, ref result, "PeekInvalidPath"); - Test.AssertThrows(SaveInvalidPath, ref result, "SaveInvalidPath"); - Test.AssertThrows(LoadInvalidPath, ref result, "LoadInvalidPath"); - Test.AssertThrows(NPZInputStreamInvalidPath, ref result, "NPZInputStreamInvalidPath"); - Test.AssertThrows(NPZInputStreamReadInvalidFilename, ref result, "NPZInputStreamReadInvalidFilename"); - Test.AssertThrows(NPZInputStreamPeekInvalidFilename, ref result, "NPZInputStreamPeekInvalidFilename"); - Test.AssertThrows(NPZOutputStreamCompression, ref result, "NPZOutputStreamCompression"); - Test.AssertThrows(TensorCopyFrom, ref result, "TensorCopyFrom"); - Test.AssertThrows(TensorIndexSize, ref result, "TensorIndexSize"); - - Test.AssertThrows(TensorIndexRange, ref result, "TensorIndexRange"); - - Test.AssertThrows(NPZInputStreamInvalidFile, ref result, "NPZInputStreamInvalidFile"); - Test.AssertThrows(NPZOutputStreamClosed, ref result, "NPZOutputStreamClosed"); - - File.Delete(TEMP_NPZ); - - return result; - } - } -} \ No newline at end of file diff --git a/test/CSharpTests/test_npy_peek.cs b/test/CSharpTests/test_npy_peek.cs deleted file mode 100644 index 2a2a032..0000000 --- a/test/CSharpTests/test_npy_peek.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using NumpyIO; - -namespace Testing -{ - class TestNPYPeek - { - static void TestPeek(ref int result, string tag, DataType dtype) - { - TestPeek(ref result, tag, dtype, Endian.LITTLE, false); - } - - static void TestPeek(ref int result, string tag, DataType dtype, Endian endianness) - { - TestPeek(ref result, tag, dtype, endianness, false); - } - - static void TestPeek(ref int result, string tag, DataType dtype, bool fortranOrder) - { - TestPeek(ref result, tag, dtype, Endian.LITTLE, fortranOrder); - } - static void TestPeek(ref int result, string tag, DataType dtype, Endian endianness, bool fortranOrder) - { - Shape shape = new Shape(new uint[]{5, 2, 5}); - HeaderInfo expected = new HeaderInfo(dtype, endianness, fortranOrder, shape); - HeaderInfo actual = NumpyIO.NumpyIO.Peek(Test.AssetPath(tag + ".npy")); - Test.AssertEqual(expected, actual, ref result, "c#_npy_peek_" + tag); - } - public static int Main() - { - int result = Test.EXIT_SUCCESS; - - TestPeek(ref result, "uint8", DataType.UINT8, Endian.NATIVE); - TestPeek(ref result, "uint8_fortran", DataType.UINT8, Endian.NATIVE,true); - TestPeek(ref result, "int8", DataType.INT8, Endian.NATIVE); - TestPeek(ref result, "uint16", DataType.UINT16); - TestPeek(ref result, "int16", DataType.INT16); - TestPeek(ref result, "uint32", DataType.UINT32); - TestPeek(ref result, "int32", DataType.INT32); - TestPeek(ref result, "int32_big", DataType.INT32, Endian.BIG); - TestPeek(ref result, "uint64", DataType.UINT64); - TestPeek(ref result, "int64", DataType.INT64); - TestPeek(ref result, "float32", DataType.FLOAT32); - TestPeek(ref result, "float64", DataType.FLOAT64); - - return result; - } - } -} \ No newline at end of file diff --git a/test/CSharpTests/test_npy_read.cs b/test/CSharpTests/test_npy_read.cs deleted file mode 100644 index e10703d..0000000 --- a/test/CSharpTests/test_npy_read.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using NumpyIO; - -namespace Testing -{ - class TestNPYRead - { - static void TestRead(ref int result, string tag) - where B : IList - where T : Tensor - { - Shape shape = new Shape(new uint[] { 5, 2, 5 }); - T expected = Test.Tensor(shape); - string path = Test.AssetPath(tag + ".npy"); - T actual = (T)Activator.CreateInstance(typeof(T), new object[] { path }); - Test.AssertEqual(expected, actual, ref result, "c#_npy_read_" + tag); - } - public static int Main() - { - int result = Test.EXIT_SUCCESS; - - TestRead(ref result, "uint8"); - TestRead(ref result, "int8"); - TestRead(ref result, "uint16"); - TestRead(ref result, "int16"); - TestRead(ref result, "uint32"); - TestRead(ref result, "int32"); - TestRead(ref result, "uint64"); - TestRead(ref result, "int64"); - - return result; - } - } -} \ No newline at end of file diff --git a/test/CSharpTests/test_npy_write.cs b/test/CSharpTests/test_npy_write.cs deleted file mode 100644 index ab43ec2..0000000 --- a/test/CSharpTests/test_npy_write.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using NumpyIO; - -namespace Testing -{ - class TestNPYWrite - { - static void TestWrite(ref int result, string tag) - where B : IList - where T : Tensor - { - Shape shape = new Shape(new uint[] { 5, 2, 5 }); - T tensor = Test.Tensor(shape); - string path = Path.GetRandomFileName(); - tensor.Save(path); - byte[] actual = File.ReadAllBytes(path); - byte[] expected = File.ReadAllBytes(Test.AssetPath(tag + ".npy")); - Test.AssertEqual(expected, actual, ref result, "c#_npy_write_" + tag); - File.Delete(path); - } - public static int Main() - { - int result = Test.EXIT_SUCCESS; - - TestWrite(ref result, "uint8"); - TestWrite(ref result, "int8"); - TestWrite(ref result, "uint16"); - TestWrite(ref result, "int16"); - TestWrite(ref result, "uint32"); - TestWrite(ref result, "int32"); - TestWrite(ref result, "uint64"); - TestWrite(ref result, "int64"); - - return result; - } - } -} \ No newline at end of file diff --git a/test/CSharpTests/test_npz_peek.cs b/test/CSharpTests/test_npz_peek.cs deleted file mode 100644 index 2f05a14..0000000 --- a/test/CSharpTests/test_npz_peek.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using NumpyIO; - -namespace Testing -{ - class TestNPZPeek - { - static void TestPeek(bool compressed, ref int result) - { - HeaderInfo expectedColor = new HeaderInfo(DataType.UINT8, Endian.NATIVE, false, new Shape(new uint[] { 5, 5, 3 })); - HeaderInfo expectedDepth = new HeaderInfo(DataType.FLOAT32, Endian.LITTLE, false, new Shape(new uint[] { 5, 5 })); - - string filename = compressed ? "test_compressed.npz" : "test.npz"; - NPZInputStream stream = new NPZInputStream(Test.AssetPath(filename)); - - HeaderInfo actualColor = stream.Peek("color.npy"); - HeaderInfo actualDepth = stream.Peek("depth.npy"); - - string tag = "c#_npz_read"; - if (compressed) - { - tag += "_compressed"; - } - - StringList keys = stream.Keys(); - Test.AssertEqual(keys[0], "color.npy", ref result, tag + " keys"); - Test.AssertEqual(keys[1], "depth.npy", ref result, tag + " keys"); - Test.AssertEqual(keys[2], "unicode.npy", ref result, tag + " keys"); - - Test.AssertEqual(false, stream.Contains("not_there.npy"), ref result, tag + " contains not_there"); - Test.AssertEqual(true, stream.Contains("color.npy"), ref result, tag + " contains color"); - Test.AssertEqual(true, stream.Contains("depth.npy"), ref result, tag + " contains depth"); - Test.AssertEqual(expectedColor, actualColor, ref result, tag + " color"); - Test.AssertEqual(expectedDepth, actualDepth, ref result, tag + " depth"); - } - public static int Main() - { - int result = Test.EXIT_SUCCESS; - - TestPeek(false, ref result); - TestPeek(true, ref result); - - return result; - } - } -} \ No newline at end of file diff --git a/test/CSharpTests/test_npz_read.cs b/test/CSharpTests/test_npz_read.cs deleted file mode 100644 index a365d62..0000000 --- a/test/CSharpTests/test_npz_read.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using NumpyIO; - -namespace Testing -{ - class TestNPZRead - { - static void TestRead(bool compressed, ref int result) - { - UInt8Tensor expectedColor = Test.Tensor(new Shape(new uint[] { 5, 5, 3 })); - Float32Tensor expectedDepth = Test.Tensor(new Shape(new uint[] { 5, 5 })); - UnicodeStringTensor expectedUnicode = Test.Tensor(new Shape(new uint[]{5, 2, 5})); - - string filename = compressed ? "test_compressed.npz" : "test.npz"; - NPZInputStream stream = new NPZInputStream(Test.AssetPath(filename)); - - UInt8Tensor actualColor = stream.ReadUInt8("color"); - Float32Tensor actualDepth = stream.ReadFloat32("depth.npy"); - UnicodeStringTensor actualUnicode = stream.ReadUnicodeString("unicode.npy"); - - string tag = "c#_npz_read"; - if (compressed) - { - tag += "_compressed"; - } - - Test.AssertEqual(expectedColor, actualColor, ref result, tag + " color"); - Test.AssertEqual(expectedDepth, actualDepth, ref result, tag + " depth"); - Test.AssertEqual(expectedUnicode, actualUnicode, ref result, tag + " unicode"); - } - public static int Main() - { - int result = Test.EXIT_SUCCESS; - - TestRead(false, ref result); - TestRead(true, ref result); - - return result; - } - } -} \ No newline at end of file diff --git a/test/CSharpTests/test_npz_write.cs b/test/CSharpTests/test_npz_write.cs deleted file mode 100644 index 748da13..0000000 --- a/test/CSharpTests/test_npz_write.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using NumpyIO; - -namespace Testing -{ - class TestNPZWrite - { - static void TestWrite(bool compressed, ref int result) - { - string filename = compressed ? "test_compressed.npz" : "test.npz"; - byte[] expected = File.ReadAllBytes(Test.AssetPath(filename)); - - UInt8Tensor color = Test.Tensor(new Shape(new uint[] { 5, 5, 3 })); - Float32Tensor depth = Test.Tensor(new Shape(new uint[] { 5, 5 })); - UnicodeStringTensor unicode = Test.Tensor(new Shape(new uint[]{5, 2, 5})); - string path = Path.GetRandomFileName(); - NPZOutputStream stream = new NPZOutputStream(path, compressed ? CompressionMethod.DEFLATED : CompressionMethod.STORED); - stream.Write("color.npy", color); - stream.Write("depth", depth); - stream.Write("unicode.npy", unicode); - stream.Close(); - - byte[] actual = File.ReadAllBytes(path); - - string tag = "c#_npz_write"; - if (compressed) - { - tag += "_compressed"; - } - Test.AssertEqual(expected, actual, ref result, tag); - - File.Delete(path); - } - - public static int Main() - { - int result = Test.EXIT_SUCCESS; - - TestWrite(false, ref result); - TestWrite(true, ref result); - - return result; - } - } -} \ No newline at end of file diff --git a/test/custom_tensor.cpp b/test/custom_tensor.cpp new file mode 100644 index 0000000..b28b22f --- /dev/null +++ b/test/custom_tensor.cpp @@ -0,0 +1,111 @@ +#include "libnpy_tests.h" +#include +#include + +template class custom_tensor { +public: + typedef T value_type; + typedef value_type &reference; + typedef const value_type &const_reference; + typedef value_type *pointer; + typedef const value_type *const_pointer; + + custom_tensor(const Shape shape, bool fortran_order = false) + : m_shape(shape), m_fortran_order(fortran_order) { + m_size = 1; + for (size_t i = 0; i < shape.size(); ++i) { + m_size *= shape[i]; + } + + m_data = new T[m_size]{}; + } + + ~custom_tensor() { delete[] m_data; } + + static custom_tensor load(std::basic_istream &input, + const npy::header_info &info) { + Shape shape; + for (size_t i = 0; i < shape.size(); ++i) { + shape[i] = info.shape[i]; + } + + custom_tensor result(shape, info.fortran_order); + npy::read_values(input, result.m_data, result.m_size, info.endianness); + } + + void save(std::basic_ostream &output, npy::endian_t endianness) const { + npy::write_values(output, m_data, m_size, endianness); + } + + T *data() { return m_data; } + + const T *data() const { return m_data; } + + std::size_t size() const { return m_size; } + + size_t shape(size_t i) const { return m_shape[i]; } + + size_t ndim() const { return m_shape.size(); } + + npy::data_type_t dtype() const { + if constexpr (std::is_same()) { + return npy::data_type_t::FLOAT32; + } + + if constexpr (std::is_same()) { + return npy::data_type_t::FLOAT64; + } + + if constexpr (std::is_same()) { + return npy::data_type_t::INT32; + } + + throw std::invalid_argument("Unsupported matrix type"); + } + + std::string dtype(npy::endian_t endianness) const { + return npy::to_dtype(dtype(), endianness); + } + + bool fortran_order() const { return m_fortran_order; } + +private: + T *m_data; + size_t m_size; + Shape m_shape; + bool m_fortran_order; +}; + +typedef custom_tensor> custom1f; +typedef custom_tensor> custom2d; +typedef custom_tensor> custom3i; + +template void populate(T &tensor) { + typename T::pointer ptr = tensor.data(); + for (size_t i = 0; i < tensor.size(); ++i, ++ptr) { + *ptr = typename T::value_type(i); + } +} + +const std::string TEMP_NPZ = "custom.npz"; + +int test_custom_tensor() { + int result = EXIT_SUCCESS; + + custom1f a({3}); + custom2d b{{3, 4}}; + custom3i c({3, 4, 5}); + + populate(a); + populate(b); + populate(c); + + { + npy::npzfilewriter npz(TEMP_NPZ); + npz.write("a", a); + } + + std::filesystem::remove(TEMP_NPZ); + + return result; +} \ No newline at end of file diff --git a/test/exceptions.cpp b/test/exceptions.cpp index b6006db..3b217ea 100644 --- a/test/exceptions.cpp +++ b/test/exceptions.cpp @@ -1,17 +1,14 @@ #include "libnpy_tests.h" -#include "npy/npy.h" -#include "npy/npz.h" -#include "npy/tensor.h" +#include namespace { -npy::tensor TENSOR(std::vector({5, 2, 5})); -void save_invalid_path() { - npy::save(test::path_join({"does_not_exist", "bad.npy"}), TENSOR); +void save_invalid_path(npy::tensor &tensor) { + npy::save(test::path_join({"does_not_exist", "bad.npy"}), tensor); } void load_invalid_path() { - npy::load( + npy::load>( test::path_join({"does_not_exist", "bad.npy"})); } @@ -19,100 +16,109 @@ void peek_invalid_path() { npy::peek(test::path_join({"does_not_exist", "bad.npy"})); } -void inpzstream_invalid_path() { - npy::inpzstream(test::path_join({"does_not_exist", "bad.npz"})); +void npzfilereader_invalid_path() { + npy::npzfilereader(test::path_join({"does_not_exist", "bad.npz"})); } -void inpzstream_read_invalid_filename() { - npy::inpzstream stream(test::path_join({"assets", "test", "test.npz"})); - npy::tensor tensor = stream.read("not_there.npy"); +void npzfilereader_read_invalid_filename() { + npy::npzfilereader stream(test::path_join({"assets", "test", "test.npz"})); + npy::tensor tensor = + stream.read>("not_there.npy"); } -void inpzstream_peek_invalid_filename() { - npy::inpzstream stream(test::path_join({"assets", "test", "test.npz"})); +void npzfilereader_peek_invalid_filename() { + npy::npzfilereader stream(test::path_join({"assets", "test", "test.npz"})); npy::header_info header = stream.peek("not_there.npy"); } -void onpzstream_compression() { +void npzfilewriter_compression(npy::tensor &tensor) { npy::compression_method_t compression_method = static_cast(99); - npy::onpzstream stream("test.npz", compression_method); - stream.write("test.npy", TENSOR); + npy::npzfilewriter stream("test.npz", compression_method); + stream.write("test.npy", tensor); } -void tensor_copy_from_0() { +void tensor_copy_from_0(npy::tensor &tensor) { std::vector buffer; - TENSOR.copy_from(buffer.data(), buffer.size()); + tensor.copy_from(buffer.data(), buffer.size()); } -void tensor_copy_from_1() { +void tensor_copy_from_1(npy::tensor &tensor) { std::vector buffer; - TENSOR.copy_from(buffer); + tensor.copy_from(buffer); } -void tensor_move_from() { +void tensor_move_from(npy::tensor &tensor) { std::vector buffer; - TENSOR.copy_from(std::move(buffer)); + tensor.move_from(std::move(buffer)); } -void tensor_index_size() { std::uint8_t value = TENSOR(0, 0); } +void tensor_index_size(npy::tensor &tensor) { + std::uint8_t value = tensor(0, 0); +} -void tensor_index_range() { std::uint8_t value = TENSOR(2, 3, 3); } +void tensor_index_range(npy::tensor &tensor) { + std::uint8_t value = tensor(2, 3, 3); +} void load_wrong_dtype() { - npy::tensor tensor = npy::load( + npy::tensor tensor = npy::load>( test::path_join({"assets", "test", "uint8.npy"})); } -void onpzstream_closed() { - npy::onpzstream stream("test.npz"); +void npzfilewriter_closed(npy::tensor &tensor) { + npy::npzfilewriter stream("test.npz"); stream.close(); - stream.write("error.npy", TENSOR); + stream.write("error.npy", tensor); } -void inpzstream_invalid_file() { - npy::inpzstream stream(test::path_join({"assets", "test", "uint8.npy"})); +void npzfilereader_invalid_file() { + npy::npzfilereader stream(test::path_join({"assets", "test", "uint8.npy"})); } +typedef npy::tensor tensor_t; + } // namespace int test_exceptions() { int result = EXIT_SUCCESS; + tensor_t tensor({5, 2, 5}); + test::assert_throws(peek_invalid_path, result, "peek_invalid_path"); - test::assert_throws(save_invalid_path, result, - "save_invalid_path"); + test::assert_throws( + save_invalid_path, tensor, result, "save_invalid_path"); test::assert_throws(load_invalid_path, result, "load_invalid_path"); - test::assert_throws(inpzstream_invalid_path, result, - "inpzstream_invalid_path"); + test::assert_throws(npzfilereader_invalid_path, result, + "npzfilereader_invalid_path"); test::assert_throws( - inpzstream_read_invalid_filename, result, - "inpzstream_read_invalid_filename"); + npzfilereader_read_invalid_filename, result, + "npzfilereader_read_invalid_filename"); test::assert_throws( - inpzstream_peek_invalid_filename, result, - "inpzstream_peek_invalid_filename"); - test::assert_throws(onpzstream_compression, result, - "onpzstream_compression"); - test::assert_throws(tensor_copy_from_0, result, - "tensor_copy_from_0"); - test::assert_throws(tensor_copy_from_1, result, - "tensor_copy_from_1"); - test::assert_throws(tensor_move_from, result, - "tensor_move_from"); - test::assert_throws(tensor_index_size, result, - "tensor_index"); - - test::assert_throws(tensor_index_range, result, - "tensor_index_range"); - - test::assert_throws(load_wrong_dtype, result, - "load_wrong_dtype"); - test::assert_throws(onpzstream_closed, result, - "onpzstream_closed"); - test::assert_throws(inpzstream_invalid_file, result, - "inpzstream_invalid_file"); + npzfilereader_peek_invalid_filename, result, + "npzfilereader_peek_invalid_filename"); + test::assert_throws( + npzfilewriter_compression, tensor, result, "npzfilewriter_compression"); + test::assert_throws( + tensor_copy_from_0, tensor, result, "tensor_copy_from_0"); + test::assert_throws( + tensor_copy_from_1, tensor, result, "tensor_copy_from_1"); + test::assert_throws( + tensor_move_from, tensor, result, "tensor_move_from"); + test::assert_throws( + tensor_index_size, tensor, result, "tensor_index"); + + test::assert_throws( + tensor_index_range, tensor, result, "tensor_index_range"); + + test::assert_throws(load_wrong_dtype, result, + "load_wrong_dtype"); + test::assert_throws( + npzfilewriter_closed, tensor, result, "npzfilewriter_closed"); + test::assert_throws(npzfilereader_invalid_file, result, + "npzfilereader_invalid_file"); std::remove("test.npz"); diff --git a/test/generate_large_test.py b/test/generate_large_test.py index 65bb4d9..960aefd 100644 --- a/test/generate_large_test.py +++ b/test/generate_large_test.py @@ -3,13 +3,15 @@ import numpy as np if __name__ == "__main__": + path = os.path.join(os.path.dirname(__file__), "..", + "assets", "test", "test_large.npz") + if os.path.exists(path): + exit() + test_int = np.arange(1000000).astype(np.int32) test_int = test_int.reshape(200, 5, 1000) test_float = np.arange(1000000).astype(np.float32) test_float = test_float.reshape(1000, 5, 20, 10) - - path = os.path.join(os.path.dirname(__file__), "..", - "assets", "test", "test_large.npz") np.savez(path, test_int=test_int, test_float=test_float) path = os.path.join(os.path.dirname(__file__), "..", diff --git a/test/libnpy_tests.cpp b/test/libnpy_tests.cpp index 8f70fa3..84bb64e 100644 --- a/test/libnpy_tests.cpp +++ b/test/libnpy_tests.cpp @@ -21,7 +21,7 @@ inline char sep() { namespace test { std::string path_join(const std::vector &parts) { - std::stringstream result; + std::ostringstream result; result << parts.front(); for (auto it = parts.begin() + 1; it < parts.end(); ++it) { @@ -59,7 +59,6 @@ int main(int argc, char **argv) { tests["crc32"] = test_crc32; tests["exceptions"] = test_exceptions; - tests["memstream"] = test_memstream; tests["npy_peek"] = test_npy_peek; tests["npy_read"] = test_npy_read; tests["npy_write"] = test_npy_write; @@ -67,6 +66,7 @@ int main(int argc, char **argv) { tests["npz_read"] = test_npz_read; tests["npz_write"] = test_npz_write; tests["tensor"] = test_tensor; + tests["custom_tensor"] = test_custom_tensor; if (argc == 2) { std::string test(argv[1]); diff --git a/test/libnpy_tests.h b/test/libnpy_tests.h index 0c96471..94f9e83 100644 --- a/test/libnpy_tests.h +++ b/test/libnpy_tests.h @@ -6,13 +6,10 @@ #include #include -#include "npy/core.h" #include "npy/npy.h" -#include "npy/tensor.h" int test_crc32(); int test_exceptions(); -int test_memstream(); int test_npy_peek(); int test_npy_read(); int test_npy_write(); @@ -20,6 +17,7 @@ int test_npz_peek(); int test_npz_read(); int test_npz_write(); int test_tensor(); +int test_custom_tensor(); namespace test { template @@ -50,8 +48,7 @@ void assert_equal(const std::vector &expected, const std::vector &actual, template void assert_equal(const npy::tensor &expected, const npy::tensor &actual, int &result, const std::string &tag) { - assert_equal(to_dtype(expected.dtype()), to_dtype(actual.dtype()), result, - tag + " dtype"); + assert_equal(expected.dtype(), actual.dtype(), result, tag + " dtype"); assert_equal(expected.fortran_order(), actual.fortran_order(), result, tag + " fortran_order"); assert_equal(expected.shape(), actual.shape(), result, tag + " shape"); @@ -112,8 +109,7 @@ inline void assert_equal>( const npy::tensor &expected, const npy::tensor &actual, int &result, const std::string &tag) { - assert_equal(to_dtype(expected.dtype()), to_dtype(actual.dtype()), result, - tag + " dtype"); + assert_equal(expected.dtype(), actual.dtype(), result, tag + " dtype"); assert_equal(expected.fortran_order(), actual.fortran_order(), result, tag + " fortran_order"); assert_equal(expected.shape(), actual.shape(), result, tag + " shape"); @@ -138,9 +134,32 @@ void assert_throws(void (*function)(), int &result, const std::string &tag) { function(); result = EXIT_FAILURE; std::cout << tag << " did not throw an exception" << std::endl; - } catch (EXCEPTION &) { - } catch (std::exception &e) { + } catch (const EXCEPTION &) { + std::cout << tag << " expected exception thrown" << std::endl; + return; + } catch (const std::exception &e) { +#ifndef __APPLE__ // macos sometimes falls through erroneously. result = EXIT_FAILURE; +#endif + std::cout << tag << " threw unexpected exception: " << e.what() + << std::endl; + } +} + +template +void assert_throws(void (*function)(Arg), Arg arg, int &result, + const std::string &tag) { + try { + function(arg); + result = EXIT_FAILURE; + std::cout << tag << " did not throw an exception" << std::endl; + } catch (const EXCEPTION &) { + std::cout << tag << " expected exception thrown" << std::endl; + return; + } catch (const std::exception &e) { +#ifndef __APPLE__ // macos sometimes falls through erroneously. + result = EXIT_FAILURE; +#endif std::cout << tag << " threw unexpected exception: " << e.what() << std::endl; } @@ -205,7 +224,7 @@ template std::string npy_stream(npy::endian_t endianness = npy::endian_t::NATIVE) { std::ostringstream actual_stream; npy::tensor tensor = test_tensor({5, 2, 5}); - npy::save(actual_stream, tensor, endianness); + npy::save(actual_stream, tensor, endianness); return actual_stream.str(); } diff --git a/test/memstream.cpp b/test/memstream.cpp deleted file mode 100644 index e042ec5..0000000 --- a/test/memstream.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include - -#include "libnpy_tests.h" - -namespace { -const size_t SIZE = 50; - -void test_read(int &result) { - std::vector values(SIZE); - std::iota(values.begin(), values.end(), 0); - std::string expected(values.begin(), values.end()); - - npy::imemstream stream(expected); - stream.read(values.data(), SIZE); - std::string actual(values.begin(), values.end()); - - test::assert_equal(expected, actual, result, "memstream_test_copy_read"); - - stream = npy::imemstream(std::move(expected)); - std::fill(actual.begin(), actual.end(), 0); - stream.read(actual.data(), SIZE); - - expected = std::move(stream.str()); - - test::assert_equal(expected, actual, result, "memstream_test_move_read"); -} - -void test_write(int &result) { - std::vector values(SIZE); - std::iota(values.begin(), values.end(), 0); - std::string expected(values.begin(), values.end()); - - npy::omemstream stream; - stream.write(expected.data(), SIZE); - - std::string actual = stream.str(); - test::assert_equal(expected, actual, result, "memstream_test_copy_write"); - - std::fill(actual.begin(), actual.end(), 0); - stream = npy::omemstream(std::move(actual)); - stream.write(expected.data(), SIZE); - actual = std::move(stream.str()); - - test::assert_equal(expected, actual, result, "memstream_test_move_write"); -} -} // namespace - -int test_memstream() { - int result = EXIT_SUCCESS; - - test_read(result); - - return result; -} \ No newline at end of file diff --git a/test/npy_read.cpp b/test/npy_read.cpp index 183edc8..aaf8b75 100644 --- a/test/npy_read.cpp +++ b/test/npy_read.cpp @@ -1,5 +1,6 @@ #include "npy_read.h" #include "libnpy_tests.h" +#include int test_npy_read() { int result = EXIT_SUCCESS; @@ -18,6 +19,8 @@ int test_npy_read() { test_read(result, "int64"); test_read(result, "float32"); test_read(result, "float64"); + test_read>(result, "complex64"); + test_read>(result, "complex128"); test_read(result, "unicode"); return result; diff --git a/test/npy_read.h b/test/npy_read.h index 8650620..f91df39 100644 --- a/test/npy_read.h +++ b/test/npy_read.h @@ -2,8 +2,6 @@ #define _NPY_READ_H_ #include "libnpy_tests.h" -#include "npy/npy.h" -#include "npy/tensor.h" template void test_read(int &result, const std::string &name, @@ -14,7 +12,7 @@ void test_read(int &result, const std::string &name, } npy::tensor actual = - npy::load(test::asset_path(name + ".npy")); + npy::load>(test::asset_path(name + ".npy")); test::assert_equal(expected, actual, result, "npy_read_" + name); } @@ -23,7 +21,7 @@ void test_read_scalar(int &result, const std::string &name) { npy::tensor expected = test::test_tensor({}); *expected.data() = static_cast(42); npy::tensor actual = - npy::load(test::asset_path(name + ".npy")); + npy::load>(test::asset_path(name + ".npy")); test::assert_equal(expected, actual, result, "npy_read_" + name); } @@ -31,7 +29,7 @@ template void test_read_array(int &result, const std::string &name) { npy::tensor expected = test::test_tensor({25}); npy::tensor actual = - npy::load(test::asset_path(name + ".npy")); + npy::load>(test::asset_path(name + ".npy")); test::assert_equal(expected, actual, result, "npy_read_" + name); } diff --git a/test/npy_write.cpp b/test/npy_write.cpp index ddc2d75..aecaeaf 100644 --- a/test/npy_write.cpp +++ b/test/npy_write.cpp @@ -1,6 +1,5 @@ #include "libnpy_tests.h" -#include "npy/npy.h" -#include "npy/tensor.h" +#include int test_npy_write() { int result = EXIT_SUCCESS; @@ -67,6 +66,14 @@ int test_npy_write() { actual = test::npy_stream(npy::endian_t::LITTLE); test::assert_equal(expected, actual, result, "npy_write_float64"); + expected = test::read_asset("complex64.npy"); + actual = test::npy_stream>(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_complex64"); + + expected = test::read_asset("complex128.npy"); + actual = test::npy_stream>(npy::endian_t::LITTLE); + test::assert_equal(expected, actual, result, "npy_write_complex128"); + expected = test::read_asset("unicode.npy"); actual = test::npy_stream(npy::endian_t::LITTLE); test::assert_equal(expected, actual, result, "npy_write_unicode"); diff --git a/test/npz_peek.cpp b/test/npz_peek.cpp index 1facf93..e7c159c 100644 --- a/test/npz_peek.cpp +++ b/test/npz_peek.cpp @@ -1,5 +1,4 @@ #include "libnpy_tests.h" -#include "npy/npz.h" namespace { void _test(int &result, const std::string &filename, bool compressed) { @@ -8,7 +7,7 @@ void _test(int &result, const std::string &filename, bool compressed) { npy::header_info expected_depth(npy::data_type_t::FLOAT32, npy::endian_t::LITTLE, false, {5, 5}); - npy::inpzstream stream(test::asset_path(filename)); + npy::npzfilereader stream(test::asset_path(filename)); const auto &keys = stream.keys(); test::assert_equal(keys[0], std::string("color.npy"), result, diff --git a/test/npz_read.cpp b/test/npz_read.cpp index 0256f57..f5ef24c 100644 --- a/test/npz_read.cpp +++ b/test/npz_read.cpp @@ -1,6 +1,4 @@ #include "libnpy_tests.h" -#include "npy/core.h" -#include "npy/npz.h" namespace { void _test(int &result, const std::string &filename, bool compressed) { @@ -8,10 +6,10 @@ void _test(int &result, const std::string &filename, bool compressed) { auto expected_depth = test::test_tensor({5, 5}); auto expected_unicode = test::test_tensor({5, 2, 5}); - npy::inpzstream stream(test::asset_path(filename)); - auto actual_color = stream.read("color.npy"); - auto actual_depth = stream.read("depth"); - auto actual_unicode = stream.read("unicode"); + npy::npzfilereader stream(test::asset_path(filename)); + auto actual_color = stream.read>("color.npy"); + auto actual_depth = stream.read>("depth"); + auto actual_unicode = stream.read>("unicode"); std::string suffix = compressed ? "_compressed" : ""; test::assert_equal(expected_color, actual_color, result, @@ -26,9 +24,9 @@ void _test_large(int &result, const std::string &filename, bool compressed) { auto expected_int = test::test_tensor({200, 5, 1000}); auto expected_float = test::test_tensor({1000, 5, 20, 10}); - npy::inpzstream stream(test::asset_path(filename)); - auto actual_int = stream.read("test_int"); - auto actual_float = stream.read("test_float"); + npy::npzfilereader stream(test::asset_path(filename)); + auto actual_int = stream.read>("test_int"); + auto actual_float = stream.read>("test_float"); std::string suffix = compressed ? "_compressed" : ""; test::assert_equal(expected_int, actual_int, result, @@ -42,13 +40,12 @@ void _test_memory(int &result, const std::string &filename) { std::ios::in | std::ios::binary); std::string contents((std::istreambuf_iterator(input)), std::istreambuf_iterator()); - auto memory = std::make_shared(contents); auto expected_color = test::test_tensor({5, 5, 3}); auto expected_depth = test::test_tensor({5, 5}); auto expected_unicode = test::test_tensor({5, 2, 5}); - npy::inpzstream stream(memory); + npy::npzstringreader stream(contents); auto actual_color = stream.read("color.npy"); auto actual_depth = stream.read("depth"); auto actual_unicode = stream.read("unicode"); diff --git a/test/npz_write.cpp b/test/npz_write.cpp index cfc8760..b111acf 100644 --- a/test/npz_write.cpp +++ b/test/npz_write.cpp @@ -1,14 +1,11 @@ #include #include #include -#include #include "libnpy_tests.h" -#include "npy/core.h" -#include "npy/npz.h" namespace { -const char *TEMP_NPZ = "temp.npz"; +const std::string TEMP_NPZ = "temp.npz"; } namespace { @@ -23,7 +20,7 @@ void _test(int &result, npy::compression_method_t compression_method) { std::string expected = test::read_asset(asset_name); { - npy::onpzstream npz(TEMP_NPZ, compression_method, npy::endian_t::LITTLE); + npy::npzfilewriter npz(TEMP_NPZ, compression_method, npy::endian_t::LITTLE); npz.write("color", test::test_tensor({5, 5, 3})); npz.write("depth.npy", test::test_tensor({5, 5})); npz.write("unicode.npy", test::test_tensor({5, 2, 5})); @@ -32,24 +29,23 @@ void _test(int &result, npy::compression_method_t compression_method) { std::string actual = test::read_file(TEMP_NPZ); test::assert_equal(expected, actual, result, "npz_write" + suffix); - std::remove(TEMP_NPZ); + std::filesystem::remove(TEMP_NPZ); } void _test_memory(int &result) { std::string asset_name = "test.npz"; std::string expected = test::read_asset(asset_name); - auto memory = std::make_shared(); + std::string actual; - { - npy::onpzstream npz(memory, npy::compression_method_t::STORED, - npy::endian_t::LITTLE); - npz.write("color", test::test_tensor({5, 5, 3})); - npz.write("depth.npy", test::test_tensor({5, 5})); - npz.write("unicode.npy", test::test_tensor({5, 2, 5})); - } + npy::npzstringwriter npz(npy::compression_method_t::STORED, + npy::endian_t::LITTLE); + npz.write("color", test::test_tensor({5, 5, 3})); + npz.write("depth.npy", test::test_tensor({5, 5})); + npz.write("unicode.npy", test::test_tensor({5, 2, 5})); + npz.close(); + actual = npz.str(); - std::string actual = memory->str(); test::assert_equal(expected, actual, result, "npz_write_memory"); } } // namespace diff --git a/test/tensor.cpp b/test/tensor.cpp index a03fb59..8fa5009 100644 --- a/test/tensor.cpp +++ b/test/tensor.cpp @@ -2,7 +2,6 @@ #include #include "libnpy_tests.h" -#include "npy/tensor.h" namespace { const char *TEMP_NPY = "temp.npy"; @@ -25,7 +24,7 @@ int test_tensor() { fortran.save(TEMP_NPY); - npy::tensor from_file(TEMP_NPY); + auto from_file = npy::tensor::from_file(TEMP_NPY); npy::tensor standard(from_file.shape(), false); for (int i = 0; i < 3; ++i) { for (int j = 0; j < 4; ++j) {