From a26a5e2fb1e76c0861a55a40aa0635c1300d2a28 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 24 Jun 2026 09:29:40 -0700 Subject: [PATCH 1/2] Refactor README and rename library to onnxruntime_ep_tensorrt - Overhaul README with comprehensive Usage section covering C/C++, Python, C#, and onnxruntime_perf_test workflows - Add full provider options reference table - Add unit test documentation with build instructions and test case table - Rename library from ORTTensorRTEp to onnxruntime_ep_tensorrt across all files (CMake, DEF, Python, C#, CI workflows, tests, docs) - Rename cmake config templates to match new naming convention Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/linux_x64_cuda.yml | 4 +- .github/workflows/windows_x64_cuda.yml | 4 +- CMakeLists.txt | 54 +-- README.md | 316 ++++++++++++++++-- cmake/ORTTensorRTEpConfig.cmake.in | 7 - cmake/onnxruntime_ep_tensorrt_config.cmake.in | 7 + ...ntime_ep_tensorrt_config_version.cmake.in} | 2 +- ...icrosoft.ML.OnnxRuntime.EP.TensorRT.csproj | 4 +- .../TensorRTEp.cs | 2 +- csharp/readme.md | 2 +- python/onnxruntime_ep_tensorrt/__init__.py | 4 +- python/readme.md | 2 +- src/tensorrt_execution_provider.def | 2 +- tests/cuda_graph_test.cc | 2 +- tests/tensorrt_basic_test.cc | 2 +- 15 files changed, 330 insertions(+), 84 deletions(-) delete mode 100644 cmake/ORTTensorRTEpConfig.cmake.in create mode 100644 cmake/onnxruntime_ep_tensorrt_config.cmake.in rename cmake/{ORTTensorRTEpConfigVersion.cmake.in => onnxruntime_ep_tensorrt_config_version.cmake.in} (84%) diff --git a/.github/workflows/linux_x64_cuda.yml b/.github/workflows/linux_x64_cuda.yml index d4f1780..4d22cb4 100644 --- a/.github/workflows/linux_x64_cuda.yml +++ b/.github/workflows/linux_x64_cuda.yml @@ -179,7 +179,7 @@ jobs: -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} \ -DCMAKE_CUDA_COMPILER=${CUDA_HOME}/bin/nvcc \ -DTENSORRT_HOME=${TRT_HOME} \ - -DORTTensorRTEp_BUILD_TESTS=ON \ + -Donnxruntime_ep_tensorrt_BUILD_TESTS=ON \ -S . -B out/build cmake --build out/build -j `nproc` cmake --install out/build @@ -359,7 +359,7 @@ jobs: - name: Setup library path run: | - echo "TRTEP_LIBRARY_PATH=$GITHUB_WORKSPACE/orttrtep/lib/libORTTensorRTEp.so" >> $GITHUB_ENV + echo "TRTEP_LIBRARY_PATH=$GITHUB_WORKSPACE/orttrtep/lib/libonnxruntime_ep_tensorrt.so" >> $GITHUB_ENV - name: Run unit tests env: diff --git a/.github/workflows/windows_x64_cuda.yml b/.github/workflows/windows_x64_cuda.yml index 08c1ff9..7196867 100644 --- a/.github/workflows/windows_x64_cuda.yml +++ b/.github/workflows/windows_x64_cuda.yml @@ -104,7 +104,7 @@ jobs: -DCMAKE_CUDA_COMPILER=${{ runner.temp }}\v${{ matrix.cuda_version }}\bin\nvcc.exe ^ -DCMAKE_CUDA_ARCHITECTURES=${{ env.CMAKE_CUDA_ARCHITECTURES }} ^ -DTENSORRT_HOME=${{ runner.temp }}\TensorRT-${{ matrix.trt_version }}.Windows.win10.cuda-${{ matrix.cuda_version }} ^ - -DORTTensorRTEp_BUILD_TESTS=ON ^ + -Donnxruntime_ep_tensorrt_BUILD_TESTS=ON ^ -S . -B out/build if %errorlevel% neq 0 exit /b %errorlevel% @@ -270,7 +270,7 @@ jobs: AZCOPY_AUTO_LOGIN_TYPE: ${{ vars.AZCOPY_AUTO_LOGIN_TYPE }} AZCOPY_MSI_CLIENT_ID: ${{ vars.AZCOPY_MSI_CLIENT_ID }} ONNXRUNTIME_TEST_GPU_DEVICE_ID: '0' - TRTEP_LIBRARY_PATH: '${{ github.workspace }}\orttrtep\bin\ORTTensorRTEp.dll' + TRTEP_LIBRARY_PATH: '${{ github.workspace }}\orttrtep\bin\onnxruntime_ep_tensorrt.dll' ARTIFACT_NAME: orttrtep_cuda${{ matrix.cuda_version }}_trt${{ matrix.trt_version }}_${{ matrix.build_type }}.results BUILD_TYPE: ${{ matrix.build_type }} CUDA_VERSION: ${{ matrix.cuda_version }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 4bda5bb..8a81031 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ cmake_minimum_required(VERSION 3.25) cmake_policy(SET CMP0091 NEW) # Use CMAKE_MSVC_RUNTIME_LIBRARY cmake_policy(SET CMP0141 NEW) # Use CMAKE_MSVC_DEBUG_INFORMATION_FORMAT -project(ORTTensorRTEp VERSION 1.0.0 LANGUAGES CXX) +project(onnxruntime_ep_tensorrt VERSION 1.0.0 LANGUAGES CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -43,11 +43,11 @@ if(MSVC) set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>DLL") endif() -option(ORTTensorRTEp_INSTALL "Install ORTTensorRTEp binaries and files" ON) -option(ORTTensorRTEp_OBJECT_CACHE "Enable object caching using either sccache/ccache" ON) +option(onnxruntime_ep_tensorrt_INSTALL "Install onnxruntime_ep_tensorrt binaries and files" ON) +option(onnxruntime_ep_tensorrt_OBJECT_CACHE "Enable object caching using either sccache/ccache" ON) option(CMAKE_CUDA_ARCHITECTURES "Version of CUDA architectures to enable" 86) -if(ORTTensorRTEp_OBJECT_CACHE) +if(onnxruntime_ep_tensorrt_OBJECT_CACHE) find_program(SCCACHE_PROGRAM sccache) find_program(CCACHE_PROGRAM ccache) # Prioritize sccache on Windows & ccache on non-Windows @@ -94,7 +94,7 @@ endif() enable_language(CUDA) # via nvcc to get the CUDA tool kit find_package(CUDAToolkit REQUIRED) -file(GLOB ORTTensorRTEp_SRC CONFIGURE_DEPENDS +file(GLOB onnxruntime_ep_tensorrt_SRC CONFIGURE_DEPENDS "./src/*.cc" "./src/kernels/*.cc" "./src/utils/*.cc" @@ -102,8 +102,8 @@ file(GLOB ORTTensorRTEp_SRC CONFIGURE_DEPENDS "./src/*.h" "./src/kernels/*.h" ) -add_library(ORTTensorRTEp SHARED ${ORTTensorRTEp_SRC}) -target_compile_definitions(ORTTensorRTEp PUBLIC +add_library(onnxruntime_ep_tensorrt SHARED ${onnxruntime_ep_tensorrt_SRC}) +target_compile_definitions(onnxruntime_ep_tensorrt PUBLIC -DONNX_NAMESPACE=onnx -DONNX_ML -DNOMINMAX @@ -169,7 +169,7 @@ if(WIN32) # Windows ) set(ORT_LIBS "${ORT_LIBRARY_DIR}/onnxruntime.lib") - target_link_options(ORTTensorRTEp PRIVATE "-DEF:${CMAKE_SOURCE_DIR}/src/tensorrt_execution_provider.def") + target_link_options(onnxruntime_ep_tensorrt PRIVATE "-DEF:${CMAKE_SOURCE_DIR}/src/tensorrt_execution_provider.def") else() set(TRT_LIBS "${TENSORRT_HOME}/libnvinfer.so.${TRT_MAJOR_VERSION}" @@ -182,15 +182,15 @@ endif() message(STATUS "ORT libs: ${ORT_LIBS}") message(STATUS "TRT libs: ${TRT_LIBS}") -target_include_directories(ORTTensorRTEp PRIVATE +target_include_directories(onnxruntime_ep_tensorrt PRIVATE "$" "$" "$" "$" - "$" + "$" ) -target_link_libraries(ORTTensorRTEp PRIVATE +target_link_libraries(onnxruntime_ep_tensorrt PRIVATE ${ORT_LIBS} ${TRT_LIBS} CUDA::cudart @@ -201,20 +201,20 @@ target_link_libraries(ORTTensorRTEp PRIVATE ) configure_file( - "${PROJECT_SOURCE_DIR}/cmake/ORTTensorRTEpConfigVersion.cmake.in" - "${CMAKE_CURRENT_BINARY_DIR}/ORTTensorRTEpConfigVersion.cmake" + "${PROJECT_SOURCE_DIR}/cmake/onnxruntime_ep_tensorrt_config_version.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_ep_tensorrt_config_version.cmake" @ONLY ) configure_file( - "${PROJECT_SOURCE_DIR}/cmake/ORTTensorRTEpConfig.cmake.in" - "${CMAKE_CURRENT_BINARY_DIR}/ORTTensorRTEpConfig.cmake" + "${PROJECT_SOURCE_DIR}/cmake/onnxruntime_ep_tensorrt_config.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_ep_tensorrt_config.cmake" @ONLY ) # ── Tests ────────────────────────────────────────────────────────────────────── -option(ORTTensorRTEp_BUILD_TESTS "Build unit tests" OFF) +option(onnxruntime_ep_tensorrt_BUILD_TESTS "Build unit tests" OFF) -if(ORTTensorRTEp_BUILD_TESTS) +if(onnxruntime_ep_tensorrt_BUILD_TESTS) enable_testing() FetchContent_Declare( @@ -265,14 +265,14 @@ if(ORTTensorRTEp_BUILD_TESTS) ) endif() -if(ORTTensorRTEp_INSTALL) +if(onnxruntime_ep_tensorrt_INSTALL) # Installation target include(GNUInstallDirs) include(CMakePackageConfigHelpers) if(WIN32 AND (CMAKE_CXX_COMPILER_ID MATCHES "MSVC")) install( - FILES "$" + FILES "$" CONFIGURATIONS Debug RelWithDebInfo DESTINATION "${CMAKE_INSTALL_BINDIR}" ) @@ -280,20 +280,20 @@ if(ORTTensorRTEp_INSTALL) install( DIRECTORY src/ - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/orttrtep" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime_ep_tensorrt" FILES_MATCHING PATTERN "*.h" ) install( - FILES "${CMAKE_CURRENT_BINARY_DIR}/ORTTensorRTEpConfig.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/ORTTensorRTEpConfigVersion.cmake" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/orttrtep" + FILES "${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_ep_tensorrt_config.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/onnxruntime_ep_tensorrt_config_version.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/onnxruntime_ep_tensorrt" ) - install(TARGETS ORTTensorRTEp GSL flatbuffers EXPORT ORTTensorRTEpTargets) - install(EXPORT ORTTensorRTEpTargets - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/orttrtep" - NAMESPACE ORTTensorRTEp:: + install(TARGETS onnxruntime_ep_tensorrt GSL flatbuffers EXPORT onnxruntime_ep_tensorrt_targets) + install(EXPORT onnxruntime_ep_tensorrt_targets + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/onnxruntime_ep_tensorrt" + NAMESPACE onnxruntime_ep_tensorrt:: ) endif() diff --git a/README.md b/README.md index fc546b5..426fd54 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,306 @@ # TensorRT Plugin Execution Provider -This TensorRT plugin EP is originally migrated from the provider-bridge [TensorRT EP](https://github.com/microsoft/onnxruntime/tree/main/onnxruntime/core/providers/tensorrt) and implements the required ORT EP interfaces (including `OrtEpFactory`, `OrtEp`, `OrtNodeComputeInfo`, `OrtDataTransferImpl`, etc.) to interact with ONNX Runtime through the EP ABI introduced in ORT 1.23.0. -TensorRT plugin EP should be built as a shared library and does not need to be built together with ONNX Runtime. It only needs to link against the ONNX Runtime shared library, i.e., `onnxruntime.dll` or `libonnxruntime.so`. +The TensorRT plugin Execution Provider (EP) implements the [ORT EP plugin ABI](https://onnxruntime.ai/docs/reference/ep-abi.html) introduced in ONNX Runtime 1.23.0, enabling NVIDIA TensorRT acceleration for ONNX models. It is migrated from the in-tree [TensorRT EP](https://github.com/microsoft/onnxruntime/tree/main/onnxruntime/core/providers/tensorrt) and exposes the same set of provider options and features. -This TensorRT plugin EP can be built on Linux and Windows with "Debug" and "Release" mode. +Unlike the legacy in-tree EP, this plugin EP is built as a **standalone shared library** (`onnxruntime_ep_tensorrt.dll` / `libonnxruntime_ep_tensorrt.so`) and does **not** need to be compiled together with ONNX Runtime. It only links against the ONNX Runtime shared library (`onnxruntime.dll` / `libonnxruntime.so`). + +Supported platforms: **Linux** and **Windows** (Debug / Release). ## Contents -- `CMakeLists.txt`: Build configuration for the TensorRT plugin EP. -- `src`: Contains source code for the TensorRT plugin EP. -- `python`: Contains example code for setting up and using a Python package. -- `csharp`: Contains example code for setting up and using a C# NuGet package. + +| Path | Description | +|------|-------------| +| `CMakeLists.txt` | Build configuration for the plugin EP and optional unit tests. | +| `src/` | C++ source code for the plugin EP. | +| `tests/` | GTest-based unit tests (basic inference, CUDA graph, engine caching, etc.). | +| `python/` | Python package and example usage script. See [`python/readme.md`](python/readme.md). | +| `csharp/` | C# NuGet package and sample application. See [`csharp/readme.md`](csharp/readme.md). | + +## Prerequisites + +- **ONNX Runtime** ≥ 1.23.0 (headers + shared library) +- **NVIDIA TensorRT** (10.x or 11.x) +- **CUDA Toolkit** (with `nvcc`) +- **CMake** ≥ 3.25 ## Build Instructions + ### On Windows + ```bash -mkdir build;cd build -``` -```bash -cmake -S ../ -B ./ -DCMAKE_BUILD_TYPE=Debug -DTENSORRT_HOME=C:/folder/to/trt -DORT_HOME=C:/folder/to/ort +mkdir build && cd build +cmake -S ../ -B ./ -DCMAKE_BUILD_TYPE=Release ^ + -DTENSORRT_HOME=C:/path/to/TensorRT ^ + -DORT_HOME=C:/path/to/onnxruntime ^ + -DTRT_MAJOR_VERSION=11 +cmake --build ./ --config Release ``` + +### On Linux + ```bash -cmake --build ./ --config Debug +mkdir build && cd build +cmake -S ../ -B ./ -DCMAKE_BUILD_TYPE=Release \ + -DTENSORRT_HOME=/path/to/TensorRT \ + -DORT_HOME=/path/to/onnxruntime \ + -DTRT_MAJOR_VERSION=11 \ + -DCMAKE_CUDA_ARCHITECTURES=80 \ + -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc +cmake --build ./ --config Release ``` -(Note: The `ORT_HOME` should contain the include and lib folders as below) + +> **Note:** `ORT_HOME` must contain `include/` and `lib/` subdirectories: +> ``` +> ORT_HOME/ +> ├── include/ +> │ ├── onnxruntime_c_api.h +> │ ├── onnxruntime_ep_c_api.h +> │ ├── onnxruntime_cxx_api.h +> │ └── ... +> └── lib/ +> ├── onnxruntime.dll (or libonnxruntime.so) +> ├── onnxruntime.lib (Windows only) +> └── ... +> ``` + +### CMake Options + +| Option | Default | Description | +|--------|---------|-------------| +| `ORT_HOME` | auto-download | Path to ONNX Runtime package (include + lib). | +| `TENSORRT_HOME` | *(required)* | Path to TensorRT installation. | +| `TRT_MAJOR_VERSION` | `10` | TensorRT major version (affects library names). | +| `CMAKE_CUDA_ARCHITECTURES` | `86` | Target CUDA architectures (e.g., `80`, `86`, `89`, `90`). | +| `onnxruntime_ep_tensorrt_BUILD_TESTS` | `OFF` | Build unit tests (requires GTest, fetched automatically). | +| `onnxruntime_ep_tensorrt_OBJECT_CACHE` | `ON` | Use sccache/ccache if available. | + +## Usage + +The plugin EP follows the ORT EP plugin ABI workflow: + +1. **Register** the plugin EP library with the ORT environment. +2. **Discover** available EP devices. +3. **Append** the EP to session options with provider-specific options. +4. **Create** an inference session and run the model. +5. **Unregister** the library after all sessions using it have been released. + +### C/C++ API + +```cpp +#include "onnxruntime_cxx_api.h" + +Ort::InitApi(); +Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "MyApp"); + +// 1. Register the plugin EP library +env.RegisterExecutionProviderLibrary("TRTPluginEP", "path/to/onnxruntime_ep_tensorrt.dll"); + +// 2. Find the EP device +auto all_devices = env.GetEpDevices(); +std::vector trt_devices; +for (const auto& d : all_devices) { + if (std::string(d.EpName()) == "TRTPluginEP") { + trt_devices.push_back(d); + break; + } +} + +// 3. Create session with EP options +Ort::SessionOptions session_options; +std::unordered_map ep_options = { + {"trt_fp16_enable", "1"}, + {"trt_engine_cache_enable", "1"}, + {"trt_engine_cache_path", "./cache"}, +}; +session_options.AppendExecutionProvider_V2(env, trt_devices, ep_options); + +// 4. Run inference +Ort::Session session(env, "model.onnx", session_options); +auto outputs = session.Run(Ort::RunOptions{}, input_names, inputs, num_inputs, output_names, num_outputs); + +// 5. Unregister (after all sessions are released) +session = Ort::Session{nullptr}; // release session first +env.UnregisterExecutionProviderLibrary("TRTPluginEP"); ``` -C:/folder/to/ort - | ----- lib - | | ----- onnxruntime.dll - | | ----- onnxruntime.lib - | | ----- onnxruntime.pdb - | ... - | - | ---- include - | | ----- onnxruntime_c_api.h - | | ----- onnxruntime_ep_c_api.h - | | ----- onnxruntime_cxx_api.h - | | ----- onnxruntime_cxx_inline_api.h - | ... + +### Python + +Install the helper package (see [`python/readme.md`](python/readme.md) for build instructions): + +```python +import numpy as np +import onnxruntime as ort +import onnxruntime_ep_tensorrt as tensorrt_ep + +# 1. Register the plugin EP library +ep_lib_path = tensorrt_ep.get_library_path() +ep_name = tensorrt_ep.get_ep_name() # "TensorRTPluginExecutionProvider" +ort.register_execution_provider_library(ep_name, ep_lib_path) + +# 2. Select an EP device +all_devices = ort.get_ep_devices() +trt_devices = [d for d in all_devices if d.ep_name == ep_name] + +# 3. Create session with EP options +sess_options = ort.SessionOptions() +ep_options = { + "trt_fp16_enable": "1", + "trt_engine_cache_enable": "1", +} +sess_options.add_provider_for_devices(trt_devices, ep_options) + +# 4. Run inference +session = ort.InferenceSession("model.onnx", sess_options=sess_options) +output = session.run([], {"input": input_data}) + +# 5. Unregister +del session +ort.unregister_execution_provider_library(ep_name) ``` - -### On Linux -```bash -mkdir build;cd build +### C\# + +Install the NuGet package (see [`csharp/readme.md`](csharp/readme.md) for build instructions): + +```csharp +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.EP.TensorRT; + +// 1. Register the plugin EP library +var env = OrtEnv.Instance(); +string epLibPath = TensorRTEp.GetLibraryPath(); +string epName = TensorRTEp.GetEpName(); +env.RegisterExecutionProviderLibrary(epName, epLibPath); + +// 2. Find the EP device +OrtEpDevice? epDevice = env.GetEpDevices() + .FirstOrDefault(d => d.EpName == epName); + +// 3. Create session with EP options +using var sessionOptions = new SessionOptions(); +sessionOptions.AppendExecutionProvider(env, new[] { epDevice }, + new Dictionary { + { "trt_fp16_enable", "1" }, + }); + +// 4. Run inference +using var session = new InferenceSession("model.onnx", sessionOptions); +using var results = session.Run(runOptions, inputNames, inputValues, session.OutputNames); + +// 5. Unregister +env.UnregisterExecutionProviderLibrary(epName); ``` + +### Quick Test with `onnxruntime_perf_test` + +For a quick smoke test without writing code, use the ORT perf test tool: + ```bash -cmake -S ../ -B ./ -DCMAKE_BUILD_TYPE=Debug -DTENSORRT_HOME=/home/to/trt/ -DORT_HOME=/home/to/ort -DCMAKE_CUDA_ARCHITECTURES=80 -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_POSITION_INDEPENDENT_CODE=ON +onnxruntime_perf_test \ + --plugin_ep_libs "TRTPluginEP|path/to/onnxruntime_ep_tensorrt.dll" \ + --plugin_eps TRTPluginEP \ + -r 1 path/to/model.onnx ``` + +## Provider Options + +Provider options are passed as key-value string pairs when creating a session. These are the same options supported by the legacy in-tree TensorRT EP. + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `device_id` | int | `0` | CUDA device ID. | +| `trt_max_partition_iterations` | int | `1000` | Maximum iterations for TensorRT graph partitioning. | +| `trt_min_subgraph_size` | int | `1` | Minimum number of nodes in a subgraph to be accelerated by TRT. | +| `trt_max_workspace_size` | size_t | `1073741824` (1 GB) | Maximum workspace size for TensorRT engine building. | +| `trt_fp16_enable` | bool | `0` | Enable FP16 precision. | +| `trt_int8_enable` | bool | `0` | Enable INT8 precision. | +| `trt_bf16_enable` | bool | `0` | Enable BF16 precision. | +| `trt_int8_calibration_table_name` | string | `""` | Path to INT8 calibration table. | +| `trt_int8_use_native_calibration_table` | bool | `0` | Use native TRT calibration table format. | +| `trt_dla_enable` | bool | `0` | Enable DLA (Deep Learning Accelerator). | +| `trt_dla_core` | int | `0` | DLA core to use. | +| `trt_engine_cache_enable` | bool | `0` | Enable TensorRT engine caching. | +| `trt_engine_cache_path` | string | `""` | Directory path for cached engines. | +| `trt_engine_cache_prefix` | string | `""` | Filename prefix for cached engines. | +| `trt_dump_subgraphs` | bool | `0` | Dump subgraphs to files for debugging. | +| `trt_force_sequential_engine_build` | bool | `0` | Build TRT engines sequentially (for debugging). | +| `trt_context_memory_sharing_enable` | bool | `0` | Share context memory across TRT subgraphs. | +| `trt_layer_norm_fp32_fallback` | bool | `0` | Force FP32 for LayerNorm (for accuracy). | +| `trt_timing_cache_enable` | bool | `0` | Enable timing cache to speed up engine building. | +| `trt_timing_cache_path` | string | `""` | Path for the timing cache file. | +| `trt_force_timing_cache` | bool | `0` | Fail if timing cache is not found. | +| `trt_detailed_build_log` | bool | `0` | Print detailed TRT engine build log. | +| `trt_build_heuristics_enable` | bool | `0` | Enable builder heuristics for faster build. | +| `trt_sparsity_enable` | bool | `0` | Enable structured sparsity. | +| `trt_builder_optimization_level` | int | `3` | TRT builder optimization level (0–5). | +| `trt_auxiliary_streams` | int | `-1` | Number of auxiliary streams (-1 = auto). | +| `trt_tactic_sources` | string | `""` | Tactic sources to enable/disable. | +| `trt_extra_plugin_lib_paths` | string | `""` | Semicolon-separated paths to extra TRT plugin libraries. | +| `trt_profile_min_shapes` | string | `""` | Min shapes for optimization profiles (e.g., `input:1x3x224x224`). | +| `trt_profile_max_shapes` | string | `""` | Max shapes for optimization profiles. | +| `trt_profile_opt_shapes` | string | `""` | Optimal shapes for optimization profiles. | +| `trt_cuda_graph_enable` | bool | `0` | Enable CUDA graph capture and replay. | +| `trt_dump_ep_context_model` | bool | `0` | Dump EPContext model with embedded engine. | +| `trt_ep_context_file_path` | string | `""` | Path for the EPContext model file. | +| `trt_ep_context_embed_mode` | int | `0` | EPContext embedding mode. | +| `trt_weight_stripped_engine_enable` | bool | `0` | Enable weight-stripped engine. | +| `trt_onnx_model_folder_path` | string | `""` | Path to original ONNX model folder (for weight-stripped engine). | +| `trt_engine_hw_compatible` | bool | `0` | Build HW-compatible engine. | +| `trt_op_types_to_exclude` | string | `""` | Op types to exclude from TRT acceleration. | + +## Building and Running Tests + +Unit tests cover basic inference, dynamic shapes, multi-threading, engine caching, EPContext models, CUDA graph capture/replay, and TRT plugin custom ops. + +### Build with Tests + ```bash +mkdir build && cd build +cmake -S ../ -B ./ -DCMAKE_BUILD_TYPE=Debug \ + -DTENSORRT_HOME=/path/to/TensorRT \ + -DORT_HOME=/path/to/onnxruntime \ + -DTRT_MAJOR_VERSION=11 \ + -DCMAKE_CUDA_ARCHITECTURES=80 \ + -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \ + -Donnxruntime_ep_tensorrt_BUILD_TESTS=ON cmake --build ./ --config Debug -```` +``` + +### Run Tests + +Set the `TRT_EP_LIBRARY_PATH` environment variable to point to the built plugin EP library, then run via CTest or the test binary directly: -## Usage -Please use `onnxruntime_perf_test` ```bash ---plugin_ep_libs "TensorRTEp|C:\repos\onnxruntime-ep-tensorrt\build\Debug\TensorRTEp.dll" --plugin_eps TensorRTEp -r 1 C:\path\to\model +# Via CTest +cd build +export TRT_EP_LIBRARY_PATH=$(pwd)/libonnxruntime_ep_tensorrt.so # or onnxruntime_ep_tensorrt.dll on Windows +ctest --output-on-failure + +# Or run the test binary directly +./trt_ep_tests ``` +### Test Cases + +| Test | Description | +|------|-------------| +| `FunctionTest` | Basic inference with a simple Add model. | +| `TestSessionOutputs_MultipleOutputs` | Verifies correct output count for multi-output models. | +| `TestSessionOutputs_UnusedNodeOutput` | Handles models with unused node outputs. | +| `DDSOutputTest` | Inference with data-dependent shapes (DDS). | +| `MultiThreadInference` | Multi-threaded inference on a single session. | +| `MnistModelTest` | End-to-end inference on the MNIST model. | +| `EngineCacheTest` | Engine caching with `trt_engine_cache_enable`. | +| `EPContextNode_ForeignSourceSkipped` | Skips EPContext nodes from other EPs. | +| `EPContextNode_NoSourceAttribute_BackwardCompat` | Backward compatibility with legacy EPContext nodes. | +| `SequentialRuns` | Multiple sequential runs for stability. | +| `DynamicInputShapes` | Dynamic shape support with optimization profiles. | +| `TRTPluginsCustomOpTest` | TRT plugin custom op registration. | +| `BasicCudaGraph` | CUDA graph capture, replay, and in-place input update. | +| `WithoutCudaGraph` | Baseline inference without CUDA graph. | +| `MultipleReplays` | Repeated CUDA graph replays for stability. | + +## License + +This project is licensed under the MIT License. See [LICENSE](LICENSE) for details. diff --git a/cmake/ORTTensorRTEpConfig.cmake.in b/cmake/ORTTensorRTEpConfig.cmake.in deleted file mode 100644 index fa507e8..0000000 --- a/cmake/ORTTensorRTEpConfig.cmake.in +++ /dev/null @@ -1,7 +0,0 @@ -# library version -set(TensortRTEp_VERSION "@TensortRTEp_VERSION@") - -# import targets -include ("${CMAKE_CURRENT_LIST_DIR}/TensortRTEpTargets.cmake") - -set(TensortRTEp_INCLUDE_DIRS "${_INSTALL_PREFIX}/include") diff --git a/cmake/onnxruntime_ep_tensorrt_config.cmake.in b/cmake/onnxruntime_ep_tensorrt_config.cmake.in new file mode 100644 index 0000000..e4e23b1 --- /dev/null +++ b/cmake/onnxruntime_ep_tensorrt_config.cmake.in @@ -0,0 +1,7 @@ +# library version +set(onnxruntime_ep_tensorrt_VERSION "@onnxruntime_ep_tensorrt_VERSION@") + +# import targets +include ("${CMAKE_CURRENT_LIST_DIR}/onnxruntime_ep_tensorrt_targets.cmake") + +set(onnxruntime_ep_tensorrt_INCLUDE_DIRS "${_INSTALL_PREFIX}/include") diff --git a/cmake/ORTTensorRTEpConfigVersion.cmake.in b/cmake/onnxruntime_ep_tensorrt_config_version.cmake.in similarity index 84% rename from cmake/ORTTensorRTEpConfigVersion.cmake.in rename to cmake/onnxruntime_ep_tensorrt_config_version.cmake.in index 9758f61..8eeb483 100644 --- a/cmake/ORTTensorRTEpConfigVersion.cmake.in +++ b/cmake/onnxruntime_ep_tensorrt_config_version.cmake.in @@ -1,4 +1,4 @@ -set(PACKAGE_VERSION "@TensortRTEp_VERSION@") +set(PACKAGE_VERSION "@onnxruntime_ep_tensorrt_VERSION@") # Check whether the requested PACKAGE_FIND_VERSION is compatible if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION) diff --git a/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/Microsoft.ML.OnnxRuntime.EP.TensorRT.csproj b/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/Microsoft.ML.OnnxRuntime.EP.TensorRT.csproj index 2d9e186..1d1760b 100644 --- a/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/Microsoft.ML.OnnxRuntime.EP.TensorRT.csproj +++ b/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/Microsoft.ML.OnnxRuntime.EP.TensorRT.csproj @@ -41,11 +41,11 @@ - + - + diff --git a/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/TensorRTEp.cs b/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/TensorRTEp.cs index df18cad..ca669ec 100644 --- a/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/TensorRTEp.cs +++ b/csharp/Microsoft.ML.OnnxRuntime.EP.TensorRT/TensorRTEp.cs @@ -18,7 +18,7 @@ public static string GetLibraryPath() string rootDir = GetNativeDirectory(); string osArch = $"{GetOSTag()}-{GetArchTag()}"; string epDllPath = Path.GetFullPath(Path.Combine(rootDir, "runtimes", osArch, - "native", "ORTTensorRTEp.dll")); + "native", "onnxruntime_ep_tensorrt.dll")); if (!File.Exists(epDllPath)) { diff --git a/csharp/readme.md b/csharp/readme.md index 05cb11c..7111ab7 100644 --- a/csharp/readme.md +++ b/csharp/readme.md @@ -14,7 +14,7 @@ Follow instructions [here](../README.md#build-instructions) to build the native ### Build the C\# NuGet Package -Set the environment variable `TENSORRT_PLUGIN_EP_LIBRARY_PATH` to the path to the native plugin EP shared library. E.g., `ORTTensorRTEp.dll`. +Set the environment variable `TENSORRT_PLUGIN_EP_LIBRARY_PATH` to the path to the native plugin EP shared library. E.g., `onnxruntime_ep_tensorrt.dll`. Run `setup.bat` from this directory. Pass the build configuration (e.g., Release or Debug) as an argument. diff --git a/python/onnxruntime_ep_tensorrt/__init__.py b/python/onnxruntime_ep_tensorrt/__init__.py index a0ba054..ccda72e 100644 --- a/python/onnxruntime_ep_tensorrt/__init__.py +++ b/python/onnxruntime_ep_tensorrt/__init__.py @@ -9,8 +9,8 @@ def get_library_path() -> str: candidate_paths = [ - module_dir / "ORTTensorRTEp.dll", - module_dir / "ORTTensorRTEp.so", + module_dir / "onnxruntime_ep_tensorrt.dll", + module_dir / "onnxruntime_ep_tensorrt.so", ] paths = [p for p in candidate_paths if p.is_file()] diff --git a/python/readme.md b/python/readme.md index 908f7e6..a89d9ac 100644 --- a/python/readme.md +++ b/python/readme.md @@ -13,7 +13,7 @@ Follow instructions [here](../README.md#build-instructions) to build the native ### Build the Python package -Set the environment variable `TENSORRT_PLUGIN_EP_LIBRARY_PATH` to the path to the native plugin EP shared library. E.g., `ORTTensorRTEp.dll` on Windows or `libtensorrt_plugin_ep.so` on Linux. +Set the environment variable `TENSORRT_PLUGIN_EP_LIBRARY_PATH` to the path to the native plugin EP shared library. E.g., `onnxruntime_ep_tensorrt.dll` on Windows or `libonnxruntime_ep_tensorrt.so` on Linux. Run `setup.py` from this directory. diff --git a/src/tensorrt_execution_provider.def b/src/tensorrt_execution_provider.def index d2589b2..3b9e9c5 100644 --- a/src/tensorrt_execution_provider.def +++ b/src/tensorrt_execution_provider.def @@ -1,4 +1,4 @@ -LIBRARY "ORTTensorRTEp.dll" +LIBRARY "onnxruntime_ep_tensorrt.dll" EXPORTS CreateEpFactories @1 ReleaseEpFactory @2 diff --git a/tests/cuda_graph_test.cc b/tests/cuda_graph_test.cc index 1b65f01..a13f515 100644 --- a/tests/cuda_graph_test.cc +++ b/tests/cuda_graph_test.cc @@ -87,7 +87,7 @@ static std::string GetEpLibraryPath() { return std::string(env); } // Fallback: try to find it relative to the test binary - GTEST_LOG_(WARNING) << "TRT_EP_LIBRARY_PATH not set. Set it to the path of ORTTensorRTEp shared library."; + GTEST_LOG_(WARNING) << "TRT_EP_LIBRARY_PATH not set. Set it to the path of onnxruntime_ep_tensorrt shared library."; return ""; } diff --git a/tests/tensorrt_basic_test.cc b/tests/tensorrt_basic_test.cc index 1de585d..1350d57 100644 --- a/tests/tensorrt_basic_test.cc +++ b/tests/tensorrt_basic_test.cc @@ -52,7 +52,7 @@ static std::string GetEpLibraryPath() { if (env && std::strlen(env) > 0) { return std::string(env); } - GTEST_LOG_(WARNING) << "TRT_EP_LIBRARY_PATH not set. Set it to the path of ORTTensorRTEp shared library."; + GTEST_LOG_(WARNING) << "TRT_EP_LIBRARY_PATH not set. Set it to the path of onnxruntime_ep_tensorrt shared library."; return ""; } From a84758d6b12fd3715aab10a23adbbbd8a96d6ac8 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 24 Jun 2026 09:35:45 -0700 Subject: [PATCH 2/2] Fix clang-format lint in onnx_ctx_model_helper.cc Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/onnx_ctx_model_helper.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/onnx_ctx_model_helper.cc b/src/onnx_ctx_model_helper.cc index 7378b5c..882ae4c 100644 --- a/src/onnx_ctx_model_helper.cc +++ b/src/onnx_ctx_model_helper.cc @@ -227,8 +227,9 @@ OrtStatus* EPContextNodeReader::GetEpContextFromGraph(const OrtGraph& graph) { if (val_status == nullptr && !source_value.empty() && source_value != "TensorrtExecutionProvider") { return ort_api.CreateStatus(ORT_EP_FAIL, - ("[TensorRT EP] EPContext node has source '" + source_value + - "' which does not match this EP. Skipping.").c_str()); + ("[TensorRT EP] EPContext node has source '" + source_value + + "' which does not match this EP. Skipping.") + .c_str()); } if (val_status != nullptr) { ort_api.ReleaseStatus(val_status);