diff --git a/Demos/Multi-model/README.md b/Demos/Multi-model/README.md new file mode 100644 index 00000000..dea320fe --- /dev/null +++ b/Demos/Multi-model/README.md @@ -0,0 +1,60 @@ + + + + +

Ryzen™ AI Multi model demo

+
+ +# Multi Model Running on NPU + +We showcase an application that loads 2 different models in a single thread. + +```text +NOTE: Models has to be initiated in proper order. The largest model needs to be initiated first. +``` + +## Prerequisites and Environment Setup + +Install Ryzen AI Software using the automatic installer [Link](https://ryzenai.docs.amd.com/en/latest/inst.html). This should create a conda environment that can be used for this example. Check the installation path variables + +Below instructions can be executed on Conda command prompt or Miniforge Prompt + +1. Create a clone of the Ryzen AI installation conda environment to add required python packages + +```python +set RYZEN_AI_CONDA_ENV_NAME=ryzen-ai- +conda create --name npu-gpu-pipeline --clone %RYZEN_AI_CONDA_ENV_NAME% +conda activate npu-gpu-pipeline +``` +2. Set RyzenAI Environment variable + +```bash +# Location of RyzenAI software installation path or default at "C:\Program Files\RyzenAI\" +set RYZEN_AI_INSTALLATION_PATH= +``` + +3. Additional configuration needed. + +```text + Running of multi models in a single app needs additional setup. + Models has to be initiated in proper order. The largest model needs to be initiated first. + Also maxSpillBufferSize (in bytes) in provider_options should be bigger than the largest model. + Please check compile_multi_model.py for reference. +``` + +## Prepare models +Download ResNet50 and Mobilenet_v2 models + +```bash +cd models +python download_2_models.py +``` + +## Running the application + +1. Run the example: + +```bash +cd .. +python compile_multi_model.py +``` diff --git a/Demos/Multi-model/compile_multi_model.py b/Demos/Multi-model/compile_multi_model.py new file mode 100755 index 00000000..e41ea013 --- /dev/null +++ b/Demos/Multi-model/compile_multi_model.py @@ -0,0 +1,28 @@ +import onnxruntime as ort + +# Model - size +# mobilenet_v2.onnx - 25153280 +# resnet50.onnx - 12548096 + +print("Init the largest model on top"); +provider_options = [{ + 'config_file': 'vaiml_config.json', + 'cache_dir': 'models_cache', + 'cache_key': 'model1', + 'maxSpillBufferSize': "26000000", # 26MB Larger than the largest model + "enable_cache_file_io_in_mem": "0" +}] + +s1 = ort.InferenceSession("models/mobilenet_v2.onnx", providers=['VitisAIExecutionProvider'], provider_options=provider_options) + + +print("Init smaller models below"); +provider_options = [{ + 'config_file': 'vaiml_config.json', + 'cache_dir': 'models_cache', + 'cache_key': 'model2', + 'maxSpillBufferSize': "26000000", # 26MB Larger than the largest model + "enable_cache_file_io_in_mem": "0" +}] + +s2 = ort.InferenceSession("models/resnet50.onnx", providers=['VitisAIExecutionProvider'], provider_options=provider_options) diff --git a/Demos/Multi-model/models/download_2_models.py b/Demos/Multi-model/models/download_2_models.py new file mode 100644 index 00000000..f8bbf48c --- /dev/null +++ b/Demos/Multi-model/models/download_2_models.py @@ -0,0 +1,44 @@ +import torch +import torchvision.models as models +import torch.onnx +from torchvision.models import resnet50, ResNet50_Weights +import onnx + +# Load a pre-trained ResNet model +model = models.resnet50(weights=ResNet50_Weights.DEFAULT) +model.eval() # Set the model to evaluation mode + +# Create a dummy input tensor with the same size as the model's input +dummy_input = torch.randn(1, 3, 224, 224) + +onnx_model_path = "resnet50.onnx" + +# Export the model to ONNX format +torch.onnx.export( + model, + dummy_input, + onnx_model_path, + opset_version=17, + export_params=True, + input_names=['input'], + output_names=['output'], + dynamic_axes={'input': {0: 'batch_size'}, 'output': {0: 'batch_size'}} +) + +mobilenet_v2_model_path = "mobilenet_v2.onnx" +mobilenet_v2 = models.mobilenet_v2(pretrained=True) +mobilenet_v2.eval() + +torch.onnx.export( + mobilenet_v2, + dummy_input, + mobilenet_v2_model_path, + export_params=True, + opset_version=17, + do_constant_folding=True, + input_names=["input"], + output_names=["output"], + dynamic_axes={"input": {0: "batch"}, "output": {0: "batch"}} +) + +print(f"Models has been successfully exported to {onnx_model_path} and {mobilenet_v2_model_path}") diff --git a/Demos/Multi-model/vaiml_config.json b/Demos/Multi-model/vaiml_config.json new file mode 100644 index 00000000..d09b13e9 --- /dev/null +++ b/Demos/Multi-model/vaiml_config.json @@ -0,0 +1,26 @@ +{ + "passes": [ + { + "name": "init", + "plugin": "vaip-pass_init" + }, + { + "name": "vaiml_partition", + "plugin": "vaip-pass_vaiml_partition", + "vaiml_config": { + "enable_f32_to_bf16_conversion": true, + "preferred_data_storage": "auto" + } + } + ], + "target": "VAIML", + "targets": [ + { + "name": "VAIML", + "pass": [ + "init", + "vaiml_partition" + ] + } + ] + }