From 7f0c39cdf8d43b3feca973ebf52cc9da41b2f51f Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Sat, 4 Oct 2025 00:22:12 +0200 Subject: [PATCH 1/3] extra oncommit tests --- ci/build_test_OnCommit.groovy | 53 ++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/ci/build_test_OnCommit.groovy b/ci/build_test_OnCommit.groovy index 68eb18b3ce..e9b3b8a486 100644 --- a/ci/build_test_OnCommit.groovy +++ b/ci/build_test_OnCommit.groovy @@ -1,4 +1,6 @@ def image_build_needed = "false" +def perf_test_needed = "false" +def docs_test_needed = "false" def win_image_build_needed = "false" def client_test_needed = "false" def shortCommit = "" @@ -43,6 +45,8 @@ pipeline { if (win_matched){ win_image_build_needed = "true" } + perf_test_needed = (CHANGE_TITLE =~ /\[PERF_TEST\]/) ? "1" : "0" + docs_test_needed = (CHANGE_TITLE =~ /\[DOCS_TEST\]/) ? "1" : "0" } } } @@ -166,7 +170,7 @@ pipeline { label "${agent_name_linux}" } steps { - sh "make release_image RUN_TESTS=0 OV_USE_BINARY=1 BASE_OS=redhat OVMS_CPP_IMAGE_TAG=${shortCommit} BUILD_IMAGE=openvino/model_server-build:${shortCommit}" + sh "make release_image RUN_TESTS=0 OV_USE_BINARY=1 GPU=${perf_test_needed} BASE_OS=redhat OVMS_CPP_IMAGE_TAG=${shortCommit} BUILD_IMAGE=openvino/model_server-build:${shortCommit}" sh "make run_lib_files_test BASE_OS=redhat OVMS_CPP_IMAGE_TAG=${shortCommit}" script { dir ('internal_tests'){ @@ -178,6 +182,12 @@ pipeline { sh "make create-venv && TT_ON_COMMIT_TESTS=True TT_XDIST_WORKERS=10 TT_BASE_OS=redhat TT_OVMS_IMAGE_NAME=openvino/model_server:${shortCommit} TT_OVMS_IMAGE_LOCAL=True make tests" } } + script { + if (perf_test_needed == "1") { + sh "docker tag openvino/model_server:${shortCommit} registry.toolbox.iotg.sclab.intel.com/model_server:${shortCommit}" + sh "docker push registry.toolbox.iotg.sclab.intel.com/model_server:${shortCommit}" + } + } } } stage('Test windows') { @@ -207,5 +217,46 @@ pipeline { } } } + stage("Additional tests on other hosts in parallel") { + when { expression { perf_test_needed == "1" || docs_test_needed == "1" } } + options { + timeout(time: 120, unit: 'MINUTES') + } + parallel { + stage("Run performance tests CPU") { + when { expression { perf_test_needed == "1" } } + steps { + sh 'echo Running performance tests...' + build job: "ovmsc/perf_test", parameters: [ + [$class: 'StringParameterValue', name: 'DOCKER_IMAGE_NAME', value: "registry.toolbox.iotg.sclab.intel.com/model_server:${shortCommit}"], + [$class: 'StringParameterValue', name: 'TARGET_ENV', value: "ov-spr-19"] + ] + } + } + stage("Run performance tests GPU") { + when { expression { perf_test_needed == "1" } } + steps { + sh 'echo Running performance tests...' + build job: "ovmsc/perf_test", parameters: [ + [$class: 'StringParameterValue', name: 'DOCKER_IMAGE_NAME', value: "registry.toolbox.iotg.sclab.intel.com/model_server:${shortCommit}"], + [$class: 'StringParameterValue', name: 'TARGET_ENV', value: "ov-spr-19"], + [$class: 'StringParameterValue', name: 'DEVICE', value: "GPU"] + ] + } + } + stage("Run documentation tests") { + when { expression { docs_test_needed == "1" } } + steps { + sh 'echo Running documentation tests...' + build job: "ovmsc/job/testing_ci_changes/job/test-demos", parameters: [ + [$class: 'StringParameterValue', name: 'TT_OVMS_IMAGE_NAME', value: "registry.toolbox.iotg.sclab.intel.com/model_server:${shortCommit}"], + [$class: 'StringParameterValue', name: 'CORE_BRANCH', value: "${BRANCH_NAME}"], + [$class: 'StringParameterValue', name: 'PATTERN', value: "demos/continuous_batching/agentic_ai/README.md"] + ] + } + } + } + } } } + From 95b7f08a821880ad80564b05bbaafa884127fa9d Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Wed, 5 Nov 2025 13:47:12 +0100 Subject: [PATCH 2/3] fixes --- ci/build_test_OnCommit.groovy | 4 ++-- ci/perf_linux.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build_test_OnCommit.groovy b/ci/build_test_OnCommit.groovy index 5f71f73b6a..bccb1b190f 100644 --- a/ci/build_test_OnCommit.groovy +++ b/ci/build_test_OnCommit.groovy @@ -220,7 +220,7 @@ pipeline { stage("Additional tests on other hosts in parallel") { when { expression { perf_test_needed == "1" || docs_test_needed == "1" } } options { - timeout(time: 120, unit: 'MINUTES') + timeout(time: 30, unit: 'MINUTES') } parallel { stage("Run performance tests CPU") { @@ -248,7 +248,7 @@ pipeline { when { expression { docs_test_needed == "1" } } steps { sh 'echo Running documentation tests...' - build job: "ovmsc/job/testing_ci_changes/job/test-demos", parameters: [ + build job: "ovmsc/testing_ci_changes/test-demos", parameters: [ [$class: 'StringParameterValue', name: 'TT_OVMS_IMAGE_NAME', value: "registry.toolbox.iotg.sclab.intel.com/model_server:${shortCommit}"], [$class: 'StringParameterValue', name: 'CORE_BRANCH', value: "${BRANCH_NAME}"], [$class: 'StringParameterValue', name: 'PATTERN', value: "demos/continuous_batching/agentic_ai/README.md"] diff --git a/ci/perf_linux.groovy b/ci/perf_linux.groovy index 4695223357..8ab3b3e155 100644 --- a/ci/perf_linux.groovy +++ b/ci/perf_linux.groovy @@ -336,7 +336,7 @@ pipeline { } sh "echo Install BFCL && \ test -d gorilla || git clone https://github.com/ShishirPatil/gorilla && \ - cd gorilla/berkeley-function-call-leaderboard && git checkout cd9429ccf3d4d04156affe883c495b3b047e6b64 -f && curl -s https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/perf-111/demos/continuous_batching/accuracy/gorilla.patch | git apply -v" + cd gorilla/berkeley-function-call-leaderboard && git checkout cd9429ccf3d4d04156affe883c495b3b047e6b64 -f && curl -s https://raw.githubusercontent.com/openvinotoolkit/model_server/refs/heads/main/demos/continuous_batching/accuracy/gorilla.patch | git apply -v" sh "test -d .venv || python3 -m venv .venv && \ . .venv/bin/activate && pip install -e ./gorilla/berkeley-function-call-leaderboard && \ echo Running agentic accuracy test && \ From e3d9277c88d5c8d49ae6e03211cfa6acf81f912a Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Fri, 7 Nov 2025 00:44:33 +0100 Subject: [PATCH 3/3] fix --- ci/perf_linux.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/perf_linux.groovy b/ci/perf_linux.groovy index 8ab3b3e155..c5849409a3 100644 --- a/ci/perf_linux.groovy +++ b/ci/perf_linux.groovy @@ -340,9 +340,8 @@ pipeline { sh "test -d .venv || python3 -m venv .venv && \ . .venv/bin/activate && pip install -e ./gorilla/berkeley-function-call-leaderboard && \ echo Running agentic accuracy test && \ - export OPENAI_BASE_URL=http://localhost:9000/v3 && \ - ${params.USE_THINKING ? 'export ENABLE_THINKING=true && \\' : ''} \ - bfcl generate --model ovms-model --test-category simple --temperature 0.0 --num-threads 100 -o --result-dir bfcl_results && bfcl evaluate --model ovms-model --result-dir bfcl_results --score-dir bfcl_scores && \ + OPENAI_BASE_URL=http://localhost:9000/v3 ENABLE_THINKING=${params.USE_THINKING} bfcl generate --model ovms-model --test-category simple --temperature 0.0 --num-threads 100 -o --result-dir bfcl_results && \ + bfcl evaluate --model ovms-model --result-dir bfcl_results --score-dir bfcl_scores && \ cat gorilla/berkeley-function-call-leaderboard/bfcl_scores/ovms-model/BFCL_v3_simple_score.json | head -1 | jq ." script { def accuracy = sh(script: "cat gorilla/berkeley-function-call-leaderboard/bfcl_scores/ovms-model/BFCL_v3_simple_score.json | head -1 | jq -r '.accuracy'", returnStdout: true).trim()