Downgrade model to gpt-4o-mini

Makefile

@@ -9,7 +9,7 @@ ARTIFACT_DIR := $(if $(ARTIFACT_DIR),$(ARTIFACT_DIR),tests/test_results)
 TEST_TAGS := $(if $(TEST_TAGS),$(TEST_TAGS),"")
 SUITE_ID := $(if $(SUITE_ID),$(SUITE_ID),"nosuite")
 PROVIDER := $(if $(PROVIDER),$(PROVIDER),"openai")
-MODEL := $(if $(MODEL),$(MODEL),"gpt-4.1-mini")
+MODEL := $(if $(MODEL),$(MODEL),"gpt-4o-mini")
 OLS_CONFIG_SUFFIX := $(if $(OLS_CONFIG_SUFFIX),$(OLS_CONFIG_SUFFIX),"default")
 PATH_TO_PLANTUML := ~/bin
 

eval/system.yaml

@@ -3,7 +3,7 @@
 # LLM Configuration
 llm:
   provider: "openai"          # Judge LLM Provider (openai, gemini etc..)
-  model: "gpt-4.1-mini"        # Model name for the provider
+  model: "gpt-4o-mini"        # Model name for the provider
   temperature: 0.0            # Generation temperature
   max_tokens: 512             # Maximum tokens in response
   timeout: 300                # Request timeout in seconds
@@ -19,7 +19,7 @@ api:
 
   # API input configuration
   provider: "openai"                   # LLM provider for queries
-  model: "gpt-4.1-mini"                 # Model to use for queries
+  model: "gpt-4o-mini"                 # Model to use for queries
   no_tools: null                       # Whether to bypass tools and MCP servers (optional)
   system_prompt: null                  # System prompt (default None)
 

eval/system_azure_openai_lseval.yaml

@@ -1,5 +1,5 @@
 # LightSpeed Evaluation Framework Configuration
-# OLS Provider: Azure OpenAI gpt-4.1-mini (model being evaluated)
+# OLS Provider: Azure OpenAI gpt-4o-mini (model being evaluated)
 # Judge LLM: OpenAI GPT-5-mini (scoring responses)
 
 # Core Evaluation Configuration
@@ -21,7 +21,7 @@ llm:
   num_retries: 3
 
 # OLS API Configuration
-# Targets the OLS /query endpoint using Azure OpenAI gpt-4.1-mini as the backend LLM.
+# Targets the OLS /query endpoint using Azure OpenAI gpt-4o-mini as the backend LLM.
 # api_base is overridden at runtime with the actual OLS service URL.
 # Requires OLS to be configured with an azure_openai provider (credentials + deployment URL in olsconfig).
 # Override the model via LSEVAL_OLS_MODEL env var if your Azure deployment uses a different model.
@@ -31,7 +31,7 @@ api:
   endpoint_type: query
   timeout: 300
   provider: "azure_openai"
-  model: "gpt-4.1-mini"
+  model: "gpt-4o-mini"
   no_tools: null
   system_prompt: null
 

eval/system_cluster_updates.yaml

@@ -21,7 +21,7 @@ api:
 
   # API input configuration
   provider: "openai"                   # LLM provider for queries
-  model: "gpt-4.1-mini"                # Model to use for queries (must be valid for OLS API)
+  model: "gpt-4o-mini"                # Model to use for queries (must be valid for OLS API)
   no_tools: null                       # Whether to bypass tools and MCP servers (optional)
   system_prompt: null                  # System prompt (default None)
 

eval/troubleshooting/olsconfig/olsconfig-azure-openai.yaml

@@ -19,7 +19,7 @@ llm_providers:
     api_version: "2024-02-15-preview"
     deployment_name: <your-deployment-name>  # replace with your deployment name
     models:
-      - name: gpt-4.1-mini
+      - name: gpt-4o-mini
 
 mcp_servers:
   - name: obs-mcp
@@ -41,7 +41,7 @@ ols_config:
     lib_log_level: warning
     uvicorn_log_level: info
   default_provider: azure_openai
-  default_model: gpt-4.1-mini
+  default_model: gpt-4o-mini
   authentication_config:
     module: "noop-with-token"
   user_data_collection:

scripts/evaluation/utils/constants.py

@@ -4,9 +4,9 @@
 INSCOPE_MODELS = {
     "watsonx+ibm/granite-3-8b-instruct": ("watsonx", "ibm/granite-3-8b-instruct"),
     "watsonx+ibm/granite-4-h-small": ("watsonx", "ibm/granite-4-h-small"),
-    "openai+gpt-4.1-mini": ("openai", "gpt-4.1-mini"),
+    "openai+gpt-4o-mini": ("openai", "gpt-4o-mini"),
     "openai+gpt-5.1": ("openai", "gpt-5.1"),
-    "azure_openai+gpt-4.1-mini": ("azure_openai", "gpt-4.1-mini"),
+    "azure_openai+gpt-4o-mini": ("azure_openai", "gpt-4o-mini"),
     "azure_openai+gpt-5.1": ("azure_openai", "gpt-5.1"),
     "ollama+llama3.1:latest": ("ollama", "llama3.1:latest"),
     "ollama+mistral": ("ollama", "mistral"),

tests/config/operator_install/olsconfig.crd.azure_openai.yaml

@@ -15,20 +15,20 @@ spec:
         type: azure_openai
         credentialsSecretRef:
           name: llmcreds
-        deploymentName: gpt-4.1-mini
+        deploymentName: gpt-4o-mini
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         url: 'https://ols-test.openai.azure.com/'
       - name: azure_openai_with_entra_id
         type: azure_openai
         credentialsSecretRef:
           name: azure-entra-id
-        deploymentName: gpt-4.1-mini
+        deploymentName: gpt-4o-mini
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         url: 'https://ols-test.openai.azure.com/'
   ols:
-    defaultModel: gpt-4.1-mini
+    defaultModel: gpt-4o-mini
     defaultProvider: azure_openai
     deployment:
       replicas: 1

tests/config/operator_install/olsconfig.crd.azure_openai_lseval.yaml

@@ -15,12 +15,12 @@ spec:
         type: azure_openai
         credentialsSecretRef:
           name: llmcreds
-        deploymentName: gpt-4.1-mini
+        deploymentName: gpt-4o-mini
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         url: 'https://ols-test.openai.azure.com/'
   ols:
-    defaultModel: gpt-4.1-mini
+    defaultModel: gpt-4o-mini
     defaultProvider: azure_openai
     deployment:
       replicas: 1

tests/config/operator_install/olsconfig.crd.azure_openai_tool_calling.yaml

@@ -15,12 +15,12 @@ spec:
         type: azure_openai
         credentialsSecretRef:
           name: llmcreds
-        deploymentName: gpt-4.1-mini
+        deploymentName: gpt-4o-mini
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         url: 'https://ols-test.openai.azure.com/'
   ols:
-    defaultModel: gpt-4.1-mini
+    defaultModel: gpt-4o-mini
     defaultProvider: azure_openai
     deployment:
       replicas: 1

tests/config/operator_install/olsconfig.crd.evaluation.yaml

@@ -15,14 +15,14 @@ spec:
         type: azure_openai
         credentialsSecretRef:
           name: azure-openaicreds
-        deploymentName: gpt-4.1-mini
+        deploymentName: gpt-4o-mini
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         url: 'https://ols-test.openai.azure.com/'
       - credentialsSecretRef:
           name: openaicreds
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         name: openai
         type: openai
       - credentialsSecretRef:
@@ -33,7 +33,7 @@ spec:
         name: watsonx
         type: watsonx
   ols:
-    defaultModel: gpt-4.1-mini
+    defaultModel: gpt-4o-mini
     defaultProvider: openai
     deployment:
       replicas: 1

tests/config/operator_install/olsconfig.crd.openai.yaml

@@ -14,11 +14,11 @@ spec:
       - credentialsSecretRef:
           name: llmcreds
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         name: openai
         type: openai
   ols:
-    defaultModel: gpt-4.1-mini
+    defaultModel: gpt-4o-mini
     defaultProvider: openai
     deployment:
       replicas: 1

tests/config/operator_install/olsconfig.crd.openai_data_export.yaml

@@ -14,11 +14,11 @@ spec:
       - credentialsSecretRef:
           name: llmcreds
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         name: openai
         type: openai
   ols:
-    defaultModel: gpt-4.1-mini
+    defaultModel: gpt-4o-mini
     defaultProvider: openai
     deployment:
       replicas: 1

tests/config/operator_install/olsconfig.crd.openai_mcp.yaml

@@ -14,11 +14,11 @@ spec:
       - credentialsSecretRef:
           name: llmcreds
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         name: openai
         type: openai
   ols:
-    defaultModel: gpt-4.1-mini
+    defaultModel: gpt-4o-mini
     defaultProvider: openai
     deployment:
       replicas: 1

tests/config/operator_install/olsconfig.crd.openai_quota.yaml

@@ -14,11 +14,11 @@ spec:
       - credentialsSecretRef:
           name: llmcreds
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         name: openai
         type: openai
   ols:
-    defaultModel: gpt-4.1-mini
+    defaultModel: gpt-4o-mini
     defaultProvider: openai
     deployment:
       replicas: 1

tests/config/operator_install/olsconfig.crd.openai_tool_calling.yaml

@@ -14,11 +14,11 @@ spec:
       - credentialsSecretRef:
           name: llmcreds
         models:
-          - name: gpt-4.1-mini
+          - name: gpt-4o-mini
         name: openai
         type: openai
   ols:
-    defaultModel: gpt-4.1-mini
+    defaultModel: gpt-4o-mini
     defaultProvider: openai
     deployment:
       replicas: 1

tests/e2e/conftest.py

@@ -186,8 +186,8 @@ def pytest_addoption(parser):
         nargs="+",
         default=[
             "watsonx+ibm/granite-4-h-small",
-            "openai+gpt-4.1-mini",
-            "azure_openai+gpt-4.1-mini",
+            "openai+gpt-4o-mini",
+            "azure_openai+gpt-4o-mini",
         ],
         type=str,
         help="Identifier for Provider/Model to be used for model eval.",

tests/e2e/evaluation/test_cluster_updates.py

@@ -1,4 +1,4 @@
-"""Cluster-updates evaluation tests using OpenAI GPT-4o-mini and GPT-4.1-mini judge."""
+"""Cluster-updates evaluation tests using OpenAI GPT-4o-mini and gpt-4o-mini judge."""
 
 import json
 import os
@@ -143,7 +143,7 @@ def _run_lseval(eval_data: Path, out_dir: Path) -> None:
 def test_cluster_updates(request: pytest.FixtureRequest) -> None:
     """Run cluster-updates eval suite (18 conversations, 35 evaluations).
 
-    Uses GPT-4o-mini and GPT-4.1-mini judge.
+    Uses GPT-4o-mini and gpt-4o-mini judge.
     """
     out_dir_base = request.config.option.eval_out_dir or str(
         EVAL_DIR / "results-cluster-updates"

tests/e2e/test_api.py

@@ -104,8 +104,8 @@ def test_model_provider():
 
     # enabled model must be one of our expected combinations
     assert model, provider in {
-        ("gpt-4.1-mini", "openai"),
-        ("gpt-4.1-mini", "azure_openai"),
+        ("gpt-4o-mini", "openai"),
+        ("gpt-4o-mini", "azure_openai"),
         ("ibm/granite-4-h-small", "watsonx"),
     }
 
@@ -431,7 +431,7 @@ def test_azure_entra_id():
         json={
             "query": "what is kubernetes?",
             "provider": "azure_openai_with_entra_id",
-            "model": "gpt-4.1-mini",
+            "model": "gpt-4o-mini",
         },
         timeout=LLM_REST_API_TIMEOUT,
     )

tests/scripts/test-cluster-updates.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# CI job: run cluster-updates eval (18 conversations, 35 evaluations) against OLS using OpenAI GPT-4o-mini + GPT-4.1-mini judge.
+# CI job: run cluster-updates eval (18 conversations, 35 evaluations) against OLS using OpenAI GPT-4o-mini + gpt-4o-mini judge.
 #
 # Input environment variables:
 #   OPENAI_PROVIDER_KEY_PATH  - path to file containing the OpenAI API key

tests/scripts/test-e2e-cluster-periodics.sh

@@ -18,7 +18,7 @@ DIR="${BASH_SOURCE%/*}"
 if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 . "$DIR/utils.sh"
 
-# install operator-sdk 
+# install operator-sdk
 export ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac)
 export OS=$(uname | awk '{print tolower($0)}')
 export OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/v1.36.1
@@ -39,10 +39,10 @@ function run_suites() {
   # empty test_tags means run all tests
   if [ -z "${DISCONNECTED:-}" ]; then
     # Tests for not disconnected environments
-    run_suite "azure_openai" "not certificates and not (tool_calling and not smoketest and not rag) and not byok1 and not byok2 and not quota_limits and not data_export" "azure_openai" "$AZUREOPENAI_PROVIDER_KEY_PATH" "gpt-4.1-mini" "$OLS_IMAGE" "default"
+    run_suite "azure_openai" "not certificates and not (tool_calling and not smoketest and not rag) and not byok1 and not byok2 and not quota_limits and not data_export" "azure_openai" "$AZUREOPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "default"
     (( rc = rc || $? ))
 
-    run_suite "openai" "not azure_entra_id and not certificates and not (tool_calling and not smoketest and not rag) and not byok1 and not byok2 and not quota_limits and not data_export" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4.1-mini" "$OLS_IMAGE" "default"
+    run_suite "openai" "not azure_entra_id and not certificates and not (tool_calling and not smoketest and not rag) and not byok1 and not byok2 and not quota_limits and not data_export" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "default"
     (( rc = rc || $? ))
 
     run_suite "google_vertex" "not azure_entra_id and not certificates and not (tool_calling and not smoketest and not rag) and not byok1 and not byok2 and not quota_limits and not data_export" "google_vertex" "$VERTEX_PROVIDER_KEY_PATH" "gemini-2.5-flash-lite" "$OLS_IMAGE" "default"
@@ -65,14 +65,14 @@ function run_suites() {
     run_suite "rhelai_vllm" "smoketest" "rhelai_vllm" "$OPENAI_PROVIDER_KEY_PATH" "gpt-3.5-turbo" "$OLS_IMAGE" "default"
     (( rc = rc || $? ))
 
-    run_suite "certificates" "certificates" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4.1-mini" "$OLS_IMAGE" "default"
+    run_suite "certificates" "certificates" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "default"
     (( rc = rc || $? ))
 
     # TODO: Reduce execution time. Sequential execution will take more time. Parallel execution will have cluster claim issue.
     # Run tool calling - Enable tool_calling
-    run_suite "azure_openai_tool_calling" "tool_calling" "azure_openai" "$AZUREOPENAI_PROVIDER_KEY_PATH" "gpt-4.1-mini" "$OLS_IMAGE" "tool_calling"
+    run_suite "azure_openai_tool_calling" "tool_calling" "azure_openai" "$AZUREOPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "tool_calling"
     (( rc = rc || $? ))
-    run_suite "openai_tool_calling" "tool_calling" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4.1-mini" "$OLS_IMAGE" "tool_calling"
+    run_suite "openai_tool_calling" "tool_calling" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "tool_calling"
     (( rc = rc || $? ))
     run_suite "watsonx_tool_calling" "tool_calling" "watsonx" "$WATSONX_PROVIDER_KEY_PATH" "ibm/granite-4-h-small" "$OLS_IMAGE" "tool_calling"
     (( rc = rc || $? ))
@@ -87,21 +87,21 @@ function run_suites() {
     run_suite "watsonx_byok2" "byok2" "watsonx" "$WATSONX_PROVIDER_KEY_PATH" "ibm/granite-4-h-small" "$OLS_IMAGE" "byok2"
 
     # quota limits tests, independent of provider therefore only testing one
-    run_suite "quota_limits" "quota_limits" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4.1-mini" "$OLS_IMAGE" "quota"
+    run_suite "quota_limits" "quota_limits" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "quota"
     (( rc = rc || $? ))
 
     # exporter test
-    run_suite "data_export" "data_export" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4.1-mini" "$OLS_IMAGE" "data_export"
+    run_suite "data_export" "data_export" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "data_export"
     (( rc = rc || $? ))
 
   else
     # Tests for disconnected environments
     # smoke tests for RHOAI VLLM-compatible provider
     run_suite "rhoai_vllm" "smoketest" "rhoai_vllm" "$OPENAI_PROVIDER_KEY_PATH" "gpt-3.5-turbo" "$OLS_IMAGE" "default"
     (( rc = rc || $? ))
-  
+
   cleanup_ols_operator
-  
+
   fi
   set -e