test-databricks-models.sh

#!/bin/bash
# Test script for Databricks models via opencode API
# Tests both basic response and tool call capabilities
# Auto-starts/restarts the dev server as needed

BASE_URL="http://localhost:4096"
PROVIDER="databricks"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"

# Representative models from each family
MODELS=(
  "databricks-claude-sonnet-4-6"
  "databricks-gpt-5-3-codex"
  "databricks-gpt-5-2-codex"
  "databricks-gpt-5"
  "databricks-gpt-5-mini"
  "databricks-gemini-2-5-flash"
  "databricks-gemini-2-5-pro"
  "databricks-llama-4-maverick"
  "databricks-meta-llama-3-3-70b-instruct"
  "databricks-qwen3-next-80b-a3b-instruct"
)

PASSED=0
FAILED=0
WARNINGS=0
ERRORS=()
WARN_LIST=()

# Auto-manage the dev server
start_server() {
  echo -n "  Starting dev server... "
  pkill -f "bun.*serve" 2>/dev/null
  sleep 1

  cd "$SCRIPT_DIR" && bun dev serve > /tmp/opencode-test-server.log 2>&1 &
  SERVER_PID=$!

  # Wait for server to be ready (up to 30s)
  for i in $(seq 1 30); do
    if curl -s "$BASE_URL/session" | jq -e . > /dev/null 2>&1; then
      echo "OK (pid: $SERVER_PID)"
      return 0
    fi
    sleep 1
  done
  echo "FAIL (timeout)"
  return 1
}

stop_server() {
  pkill -f "bun.*serve" 2>/dev/null
}

test_model() {
  local model="$1"
  echo ""
  echo "============================================"
  echo "Testing: $model"
  echo "============================================"

  # 1. Create session
  echo -n "  Creating session... "
  SESSION_RESP=$(curl -s -X POST "$BASE_URL/session" \
    -H "Content-Type: application/json" \
    -d '{"title":"test-'"$model"'"}')

  SESSION_ID=$(echo "$SESSION_RESP" | jq -r '.id // empty')
  if [ -z "$SESSION_ID" ]; then
    echo "FAIL"
    FAILED=$((FAILED + 1))
    ERRORS+=("$model: session creation failed")
    return 1
  fi
  echo "$SESSION_ID"

  # 2. Test basic response
  echo -n "  Basic response... "
  MSG_RESP=$(curl -s --max-time 120 -X POST "$BASE_URL/session/$SESSION_ID/message" \
    -H "Content-Type: application/json" \
    -d '{
      "model": {"providerID": "'"$PROVIDER"'", "modelID": "'"$model"'"},
      "parts": [{"type": "text", "text": "What is 2+2? Answer with just the number."}]
    }')

  # Check for API-level errors
  API_ERROR=$(echo "$MSG_RESP" | jq -r '.info.error.name // empty' 2>/dev/null)
  API_ERROR_MSG=$(echo "$MSG_RESP" | jq -r '.info.error.data.message // empty' 2>/dev/null | head -c 200)

  if [ -n "$API_ERROR" ]; then
    echo "FAIL ($API_ERROR: ${API_ERROR_MSG:0:100})"
    FAILED=$((FAILED + 1))
    ERRORS+=("$model: $API_ERROR - ${API_ERROR_MSG:0:150}")
    return 1
  fi

  # Check for text parts
  TEXT_CONTENT=$(echo "$MSG_RESP" | jq -r '[.parts[] | select(.type == "text") | .text] | join(" ")' 2>/dev/null)
  INPUT_TOKENS=$(echo "$MSG_RESP" | jq '.info.tokens.input // 0' 2>/dev/null)
  OUTPUT_TOKENS=$(echo "$MSG_RESP" | jq '.info.tokens.output // 0' 2>/dev/null)

  if [ -z "$TEXT_CONTENT" ] || [ "$TEXT_CONTENT" = "null" ]; then
    echo "FAIL (no text in response)"
    FAILED=$((FAILED + 1))
    ERRORS+=("$model: no text in basic response")
    return 1
  fi
  echo "OK (\"${TEXT_CONTENT:0:50}\") [${INPUT_TOKENS}/${OUTPUT_TOKENS} tokens]"

  # 3. Test tool call - ask it to use the read tool
  echo -n "  Tool call... "
  TOOL_RESP=$(curl -s --max-time 300 -X POST "$BASE_URL/session/$SESSION_ID/message" \
    -H "Content-Type: application/json" \
    -d '{
      "model": {"providerID": "'"$PROVIDER"'", "modelID": "'"$model"'"},
      "parts": [{"type": "text", "text": "Use the read tool to read the file at /Users/david.okeeffe/Repos/opencode/opencode.json and tell me the provider name configured in it."}]
    }')

  # Check for API-level errors
  TOOL_API_ERROR=$(echo "$TOOL_RESP" | jq -r '.info.error.name // empty' 2>/dev/null)
  TOOL_API_ERROR_MSG=$(echo "$TOOL_RESP" | jq -r '.info.error.data.message // empty' 2>/dev/null | head -c 200)

  if [ -n "$TOOL_API_ERROR" ]; then
    echo "FAIL ($TOOL_API_ERROR: ${TOOL_API_ERROR_MSG:0:100})"
    FAILED=$((FAILED + 1))
    ERRORS+=("$model: tool test - $TOOL_API_ERROR")
    return 1
  fi

  TOOL_TEXT=$(echo "$TOOL_RESP" | jq -r '[.parts[] | select(.type == "text") | .text] | join(" ")' 2>/dev/null | head -c 150)

  # The prompt endpoint returns only the final assistant message.
  # Check full message history for tool calls in intermediate steps.
  HISTORY=$(curl -s "$BASE_URL/session/$SESSION_ID/message" 2>/dev/null)
  HAS_TOOL_IN_HISTORY=$(echo "$HISTORY" | jq '[.[] | .parts[] | select(.type == "tool")] | length' 2>/dev/null)

  if [ "$HAS_TOOL_IN_HISTORY" -gt 0 ]; then
    TOOL_NAMES=$(echo "$HISTORY" | jq -r '[.[] | .parts[] | select(.type == "tool") | .tool] | unique | join(", ")' 2>/dev/null)
    echo "OK (tools: $TOOL_NAMES) text: \"${TOOL_TEXT:0:80}\""
    PASSED=$((PASSED + 1))
  else
    echo "WARN (no tool call, but text: \"${TOOL_TEXT:0:80}\")"
    WARNINGS=$((WARNINGS + 1))
    WARN_LIST+=("$model: responded without using tools")
    if [ -n "$TOOL_TEXT" ]; then
      PASSED=$((PASSED + 1))
    else
      FAILED=$((FAILED + 1))
      ERRORS+=("$model: no tool call and no text in tool test")
    fi
  fi
}

echo "====================================================="
echo "  Databricks Model Integration Test Suite"
echo "====================================================="
echo "  Server: $BASE_URL"
echo "  Models: ${#MODELS[@]}"
echo "====================================================="

# Auto-start server
if ! curl -s "$BASE_URL/session" | jq -e . > /dev/null 2>&1; then
  start_server || exit 1
else
  echo "  Server already running"
fi

for model in "${MODELS[@]}"; do
  test_model "$model"
done

echo ""
echo "====================================================="
echo "  RESULTS"
echo "====================================================="
echo "  Passed:   $PASSED / ${#MODELS[@]}"
echo "  Failed:   $FAILED / ${#MODELS[@]}"
echo "  Warnings: $WARNINGS"
if [ ${#ERRORS[@]} -gt 0 ]; then
  echo ""
  echo "  Failures:"
  for err in "${ERRORS[@]}"; do
    echo "    - $err"
  done
fi
if [ ${#WARN_LIST[@]} -gt 0 ]; then
  echo ""
  echo "  Warnings:"
  for w in "${WARN_LIST[@]}"; do
    echo "    - $w"
  done
fi
echo "====================================================="

exit $FAILED