api_test_effect.yml

name: 07. API Effect Tests

on:
  workflow_dispatch:
  schedule:
    - cron: '0 1,10 * * *'
  release:
    types: [prereleased]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  effect-tests:
    name: API Effect Tests (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 120
    if: github.event_name == 'workflow_dispatch' || github.repository == 'volcengine/OpenViking'
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        os: [ubuntu-24.04]

    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive
          fetch-depth: 100

      - name: Set up Python 3.10
        uses: actions/setup-python@v6
        with:
          python-version: '3.10'

      - name: Cache Python dependencies
        uses: actions/cache@v5
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-effect-${{ hashFiles('**/requirements.txt', '**/pyproject.toml') }}
          restore-keys: |
            ${{ runner.os }}-pip-effect-

      - name: Install system dependencies
        run: |
          sudo apt-get update -y && sudo apt-get install -y cmake build-essential ffmpeg --no-install-recommends

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true

      - name: Install build dependencies (system pip)
        run: |
          python -m pip install --upgrade pip
          pip install "setuptools>=70.1" setuptools_scm cmake wheel

      - name: Install dependencies using uv sync
        run: uv sync --frozen

      - name: Install build dependencies (uv pip)
        run: uv pip install setuptools setuptools_scm cmake wheel maturin

      - name: Set up Rust toolchain
        uses: dtolnay/rust-toolchain@v1
        with:
          toolchain: stable

      - name: Build C++ extensions
        shell: bash
        run: |
          export OV_SKIP_OV_BUILD=1
          mkdir -p openviking/bin
          touch openviking/bin/ov
          chmod +x openviking/bin/ov
          if [ -z "$SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENVIKING" ]; then
            PRETEND_VERSION=$(git describe --tags --always 2>/dev/null | sed -E 's/^v?([0-9]+\.[0-9]+\.[0-9]+).*/\1.dev0/' || echo "0.0.0.dev0")
            echo "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENVIKING=$PRETEND_VERSION" >> $GITHUB_ENV
          fi
          uv run python setup.py build_ext --inplace

      - name: Build ragfs-python native extension
        shell: bash
        run: |
          RAGFS_LIB_DIR="openviking/lib"
          RAGFS_SO_COUNT=$(ls -1 "$RAGFS_LIB_DIR"/ragfs_python*.so "$RAGFS_LIB_DIR"/ragfs_python*.pyd "$RAGFS_LIB_DIR"/ragfs_python*.dylib 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0")
          if [ "$RAGFS_SO_COUNT" -gt 0 ]; then exit 0; fi
          TMPDIR_RAGFS=$(mktemp -d)
          cd crates/ragfs-python
          uv run --no-project maturin build --release --features s3 --out "$TMPDIR_RAGFS"
          cd ../..
          WHL_FILE=$(ls -1 "$TMPDIR_RAGFS"/ragfs_python-*.whl 2>/dev/null | head -1)
          if [ -z "$WHL_FILE" ]; then exit 1; fi
          mkdir -p "$RAGFS_LIB_DIR"
          WHL_FILE_PY=$(echo "$WHL_FILE" | sed 's/\\/\\\\/g')
          RAGFS_LIB_DIR_PY=$(echo "$RAGFS_LIB_DIR" | sed 's/\\/\\\\/g')
          uv run python -c "
          import zipfile, os, sys
          with zipfile.ZipFile('${WHL_FILE_PY}') as zf:
              for name in zf.namelist():
                  bn = os.path.basename(name)
                  if bn.startswith('ragfs_python') and (bn.endswith('.so') or bn.endswith('.pyd') or bn.endswith('.dylib')):
                      dst = os.path.join('${RAGFS_LIB_DIR_PY}', bn)
                      with zf.open(name) as src, open(dst, 'wb') as f:
                          f.write(src.read())
                      os.chmod(dst, 0o755)
                      sys.exit(0)
          sys.exit(1)
          "
          rm -rf "$TMPDIR_RAGFS"

      - name: Install API test dependencies
        run: |
          cd tests/api_test
          uv pip install -r requirements.txt

      - name: Create OpenViking config file
        env:
          VLM_API_KEY: ${{ secrets.VLM_API_KEY }}
          EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
        run: |
          mkdir -p $HOME/.openviking
          if [ -n "$VLM_API_KEY" ] && [ -n "$EMBEDDING_API_KEY" ]; then
            echo "HAS_SECRETS=true" >> $GITHUB_ENV
            cat > $HOME/.openviking/ov.conf << EOF
          {
            "server": { "root_api_key": "test-root-api-key" },
            "vlm": {
              "provider": "volcengine",
              "api_key": "$VLM_API_KEY",
              "model": "doubao-seed-2-0-mini-260215",
              "api_base": "https://ark.cn-beijing.volces.com/api/v3",
              "temperature": 0.1, "max_retries": 3
            },
            "embedding": {
              "dense": {
                "provider": "volcengine",
                "api_key": "$EMBEDDING_API_KEY",
                "model": "doubao-embedding-vision-251215",
                "api_base": "https://ark.cn-beijing.volces.com/api/v3",
                "dimension": 1024, "input": "multimodal"
              }
            }
          }
          EOF
          else
            echo "HAS_SECRETS=false" >> $GITHUB_ENV
            cat > $HOME/.openviking/ov.conf << EOF
          {
            "server": { "root_api_key": "test-root-api-key" },
            "vlm": {
              "provider": "volcengine",
              "api_key": "dummy-vlm-api-key",
              "model": "doubao-seed-2-0-mini-260215",
              "api_base": "https://ark.cn-beijing.volces.com/api/v3",
              "temperature": 0.1, "max_retries": 3
            },
            "embedding": {
              "dense": {
                "provider": "volcengine",
                "api_key": "dummy-embedding-api-key",
                "model": "doubao-embedding-vision-251215",
                "api_base": "https://ark.cn-beijing.volces.com/api/v3",
                "dimension": 1024, "input": "multimodal"
              }
            }
          }
          EOF
          fi

      - name: Start OpenViking Server
        run: |
          SERVER_PORT=1933
          echo "SERVER_PORT=$SERVER_PORT" >> $GITHUB_ENV
          export ROOT_API_KEY=test-root-api-key
          export SERVER_PORT=$SERVER_PORT
          nohup uv run python -m openviking.server.bootstrap > openviking-server.log 2>&1 &
          echo $! > openviking-server.pid
          echo "Waiting for server to start..."
          for i in {1..30}; do
            if curl -s http://127.0.0.1:$SERVER_PORT/health | grep -q '"healthy":true'; then
              echo "Server is ready!"
              break
            fi
            echo "Waiting... ($i/30)"
            sleep 2
          done
          if ! curl -s http://127.0.0.1:$SERVER_PORT/health | grep -q '"healthy":true'; then
            echo "Server failed to start!"
            cat openviking-server.log
            exit 1
          fi

      - name: Run Effect Tests
        id: run-effect-tests
        run: |
          cd tests/api_test
          export OPENVIKING_API_KEY=test-root-api-key
          export SERVER_URL=http://127.0.0.1:${{ env.SERVER_PORT }}
          echo "Running lightweight slow tests (parallel with reduced workers)..."
          uv run python -m pytest \
            common/slow/ tasks/slow/ skills/slow/ retrieval/slow/ \
            -v -n 2 --tb=short --durations=0 \
            --html=api-effect-light-report.html --self-contained-html || true
          echo "Running heavy slow tests (serial to avoid resource conflicts)..."
          uv run python -m pytest \
            content/slow/ resources/slow/ sessions/slow/ \
            scenarios/slow/ scenarios/stability_error/slow/ \
            scenarios/resources_retrieval_slow/ \
            -v --tb=short --durations=0 \
            --html=api-effect-heavy-report.html --self-contained-html
        continue-on-error: true

      - name: Upload test reports
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: api-effect-test-reports-${{ github.run_id }}
          path: |
            tests/api_test/api-effect-light-report.html
            tests/api_test/api-effect-heavy-report.html
            openviking-server.log

      - name: Stop OpenViking Server
        if: always()
        run: |
          if [ -f openviking-server.pid ]; then
            kill $(cat openviking-server.pid) 2>/dev/null || true
            pkill -f "openviking.server.bootstrap" 2>/dev/null || true
          fi

      - name: Check test results
        if: steps.run-effect-tests.outcome != 'success'
        run: |
          echo "Effect tests failed!"
          exit 1