Cicd formatting data

.github/workflows/bfcl_check-illegal-params.yml

@@ -0,0 +1,65 @@
+name: BFCL Illegal Parameter Check
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  check-illegal-params:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd berkeley-function-call-leaderboard
+          pip install -e .
+
+      - name: Check for illegal parameter names
+        id: check_params
+        working-directory: berkeley-function-call-leaderboard
+        run: |
+          # Capture the output of the script
+          OUTPUT=$(python utils/check_illegal_python_param_name.py)
+          echo "$OUTPUT"
+
+          # If the output contains "Illegal parameter name", fail the check
+          if echo "$OUTPUT" | grep -q "Illegal parameter name"; then
+            echo "::error::Found illegal Python parameter names!"
+            echo "ILLEGAL_PARAMS_FOUND=true" >> $GITHUB_ENV
+            echo "$OUTPUT" > illegal_params.txt
+          else
+            echo "ILLEGAL_PARAMS_FOUND=false" >> $GITHUB_ENV
+          fi
+
+      - name: Comment on PR with results
+        if: github.event_name == 'pull_request'
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ## BFCL Illegal Parameter Check Results
+
+            ${{ env.ILLEGAL_PARAMS_FOUND == 'true' && '❌ Failed: Illegal Python parameter names detected!' || '✅ Passed: No illegal parameters found.' }}
+
+            ${{ env.ILLEGAL_PARAMS_FOUND == 'true' && '### How to fix:
+            1. Run this script locally to automatically fix the illegal parameters:
+               ```bash
+               cd berkeley-function-call-leaderboard
+               python utils/check_illegal_python_param_name.py
+               ```
+            2. Commit and push the changes
+            3. Update your pull request' || '' }}
+
+      - name: Fail if illegal parameters found
+        if: env.ILLEGAL_PARAMS_FOUND == 'true'
+        run: |
+          cat illegal_params.txt
+          exit 1

.github/workflows/bfcl_data_format_check.yml

@@ -0,0 +1,129 @@
+name: BFCL Data Format Check
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  check-data-format:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install jsonschema
+
+      - name: Create schema validation script
+        run: |
+          cat > validate_data.py << 'EOF'
+          import json
+          import sys
+          import os
+          from jsonschema import validate
+          from glob import glob
+
+          # Print current working directory and list files for debugging
+          print(f"Current working directory: {os.getcwd()}")
+          print("Contents of current directory:", os.listdir())
+
+          schema = {
+              "type": "object",
+              "required": ["id", "question", "initial_config", "path", "involved_classes"],
+              "properties": {
+                  "id": {"type": "string"},
+                  "question": {
+                      "type": "array",
+                      "items": {
+                          "type": "array",
+                          "items": {
+                              "type": "object",
+                              "required": ["role", "content"],
+                              "properties": {
+                                  "role": {"type": "string"},
+                                  "content": {"type": "string"}
+                              }
+                          }
+                      }
+                  },
+                  "initial_config": {"type": "object"},
+                  "path": {
+                      "type": "array",
+                      "items": {"type": "string"}
+                  },
+                  "involved_classes": {
+                      "type": "array",
+                      "items": {"type": "string"}
+                  }
+              }
+          }
+
+          def validate_file(filepath):
+              print(f"Validating file: {filepath}")  # Debug print
+              with open(filepath, 'r') as f:
+                  for line_num, line in enumerate(f, 1):
+                      try:
+                          data = json.loads(line.strip())
+                          validate(instance=data, schema=schema)
+                      except Exception as e:
+                          print(f"Error in file {filepath} at line {line_num}:")
+                          print(f"Line content: {line.strip()}")
+                          print(f"Error: {str(e)}")
+                          return False
+              return True
+
+          # Find all multi-turn files
+          pattern = "berkeley-function-call-leaderboard/data/*multi_turn*.json"
+          files = glob(pattern)
+          print(f"Found files matching pattern '{pattern}': {files}")  # Debug print
+
+          if not files:
+              print("No files found! Check the path and pattern.")
+              sys.exit(1)
+
+          success = True
+          for filepath in files:
+              if not validate_file(filepath):
+                  success = False
+
+          sys.exit(0 if success else 1)
+          EOF
+
+      - name: Run format validation
+        id: validate
+        run: python validate_data.py
+
+      - name: Comment on PR with results
+        if: github.event_name == 'pull_request'
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ## BFCL Data Format Check Results
+
+            ${{ steps.validate.outcome == 'success' && '✅ Passed: All data files are correctly formatted.' || '❌ Failed: Data format validation failed. Please check the workflow logs for details.' }}
+
+            ${{ steps.validate.outcome != 'success' && '### How to fix:
+            1. Review the workflow logs to see which files failed validation
+            2. Ensure all data entries follow the required format:
+               ```json
+               {
+                 "id": "string",
+                 "question": [[{"role": "string", "content": "string"}]],
+                 "initial_config": {},
+                 "path": ["string"],
+                 "involved_classes": ["string"]
+               }
+               ```
+            3. Update the files and push the changes' || '' }}
+
+      - name: Fail if validation failed
+        if: steps.validate.outcome != 'success'
+        run: exit 1

.github/workflows/bfcl_evaluation.yml

@@ -0,0 +1,73 @@
+name: BFCL Evaluation Check
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  evaluate:
+    runs-on: ubuntu-latest
+
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      MIN_ACCEPTABLE_SCORE: 0.60
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Check for OPENAI_API_KEY
+        run: |
+          if [ -z "$OPENAI_API_KEY" ]; then
+            echo "Error: OPENAI_API_KEY is not set"
+            exit 1
+          fi
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          cd berkeley-function-call-leaderboard
+          pip install -e .
+
+      - name: Run BFCL generate
+        working-directory: berkeley-function-call-leaderboard
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          bfcl generate \
+            --model gpt-4o-2024-08-06-FC \
+            --test-category live_parallel
+
+      - name: Run BFCL evaluate and extract score
+        working-directory: berkeley-function-call-leaderboard
+        run: |
+          bfcl evaluate \
+            --model gpt-4o-2024-08-06-FC \
+            --test-category live_parallel
+
+          # Read score from the JSON file - get the first line only and parse accuracy
+          score=$(head -n 1 score/gpt-4o-2024-08-06-FC/BFCL_v3_live_parallel_score.json | jq -r '.accuracy')
+          echo "EVALUATION_SCORE=${score}" >> $GITHUB_ENV
+
+          if (( $(echo "$score < $MIN_ACCEPTABLE_SCORE" | bc -l) )); then
+            echo "Score ($score) is below minimum threshold ($MIN_ACCEPTABLE_SCORE)"
+            exit 1
+          else
+            echo "Score ($score) meets or exceeds minimum threshold ($MIN_ACCEPTABLE_SCORE)"
+          fi
+
+      - name: Comment on PR with results
+        if: github.event_name == 'pull_request'
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ## BFCL Evaluation Results
+
+            - Score: ${{ env.EVALUATION_SCORE }}
+            - Minimum Threshold: ${{ env.MIN_ACCEPTABLE_SCORE }}
+            - Status: ${{ env.EVALUATION_SCORE >= env.MIN_ACCEPTABLE_SCORE && '✅ Passed' || '❌ Failed' }}