diff --git a/.github/workflows/bfcl_check-illegal-params.yml b/.github/workflows/bfcl_check-illegal-params.yml new file mode 100644 index 000000000000..cb98dc676314 --- /dev/null +++ b/.github/workflows/bfcl_check-illegal-params.yml @@ -0,0 +1,65 @@ +name: BFCL Illegal Parameter Check + +on: + pull_request: + branches: [ main ] + +jobs: + check-illegal-params: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + cd berkeley-function-call-leaderboard + pip install -e . + + - name: Check for illegal parameter names + id: check_params + working-directory: berkeley-function-call-leaderboard + run: | + # Capture the output of the script + OUTPUT=$(python utils/check_illegal_python_param_name.py) + echo "$OUTPUT" + + # If the output contains "Illegal parameter name", fail the check + if echo "$OUTPUT" | grep -q "Illegal parameter name"; then + echo "::error::Found illegal Python parameter names!" + echo "ILLEGAL_PARAMS_FOUND=true" >> $GITHUB_ENV + echo "$OUTPUT" > illegal_params.txt + else + echo "ILLEGAL_PARAMS_FOUND=false" >> $GITHUB_ENV + fi + + - name: Comment on PR with results + if: github.event_name == 'pull_request' + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + ## BFCL Illegal Parameter Check Results + + ${{ env.ILLEGAL_PARAMS_FOUND == 'true' && '❌ Failed: Illegal Python parameter names detected!' || '✅ Passed: No illegal parameters found.' }} + + ${{ env.ILLEGAL_PARAMS_FOUND == 'true' && '### How to fix: + 1. Run this script locally to automatically fix the illegal parameters: + ```bash + cd berkeley-function-call-leaderboard + python utils/check_illegal_python_param_name.py + ``` + 2. Commit and push the changes + 3. Update your pull request' || '' }} + + - name: Fail if illegal parameters found + if: env.ILLEGAL_PARAMS_FOUND == 'true' + run: | + cat illegal_params.txt + exit 1 diff --git a/.github/workflows/bfcl_data_format_check.yml b/.github/workflows/bfcl_data_format_check.yml new file mode 100644 index 000000000000..fdd527fc7b4a --- /dev/null +++ b/.github/workflows/bfcl_data_format_check.yml @@ -0,0 +1,129 @@ +name: BFCL Data Format Check + +on: + pull_request: + branches: [ main ] + +jobs: + check-data-format: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install jsonschema + + - name: Create schema validation script + run: | + cat > validate_data.py << 'EOF' + import json + import sys + import os + from jsonschema import validate + from glob import glob + + # Print current working directory and list files for debugging + print(f"Current working directory: {os.getcwd()}") + print("Contents of current directory:", os.listdir()) + + schema = { + "type": "object", + "required": ["id", "question", "initial_config", "path", "involved_classes"], + "properties": { + "id": {"type": "string"}, + "question": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "object", + "required": ["role", "content"], + "properties": { + "role": {"type": "string"}, + "content": {"type": "string"} + } + } + } + }, + "initial_config": {"type": "object"}, + "path": { + "type": "array", + "items": {"type": "string"} + }, + "involved_classes": { + "type": "array", + "items": {"type": "string"} + } + } + } + + def validate_file(filepath): + print(f"Validating file: {filepath}") # Debug print + with open(filepath, 'r') as f: + for line_num, line in enumerate(f, 1): + try: + data = json.loads(line.strip()) + validate(instance=data, schema=schema) + except Exception as e: + print(f"Error in file {filepath} at line {line_num}:") + print(f"Line content: {line.strip()}") + print(f"Error: {str(e)}") + return False + return True + + # Find all multi-turn files + pattern = "berkeley-function-call-leaderboard/data/*multi_turn*.json" + files = glob(pattern) + print(f"Found files matching pattern '{pattern}': {files}") # Debug print + + if not files: + print("No files found! Check the path and pattern.") + sys.exit(1) + + success = True + for filepath in files: + if not validate_file(filepath): + success = False + + sys.exit(0 if success else 1) + EOF + + - name: Run format validation + id: validate + run: python validate_data.py + + - name: Comment on PR with results + if: github.event_name == 'pull_request' + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + ## BFCL Data Format Check Results + + ${{ steps.validate.outcome == 'success' && '✅ Passed: All data files are correctly formatted.' || '❌ Failed: Data format validation failed. Please check the workflow logs for details.' }} + + ${{ steps.validate.outcome != 'success' && '### How to fix: + 1. Review the workflow logs to see which files failed validation + 2. Ensure all data entries follow the required format: + ```json + { + "id": "string", + "question": [[{"role": "string", "content": "string"}]], + "initial_config": {}, + "path": ["string"], + "involved_classes": ["string"] + } + ``` + 3. Update the files and push the changes' || '' }} + + - name: Fail if validation failed + if: steps.validate.outcome != 'success' + run: exit 1 \ No newline at end of file diff --git a/.github/workflows/bfcl_evaluation.yml b/.github/workflows/bfcl_evaluation.yml new file mode 100644 index 000000000000..6d5780379e1b --- /dev/null +++ b/.github/workflows/bfcl_evaluation.yml @@ -0,0 +1,73 @@ +name: BFCL Evaluation Check + +on: + pull_request: + branches: [ main ] + +jobs: + evaluate: + runs-on: ubuntu-latest + + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + MIN_ACCEPTABLE_SCORE: 0.60 + + steps: + - uses: actions/checkout@v3 + + - name: Check for OPENAI_API_KEY + run: | + if [ -z "$OPENAI_API_KEY" ]; then + echo "Error: OPENAI_API_KEY is not set" + exit 1 + fi + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + cd berkeley-function-call-leaderboard + pip install -e . + + - name: Run BFCL generate + working-directory: berkeley-function-call-leaderboard + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + bfcl generate \ + --model gpt-4o-2024-08-06-FC \ + --test-category live_parallel + + - name: Run BFCL evaluate and extract score + working-directory: berkeley-function-call-leaderboard + run: | + bfcl evaluate \ + --model gpt-4o-2024-08-06-FC \ + --test-category live_parallel + + # Read score from the JSON file - get the first line only and parse accuracy + score=$(head -n 1 score/gpt-4o-2024-08-06-FC/BFCL_v3_live_parallel_score.json | jq -r '.accuracy') + echo "EVALUATION_SCORE=${score}" >> $GITHUB_ENV + + if (( $(echo "$score < $MIN_ACCEPTABLE_SCORE" | bc -l) )); then + echo "Score ($score) is below minimum threshold ($MIN_ACCEPTABLE_SCORE)" + exit 1 + else + echo "Score ($score) meets or exceeds minimum threshold ($MIN_ACCEPTABLE_SCORE)" + fi + + - name: Comment on PR with results + if: github.event_name == 'pull_request' + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + ## BFCL Evaluation Results + + - Score: ${{ env.EVALUATION_SCORE }} + - Minimum Threshold: ${{ env.MIN_ACCEPTABLE_SCORE }} + - Status: ${{ env.EVALUATION_SCORE >= env.MIN_ACCEPTABLE_SCORE && '✅ Passed' || '❌ Failed' }}