diff --git a/.gitattributes b/.gitattributes index 5378fe089b..5b82ef218b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,4 @@ -* -text \ No newline at end of file +* -text + +# Explicitly ensure shell scripts use LF for CI compatibility +*.sh text eol=lf \ No newline at end of file diff --git a/.github/scripts/clang-query.sh b/.github/scripts/clang-query.sh new file mode 100644 index 0000000000..5b915e43a4 --- /dev/null +++ b/.github/scripts/clang-query.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +TMP_DIR="$REPO_ROOT/.ci_tmp" +mkdir -p "$TMP_DIR" + +# Define the cleanup function +cleanup() { + echo "Cleaning up temporary files..." + rm -rf "$TMP_DIR" +} + +# Register the trap: run cleanup on EXIT, plus common signals like INT (Ctrl+C) or TERM +# trap cleanup EXIT INT TERM + + +cd "$REPO_ROOT" + +# find path to compile_commands.json +if [ -f "$REPO_ROOT/SerialPrograms/bin/compile_commands.json" ]; then + DB_PATH="$REPO_ROOT/SerialPrograms/bin/compile_commands.json" +elif [ -f "$REPO_ROOT/build/RelWithDebInfo/compile_commands.json" ]; then + DB_PATH="$REPO_ROOT/build/RelWithDebInfo/compile_commands.json" +else + echo "Error: compile_commands.json not found!" + exit 1 +fi + + +echo "Generating clang-scan-deps experimental-full > deps.json." + +# in ubuntu, the command is clang-scan-deps-18. in Windows, it is clang-scan-deps +SCAN_DEPS=$(command -v clang-scan-deps-18 || command -v clang-scan-deps) + +# Safety check: Exit if the tool isn't found +if [ -z "$SCAN_DEPS" ]; then + echo "Error: clang-scan-deps (or version -18) not found in PATH." + exit 1 +fi + + +# filter compile_commands.json, to remove .rc files, since clang-scan-deps doesn't recognize this format +jq '[.[] | select(.file | endswith(".rc") | not)]' "$DB_PATH" > "$TMP_DIR/compile_commands_filtered.json" + +# get dependency graph +"$SCAN_DEPS" -compilation-database "$TMP_DIR/compile_commands_filtered.json" -format experimental-full > "$TMP_DIR/deps.json" + +# normalize slashes +# sed 's|\\\\|/|g' deps.json > normalized_deps.json +sed -i 's|\\\\|/|g' "$TMP_DIR/deps.json" + +# check if deps.json has the expected keys +# because we are relying on clang-scan-deps experimental-full, where the names of the keys can change. +TU_KEY="translation-units" +CMD_KEY="commands" +DEPS="file-deps" +INPUT="input-file" + +JQ_SCRIPT=$(cat << 'EOF' + # 1. Access the target object + (.[$TU][0][$CMD][0]) as $target + + # 2. Define the required keys + + | [$DEPS, $INPUT] as $required + + | ( + if .[$TU] == null then + "Missing: \($TU). Keys found at top-level: \(keys_unsorted)" + elif .[$TU][0] == null then + "Missing: \($TU)[0]" + elif .[$TU][0].[$CMD] == null then + "Missing: \($TU)[0].\($CMD). Keys found from \($TU)[0]: \(.[$TU][0] | keys_unsorted)" + elif $target == null then + "Missing: \($TU)[0].[$CMD][0]" + elif ($required | all(. as $req | $target | has($req)) | not) then + "Missing: One or more required keys \($required). Found: \($target | keys_unsorted)" + # elif (.[$TU][0].[$CMD][0] | keys_unsorted | any(. == [$DEPS] or . == [$INPUT]) | not) then + # "Missing: both \($DEPS) and \($INPUT). Found: \(.[$TU][0][$CMD][0] | keys_unsorted)" + else + "All keys \($required) found in \($TU)[0].\($CMD)[0]" + end + ) as $result + + | if ($result | type == "string" and startswith("Missing:")) then + ("\($result). The keys within the experimental-full format from clang-scan-deps can change over time. Please fix the CI to use the correct keys.") | halt_error + else $result end +EOF +) + +echo "Checking keys in deps.json." +jq -r "$JQ_SCRIPT" \ + --arg TU "$TU_KEY" \ + --arg CMD "$CMD_KEY" \ + --arg DEPS "$DEPS" \ + --arg INPUT "$INPUT" \ + "$TMP_DIR/deps.json" + +echo "Generating changed_files.txt from git diff." + +# git diff with relative paths +git diff --name-only origin/main...HEAD > "$TMP_DIR/changed_files.txt" + +echo "Generating files_to_query.txt, based on changed_files.txt and deps.json." + +# for each line in changed_files_unix.txt, search deps.json to find all their dependants +jq -r --rawfile mod "$TMP_DIR/changed_files.txt" \ + --arg TU "$TU_KEY" \ + --arg CMD "$CMD_KEY" \ + --arg DEPS "$DEPS" \ + --arg INPUT "$INPUT" ' + # 1. Clean the list of changed files + ($mod | split("\n") | map(select(length > 0))) as $changes | + + # 2. Access the translation-units array + [ .[$TU][] | .[$CMD][] | + select( + # 3. Check "file-deps" for matches + .[$DEPS][] | . as $dp | + any($changes[]; . as $c | $dp | endswith($c)) + ) | + # 4. Get the source file path + .[$INPUT] + ] | unique[] +' "$TMP_DIR/deps.json" | tr -d '\r' > "$TMP_DIR/files_to_query.txt" + + + +cat << 'EOF' > "$TMP_DIR/query.txt" +set output dump +match invocation( + isExpansionInFileMatching("SerialPrograms/"), + hasDeclaration(cxxConstructorDecl(ofClass(hasName("std::filesystem::path")))), + hasArgument(0, hasType(asString("std::string"))) +) +EOF + +echo "Running clang-query." + +# files=$(jq -r '.[].file' "$DB_PATH") +DB_DIR=$(dirname "$DB_PATH") +#echo "$files" | xargs --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> output.txt + +# jq -r '.[].file' "$DB_PATH" | sed 's/\\/\//g' | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" -- -Wno-unused-command-line-argument >> "$TMP_DIR/output.txt" + +# this works +# jq -r '.[].file' "$DB_PATH" | sed 's/\\/\//g' | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" +# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" + +# also works +# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | xargs -d '\n' --max-args=150 \ +# clang-query -p "$DB_DIR" \ +# --extra-arg="-Wno-unused-command-line-argument" \ +# -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" + +# also works +# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' | \ +# xargs -d '\n' --max-args=150 \ +# clang-query -p "$DB_DIR" \ +# --extra-arg="-Wno-unused-command-line-argument" \ +# --extra-arg="-Wno-unused-function" \ +# -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" + +CLANG_QUERY=$(command -v clang-query-18 || command -v clang-query) + +if [ -z "$CLANG_QUERY" ]; then + echo "Error: clang-query (or version -18) not found!" + exit 1 +fi + +ONLY_CHECK_CHANGED_FILES=true +if [ "$ONLY_CHECK_CHANGED_FILES" = "true" ]; then + LIST_FILE="$TMP_DIR/files_to_query.txt" +else # check all files + LIST_FILE="$TMP_DIR/file_list.txt" + jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' > "$LIST_FILE" +fi + +> "$TMP_DIR/output.txt" + +# Run clang-query using the list file +# check if LIST_FILE has any data to analyze +if [ ! -s "$LIST_FILE" ]; then + echo "No files found to analyze. Skipping Clang-Query." +else + xargs -d '\n' -a "$LIST_FILE" --max-args=150 \ + "$CLANG_QUERY" -p "$DB_DIR" \ + --extra-arg="-Wno-unused-command-line-argument" \ + --extra-arg="-Wno-unused-function" \ + -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" +fi + + + +cat "$TMP_DIR/output.txt" +if grep --silent "Match #" "$TMP_DIR/output.txt"; then + echo "::error Forbidden std::filesystem::path construction detected!" + exit 1 +fi \ No newline at end of file diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index 09d52a5ba1..dac82ba996 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -51,6 +51,8 @@ jobs: with: path: 'Arduino-Source' submodules: 'recursive' + fetch-depth: 0 + filter: blob:none - name: Install Qt uses: jurplel/install-qt-action@v4 @@ -64,7 +66,7 @@ jobs: cd Arduino-Source sudo apt update sudo apt upgrade - sudo apt install clang-tools libopencv-dev + sudo apt install clang-tools-18 libopencv-dev sudo apt install ./3rdPartyBinaries/libdpp-10.0.28-linux-x64.deb @@ -85,9 +87,26 @@ jobs: cd Arduino-Source/SerialPrograms mkdir bin cd bin - cmake .. -DQT_MAJOR:STRING=6 ${{env.CMAKE_ADDITIONAL_FLAGS}} - cmake --build . --config RelWithDebInfo --parallel 10 - + cmake .. -DQT_MAJOR:STRING=6 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ${{env.CMAKE_ADDITIONAL_FLAGS}} + # cmake --build . --config RelWithDebInfo --parallel 10 + - name: Fake missing Qt generated files + if: inputs.run-clang-query + run: | + # Create the bin directory if it doesn't exist + mkdir -p Arduino-Source/SerialPrograms/bin + + # Fake the RCC (resource) files + touch Arduino-Source/SerialPrograms/bin/qrc_darkstyle.cpp + + # Fake the MOC/Autogen files for all components + mkdir -p Arduino-Source/SerialPrograms/bin/SerialProgramsLib_autogen + touch Arduino-Source/SerialPrograms/bin/SerialProgramsLib_autogen/mocs_compilation.cpp + + mkdir -p Arduino-Source/SerialPrograms/bin/SerialPrograms_autogen + touch Arduino-Source/SerialPrograms/bin/SerialPrograms_autogen/mocs_compilation.cpp + + mkdir -p Arduino-Source/SerialPrograms/bin/SerialProgramsCommandLine_autogen + touch Arduino-Source/SerialPrograms/bin/SerialProgramsCommandLine_autogen/mocs_compilation.cpp - name: Prepare upload build if: inputs.upload-build shell: bash @@ -128,22 +147,6 @@ jobs: - name: Run clang query if: inputs.run-clang-query - run : | - cd Arduino-Source - - cat << 'EOF' > query.txt - set output dump - match invocation( - isExpansionInFileMatching("SerialPrograms/"), - hasDeclaration(cxxConstructorDecl(ofClass(hasName("std::filesystem::path")))), - hasArgument(0, hasType(asString("std::string"))) - ) - EOF - - files=$(jq -r '.[].file' SerialPrograms/bin/compile_commands.json) - echo "$files" | xargs --max-args=150 clang-query -p SerialPrograms/bin/ -f query.txt >> output.txt - cat output.txt - if grep --silent "Match #" output.txt; then - echo "::error Forbidden std::filesystem::path construction detected!" - exit 1 - fi + working-directory: ./Arduino-Source + run : bash ./.github/scripts/clang-query.sh + diff --git a/.github/workflows/cpp-ci-serial-programs-mac-intel.yml b/.github/workflows/cpp-ci-serial-programs-mac-intel.yml deleted file mode 100644 index d3664a0b22..0000000000 --- a/.github/workflows/cpp-ci-serial-programs-mac-intel.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: C++ CI Serial Programs Mac Intel -on: [push, pull_request, workflow_dispatch] - -jobs: - build: - uses: ./.github/workflows/cpp-ci-serial-programs-base.yml - with: - os: macos-15-intel - compiler: default - run-tests: true \ No newline at end of file diff --git a/.github/workflows/cpp-ci-serial-programs-mac.yml b/.github/workflows/cpp-ci-serial-programs-mac.yml deleted file mode 100644 index a021d44c13..0000000000 --- a/.github/workflows/cpp-ci-serial-programs-mac.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: C++ CI Serial Programs Mac -on: [push, pull_request, workflow_dispatch] - -jobs: - build: - uses: ./.github/workflows/cpp-ci-serial-programs-base.yml - with: - os: macos-15 - compiler: default - run-tests: true \ No newline at end of file diff --git a/.github/workflows/cpp-ci-serial-programs-ubuntu-default.yml b/.github/workflows/cpp-ci-serial-programs-ubuntu-default.yml deleted file mode 100644 index 5c4a9b2fcb..0000000000 --- a/.github/workflows/cpp-ci-serial-programs-ubuntu-default.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: C++ CI Serial Programs Ubuntu Default -on: [push, pull_request, workflow_dispatch] - -jobs: - build: - uses: ./.github/workflows/cpp-ci-serial-programs-base.yml - with: - os: ubuntu-24.04 - compiler: default - run-clang-query: true \ No newline at end of file diff --git a/.github/workflows/cpp-ci-serial-programs-windows-clang.yml b/.github/workflows/cpp-ci-serial-programs-windows-clang.yml deleted file mode 100644 index 7c616ff1ac..0000000000 --- a/.github/workflows/cpp-ci-serial-programs-windows-clang.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: C++ CI Serial Programs Windows Clang -on: [push, pull_request, workflow_dispatch] - -jobs: - build: - uses: ./.github/workflows/cpp-ci-serial-programs-base.yml - with: - os: windows-2025 - compiler: clang - upload-build: true - run-tests: true \ No newline at end of file diff --git a/.github/workflows/cpp-ci-serial-programs-windows-default.yml b/.github/workflows/cpp-ci-serial-programs-windows-default.yml deleted file mode 100644 index 4e835a3f94..0000000000 --- a/.github/workflows/cpp-ci-serial-programs-windows-default.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: C++ CI Serial Programs Windows Default -on: [push, pull_request, workflow_dispatch] - -jobs: - build: - uses: ./.github/workflows/cpp-ci-serial-programs-base.yml - with: - os: windows-2025 - compiler: default - upload-build: true - run-tests: true \ No newline at end of file diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index 409b205636..1a65c3c3bc 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -532,6 +532,13 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& detector.process_frame(snapshot, current_time()); #endif + std::string test = "hello"; + std::filesystem::path p{test}; + + std::string out_path = "hello/"; + std::filesystem::path const parent_dir{std::filesystem::path(out_path).parent_path()}; + + #if 0 UpdateMenuWatcher update_menu(console, COLOR_PURPLE); CheckOnlineWatcher check_online(COLOR_CYAN);