diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..46835ca7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +.github/ +.ruby-lsp/ +coverage/ +tmp/ +# Keep .git/ — needed for gemspec git ls-files and VCS checkout in tests +.idea/ +.claude/ +.bundle/ +vendor/ +.specs/ +*.gem +*.rbc +*.log + +# Ignore report files +*.attempt_*.png +*.diff.png +*.base.png +*.attempt_*.webp +*.diff.webp +*.base.webp + +# OS files +.DS_Store +Thumbs.db diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..2958c4ad --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,64 @@ +--- +name: Bug report +about: Create a report to help us improve screenshot comparisons +title: '' +labels: bug +assignees: '' + +--- + +## Describe the bug + +A clear and concise description of what the bug is. + +## To Reproduce + +Steps to reproduce the behavior: + +1. Go to '...' +2. Click on '...' +3. Run `....` +4. See error + +## Expected behavior + +A clear and concise description of what you expected to happen. + +## Actual behavior + +What actually happened. Include the full error message and stack trace if applicable. + +## Screenshots + +If applicable, add the `.diff.png` or `.heatmap.diff.png` files to help explain the problem. + +## Environment + +- **Ruby version:** (e.g., 3.4.1) +- **Rails version:** (e.g., 8.0) or N/A (non-Rails project) +- **`capybara-screenshot-diff` version:** (e.g., 1.12.0) +- **Image processing driver:** (`:vips` or `:chunky_png`) +- **Capybara driver:** (e.g., `selenium_chrome_headless`, `cuprite`) +- **Operating system:** (e.g., macOS 14, Ubuntu 24.04) +- **CI environment:** (e.g., GitHub Actions, local only) + +## Configuration + +```ruby +# Your test helper configuration (omit sensitive parts) +``` + +## Debug output + +If applicable, run with `DEBUG=1` and paste the output: + +``` +DEBUG=1 bundle exec rake test +``` + +## Additional context + +Add any other context about the problem here. For example: +- Is it specific to CI vs local? +- Does it reproduce with both VIPS and ChunkyPNG drivers? +- Is this a regression from a previous version? diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..9d0a3af1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Questions & Discussion + url: https://github.com/snap-diff/snap_diff-capybara/discussions + about: Ask questions and discuss usage with the community + - name: Documentation + url: https://github.com/snap-diff/snap_diff-capybara#readme + about: Check the README and docs/ directory first diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..6b05fcea --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,36 @@ +--- +name: Feature request +about: Suggest an idea for improving screenshot diff testing +title: '' +labels: enhancement +assignees: '' + +--- + +## Is your feature request related to a problem? + +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +## Describe the solution you'd like + +A clear and concise description of what you want to happen. + +## Describe alternatives you've considered + +A clear and concise description of any alternative solutions or features you've considered. + +## How would this benefit visual regression testing? + +Explain how this feature would help improve the screenshot comparison workflow for you or your team. + +## Example usage + +If applicable, show how you'd expect to use this feature: + +```ruby +# Example configuration or test code +``` + +## Additional context + +Add any other context, screenshots, or references about the feature request here. diff --git a/.github/actions/setup-ruby-and-dependencies/action.yml b/.github/actions/setup-ruby-and-dependencies/action.yml new file mode 100644 index 00000000..ed27ed77 --- /dev/null +++ b/.github/actions/setup-ruby-and-dependencies/action.yml @@ -0,0 +1,38 @@ +--- +name: 'Setup Ruby and Cache' +description: 'Set up Ruby environment and cache apt packages' +inputs: + ruby-version: + description: 'Ruby version to set up' + required: true + ruby-cache-version: + description: 'Bundler cache version' + required: false + cache-apt-packages: + description: 'Whether to cache apt packages' + required: false + default: 'false' +runs: + using: 'composite' + steps: + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ inputs.ruby-version }} + bundler-cache: true + cache-version: ${{ inputs.ruby-cache-version }}-v1 + + - name: Install and cache vips + if: ${{ inputs.cache-apt-packages == 'true' }} + uses: jetthoughts/cache-apt-pkgs-action@fix/upgrade-actions-cache-v5 + with: + packages: libvips libglib2.0-0 libglib2.0-dev libwebp-dev libvips42 libpng-dev + version: tests-v2 + + - name: Install vips (fallback) + if: ${{ inputs.cache-apt-packages != 'true' }} + run: sudo apt-get -qq update && sudo apt-get -qq install -y libvips + shell: bash + + - run: sudo sed -i 's/true/false/g' /etc/fonts/conf.d/10-yes-antialias.conf + shell: bash diff --git a/.github/actions/upload-screenshots/action.yml b/.github/actions/upload-screenshots/action.yml new file mode 100644 index 00000000..7a937a8e --- /dev/null +++ b/.github/actions/upload-screenshots/action.yml @@ -0,0 +1,113 @@ +--- +name: 'Upload SnapDiff screenshots' +description: 'Upload screenshot diffs, HTML report, and optionally comment on PR' +inputs: + name: + description: 'Artifact name prefix' + required: true + report-path: + description: 'Path to the HTML report directory' + default: 'doc/screenshots' + retention-days: + description: 'Number of days to retain artifacts' + default: '2' + pr-comment: + description: 'Post PR comment with report link (requires pull-requests: write)' + default: 'false' +outputs: + report-url: + description: 'Direct URL to the inline HTML report artifact' + value: ${{ steps.upload-report.outputs.artifact-url }} + report-full-url: + description: 'Direct URL to the full report artifact (with images)' + value: ${{ steps.upload-report-full.outputs.artifact-url }} +runs: + using: 'composite' + steps: + - name: Upload screenshot diffs + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.name }}-diffs + retention-days: ${{ inputs.retention-days }} + path: test/fixtures/app/doc/screenshots/ + if-no-files-found: ignore + + - name: Upload Capybara failure screenshots + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.name }}-capybara-fails + retention-days: ${{ inputs.retention-days }} + path: tmp/capybara/screenshots-diffs/ + if-no-files-found: ignore + + - name: Check for HTML report + id: check-report + shell: bash + run: | + if [ -f "${{ inputs.report-path }}/snap_diff_report.html" ]; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Prepare HTML report for inline preview + if: steps.check-report.outputs.exists == 'true' + shell: bash + run: cp "${{ inputs.report-path }}/snap_diff_report.html" "${{ inputs.report-path }}/${{ inputs.name }}-snap_diff-report.html" + + - name: Upload HTML report (inline preview) + id: upload-report + if: steps.check-report.outputs.exists == 'true' + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.name }}-snap_diff-report + retention-days: ${{ inputs.retention-days }} + path: ${{ inputs.report-path }}/${{ inputs.name }}-snap_diff-report.html + archive: false + + - name: Upload HTML report with images (full download) + id: upload-report-full + if: steps.check-report.outputs.exists == 'true' + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.name }}-report-full + retention-days: ${{ inputs.retention-days }} + path: ${{ inputs.report-path }}/ + + - name: Job summary + if: steps.check-report.outputs.exists == 'true' + shell: bash + run: | + cat >> "$GITHUB_STEP_SUMMARY" </dev/null || true + if EXISTING=$(git rev-parse -q --verify "refs/tags/$TAG^{commit}"); then + if [ "$EXISTING" = "$(git rev-parse HEAD)" ]; then + echo "Tag $TAG already exists at HEAD; skipping tag creation." + else + echo "::error::Tag $TAG already exists at $EXISTING, but HEAD is $(git rev-parse HEAD). Refusing to retag." + exit 1 + fi + else + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG" -m "$TAG" + git push origin "$TAG" + fi + + - name: Publish to RubyGems + uses: rubygems/release-gem@v1 + + # Mirror release: the same content ships under the forward-looking gem + # name snap_diff-capybara with an identical version. The mirror gemspec + # is generated here (not committed) so local `gem build` and the + # Gemfile's `gemspec` directive stay unambiguous. Requires the + # snap_diff-capybara gem on rubygems.org to trust this repo+workflow + # as a trusted publisher. + # No separate credential setup: release-gem above already configured + # trusted-publishing credentials, and the temporary key covers any gem + # whose rubygems.org settings trust this repo+workflow. + - name: Publish snap_diff-capybara mirror to RubyGems + run: | + ruby -e ' + spec = Gem::Specification.load("capybara-screenshot-diff.gemspec") + spec.name = "snap_diff-capybara" + File.write("snap_diff-capybara.gemspec", spec.to_ruby) + ' + gem build snap_diff-capybara.gemspec + gem push "snap_diff-capybara-${{ github.event.inputs.version }}.gem" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v3 + with: + tag_name: "v${{ github.event.inputs.version }}" + name: "v${{ github.event.inputs.version }}" + body: | + ## What's Changed + + See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md#v${{ github.event.inputs.version }}) for full details. + + **Upgrade Guide:** [docs/UPGRADING.md](https://github.com/${{ github.repository }}/blob/main/docs/UPGRADING.md) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8cc017b6..9f1f77d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,129 +1,234 @@ +--- + name: Test on: push: - branches: [ $default-branch ] + branches: [ master ] + paths: + - '**.gemfile' + - '**.rb' + - '.github/workflows/**' + - '.github/actions/**' + - 'Gemfile*' + - '!bin/**' pull_request: + types: [ opened, synchronize, reopened, labeled ] + paths: + - '**.gemfile' + - '**.rb' + - '.github/workflows/**' + - '.github/actions/**' + - 'Gemfile*' + - '!bin/**' + schedule: + # Weekly full-suite drift check on master, quiet slot. + - cron: '0 6 * * 1' + workflow_dispatch: env: - BUNDLE_GEMFILE: 'gemfiles/rails61_gems.rb' - FERRUM_PROCESS_TIMEOUT: '15' - WD_CACHE_TIME: '864000' # 10 days + BUNDLE_GEMFILE: gemfiles/rails81_gems.rb + DEBIAN_FRONTEND: noninteractive + FERRUM_PROCESS_TIMEOUT: 40 + JAVA_OPTS: -Xmn2g -Xms6g -Xmx6g -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m -Xss1m + -XX:+UseG1GC -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=256m + -XX:+UseCodeCacheFlushing + JRUBY_OPTS: --dev -J-Djruby.thread.pool.enabled=true + MALLOC_ARENA_MAX: 2 + RUBY_GC_HEAP_FREE_SLOTS: 600000 + RUBY_GC_HEAP_GROWTH_FACTOR: 1.1 + RUBY_YJIT_ENABLE: 1 + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: - lint: - name: Lint + # Test that new contributors can run the tests directly after checkout. + test-minimal-setup: + name: Test with minimal setup runs-on: ubuntu-latest - + timeout-minutes: 5 steps: - - name: Checkout code - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - - name: Set up Ruby - uses: ruby/setup-ruby@v1 + - uses: ./.github/actions/setup-ruby-and-dependencies with: - ruby-version: '2.7' - bundler-cache: true + ruby-version: "4.0" - - name: Run Standard Ruby linter - run: bin/standardrb --no-fix --fail-fast + - run: bin/rake test - test: - name: Functional Testing - runs-on: ubuntu-20.04 # In order to install libvips 8.9+ version + functional-test: + name: Functional Test + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + pull-requests: write steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v7 - - name: Set up Ruby - uses: ruby/setup-ruby@v1 + - uses: ./.github/actions/setup-ruby-and-dependencies with: - ruby-version: '2.7' - bundler-cache: true + ruby-version: "4.0" + cache-apt-packages: true - - name: Install libvips - run: sudo apt install libvips libvips-dev libvips-tools - - - name: Run Tests with coverage - run: bundle exec rake test + - run: bin/rake test env: COVERAGE: enabled + DISABLE_SKIP_TESTS: 1 - - name: Upload Screenshots - if: ${{ always() }} - uses: actions/upload-artifact@v2 + - uses: ./.github/actions/upload-screenshots + if: failure() with: - path: test/fixtures/app/doc/screenshots/ + name: base-screenshots + pr-comment: 'true' - - name: Upload Coverage - uses: actions/upload-artifact@v2 + - name: Uploading Coverage Report + uses: actions/upload-artifact@v7 with: name: coverage + retention-days: 1 path: coverage matrix: - name: Test Integration - needs: [ 'test', 'lint' ] - runs-on: ubuntu-20.04 - + name: Test Ruby & Rails + # Cost-intentional: full matrix stays off PRs by default (free-tier + # Actions minutes). Runs on master pushes, manual dispatch, the weekly + # scheduled drift check, and PRs opted in via the 'full-ci' label. + if: > + github.ref == 'refs/heads/master' || + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + contains(github.event.pull_request.labels.*.name, 'full-ci') + needs: [ functional-test ] + runs-on: ubuntu-latest + # Must fit `max_attempts * timeout_minutes` below, plus ~1 min of setup, + # or the last attempt gets killed mid-run and the cell reports `cancelled` + # -- a dead gate. JRuby: 1 + 15 + 15 = 31. MRI: 1 + 3 + 3 = 7, under 8. + timeout-minutes: ${{ contains(matrix.ruby-version, 'jruby') && 31 || 8 }} + continue-on-error: ${{ matrix.experimental }} strategy: matrix: - ruby-version: [ '2.7', '2.6', '2.5', 'jruby' ] + ruby-version: [ "4.0", "3.4", "3.3", "jruby-10.0" ] gemfile: - - 'rails61_gems.rb' - - 'rails60_gems.rb' - - 'rails52.gemfile' - - 'rails51.gemfile' - - 'rails50.gemfile' - - 'rails42.gemfile' + - rails71_gems.rb + - rails72_gems.rb + - rails80_gems.rb + - rails81_gems.rb + experimental: [ false ] + exclude: + # We already tested last version in the functional-test job + - ruby-version: "4.0" + gemfile: rails81_gems.rb + experimental: false + include: + - ruby-version: "4.0" + gemfile: edge_gems.rb + experimental: true + - ruby-version: jruby-head + gemfile: rails81_gems.rb + experimental: true + env: BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }} + # SCREENSHOT_DRIVER is gone: 2.1 removed the driver abstraction, so every + # cell -- JRuby included -- runs libvips. ruby-vips' FFI path on JRuby is + # now a property of the whole matrix rather than of one variable. steps: - - name: Checkout code - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - - name: Set up Ruby - uses: ruby/setup-ruby@v1 + - uses: ./.github/actions/setup-ruby-and-dependencies with: ruby-version: ${{ matrix.ruby-version }} - bundler-cache: true - - - name: Install libvips - run: sudo apt install libvips libvips-dev libvips-tools - - - name: Run tests - run: bundle exec rake test + ruby-cache-version: ${{ matrix.ruby-version }}-${{ matrix.gemfile }}-1 + cache-apt-packages: true - matrix_screenshot_driver: - name: Test Integration - needs: [ 'test', 'lint' ] - runs-on: ubuntu-20.04 + - name: Run tests (with 1 retry) + uses: nick-fields/retry@v4 + with: + # Measured on master run 32643567648: a clean JRuby attempt is + # 545-713s depending on the gemfile, so 15 min is ~26% headroom over + # the slowest. MRI is 128s against 3 min. + timeout_minutes: ${{ contains(matrix.ruby-version, 'jruby') && 15 || 3 }} + # Two, not three. A third attempt never fit inside the job cap on + # either engine (JRuby 3x15=45 > 25; MRI 3x3=9 > 8), so it only ever + # got killed partway and reported the cell as `cancelled`. Keeping + # the retry budget inside the cap is what makes the cell a real gate. + # + # The retry exists solely for the intermittent JRuby teardown hang + # (#244), which strands the process for minutes *after* the suite + # prints `Finished in ...`. Once #244 is fixed, drop this to a single + # attempt and the JRuby cap to 16 -- that halves the JRuby bill. + max_attempts: 2 + command: bin/rake test + + # Was `matrix-screenshot-driver`, a capybara-driver x screenshot-driver grid. + # 2.1 removed the screenshot-driver axis, so this is a capybara-driver matrix + # and nothing else -- half the cells, same coverage. + matrix-capybara-driver: + name: Test Drivers + # Cost-intentional: full matrix stays off PRs by default (free-tier + # Actions minutes). Runs on master pushes, manual dispatch, the weekly + # scheduled drift check, and PRs opted in via the 'full-ci' label. + if: > + github.ref == 'refs/heads/master' || + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + contains(github.event.pull_request.labels.*.name, 'full-ci') + needs: [ 'functional-test' ] strategy: matrix: - screenshot-driver: [ 'vips', 'chunky_png' ] - capybara-driver: [ 'selenium_headless', 'selenium_chrome_headless' ] - include: - - screenshot-driver: 'chunky_png' - capybara-driver: 'cuprite' + capybara-driver: [ selenium_headless, selenium_chrome_headless, cuprite ] + + runs-on: ubuntu-latest + + timeout-minutes: 10 steps: - - name: Checkout code - uses: actions/checkout@v2 + - uses: actions/checkout@v7 - - name: Set up Ruby - uses: ruby/setup-ruby@v1 + - uses: ./.github/actions/setup-ruby-and-dependencies with: - ruby-version: '2.7' - bundler-cache: true + ruby-version: "4.0" + # libvips is required now, not one of two options. + cache-apt-packages: true - - name: Install libvips - run: sudo apt install libvips libvips-dev libvips-tools + - name: Cache Selenium + uses: actions/cache@v6 + with: + path: ~/.cache/selenium + key: ${{ runner.os }}-selenium-${{ matrix.capybara-driver }} - - name: Run tests - run: bundle exec rake test:integration + - run: bin/rake test:integration env: - SCREENSHOT_DRIVER: ${{ matrix.screenshot-driver }} CAPYBARA_DRIVER: ${{ matrix.capybara-driver }} + + - uses: ./.github/actions/upload-screenshots + if: failure() + with: + name: screenshots-${{ matrix.capybara-driver }} + + test-report-upload: + name: Test Report Upload + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + + - uses: ./.github/actions/setup-ruby-and-dependencies + with: + ruby-version: "4.0" + cache-apt-packages: true + + - name: Generate sample report + run: bin/rake 'report:sample[embed]' + + - uses: ./.github/actions/upload-screenshots + with: + name: test-report diff --git a/.gitignore b/.gitignore index ae563352..d2a6596e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *~ /.bundle/ +/.claude /.idea +/.windsurf /.yardoc /_yardoc/ /coverage/ @@ -10,3 +12,24 @@ /pkg/ /spec/reports/ /tmp/ +/vendor/sigs/ + +# Ignore report files +*.attempt_*.png +*.diff.png +*.base.png +snap_diff_report.html + +*.attempt_*.webp +*.diff.webp +*.base.webp +.ruby-version +.ai/ +.qwen/ +.claude/ +.specs/ +.emdash.json +CLAUDE.md +AGENTS.md +GEMINI.md +QWEN.md diff --git a/.standard.yml b/.standard.yml index 34b94d1f..a2dae368 100644 --- a/.standard.yml +++ b/.standard.yml @@ -1,11 +1,17 @@ +--- fix: true # default: false parallel: true # default: false format: progress # default: Standard::Formatter -ruby_version: 2.5 # default: RUBY_VERSION +ruby_version: 3.1 # to support JRuby 9.4 default_ignores: false # default: true -ignore: # default: [] +ignore: # default: [] + - '.*' + - 'bin/**/*' + - 'coverage/**/*' - 'gemfiles/**/*': - - Security/Eval + - 'Security/Eval' - 'gemfiles/vendor/**/*' + - 'sig/**/*' + - 'tmp/**/*' - 'vendor/**/*' diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 00000000..93f08e58 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,19 @@ +--- + +extends: relaxed + +rules: + line-length: + max: 120 + indentation: + indent-sequences: whatever + +ignore: | + bin + coverage + gemfiles + scripts + sig + test + tmp + vendor diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..ad54e1dc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,279 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [v2.0.0.beta3] - 2026-08-23 + +Fixes the canonical `SnapDiff` entry points, which were incomplete in beta2. +If you are trying the 2.0 experiment on the new API, upgrade — beta2's +documented quick start does not work. + +### Fixed +- **`require "snap_diff/integrations/minitest"` (and rspec/cucumber/dsl/static) + now load the full `SnapDiff` surface.** In beta2 those entry points left + `SnapDiff.configure`, `.start`, `.compare` and `SnapDiff::VERSION` undefined, + so the quick start in the canonical guide raised `NoMethodError` +- **The dual-install guard now runs on every canonical entry point.** In beta2 + it lived only in `require "snap_diff"`, so the recommended path was the one + place it never fired +- **Legacy entry points keep the full `CapybaraScreenshotDiff` session surface** — + `require "capybara/screenshot/diff/cucumber"` and + `require "capybara_screenshot_diff/static"` lost `verify`/`reset`/`reporters` + and friends in beta2 while the module stayed defined, so calls failed late +- **`Capybara::Screenshot::Diff::Reporters::Default` and + `…::Diff::Comparison` are eager aliases again**, so `defined?` and + `const_defined?` report them as they did in beta1. beta2 made them lazy, + which silently broke feature detection — and made beta2's own compatibility + note inaccurate for those two constants +- **`SnapDiff::Error` is now the catch-all the docs promise** — + `WindowSizeMismatchError` and `DualInstallError` inherit it, so + `rescue SnapDiff::Error` catches every error the gem raises +- **`gem "snap_diff-capybara"` works with `Bundler.require`** — the gem had no + matching entry file, so Rails users got nothing loaded and a confusing + `NameError` later +- Reporter failure warnings use one brand and name the failing reporter class + +--- + +## [v2.0.0.beta2] - 2026-08-23 + +Third prerelease of the 2.0 opt-in experiment. This one finishes the namespace +move: `SnapDiff` no longer depends on the namespaces it deprecates. Legacy names +keep working exactly as before — see the v2.0.0.alpha1 notes below for the +opt-in and silencing basics, and [docs/snapdiff.md](docs/snapdiff.md) for the +canonical API. + +### Changed +- **Everything canonical now lives in `SnapDiff`** — configuration storage + (`SnapDiff::Config`, one storage behind every settings surface), error classes + (`SnapDiff::Error`, `ExpectationNotMet`, `UnstableImage`, + `WindowSizeMismatchError`), `SnapDiff::Region`, `SnapDiff::Reporters::Default`, + the driver registry (`SnapDiff::Drivers.loaded`/`.available`), the per-test + session (`SnapDiff.session`), and reporter registration + (`SnapDiff::Reporting.register`, mutex-guarded). Old constants remain + same-object aliases; `rescue`, `is_a?`, and `defined?` on them are unchanged + ([#224](https://github.com/snap-diff/snap_diff-capybara/pull/224)–[#230](https://github.com/snap-diff/snap_diff-capybara/pull/230)) +- **Error class names in failure output** now print as `SnapDiff::…` (the class + objects are identical, so `rescue CapybaraScreenshotDiff::ExpectationNotMet` + still catches them — only the printed name differs). CI jobs that + string-match on the old class name in output need updating. +- The images-holder struct is now `SnapDiff::Comparison::Images`, ending the + two-classes-one-name collision with the comparator + ([#227](https://github.com/snap-diff/snap_diff-capybara/pull/227)) + +### Added +- **Deprecation warnings now name your call site** — `(called from + your_file.rb:42)`, so migration is warning-driven instead of grep-driven + ([#222](https://github.com/snap-diff/snap_diff-capybara/pull/222)) +- **Dual-install guard** — installing both `capybara-screenshot-diff` and + `snap_diff-capybara` now raises a clear error instead of silently loading + files from whichever gem activated first + ([#222](https://github.com/snap-diff/snap_diff-capybara/pull/222)) +- **[docs/snapdiff.md](docs/snapdiff.md)** — the SnapDiff-native guide: quick + start for all four integrations, configuration, custom drivers and reporters, + standalone comparison ([#231](https://github.com/snap-diff/snap_diff-capybara/pull/231)) + +### Removed +- The unused `anchor:` keyword on the internal viewport seam; v3's + scroll-preservation work will design its real contract + ([#229](https://github.com/snap-diff/snap_diff-capybara/pull/229)) + +### Internal +- A test now mechanically enforces that the legacy namespace trees contain only + requires, aliases, and one-line forwarders — no real logic — so removing them + in 3.0 is a deletion, not a refactor + ([#229](https://github.com/snap-diff/snap_diff-capybara/pull/229), + [#230](https://github.com/snap-diff/snap_diff-capybara/pull/230)) +- Release workflow is idempotent on re-run; config default-eval timing is + pinned by guards across every entry point + ([#222](https://github.com/snap-diff/snap_diff-capybara/pull/222), + [#223](https://github.com/snap-diff/snap_diff-capybara/pull/223)) + +--- + +## [v2.0.0.beta1] - 2026-08-22 + +Second prerelease of the 2.0 opt-in experiment (see the v2.0.0.alpha1 notes +below for the namespace change, deprecation warnings, and caveats). Final +2.0.0 remains gated on adopter feedback — [#166](https://github.com/snap-diff/snap_diff-capybara/issues/166). + +### Added +- **Dual gem names** — releases now also publish as + [`snap_diff-capybara`](https://rubygems.org/gems/snap_diff-capybara): + identical content and versions under the forward-looking name matching this + repository (the alpha1 mirror was backfilled). Install either, not both. +- **Migration guide** — [docs/UPGRADING.md](docs/UPGRADING.md) covers the v2 + namespace move, every renamed constant, silencing, and the alpha caveats + ([#220](https://github.com/snap-diff/snap_diff-capybara/pull/220)) + +### Internal +- Test suite exercises the canonical `SnapDiff` names; legacy names remain + covered by dedicated forwarding/deprecation tests, and a strict warning + guard now fails the suite on any accidental legacy-name use + ([#221](https://github.com/snap-diff/snap_diff-capybara/pull/221)) + +--- + +## [v2.0.0.alpha1] - 2026-08-22 + +**Opt-in experiment prerelease.** RubyGems never installs prereleases by default +resolution — to try it: `gem "capybara-screenshot-diff", "2.0.0.alpha1"`. The +final 2.0.0 ships only after adopter feedback on this train; please report +anything surprising on [#166](https://github.com/snap-diff/snap_diff-capybara/issues/166). + +### Changed +- **`SnapDiff` is the canonical namespace.** The implementation lives in + `lib/snap_diff/`; every legacy constant (`Capybara::Screenshot::Diff::*`, + `CapybaraScreenshotDiff::*`) still resolves to the *same object* and keeps + working ([#208](https://github.com/snap-diff/snap_diff-capybara/pull/208), + [#209](https://github.com/snap-diff/snap_diff-capybara/pull/209), + [#210](https://github.com/snap-diff/snap_diff-capybara/pull/210)): + `ImageCompare` → `SnapDiff::Comparison`, `Difference` → + `SnapDiff::ComparisonResult`, `Drivers::BaseDriver` → `SnapDiff::Driver` (mixin). +- **Legacy constants warn on first use** — once per constant per process, naming + the replacement ([#218](https://github.com/snap-diff/snap_diff-capybara/pull/218)). + Silence with `SnapDiff.silence_deprecations = true` or + `SNAP_DIFF_SILENCE_DEPRECATIONS=1`. A few constants stay silent by design + (`Os`, `DSL`, `VERSION`, driver leaf classes — see `lib/snap_diff/legacy_shims.rb`). + +### Fixed +- Annotation color constants resolve under the bare `require "snap_diff"` entry; + previously a differing comparison raised `NameError` there + ([#219](https://github.com/snap-diff/snap_diff-capybara/pull/219)) + +### Known caveats (deliberate, alpha) +- `defined?(...)` / `const_defined?` on lazily-shimmed legacy names returns + false/nil — feature detection like `defined?(Capybara::Screenshot::Diff::Drivers::VipsDriver)` + must move to the `SnapDiff::` name. Rescuing legacy error classes is unaffected + (they remain eagerly defined). +- Reopening `module Capybara::Screenshot::Diff::Drivers` (historical custom-driver + monkey-patch pattern) defines a fresh module that shadows the shim; define custom + drivers under `SnapDiff::Drivers` instead. + +### Internal +- Core simplification, behavior-preserving (the "5.5-lite" pass): pure + capture/comparison option partition ([#212](https://github.com/snap-diff/snap_diff-capybara/pull/212)), + stability failure as data ([#213](https://github.com/snap-diff/snap_diff-capybara/pull/213)), + explicit `ScreenshotAssertion#archive_baseline!` + `#inspect` on + assertion/result ([#214](https://github.com/snap-diff/snap_diff-capybara/pull/214)), + `SnapDiff::Capture::Viewport` seam with the future `anchor:` hook + ([#215](https://github.com/snap-diff/snap_diff-capybara/pull/215)), reporter vs + session lifecycle separation ([#216](https://github.com/snap-diff/snap_diff-capybara/pull/216)), + all behind a guard-test net ([#211](https://github.com/snap-diff/snap_diff-capybara/pull/211)) + +--- + +## [v1.15.1] - 2026-08-22 + +### Fixed +- **`VipsDriver#resize_image_to` resizes to the requested dimensions** — the vips driver passed a target aspect ratio where libvips expects a scale factor, so retina-halving on macOS/Selenium could enlarge screenshots (2560×1600 → 4096×2560) and save wrongly-sized baselines; now scales width and height independently via `resize(scale, vscale:)` ([#205](https://github.com/snap-diff/snap_diff-capybara/pull/205)) + +### Internal +- Driver contract tests pinning the shared ChunkyPNG/Vips driver seam (method surface, load/compare behavior, option handling) — the net that caught the resize bug ([#204](https://github.com/snap-diff/snap_diff-capybara/pull/204)) + +--- + +## [v1.15.0] - 2026-08-22 + +### Added +- **`SnapDiff::Config`** — one flat object consolidating all 27 settings behind `SnapDiff.config` / `SnapDiff.configure { |config| ... }`; purely additive, the existing accessors stay canonical and both views share one source of truth ([#200](https://github.com/snap-diff/snap_diff-capybara/pull/200)) + +### Fixed +- **`pending_if_new` no longer masks real failures** — a raising Minitest `teardown` or a raising RSpec `after` hook registered before the gem could previously be reported as a skipped/pending test with exit 0; the pending marker now defers until the framework's teardown chain has run and yields to any failure. Known residual: an RSpec `config.append_after` registered after this gem still runs later than the gem's hook — require the gem last if you use appended after-hooks ([#203](https://github.com/snap-diff/snap_diff-capybara/pull/203)) +- **`BacktraceFilter` path-boundary matching** — a custom `lib_directory` of `/app/lib` no longer also filters locations under `/app/library` ([#202](https://github.com/snap-diff/snap_diff-capybara/pull/202)) + +### Internal +- Failure-masking guard tests for the `pending_if_new` teardown paths via subprocess RSpec runs ([#199](https://github.com/snap-diff/snap_diff-capybara/pull/199)) +- Hygiene pass — skip-area/VCS-baseline and `BacktraceFilter` guard tests, entry-point forwarder dedupe, `Comparison` and `BacktraceFilter` merged into their sole consumers (−2 files, constant paths preserved) ([#201](https://github.com/snap-diff/snap_diff-capybara/pull/201)) + +--- + +## [v1.14.0] - 2026-08-22 + +### Added +- **`SnapDiff` namespace aliases** — `SnapDiff::Comparison`, `SnapDiff.compare`, and `SnapDiff.start` provide a forward-looking entry point onto the existing `Capybara::Screenshot::Diff` API, no behavior changes ([#166](https://github.com/snap-diff/snap_diff-capybara/issues/166)) + +### Fixed +- **Standalone require path** — `require "capybara_screenshot_diff"` (or `require "snap_diff"`) followed by `Diff.compare` no longer raises `NameError` for the missing drivers require ([#194](https://github.com/snap-diff/snap_diff-capybara/pull/194)) + +### Internal +- Merged `DifferenceFinder` into `ImageCompare` (ADR-004 step) ([#192](https://github.com/snap-diff/snap_diff-capybara/pull/192)) +- Extracted `AnnotationService` from `Reporters::Default`; full public surface preserved via delegation ([#193](https://github.com/snap-diff/snap_diff-capybara/pull/193)) +- Consolidated `pending_if_new` teardown logic into one shared helper across Minitest/RSpec/Cucumber adapters ([#197](https://github.com/snap-diff/snap_diff-capybara/pull/197)) +- Test-suite driver-coverage banner and CI guard against silently missing drivers ([#198](https://github.com/snap-diff/snap_diff-capybara/pull/198)) + +--- + +## [v1.13.0] - 2026-08-22 + +### Added +- **`capture_screenshot` DSL method** — take a screenshot without ever comparing or asserting ([#191](https://github.com/snap-diff/snap_diff-capybara/issues/191)) +- **`compare:` option on `screenshot`** — pass `compare: true` (default) to assert, or `compare: false` to capture only ([#191](https://github.com/snap-diff/snap_diff-capybara/issues/191)) +- **`Capybara::Screenshot::Diff.pending_if_new` config** — mark tests as skipped in teardown when a baseline does not exist, complementing `fail_if_new` ([#191](https://github.com/snap-diff/snap_diff-capybara/issues/191)) + +### Changed +- **`assert_matches_screenshot` is now the primary assertion method** — captures a screenshot and compares against baseline ([#191](https://github.com/snap-diff/snap_diff-capybara/issues/191)) +- **`screenshot` is now a convenience wrapper** — safe to override in user test classes; the gem no longer calls it internally ([#191](https://github.com/snap-diff/snap_diff-capybara/issues/191)) + +--- + +## [v1.12.0] - 2026-04-12 + +### Added +- **HTML reporter** — interactive dashboard with 4 comparison modes (both/base/new/heatmap), per-image zoom, keyboard navigation, and search ([#170](https://github.com/snap-diff/snap_diff-capybara/pull/170)) +- **GitHub Actions integration** — inline HTML preview with base64 embedded images + reusable composite action ([#171](https://github.com/snap-diff/snap_diff-capybara/pull/171)) +- **`disable_animations` helper** — inject CSS to stop all animations/transitions during screenshots ([#174](https://github.com/snap-diff/snap_diff-capybara/pull/174)) +- **`snap_diff:clean` rake task** — remove diff artifacts while keeping baselines ([#177](https://github.com/snap-diff/snap_diff-capybara/pull/177)) +- `Diff.compare` for standalone image comparison without Capybara or browser +- Perceptual color distance (dE00) for anti-aliasing tolerance in VipsDriver +- `assert_no_screenshot_changes` DSL assertion +- `Diff.configure` block helper for simplified configuration +- Ruby 3.5 and 4.0 support + +### Changed +- `blur_active_element` now defaults to `true` — prevents cursor blinking artifacts +- `hide_caret` now defaults to `true` — stable screenshots without caret +- `fail_if_new` now defaults to `true` in CI (when `ENV['CI']` is set) +- Thread-safe reporter notification with mutex ([#175](https://github.com/snap-diff/snap_diff-capybara/pull/175)) + +### Removed +- **SVN support** — Git only +- **ActiveSupport runtime dependency** — pure Ruby, lighter installations + +### Fixed +- VCS path resolution rewritten for thread safety (Open3 + array-form system) +- Reporter diff artifacts cleaned up properly on `Snap#delete!` ([#173](https://github.com/snap-diff/snap_diff-capybara/pull/173)) +- RSpec matcher now provides `failure_message` and description +- Missing baseline error includes recording instructions +- Ruby 4.0 DSLStub ordering compatibility +- ChunkyPNG `filter_image_with_median` incorrect behavior +- Tolerance calculation no longer skips large changes + +### Performance +- ChunkyPNG shift-detection: eliminated array allocations (~30% faster for large images) +- VIPS: cached computations at construction (~15% faster) +- Memoized region area size, replaced closures with blocks + +### Documentation +- README restructured from 970 to 149 lines with 7 dedicated `docs/` files ([#171](https://github.com/snap-diff/snap_diff-capybara/pull/171)) +- CI integration guide with artifact upload and PR commenting ([docs/ci-integration.md](docs/ci-integration.md)) +- Upgrade guide ([docs/UPGRADING.md](docs/UPGRADING.md)) +- Color comparison guide — tolerance vs perceptual_threshold vs color_distance_limit ([#176](https://github.com/snap-diff/snap_diff-capybara/pull/176)) + +### Internal +- Simplified internals: inlined CaptureStrategy, ComparisonLoader, ScreenshotNamerDSL, VipsUtil +- Unified Screenshoter constructors, consolidated skip_area accessor +- Upgraded CI dependencies (actions/checkout v5, upload-artifact v7) + +--- + +## [v1.11.0] - Previous Release + +[Unreleased]: https://github.com/snap-diff/snap_diff-capybara/compare/v1.12.0...HEAD +[v1.12.0]: https://github.com/snap-diff/snap_diff-capybara/releases/tag/v1.12.0 +[v1.11.0]: https://github.com/snap-diff/snap_diff-capybara/releases/tag/v1.11.0 + +**Upgrade Guide:** See [docs/UPGRADING.md](docs/UPGRADING.md) for detailed migration instructions. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..431f1f35 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,129 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement by +[opening a GitHub Issue](https://github.com/snap-diff/snap_diff-capybara/issues). + +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact:** Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence:** A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact:** A violation through a single incident or series +of actions. + +**Consequence:** A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact:** A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence:** A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact:** Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence:** A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1eb58e11..581f276e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,22 +1,168 @@ -Contributing -============ +# Contributing -Bug reports and pull requests are welcome on GitHub at https://github.com/donv/capybara-screenshot-diff. +Bug reports and pull requests are welcome on GitHub at https://github.com/snap-diff/snap_diff-capybara. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected -to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. +to adhere to the [Contributor Covenant](CODE_OF_CONDUCT.md) code of conduct. -## Testing +## Development Setup -Run the tests before committing using Rake +### Prerequisites - rake +- **Ruby 3.2+** (the project tests against 3.2–4.0) +- **libvips 8.9+** (optional, for the VIPS driver). Install with: + - macOS: `brew install vips` + - Ubuntu: `sudo apt-get install libvips-dev` +- **Chrome** (for integration tests with `selenium_chrome_headless` or `cuprite`) -### Matrix testing +### First-time setup -Run the tests for a matrix of configurations of Ruby implementations and Rails versions +```bash +bin/setup +``` - ./matrix_text.rb +This installs gem dependencies and prepares the development environment. -## Merging to master +### Running tests -Before merging to master, please have a member of the project review your changes, and make sure the tests are green in travis-ci. +```bash +# All tests +rake test + +# Unit tests only (faster, no browser required) +rake test:unit + +# Integration tests (require a browser) +rake test:integration + +# Run specific test file +ruby -Ilib:test test/unit/image_compare_test.rb + +# Run with a specific screenshot driver +SCREENSHOT_DRIVER=vips rake test + +# Run with a specific Capybara driver (integration tests) +CAPYBARA_DRIVER=cuprite rake test:integration + +# Record new baseline screenshots (integration tests) +RECORD_SCREENSHOTS=1 bin/dtest +``` + +### Docker testing + +Use Docker for reproducible CI-matching test runs: + +```bash +bin/dtest # Run all tests in Docker +bin/dtest test/integration/ # Run specific directory +``` + +See [docs/docker-testing.md](docs/docker-testing.md) for details. + +### Linting + +```bash +# Ruby linting (Standard Ruby) +bundle exec standardrb + +# Auto-fix +bundle exec standardrb --fix +``` + +## Coding Conventions + +### Style + +This project uses [Standard Ruby](https://github.com/standardrb/standard) for consistent formatting. Run `bundle exec standardrb --fix` before committing. + +### Ruby version compatibility + +The gem supports Ruby 3.2 through 4.0 (including JRuby). When adding features: +- Avoid syntax or APIs that are only available in newer Ruby versions +- Test with the full matrix (see `.github/workflows/test.yml`) +- Be mindful of JRuby compatibility (no C extensions, avoid platform-specific code) + +### Naming conventions + +- **Classes/modules:** `CamelCase` (Ruby convention) +- **Methods:** `snake_case` +- **Test files:** `snake_case_test.rb` (matching the class they test) +- **Screenshot names:** descriptive, kebab-case or snake_case + +### Architecture patterns + +- **Value objects** — immutable data carriers (e.g., `Difference`, `Comparison`, `Region`) +- **Strategy pattern** — interchangeable algorithms (e.g., `VipsDriver`/`ChunkyPNGDriver`) +- **Layered comparison** — fast-then-slow strategy in `ImageCompare` (byte → pixel → region) +- **Thread safety** — thread-local state for per-test data, mutex for shared state +- **Test doubles** — use `TestDoubles::TestDriver` and `TestDoubles::TestPath` (see `test/support/test_doubles.rb`) + +### What to avoid + +- **Service objects** — this codebase prefers explicit method calls over service object wrappers +- **Global state mutation at runtime** — configuration should be set once before tests +- **Monkey-patching** — prefer composition over patching Capybara internals + +## Pull Request Guidelines + +### Before submitting + +1. **Run the full test suite** — `rake test` should pass +2. **Run the linter** — `bundle exec standardrb` should pass +3. **Add tests** for new functionality or bug fixes +4. **Update docs** if changing behavior or adding features +5. **Update CHANGELOG.md** under the `[Unreleased]` section + +### PR description + +Include: +- **What** changed (one-line summary) +- **Why** it changed (motivation, related issue) +- **How** it was tested +- **Screenshots** if the change affects visual output + +### Review process + +1. A maintainer will review your PR +2. Address review feedback with additional commits (no force-pushing) +3. Once approved, a maintainer will merge + +### Testing guidelines + +- **Unit tests** go in `test/unit/` and test a single class in isolation. Use test doubles from `test/support/test_doubles.rb` rather than testing with real image files or browsers. +- **Integration tests** go in `test/integration/` and exercise the full capture → compare → report pipeline with a real browser. These are slower and require Chrome. +- **Driver contract tests** (`test/support/driver_contract_tests.rb`) verify that all image processing drivers meet the same interface. Add a contract test when adding a new driver method. +- **Test environment isolation:** each unit test snapshots and restores `Capybara::Screenshot` and `Capybara::Screenshot::Diff` global state. Don't mutate globals outside of setup/teardown. + +## Adding a New Driver + +1. Create `lib/capybara/screenshot/diff/drivers/new_driver.rb` inheriting from `BaseDriver` +2. Implement required methods: `load_images`, `from_file`, `save_image_to`, `same_pixels?`, `find_difference_region`, `crop`, `add_black_box`, `draw_rectangles`, `resize_image_to` +3. Register in `Utils.detect_available_drivers` and `Utils.find_driver_class_for` +4. Add driver contract tests in `test/unit/drivers/new_driver_test.rb` +5. Add integration tests exercising the new driver + +## Adding a New Reporter + +1. Create `lib/capybara_screenshot_diff/reporters/new_reporter.rb` +2. Implement `record(assertions)` and `finalize` methods +3. Register with `CapybaraScreenshotDiff.reporters << MyReporter.new` +4. Add tests in `test/unit/reporters/` + +## Releasing + +To release a new version: + +1. Update the version number in [lib/capybara/screenshot/diff/version.rb](lib/capybara/screenshot/diff/version.rb) +2. Update [CHANGELOG.md](CHANGELOG.md) with the new version and date +3. Create a GitHub Release: + - Go to [Actions → Release](https://github.com/snap-diff/snap_diff-capybara/actions/workflows/release.yml) + - Click **Run workflow**, enter the version number + - The workflow will: test → tag → publish to RubyGems → create GitHub Release + +Or manually: + +```bash +bundle exec rake release +``` + +This creates a git tag, pushes commits and tags, and pushes the `.gem` file to [rubygems.org](https://rubygems.org). diff --git a/Dockerfile b/Dockerfile index 00ab718b..788f5574 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,58 +3,55 @@ # $ docker build . -t csd # $ docker run -v $(pwd):/app -ti csd rake test -ARG RUBY_VERSION=2.7.2 - -FROM circleci/ruby:2.7.2-node-browsers - -RUN \ - # Install dependencies - sudo apt-get update && \ - DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \ - automake \ - build-essential \ - curl \ - fftw3-dev \ - gettext \ - gobject-introspection \ - gtk-doc-tools \ - libexif-dev \ - libfftw3-dev \ - libgif-dev \ - libglib2.0-dev \ - libgsf-1-dev \ - libgtk2.0-dev \ - libmagickwand-dev \ - libmatio-dev \ - libopenexr-dev \ - libopenslide-dev \ - liborc-0.4-dev \ - libpango1.0-dev \ - libpoppler-glib-dev \ - librsvg2-dev \ - libtiff5-dev \ - libwebp-dev \ - libxml2-dev \ - swig - +FROM jetthoughts/cimg-ruby:4.0-chrome + +ENV DEBIAN_FRONTEND=noninteractive \ + BUNDLE_PATH=/bundle + +# Install system dependencies with cached apt +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + sudo sed -i 's|http://security.ubuntu.com/ubuntu|http://archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \ + sudo apt-get update -qq && \ + sudo apt-get install -y --no-install-recommends \ + automake \ + build-essential \ + curl \ + gettext \ + gobject-introspection \ + gtk-doc-tools \ + libexif-dev \ + libfftw3-dev \ + libgif-dev \ + libglib2.0-dev \ + libgsf-1-dev \ + libgtk2.0-dev \ + libmagickwand-dev \ + libmatio-dev \ + libopenexr-dev \ + libopenslide-dev \ + liborc-0.4-dev \ + libpango1.0-dev \ + libpoppler-glib-dev \ + librsvg2-dev \ + libtiff5-dev \ + libvips-dev \ + libwebp-dev \ + libxml2-dev \ + swig && \ + sudo rm -rf /var/lib/apt/lists/* + +# Setup directories and fix font config if file exists +RUN sudo mkdir -p /bundle /tmp/.X11-unix && \ + sudo chmod 1777 /bundle /tmp/.X11-unix && \ + (sudo test -f /etc/fonts/conf.d/10-yes-antialias.conf && sudo sed -i 's/true/false/g' /etc/fonts/conf.d/10-yes-antialias.conf || echo "Font config file not found, skipping") WORKDIR /app -RUN sudo chmod a+w -R /app - -ADD ./bin/install-vips /app/bin/ -RUN sudo /app/bin/install-vips - -ADD ./lib/capybara/screenshot/diff/version.rb /app/lib/capybara/screenshot/diff/ -ADD ./capybara-screenshot-diff.gemspec /app/ -ADD ./gems.rb /app/ - -RUN bundle install -RUN \ - # Clean up - sudo apt-get remove -y curl automake build-essential && \ - sudo apt-get autoremove -y && \ - sudo apt-get autoclean && \ - sudo apt-get clean && \ - sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# Copy entire project (needed for git dependencies in gems.rb) +COPY --chown=circleci:circleci . . +# Install gems +RUN sudo chown -R circleci:circleci /bundle && \ + bundle config set without 'tools' && \ + bundle install diff --git a/README.md b/README.md index 30212dd5..181876e5 100644 --- a/README.md +++ b/README.md @@ -1,538 +1,223 @@ -[![Build Status](https://travis-ci.org/donv/capybara-screenshot-diff.svg?branch=master)](https://travis-ci.org/donv/capybara-screenshot-diff) +[![Gem Version](https://badge.fury.io/rb/capybara-screenshot-diff.svg)](https://rubygems.org/gems/capybara-screenshot-diff) +[![Gem Downloads](https://img.shields.io/gem/dt/capybara-screenshot-diff.svg)](https://rubygems.org/gems/capybara-screenshot-diff) +[![Test](https://github.com/snap-diff/snap_diff-capybara/actions/workflows/test.yml/badge.svg)](https://github.com/snap-diff/snap_diff-capybara/actions/workflows/test.yml) +[![DeepWiki](https://img.shields.io/badge/DeepWiki-snap--diff%2Fsnap__diff--capybara-blue.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0IiBmaWxsPSJub25lIiBzdHJva2U9IndoaXRlIiBzdHJva2Utd2lkdGg9IjIiIHN0cm9rZS1saW5lY2FwPSJyb3VuZCIgc3Ryb2tlLWxpbmVqb2luPSJyb3VuZCI+PHBhdGggZD0iTTEyIDJhMTAgMTAgMCAxIDAgMCAyMCAxMCAxMCAwIDAgMCAwLTIweiIvPjxwYXRoIGQ9Ik0xMiA2djEyIi8+PHBhdGggZD0iTTYgMTJoMTIiLz48L3N2Zz4=)](https://deepwiki.com/snap-diff/snap_diff-capybara) # Capybara::Screenshot::Diff -Ever wondered what your project looked like two years ago? To answer that, you -start taking screen shots during your tests. Capybara provides the -`save_screenshot` method for this. Very good. +Stop shipping UI bugs. Take screenshots in your Capybara tests, commit baselines to git, and let CI catch visual regressions in pull requests — no cloud service, no subscription, runs entirely in your test suite. -Ever introduced a graphical change unintended? Never want it to happen again? -Then this gem is for you! Use this gem to detect changes in your pages by -taking screen shots and comparing them to the previous revision. +[![SnapDiff Web UI](docs/images/snap_diff_web_ui.png)](#web-ui-for-reviewing-screenshot-changes) -## Installation - -Add these lines to your application's Gemfile: - -```ruby -gem 'capybara-screenshot-diff' -gem 'oily_png', platform: :ruby -``` - -And then execute: - - $ bundle - -Or install it yourself as: - - $ gem install capybara-screenshot-diff - -### Requirements +**Why this gem?** Baselines live in git — review UI changes in pull requests like you review code. Runs offline, works in CI, zero vendor lock-in. Unlike Percy/Chromatic (paid SaaS), nothing to sign up for. Unlike BackstopJS, no Node required. -* [for :vips driver] libvips 8.9 or later, see the [libvips install instructions](https://libvips.github.io/libvips/install.html) +> **2.0 experiment (beta):** the gem is moving to a `SnapDiff` canonical namespace. Opt in with `gem "capybara-screenshot-diff", "2.0.0.beta3"` (or the latest 2.0.0 prerelease; prereleases are never installed by default — normal installs stay on 1.x). Legacy names keep working; the first legacy API a process touches prints one migration notice (lazily shimmed constants also warn once each — see [which names warn](docs/UPGRADING.md#deprecation-warnings)), silenceable via `SnapDiff.silence_deprecations = true` or `SNAP_DIFF_SILENCE_DEPRECATIONS=1`. Writing new code? Start from [SnapDiff — the canonical API](docs/snapdiff.md), which uses canonical names only. Migrating an existing suite? See the [upgrade guide](docs/UPGRADING.md). Share feedback on [#166](https://github.com/snap-diff/snap_diff-capybara/issues/166). +> +> Starting with the 2.0 prereleases the gem is also published as [`snap_diff-capybara`](https://rubygems.org/gems/snap_diff-capybara) — identical content and versions under the forward-looking name, matching this repository. Install either; don't install both. -## Usage - -### Minitest - -In your test class, include the `Capybara::Screenshot::Diff` module: - -```ruby -class FeatureTest < ActionDispatch::IntegrationTest - include Capybara::Screenshot::Diff - ... -end -``` - -or if you use the integration test directly: - -```ruby -class ActionDispatch::IntegrationTest - include Capybara::Screenshot::Diff - ... -end -``` +## Quick Start (5 minutes) -### rspec +> Already using Capybara for system tests? Add the gem and you're ready. New to system tests? See [Rails System Testing guide](https://guides.rubyonrails.org/testing.html#system-testing). ```ruby -describe 'Permissions admin', :type => :feature, :js => true do - - include Capybara::Screenshot::Diff - - it 'works with permissions' do - visit('/') - screenshot 'home_page' - end - -end +# Gemfile +gem 'capybara-screenshot-diff' +# ruby-vips comes with the gem since 2.1; libvips itself is a system package +# (brew install vips / apt-get install libvips). ``` -### Taking screenshots - -Add `screenshot ''` to your tests. The screenshot will be saved in -the `doc/screenshots` directory. - -Change your existing `save_screenshot` calls to `screenshot` - ```ruby -test 'my useful feature' do - visit '/' - screenshot 'welcome_index' - click_button 'Useful feature' - screenshot 'feature_index' - click_button 'Perform action' - screenshot 'action_performed' -end +# test/test_helper.rb +require 'capybara_screenshot_diff/minitest' ``` -This will produce a sequence of images like this - -``` -doc - screenshots - action_performed - feature_index - welcome_index -``` - -To store the screen shot history, add the `doc/screenshots` directory to your - version control system (git, svn, etc). - - Screen shots are compared to the previously COMMITTED version of the same screen shot. - -### Screenshot groups - -Commonly it is useful to group screenshots around a feature, and record them as -a sequence. To do this, add a `screenshot_group` call to the start of your -test. - ```ruby -test 'my useful feature' do - screenshot_group 'useful_feature' - visit '/' - screenshot 'welcome_index' - click_button 'Useful feature' - screenshot 'feature_index' - click_button 'Perform action' - screenshot 'action_performed' -end -``` - -This will produce a sequence of images like this - -``` -doc - screenshots - useful_feature - 00-welcome_index - 01-feature_index - 02-action_performed -``` - -**All files in the screenshot group directory will be deleted when -`screenshot_group` is called.** - - -#### Screenshot sections - -You can introduce another level above the screenshot group called a -`screenshot_section`. The section name is inserted just before the group name -in the save path. If called in the setup of the test, all screenshots in -that test will get the same prefix: - -```ruby -setup do - screenshot_section 'my_feature' -end - -test 'my subfeature' do - screenshot_group 'subfeature' - visit '/feature' - click_button 'Interesting button' - screenshot 'subfeature_index' - click_button 'Perform action' - screenshot 'action_performed' +# test/application_system_test_case.rb +class ApplicationSystemTestCase < ActionDispatch::SystemTestCase + include CapybaraScreenshotDiff::Minitest::Assertions end ``` -This will produce a sequence of images like this - -``` -doc - screenshots - my_feature - subfeature - 00-subfeature_index - 01-action_performed -``` - - -#### Setting `screenshot_section` and/or `screenshot_group` for all tests - -Setting the `screenshot_section` and/or `screenshot_group` for all tests can be -done in the super class setup: - ```ruby -class ApplicationSystemTestCase < ActionDispatch::SystemTestCase - setup do - screenshot_section class_name.underscore.sub(/(_feature|_system)?_test$/, '') - screenshot_group name[5..-1] +# test/system/homepage_test.rb +class HomepageTest < ApplicationSystemTestCase + test "homepage" do + visit "/" + assert_matches_screenshot "homepage" end end ``` -`screenshot_section` and/or `screenshot_group` can still be overridden in each -test. - - +(`screenshot` still works as a shorthand, and is safe to override in your own helpers — the gem no longer calls it internally.) -### Multiple Capybara drivers +Then run these steps in order: -Often it is useful to test your app using different browsers. To avoid the -screenshots for different Capybara drivers to overwrite each other, set +```bash +# Step 1: Save baselines (first run always passes) +bundle exec rake test -```ruby -Capybara::Screenshot.add_driver_path = true -``` - -The example above will then save your screenshots like this -(for poltergeist and selenium): +# Step 2: Commit baselines to git +git add doc/screenshots/ +git commit -m "chore: add screenshot baselines" +# Step 3: Now comparisons work — change your UI and re-run +bundle exec rake test ``` -doc - screenshots - poltergeist - useful_feature - 00-welcome_index - 01-feature_index - 02-action_performed - selenium - useful_feature - 00-welcome_index - 01-feature_index - 02-action_performed -``` - -### Multiple OSs - -If you run your tests on multiple operating systems, you will most likely find -the screen shots differ. To avoid the screenshots for different OSs to -overwrite each other, set -```ruby -Capybara::Screenshot.add_os_path = true +After Step 1, you'll see: +```text +doc/screenshots/ + homepage.png <- your baseline (commit this) ``` -The example above will then save your screenshots like this -(for Linux and Windows): - -``` -doc - screenshots - linux - useful_feature - 00-welcome_index - 01-feature_index - 02-action_performed - windows - useful_feature - 00-welcome_index - 01-feature_index - 02-action_performed +Add diff artifacts to `.gitignore` — these are generated at runtime and should not be committed: +```gitignore +# Screenshot diff artifacts (generated, not committed) +*.diff.png +*.base.png +*.diff.webp +*.base.webp +snap_diff_report.html ``` -If you combine this config with the `add_driver_path` config, the driver will be -put in front of the OS name. +If you skip Step 2 and push to CI, the build will fail — `fail_if_new` is `true` by default in CI. -### Screen size +For RSpec, Cucumber, or non-Rails setup, see [Framework Setup](docs/framework-setup.md). -You can specify the desired screen size using +### For Non-Rails Projects (Hugo, Jekyll, Static Sites) ```ruby -Capybara::Screenshot.window_size = [1024, 768] +require 'capybara_screenshot_diff/static' +CapybaraScreenshotDiff.serve("_site") # or "public", "build", "dist" ``` -This will force the screen shots to the given size, and skip taking screen shots -unless the desired window size can be achieved. +Then commit baselines to git just like Rails. [Full setup](docs/ci-integration.md#non-rails-projects-hugo-jekyll-static-sites). -### Disabling screen shots +## What Happens When a Screenshot Changes -If you want to skip taking screen shots, set +The test fails with a clear message and generates diff files: -```ruby -Capybara::Screenshot.enabled = false +```text +Screenshot does not match for 'homepage': +({"area_size":1250,"region":[0,19,199,83],"max_color_distance":42.5}) ``` -You can of course set this by an environment variable +Open `doc/screenshots/homepage.diff.png` to see exactly what changed. If the change is intentional, delete the baseline and re-run to update it. -```ruby -Capybara::Screenshot.enabled = ENV['TAKE_SCREENSHOTS'] -``` +| File | Description | +|------|-------------| +| `homepage.png` | Committed baseline | +| `homepage.diff.png` | Visual diff with changes highlighted in red | +| `homepage.heatmap.diff.png` | Heatmap of pixel differences | -### Disabling diff +## Web UI for Reviewing Screenshot Changes -If you want to skip the assertion for change in the screen shot, set +Add one line to get an interactive dashboard for reviewing all screenshot differences: ```ruby -Capybara::Screenshot::Diff.enabled = false +# test/test_helper.rb +require 'capybara_screenshot_diff/reporters/html' ``` -Using an environment variable +After tests run, open `doc/screenshots/snap_diff_report.html`: -```ruby -Capybara::Screenshot::Diff.enabled = ENV['COMPARE_SCREENSHOTS'] -``` +![SnapDiff Web UI — annotated diff showing changed regions highlighted in red](docs/images/snap_diff_annotated.png) -### Screen shot save path +See [Web UI & Custom Reporters](docs/reporters.md) for full feature details and [CI Integration](docs/ci-integration.md) for GitHub Actions setup. -By default, `Capybara::Screenshot::Diff` saves screenshots to a -`doc/screenshots` folder, relative to either `Rails.root` (if you're in Rails), - or your current directory otherwise. +## Compare Any Two Images -If you want to change where screenshots are saved to, then there are two -configuration options that that are relevant. - -The most likely one you'll want to modify is ... +Works without a browser — PDFs, generated images, CI artifacts: ```ruby -Capybara::Screenshot.save_path = "other/path" +result = Capybara::Screenshot::Diff.compare("baseline.png", "current.png") +result.different? # => true if visually different +result.quick_equal? # => true if byte-identical ``` -The `save_path` option is relative to `Capybara::Screenshot.root`. - -`Capybara::Screenshot.root` defaults to either `Rails.root` (if you're in -Rails) or your current directory. You can change it to something entirely -different if necessary, such as when using an alternative web framework. +## Next Steps -```ruby -Capybara::Screenshot.root = Hanami.root -``` - -### Screen shot stability +- **Crop to element:** `screenshot "form", crop: "#main-form"` +- **Ignore regions:** `screenshot "dashboard", skip_area: [".timestamp"]` +- **Disable animations:** `Capybara::Screenshot.disable_animations = true` +- **Set window size:** `Capybara::Screenshot.window_size = [1280, 1024]` -To ensure that animations are finished before saving a screen shot, you can add -a stability time limit. If the stability time limit is set, a second screen -shot will be taken and compared to the first. This is repeated until two -subsequent screen shots are identical. +## Handling Flaky Tests -```ruby -Capybara::Screenshot.stability_time_limit = 0.1 -``` +Defaults work for most Rails apps — `blur_active_element`, `hide_caret`, and `fail_if_new` (in CI) are enabled automatically. -This can be overridden on a single screenshot: +If screenshots differ between CI and local, set a comparison threshold: ```ruby -test 'stability_time_limit' do - visit '/' - screenshot 'index', stability_time_limit: 0.5 +Capybara::Screenshot::Diff.configure do |screenshot, diff| + screenshot.window_size = [1280, 1024] # consistent viewport + diff.perceptual_threshold = 2.0 # ignore anti-aliasing (VIPS only) + # or: diff.tolerance = 0.001 # percentage-based (default for VIPS) end ``` -### Maximum wait limit +See [Choosing the Right Method](docs/configuration.md#choosing-the-right-color-comparison-method) for detailed comparison options. -When the `stability_time_limit` is set, but no stable screenshot can be taken, a timeout occurs. -The timeout occurs after `Capybara.default_max_wait_time`, but can be overridden by an option. +## FAQ -```ruby -test 'max wait time' do - visit '/' - screenshot 'index', wait: 20.seconds -end -``` - -### Removing focus from the active element - -In Chrome the screenshot includes the blinking input cursor. This can make it impossible to get a -stable screenshot. To get around this you can set the `hide caret` option: - -```ruby -Capybara::Screenshot.hide_caret = true -``` - -This will make the cursor (caret) transparent (invisible), so the blinking does not delay the screen shot. - - - -### Removing focus from the active element - -Another way to avoid the cursor blinking is to set the `blur_active_element` option: - -```ruby -Capybara::Screenshot.blur_active_element = true -``` - -This will remove the focus from the active element, removing the blinking cursor. - - - -### Allowed color distance - -Sometimes you want to allow small differences in the images. For example, Chrome renders the same -page slightly differently sometimes. You can set set the color difference threshold for the -comparison using the `color_distance_limit` option to the `screenshot` method: - -```ruby -test 'color threshold' do - visit '/' - screenshot 'index', color_distance_limit: 30 -end -``` - -The difference is calculated as the eucledian distance. You can also set this globally: - -```ruby -Capybara::Screenshot::Diff.color_distance_limit = 42 -``` - - -### Allowed shift distance - -Sometimes you want to allow small movements in the images. For example, jquer-tablesorter -renders the same table slightly differently sometimes. You can set set the shift distance -threshold for the comparison using the `shift_distance_limit` option to the `screenshot` -method: - -```ruby -test 'color threshold' do - visit '/' - screenshot 'index', shift_distance_limit: 2 -end -``` - -The difference is calculated as maximum distance in either the X or the Y axis. -You can also set this globally: - -```ruby -Capybara::Screenshot::Diff.shift_distance_limit = 1 -``` - -**Note:** For each increase in `shift_distance_limit` more pixels are searched for a matching color value, and -this will impact performance **severely** if a match cannot be found. - -If `shift_distance_limit` is `nil` shift distance is not measured. If `shift_distance_limit` is set, -even to `0`, shift distabnce is measured and reported on image differences. +
+The test passed on first run. Did it work? -### Allowed difference size +Yes. First run saves baselines and always passes. Run tests again to compare against committed baselines. +
-You can set set a threshold for the differing area size for the comparison -using the `area_size_limit` option to the `screenshot` method: +
+How do I update baselines after intentional UI changes? -```ruby -test 'area threshold' do - visit '/' - screenshot 'index', area_size_limit: 17 -end -``` - -The difference is calculated as `width * height`. You can also set this globally: - -```ruby -Capybara::Screenshot::Diff.area_size_limit = 42 -``` - - -### Skipping an area - -Sometimes you have expected change that you want to ignore. -You can use the `skip_area` option to the `screenshot` method to ignore an area: - -```ruby -test 'unstable area' do - visit '/' - screenshot 'index', skip_area: [17, 6, 27, 16] -end -``` - -The arguments are [x1, y1, x2, y2] for the area you want to ignore. You can also set this globally: - -```ruby -Capybara::Screenshot::Diff.skip_area = [0, 0, 64, 48] -``` - -If you need to ignore multiple areas, you can supply an array of arrays: - -```ruby -screenshot 'index', skip_area: [[0, 0, 64, 48], [17, 6, 27, 16]] -``` - -### Available Image Processing Drivers - -There are several image processing supported by this gem. -There are several options to setup active driver: `:auto`, `:chunky_png` and `:vips`. +Delete the baseline file and re-run tests: `rm doc/screenshots/homepage.png && bundle exec rake test`. Or update all: `rm -rf doc/screenshots/ && bundle exec rake test`. +
-* `:auto` - will try to load `:vips` if there is gem `ruby-vips`, in other cases will load `:chunky_png` -* `:chunky_png` and `:vips` will load correspondent driver +
+CSS animations make my screenshots flaky -### Enable VIPS image processing +Enable `Capybara::Screenshot.disable_animations = true` to freeze CSS animations/transitions before each capture. Or use `stability_time_limit: 1` to wait for animations to finish. +
-[Vips](https://www.rubydoc.info/gems/ruby-vips/Vips/Image) driver provides a faster comparison, -and could be enabled by adding `ruby-vips` to `Gemfile`. +
+CI screenshots differ from local -If need to setup explicitly Vips driver, there are several ways to do this: +Set `window_size` for consistent dimensions and use `perceptual_threshold: 2.0` to ignore anti-aliasing differences across environments. +
- * Globally: `Capybara::Screenshot::Diff.driver = :vips` - * Per screenshot option: `screenshot 'index', driver: :vips` +
+Will this slow down my tests? -With enabled VIPS there are new alternatives to process differences, which easier to find and support. -For example, `shift_distance_limit` is very heavy operation. Instead better to use `median_filter_window_size`. +Comparisons add ~50ms per image. `stability_time_limit` adds wait time — keep it low (0.1-0.5s) or use `disable_animations` instead. +
-#### Tolerance level (vips only) +
+Debug mode -You can set a “tolerance” anywhere from 0% to 100%. This is the amount of change that's allowable. -If the screenshot has changed by more than that amount, it'll flag it as a failure. +`DEBUG=1 bundle exec rake test` keeps `.diff.png` files for inspection. +
-This is alternative to "Allowed difference size", only the difference that area calculates including valid pixels. -But "tolerance" compares only different pixels. - -You can use the `tolerance` option to the `screenshot` method to set level: - -```ruby -test 'unstable area' do - visit '/' - screenshot 'index', tolerance: 0.3 -end -``` - -You can also set this globally: - -```ruby -Capybara::Screenshot::Diff.tolerance = 0.3 -``` - -#### Median filter size (vips only) +## Installation -This is an alternative to "Allowed shift distance", but much faster. -You can find more about this strategy on [Median Filter](https://en.wikipedia.org/wiki/Median_filter). -Think about this like smoothing of the image, before comparison. +**Requirements:** Ruby 3.2+. Rails 7.1+ for Rails integration; non-Rails projects supported via `CapybaraScreenshotDiff.serve()`. Comparison runs on [libvips](https://libvips.github.io/libvips/install.html) (8.9+), a system package: `brew install vips` on macOS, `apt-get install libvips-dev` on Ubuntu. The `ruby-vips` binding is a runtime dependency of this gem since 2.1, so Bundler installs it for you. -You can use the `median_filter_window_size` option to the `screenshot` method to set level: +## Docs -```ruby -test 'unstable area' do - visit '/' - screenshot 'index', median_filter_window_size: 2 -end -``` +- [SnapDiff — the canonical API](docs/snapdiff.md) — setup, config, object map, custom drivers & reporters, canonical names only +- [Framework Setup](docs/framework-setup.md) — Minitest, RSpec, Cucumber +- [CI & Non-Rails Integration](docs/ci-integration.md) — GitHub Actions, reusable action, static sites, baseline updates +- [Configuration Reference](docs/configuration.md) — all options explained +- [Image Processing](docs/drivers.md) — libvips, perceptual threshold, tolerance +- [Screenshot Organization](docs/organization.md) — groups, sections, cropping, multi-browser +- [Web UI & Custom Reporters](docs/reporters.md) — interactive report, custom reporters ## Development -After checking out the repo, run `bin/setup` to install dependencies. -Then, run `rake test` to run the tests. -You can also run `bin/console` for an interactive prompt that will allow you to experiment. - -To install this gem onto your local machine, run `bundle exec rake install`. - -To release a new version, update the version number in `lib/capybara/screenshot/diff/version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). +After checking out the repo, run `bin/setup` then `rake test`. See [Docker Testing](docs/docker-testing.md) for reproducible CI-matching test runs. ## Contributing -Bug reports and pull requests are welcome on GitHub at https://github.com/donv/capybara-screenshot-diff. -This project is intended to be a safe, welcoming space for collaboration, -and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. - +See [CONTRIBUTING.md](CONTRIBUTING.md) ## License The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). - diff --git a/Rakefile b/Rakefile index b6538e1a..60cafba5 100644 --- a/Rakefile +++ b/Rakefile @@ -5,14 +5,59 @@ require "rake/testtask" task default: :test +# `test:canonical` was "everything except test/legacy/", i.e. what had to +# still pass once the v1 surface was deleted. 2.1 deleted it, test/legacy/ +# went with it, and the two tasks converged on the same file list -- so the +# second name is gone rather than kept as an alias for one thing. +# +# `rake test` -- THE gate. +# `rake test:unit` -- unit-sized tests. +# `rake test:integration` -- browser-driven tests. Rake::TestTask.new(:test) do |t| t.libs << "test" t.libs << "lib" t.test_files = FileList["test/**/*_test.rb"] end +Rake::TestTask.new("test:unit") do |t| + t.libs << "test" + t.libs << "lib" + t.test_files = FileList["test/unit/**/*_test.rb"] +end + Rake::TestTask.new("test:integration") do |t| t.libs << "test" t.libs << "lib" t.test_files = FileList["test/integration/**/*_test.rb"] end + +desc "Run all tests with coverage" +task :coverage do + ENV["COVERAGE"] = "true" + Rake::Task["test"].invoke +end + +desc "Generate sample HTML report. Use bin/rake 'report:sample[embed]' for base64 images" +task "report:sample", [:embed] do |_t, args| + embed_arg = args[:embed] ? "--embed" : "" + ruby "scripts/generate_sample_report.rb #{embed_arg}" +end + +desc "Remove screenshot diff artifacts (keeps baselines)" +task "snap_diff:clean" do + patterns = ["**/*.diff.png", "**/*.base.png", "**/*.base.diff.png", "**/*.heatmap.diff.png", + "**/*.diff.webp", "**/*.base.webp", "**/*.base.diff.webp", "**/*.heatmap.diff.webp", + "**/snap_diff_report.html"] + removed = patterns.flat_map { |p| Dir.glob("tmp/#{p}") + Dir.glob("doc/screenshots/#{p}") }.uniq + removed.each { |f| FileUtils.rm_f(f) } + puts "Removed #{removed.size} diff artifacts" +end + +task "clobber" do + puts "Cleanup tmp/" + FileUtils.rm_rf(Dir["./tmp/*"]) +end +# `test:benchmark` is deleted rather than repointed: it required +# scripts/benchmark/find_region_benchmark, which is not in this repo, so the +# task raised LoadError on every invocation -- and its body named a v1 +# constant this release removes. diff --git a/bin/bundle b/bin/bundle new file mode 100755 index 00000000..be29f10b --- /dev/null +++ b/bin/bundle @@ -0,0 +1,109 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# +# This file was generated by Bundler. +# +# The application 'bundle' is installed as part of a gem, and +# this file is here to facilitate running it. +# + +require "rubygems" + +m = Module.new do + module_function + + def invoked_as_script? + File.expand_path($0) == File.expand_path(__FILE__) + end + + def env_var_version + ENV["BUNDLER_VERSION"] + end + + def cli_arg_version + return unless invoked_as_script? # don't want to hijack other binstubs + return unless "update".start_with?(ARGV.first || " ") # must be running `bundle update` + bundler_version = nil + update_index = nil + ARGV.each_with_index do |a, i| + if update_index && update_index.succ == i && a =~ Gem::Version::ANCHORED_VERSION_PATTERN + bundler_version = a + end + next unless a =~ /\A--bundler(?:[= ](#{Gem::Version::VERSION_PATTERN}))?\z/ + bundler_version = $1 + update_index = i + end + bundler_version + end + + def gemfile + gemfile = ENV["BUNDLE_GEMFILE"] + return gemfile if gemfile && !gemfile.empty? + + File.expand_path("../gems.rb", __dir__) + end + + def lockfile + lockfile = + case File.basename(gemfile) + when "gems.rb" then gemfile.sub(/\.rb$/, gemfile) + else "#{gemfile}.lock" + end + File.expand_path(lockfile) + end + + def lockfile_version + return unless File.file?(lockfile) + lockfile_contents = File.read(lockfile) + return unless lockfile_contents =~ /\n\nBUNDLED WITH\n\s{2,}(#{Gem::Version::VERSION_PATTERN})\n/ + Regexp.last_match(1) + end + + def bundler_requirement + @bundler_requirement ||= + env_var_version || + cli_arg_version || + bundler_requirement_for(lockfile_version) + end + + def bundler_requirement_for(version) + return "#{Gem::Requirement.default}.a" unless version + + bundler_gem_version = Gem::Version.new(version) + + bundler_gem_version.approximate_recommendation + end + + def load_bundler! + ENV["BUNDLE_GEMFILE"] ||= gemfile + + activate_bundler + end + + def activate_bundler + gem_error = activation_error_handling do + gem "bundler", bundler_requirement + end + return if gem_error.nil? + require_error = activation_error_handling do + require "bundler/version" + end + return if require_error.nil? && Gem::Requirement.new(bundler_requirement).satisfied_by?(Gem::Version.new(Bundler::VERSION)) + warn "Activating bundler (#{bundler_requirement}) failed:\n#{gem_error.message}\n\nTo install the version of bundler this project requires, run `gem install bundler -v '#{bundler_requirement}'`" + exit 42 + end + + def activation_error_handling + yield + nil + rescue StandardError, LoadError => e + e + end +end + +m.load_bundler! + +if m.invoked_as_script? + load Gem.bin_path("bundler", "bundle") +end diff --git a/bin/ci b/bin/ci new file mode 100755 index 00000000..ab70c0ab --- /dev/null +++ b/bin/ci @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +echo "Running linter..." +if ! bin/standardrb; then + echo "Lint errors found. Auto-fixing..." + bin/standardrb -a + echo "Lint errors were auto-fixed. Please review and re-commit." + exit 1 +fi + +echo "Running unit tests..." +bin/rake test:unit diff --git a/bin/console b/bin/console index 0c77c72b..85227017 100755 --- a/bin/console +++ b/bin/console @@ -2,7 +2,7 @@ # frozen_string_literal: true require "bundler/setup" -require "capybara/screenshot/diff" +require "snap_diff" # You can add fixtures and/or initialization code here to make experimenting # with your gem easier. You can also use a different console, if you like. diff --git a/bin/dtest b/bin/dtest new file mode 100755 index 00000000..101b1e0e --- /dev/null +++ b/bin/dtest @@ -0,0 +1,35 @@ +#!/bin/bash + +set -o pipefail + +export DOCKER_DEFAULT_PLATFORM=linux/amd64 + +# Define allowed environment variables to pass to Docker +ALLOWED_ENV_VARS=( + "CI" "DEBUG" "TEST_ENV" "RAILS_ENV" "RACK_ENV" "COVERAGE" "DISABLE_ROLLBACK_COMPARISON_RUNTIME_FILES" + "RECORD_SCREENSHOTS" "TEST" "TESTOPTS" +) + +# Build the Docker env args string +DOCKER_ENV_ARGS="" +for var in "${ALLOWED_ENV_VARS[@]}"; do + if [[ -n "${!var}" ]]; then + DOCKER_ENV_ARGS="$DOCKER_ENV_ARGS -e $var=${!var}" + fi +done + +# Build the Docker image +docker build . -t csd:test || exit 1 + +# Run setup +(docker run $DOCKER_ENV_ARGS -v ${PWD}:/app -v csd-bundle-cache:/bundle --rm csd:test bin/setup) || exit 1 + +# Run tests with different drivers +echo "Running tests..." +DRIVERS=("cuprite" "selenium_chrome_headless" "selenium_headless") +for driver in "${DRIVERS[@]}"; do + echo "Running tests with $driver driver..." + docker run $DOCKER_ENV_ARGS -e CAPYBARA_DRIVER="$driver" \ + -v ${PWD}:/app -v csd-bundle-cache:/bundle --rm -t csd:test \ + bin/rake test "$@" || exit 1 +done diff --git a/bin/install-vips b/bin/install-vips index 5aae584b..21236bb9 100755 --- a/bin/install-vips +++ b/bin/install-vips @@ -1,8 +1,8 @@ -#!/bin/bash +#!/bin/env bash -set -e +set -eo pipefail -version=${VIPS_VERSION:-8.10.0} +version=${VIPS_VERSION:-8.15.2} wget "https://github.com/libvips/libvips/releases/download/v$version/vips-$version.tar.gz" tar xf "vips-$version.tar.gz" diff --git a/bin/rake b/bin/rake new file mode 100755 index 00000000..0cd6e0d1 --- /dev/null +++ b/bin/rake @@ -0,0 +1,27 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# +# This file was generated by Bundler. +# +# The application 'rake' is installed as part of a gem, and +# this file is here to facilitate running it. +# + +ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../gems.rb", __dir__) + +bundle_binstub = File.expand_path("bundle", __dir__) + +if File.file?(bundle_binstub) + if File.read(bundle_binstub, 300).include?("This file was generated by Bundler") + load(bundle_binstub) + else + abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. +Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") + end +end + +require "rubygems" +require "bundler/setup" + +load Gem.bin_path("rake", "rake") diff --git a/bin/setup b/bin/setup index dce67d86..edd6aca0 100755 --- a/bin/setup +++ b/bin/setup @@ -3,6 +3,7 @@ set -euo pipefail IFS=$'\n\t' set -vx -bundle install +bundle update --all +bundle update --bundler --ruby # Do any other automated setup that you need to do here diff --git a/capybara-screenshot-diff.gemspec b/capybara-screenshot-diff.gemspec index acdea655..495c5b4c 100644 --- a/capybara-screenshot-diff.gemspec +++ b/capybara-screenshot-diff.gemspec @@ -2,25 +2,39 @@ lib = File.expand_path("lib", __dir__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require "capybara/screenshot/diff/version" +require "snap_diff/version" Gem::Specification.new do |spec| spec.name = "capybara-screenshot-diff" - spec.version = Capybara::Screenshot::Diff::VERSION + spec.version = SnapDiff::VERSION spec.authors = ["Uwe Kubosch"] spec.email = ["uwe@kubosch.no"] spec.summary = "Track your GUI changes with diff assertions" spec.description = "Save screen shots and track changes with graphical diff" - spec.homepage = "https://github.com/donv/capybara-screenshot-diff" - spec.required_ruby_version = "~> 2.5" + spec.homepage = "https://github.com/snap-diff/snap_diff-capybara" + spec.required_ruby_version = ">= 3.2" spec.license = "MIT" spec.metadata["allowed_push_host"] = "https://rubygems.org/" - spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + # Allow-list: everything a consumer needs at runtime plus the shipped docs. + # Build/dev files (gems.rb, Rakefile, the gemspec itself, tests, CI) stay out. + spec.files = `git ls-files -z`.split("\x0") + .grep(%r{\A(lib/|docs/|README\.md\z|LICENSE\.txt\z|CHANGELOG\.md\z)}) + .grep_v(%r{\Adocs/RELEASE_PREP\.md\z}) # maintainer-only, not user documentation + spec.bindir = "exe" spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] - spec.add_runtime_dependency "actionpack", ">= 4.2", "< 7" + spec.add_development_dependency "actionpack", ">= 7.1", "< 9" + spec.add_development_dependency "activesupport", ">= 7.1", "< 9" spec.add_runtime_dependency "capybara", ">= 2", "< 4" - spec.add_runtime_dependency "chunky_png", "~> 1.3" + # 2.1 removed the driver abstraction: libvips is the only backend, so the + # gem that binds it is a hard dependency rather than something the user is + # told to add. Without this an install resolves fine and then dies at the + # first comparison -- a resolver error is the better failure. + # + # ruby-vips 2.x is the current major line; the gem itself needs system + # libvips >= 8.2, which no gemspec constraint can express -- see + # docs/drivers.md for the system package. + spec.add_runtime_dependency "ruby-vips", ">= 2.0", "< 3" end diff --git a/docs/RELEASE_PREP.md b/docs/RELEASE_PREP.md new file mode 100644 index 00000000..a50b1ddc --- /dev/null +++ b/docs/RELEASE_PREP.md @@ -0,0 +1,44 @@ +# Release Preparation — v1.15.1 + +## Summary + +HIGH-severity VipsDriver resize fix (retina halving could enlarge screenshots +and corrupt baselines) plus the driver contract tests that caught it. + +## Release Checklist + +### Pre-Release + +- [x] Update version to `1.15.1` +- [x] Run tests: `bundle exec rake test:unit` (376 runs, 0 failures) +- [x] Update CHANGELOG.md + +### Release (One Click) + +1. Push to GitHub +2. Go to [Actions → Release](https://github.com/snap-diff/snap_diff-capybara/actions/workflows/release.yml) +3. Click **Run workflow**, enter `1.15.1` +4. Workflow will: test → tag → publish to RubyGems → create GitHub Release + +### Post-Release + +- [ ] Verify on [RubyGems](https://rubygems.org/gems/capybara-screenshot-diff) +- [ ] Verify GitHub Release created + +## What Changed + +### Added +- `SnapDiff::Config` — flat, additive consolidation of all 27 settings + (`SnapDiff.config` / `SnapDiff.configure`); old accessors stay canonical + +### Fixed +- `pending_if_new` no longer converts real teardown/after-hook failures into + pending tests (Minitest defers to `after_teardown`; RSpec uses `append_after`; + known residual for consumer `append_after` hooks documented) +- `BacktraceFilter` custom `lib_directory` matches on a path boundary + +### Internal +- Guard tests for failure masking and skip-area/VCS-baseline regressions; + two files merged into sole consumers (constant paths preserved) + +See [CHANGELOG.md](../CHANGELOG.md) for full details. diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md new file mode 100644 index 00000000..e7b623e0 --- /dev/null +++ b/docs/UPGRADING.md @@ -0,0 +1,815 @@ +# Upgrading + +## Upgrading to v2.0 (alpha) + +### Overview + +Version 2.0 introduces a new canonical namespace (`SnapDiff`) for cleaner, more discoverable code. The public DSL remains unchanged — your existing `screenshot` and `assert_matches_screenshot` calls work without modification. This guide covers the optional migration path for settings and the new namespace. + +**Status:** `2.0.0.beta3` is an opt-in prerelease. RubyGems never installs prereleases by default resolution — normal `bundle update` keeps you on the 1.x line. The final 2.0.0 ships only after adopter feedback; please report anything surprising on [#166](https://github.com/snap-diff/snap_diff-capybara/issues/166). + +**Estimated upgrade time:** 5–15 minutes (most users need only the Gemfile pin) + +**Writing new code rather than migrating?** Skip this guide and read +[SnapDiff — the canonical API](snapdiff.md): the same setup, configuration, and extension points +with canonical names only, no legacy shapes to unlearn. + +**Breaking changes:** None for the DSL; one migration notice per process plus a deprecation warning per legacy constant you reference (both suppressible), plus two known alpha caveats (see below) + +--- + +### The Short Version (Most Users) + +```ruby +# In your Gemfile — the exact prerelease version is required to opt in +gem "capybara-screenshot-diff", "2.0.0.beta3" # or the latest 2.0.0 prerelease +``` + +```bash +bundle install +bundle exec rake test +``` + +**That's it.** Your existing code works unchanged. The old namespaces (`Capybara::Screenshot::Diff`, `CapybaraScreenshotDiff`) are shimmed with deprecation warnings; the new one (`SnapDiff`) is available if you want to modernize. + +--- + +### What Changed + +#### 1. New Canonical Namespace: `SnapDiff` + +The implementation now lives in `lib/snap_diff/` under the `SnapDiff` namespace. Every legacy constant still resolves to the *same object*. Most do so lazily and warn once each; a documented set stays eagerly defined and silent — see [Deprecation Warnings](#deprecation-warnings) for exactly which. The main renames: + +| Legacy name | v2 canonical name | +|-------------|-------------------| +| `Capybara::Screenshot::Diff::ImageCompare` | `SnapDiff::Comparison` | +| `Capybara::Screenshot::Diff::Difference` | `SnapDiff::ComparisonResult` | +| `Capybara::Screenshot::Diff::Drivers::BaseDriver` | `SnapDiff::Driver` (now a mixin — see below) | +| `CapybaraScreenshotDiff::SnapManager` / `::Snap` | `SnapDiff::SnapManager` / `SnapDiff::Snap` | +| `CapybaraScreenshotDiff::RED_RGBA` / `::ORANGE_RGBA` | `SnapDiff::RED_RGBA` / `SnapDiff::ORANGE_RGBA` | +| `CapybaraScreenshotDiff::Minitest::Assertions` | `SnapDiff::Minitest::Assertions` | +| `require "capybara_screenshot_diff/minitest"` | `require "snap_diff/integrations/minitest"` | +| `require "capybara_screenshot_diff/rspec"` | `require "snap_diff/integrations/rspec"` | +| `require "capybara_screenshot_diff/cucumber"` | `require "snap_diff/integrations/cucumber"` | +| `require "capybara_screenshot_diff/reporters/html"` | `require "snap_diff/reporters/html"` | +| `CapybaraScreenshotDiff.serve` (`…/static`) | `SnapDiff.serve` (`require "snap_diff/static"`) | +| `CapybaraScreenshotDiff.reporters <<` | `SnapDiff::Reporting.register` | +| `CapybaraScreenshotDiff.finalize_reporters!` | `SnapDiff::Reporting.finalize!` | + +Note the integration require paths gain an `integrations/` segment — `require "snap_diff/minitest"` +is a `LoadError`. + +**What stays the same:** +- `screenshot(name)` — still works +- `assert_matches_screenshot(name)` — still works, still the recommended form +- `capture_screenshot(name)` — still works +- All `compare: false/true` flags and overrides work identically + +**What's new (optional):** + +```ruby +# Old (still works; constant access now warns once per process) +Capybara::Screenshot::Diff.compare("baseline.png", "current.png") +Capybara::Screenshot::Diff.configure { |screenshot, diff| ... } + +# New (recommended for new code) +SnapDiff.compare("baseline.png", "current.png") +SnapDiff.start { |screenshot, diff| ... } # same shape as old configure +SnapDiff.configure { |config| ... } # consolidated config object +``` + +#### 2. Consolidated Configuration: `SnapDiff.config` + +Instead of scattering settings across `Capybara::Screenshot` and `Capybara::Screenshot::Diff`, v2.0 offers a single `SnapDiff::Config` object. Both the old and new paths read and write the same underlying storage — writes through either are visible through the other. + +**The DSL never changes.** `screenshot` and `assert_matches_screenshot` work exactly as before. + +--- + +### Settings Migration Table + +The most commonly-used settings and how to update them: + +| Setting | v1.x (still works in v2) | v2.0 (recommended) | What it does | +|---------|----------------------|------------------|------| +| `blur_active_element` | `Capybara::Screenshot.blur_active_element = true` | `SnapDiff.config.blur_active_element = true` | Hide cursor/focus indicator in screenshots (default: `true`) | +| `hide_caret` | `Capybara::Screenshot.hide_caret = true` | `SnapDiff.config.hide_caret = true` | Make input caret transparent for stable comparisons (default: `true`) | +| `tolerance` | `Capybara::Screenshot::Diff.tolerance = 0.0005` | `SnapDiff.config.tolerance = 0.0005` | Pixel-level color difference threshold (higher = less strict) | +| `save_path` | `Capybara::Screenshot.save_path = "doc/screenshots"` | `SnapDiff.config.save_path = "doc/screenshots"` | Where baseline screenshots are stored | +| `window_size` | `Capybara::Screenshot.window_size = [1280, 1024]` | `SnapDiff.config.window_size = [1280, 1024]` | Browser viewport size for consistent screenshots | + +**All 27 settings** from both legacy namespaces are available via `SnapDiff.config.` — see the [Configuration Reference](configuration.md) for the full list. One rename to note: `Capybara::Screenshot.enabled` becomes `SnapDiff.config.screenshot_enabled` (it would otherwise collide with `Capybara::Screenshot::Diff.enabled`, which keeps the bare `enabled` name). + +--- + +### Three Ways to Configure + +All three are equivalent and use the same underlying storage. Pick the one that fits your style. + +#### Option 1: Traditional block (v1 shape, still works) + +```ruby +# In test_helper.rb or spec_helper.rb +Capybara::Screenshot::Diff.configure do |screenshot, diff| + screenshot.window_size = [1280, 1024] + screenshot.blur_active_element = false + diff.tolerance = 0.0005 + diff.driver = :vips +end +``` + +#### Option 2: SnapDiff block with old shape (backward-compatible) + +```ruby +SnapDiff.start do |screenshot, diff| + screenshot.window_size = [1280, 1024] + screenshot.blur_active_element = false + diff.tolerance = 0.0005 + diff.driver = :vips +end +``` + +#### Option 3: Consolidated config (cleanest) + +```ruby +SnapDiff.configure do |config| + config.window_size = [1280, 1024] + config.blur_active_element = false + config.tolerance = 0.0005 + config.driver = :vips +end +``` + +--- + +### Prepare Today on v1.x (Zero Risk) + +You don't have to wait for v2.0 to start using the new namespace. `SnapDiff.compare` and `SnapDiff.start` were added in v1.14; `SnapDiff.config` / `SnapDiff.configure` in v1.15. All of them work on the current 1.x line: + +```ruby +# Works TODAY on v1.15+, zero risk +SnapDiff.compare("baseline.png", "current.png") +SnapDiff.start { |screenshot, diff| ... } +SnapDiff.configure { |config| ... } +``` + +This means you can migrate your codebase incrementally **now**, before opting into 2.0. + +--- + +### Deprecation Warnings + +v2.0 emits three different things, and it is worth knowing which is which. The first two are +about the old namespaces; the third is about the driver features 2.1 removes. + +#### 1. The migration notice — one line per process + +The first time a process touches *any* hookable legacy API, you get a single line: + +``` +[snap_diff deprecation] This process uses the v1 `Capybara::Screenshot*` / `CapybaraScreenshotDiff*` API. It still works in 2.0 and is REMOVED in 2.1 -- see docs/UPGRADING.md for the SnapDiff replacements. Silence with `SnapDiff.silence_deprecations = true` or SNAP_DIFF_SILENCE_DEPRECATIONS=1. (shown once per process) +``` + +It fires once and never again, whichever door you came through: + +- a legacy config accessor — `Capybara::Screenshot.window_size = ...`, `Capybara::Screenshot::Diff.tolerance` +- a lazily shimmed legacy constant (see below) +- `include Capybara::Screenshot` / `include Capybara::Screenshot::Diff` + +It exists because most of the v1 surface **cannot** warn per use, so without it a 2.x app could +be entirely silent right up to the bare `NameError` it would get on 2.1. + +#### 2. Per-constant warnings — one line per lazily shimmed constant + +Resolving a legacy constant that is shimmed through `const_missing` also warns, once per constant +per process: + +``` +[snap_diff deprecation] `Capybara::Screenshot::Diff::ImageCompare` is deprecated (constant); use `SnapDiff::Comparison` instead. +``` + +**These appear for:** `Capybara::Screenshot::{BrowserHelpers, Screenshoter}`; +`Capybara::Screenshot::Diff::{Vcs, StableScreenshoter, ImagePreprocessor, AreaCalculator, +AnnotationService, Utils, ScreenshotMatcher, Drivers, ImageCompare, Difference}`; +`Capybara::Screenshot::Diff::Drivers::BaseDriver`; `CapybaraScreenshotDiff::{RED_RGBA, +ORANGE_RGBA, SnapManager, Snap, ScreenshotNamer, AttemptsReporter, BacktraceFilter, +ErrorWithFilteredBacktrace, ScreenshotAssertion, AssertionRegistry}`; +`CapybaraScreenshotDiff::Reporters::HTML`. + +`CapybaraScreenshotDiff::DSL` and `::Minitest::Assertions` are shimmed this way **only under a +canonical `snap_diff*` require**. Under the v1 entry points — what an unmigrated app actually +uses — they are eagerly defined and silent, like everything in the next section. + +#### 3. Removal warnings — the driver half, removed in 2.1 + +The warnings above are about *names*. These are about *features*: 2.1 makes **libvips the only +image backend** and deletes the rest of the driver machinery. 2.0 still supports all of it and +warns once per process per subject, through the same channel and the same silencing switches. + +| You will see it when you… | Removed in 2.1 | Do this instead | +|---|---|---| +| select the ChunkyPNG driver — `driver: :chunky_png`, `SnapDiff.config.driver = :chunky_png`, or the legacy `Capybara::Screenshot::Diff.driver =` | the `:chunky_png` driver | add `gem "ruby-vips"` (plus the libvips system package) and drop the option | +| run on `driver: :auto` **without `ruby-vips` installed** | the `:auto` fallback to ChunkyPNG | same — install libvips + `ruby-vips`. This is the case worth reading twice: nothing in your setup says `chunky_png`, so the warning is the only sign that 2.1 will break this process | +| set `shift_distance_limit` — globally or per screenshot | `shift_distance_limit` (ChunkyPNG-only) | `median_filter_window_size`, `tolerance`, or `color_distance_limit` — see [Configuration](configuration.md#allowed-shift-distance) | +| read `SnapDiff::Drivers.loaded` (the custom-driver registry) | the registry | nothing — custom drivers are removed, see below | +| read `SnapDiff::Drivers.available` | driver detection | require `ruby-vips` instead of branching on a detected list | +| `include SnapDiff::Driver` in your own driver class | the driver mixin | nothing — see below | + +``` +[snap_diff deprecation] `driver: :auto` selected chunky_png because libvips is not available in this process. The chunky_png driver is REMOVED in 2.1, when libvips (the `ruby-vips` gem) becomes required -- install it now, or this setup stops comparing on 2.1. See docs/drivers.md. Silence with `SnapDiff.silence_deprecations = true` or SNAP_DIFF_SILENCE_DEPRECATIONS=1. (shown once per process) (called from /app/test/test_helper.rb:12) +``` + +**Custom drivers have no migration path.** The whole abstraction goes: the `SnapDiff::Driver` +mixin, the `SnapDiff::Drivers.loaded` registry, `SnapDiff::Drivers.available` / +`SnapDiff::Utils.detect_available_drivers`, and selecting a driver by name. Nothing replaces +them, and this guide is not going to pretend otherwise — if you maintain a third-party driver, +say so on [#166](https://github.com/snap-diff/snap_diff-capybara/issues/166) before 2.1 ships. + +Three spots on the same chopping block stay silent: the legacy +`Capybara::Screenshot::Diff::LOADED_DRIVERS` / `::AVAILABLE_DRIVERS` aliases are plain constants +with nothing to hook (use `SnapDiff::Drivers.loaded` / `.available` to hear the warning); +`SnapDiff::Drivers.for` is not warned on at all — the gem itself calls it for every comparison, +so warning there would fire on setups that are not affected by anything on this list; and +detection (`SnapDiff::Drivers.detect_available` / `SnapDiff::Utils.detect_available_drivers`) +runs at load, before any user code. + +#### Silent by design + +Some legacy names never warn individually, and that is deliberate — the migration notice above is +the signal for all of them: + +- **Requiring the gem.** `require "capybara_screenshot_diff/minitest"` etc. is not deprecated. +- **The DSL.** `screenshot`, `assert_matches_screenshot`, `capture_screenshot` are never deprecated. +- **Settings access.** `Capybara::Screenshot.blur_active_element`, `Capybara::Screenshot::Diff.tolerance=` + and the `Diff.configure` block are plain delegators onto `SnapDiff.config`. There is no + `const_missing` to hook, so they cannot warn per call without adding one on every read. +- **Eagerly defined constants.** `Capybara::Screenshot::Os`, `Capybara::Screenshot::Diff::VERSION`, + `::Comparison`, `::LOADED_DRIVERS`, `::AVAILABLE_DRIVERS`, `::Reporters::Default`, the top-level + `Region`, the `CapybaraScreenshotDiff` error classes, and — under the v1 entry points — + `CapybaraScreenshotDiff::DSL` / `::Minitest::Assertions`. `const_defined?` never triggers + `const_missing`, so these have to be real constants for adopter feature detection and `rescue` + clauses to keep working — which means nothing is left to hook. +- **The driver leaf classes.** `Drivers::VipsDriver` / `Drivers::ChunkyPNGDriver` are autoloaded on + `SnapDiff::Drivers`, so the leaf name itself never warns. Reaching them through the old path still + warns once for `Capybara::Screenshot::Diff::Drivers` — that part is a `const_missing` shim. + Each leaf is only declared when its gem is actually installed, so + `defined?(...Drivers::VipsDriver)` stays `nil` without `ruby-vips`, exactly as in v1. + +Every one of those names resolves under a canonical `snap_diff*` require too, so migrating your +`require` line first (as this guide recommends) never breaks a constant you have not renamed yet. + +Warnings go through `Kernel#warn`, so test suites that hook `Warning.warn` (e.g. raise-on-warning setups) see them like any other Ruby warning. + +#### Silencing Warnings + +If warnings appear in a test run and you're not ready to migrate yet: + +```ruby +# In test_helper.rb, before running tests +SnapDiff.silence_deprecations = true +``` + +```bash +# Or as an environment variable +export SNAP_DIFF_SILENCE_DEPRECATIONS=1 +``` + +--- + +### Known Alpha Caveats + +Two deliberate consequences of the lazy shim design — both flagged for feedback on [#166](https://github.com/snap-diff/snap_diff-capybara/issues/166): + +1. **`defined?` / `const_defined?` on lazily-shimmed legacy names returns `false`/`nil`.** The shims resolve via `const_missing`, which those checks never trigger. Feature detection like `defined?(Capybara::Screenshot::Diff::ImageCompare)` must move to the `SnapDiff::` name. Everything in [Silent by design](#silent-by-design) is unaffected — those names are real constants, so `defined?`, `const_defined?` and `rescue` all behave as they always did. + +2. **Reopening `module Capybara::Screenshot::Diff::Drivers` shadows the shim.** The historical custom-driver monkey-patch pattern defines a fresh, empty `Drivers` module instead of reaching the real one. Define custom drivers under `SnapDiff::Drivers` instead — and note `BaseDriver` is gone as a superclass: `class MyDriver < BaseDriver` becomes `include SnapDiff::Driver` (it's a mixin now). + +#### Two moves that fail *silently* if you miss them + +**Stubbing the detected-drivers list.** The value moved to `SnapDiff::Drivers::AVAILABLE_DRIVERS`, and `Capybara::Screenshot::Diff::AVAILABLE_DRIVERS` is now an eager alias of it. *Reading* either is identical, but **stubbing the legacy name only rebinds the alias** — the gem keeps reading the canonical constant, so a test that stubs it to `[]` no longer exercises the no-drivers path and just passes for the wrong reason: + +```ruby +# before +Capybara::Screenshot::Diff.stub_const(:AVAILABLE_DRIVERS, []) { ... } +# now +SnapDiff::Drivers.stub_const(:AVAILABLE_DRIVERS, []) { ... } +``` + +**`SnapDiff::Config::MAPPING` is gone.** It split in two: `SnapDiff::Config::SETTINGS` (the setting names, no legacy knowledge) and `SnapDiff::LegacyShims::CONFIG_MAPPING` (which legacy holder each name hangs off). If you referenced `MAPPING` — iterating settings in a test helper, say — use `SETTINGS`; `CONFIG_MAPPING` is `@api private` and disappears in 2.1 with the rest of the v1 surface. + +--- + +### FAQ + +#### "My tests pass but I see warnings. Should I worry?" + +No. Warnings are informational and fully suppressible. They're designed to catch legacy namespace references, not break existing CI. If silence is preferable for now, set `SNAP_DIFF_SILENCE_DEPRECATIONS=1` and migrate at your pace. + +#### "Does the DSL change at all?" + +No. `screenshot`, `assert_matches_screenshot`, and `capture_screenshot` are stable and unchanged. All overrides (`:compare`, `:tolerance`, etc.) work identically. + +#### "Can I mix old and new config in the same suite?" + +Yes. Both paths write to the same underlying storage: + +```ruby +Capybara::Screenshot::Diff.configure do |screenshot, diff| + screenshot.window_size = [1280, 1024] +end + +SnapDiff.configure do |config| + config.tolerance = 0.0005 # Same storage, visible to the old path too +end +``` + +#### "What if I need to roll back?" + +All settings and baselines are compatible with v1.x. Simply pin your Gemfile back to `"~> 1.15"` and `bundle update capybara-screenshot-diff`. + +--- + +### Summary Checklist + +- [ ] Pin `gem "capybara-screenshot-diff", "2.0.0.beta3"` (or the latest 2.0.0 prerelease) in your Gemfile +- [ ] Run `bundle install` +- [ ] Run your test suite to verify no regressions +- [ ] (Optional) Migrate config to the `SnapDiff` namespace +- [ ] (Optional) Silence deprecation warnings if not ready to migrate +- [ ] Report anything surprising on [#166](https://github.com/snap-diff/snap_diff-capybara/issues/166) + +--- + +## Upgrading to v1.13.0 + +### Overview + +Version 1.13.0 is a **minor release** clarifying API terminology and adding new capture methods. No breaking changes — your existing code continues to work. + +**Estimated upgrade time:** 0 minutes (no action required for most users) + +--- + +### Quick Upgrade Path (Most Users) + +```ruby +# In your Gemfile +gem 'capybara-screenshot-diff', '~> 1.13.0' +``` + +```bash +bundle update capybara-screenshot-diff +bundle exec rake test +``` + +**That's it!** Existing `screenshot` calls work unchanged. New methods available if needed. + +--- + +### What Changed + +#### API Clarification: Primary Method is `assert_matches_screenshot` + +**v1.12.0 and earlier:** `screenshot` was the primary method +**v1.13.0+:** `assert_matches_screenshot` is the primary method + +**Action required:** None. `screenshot` continues to work as-is. + +The method names now better reflect their behavior: +- `assert_matches_screenshot(name)` — takes screenshot and asserts it matches baseline +- `screenshot(name, compare: true)` — convenience wrapper (same behavior as above when `compare: true`) +- `capture_screenshot(name)` — new: captures without asserting + +```ruby +# All three work and are safe to use: +assert_matches_screenshot "homepage" # Primary: explicit intent +screenshot "homepage" # Shorthand (familiar) +screenshot "homepage", compare: false # Capture only +capture_screenshot "homepage" # Also capture only +``` + +**Safe to override:** You can safely define your own `screenshot` method in your test base class — the gem's implementation won't interfere. + +--- + +### New: `capture_screenshot` Method + +Capture without comparing to baseline: + +```ruby +capture_screenshot "dynamic_page" # No assertion +``` + +Equivalent to: `screenshot "dynamic_page", compare: false` + +--- + +### New: `Diff.pending_if_new` Helper + +Mark baseline-less tests pending instead of failing during initial CI runs: + +```ruby +# In test_helper.rb — before running tests +Capybara::Screenshot::Diff.pending_if_new = true +``` + +**CI requirement:** When using `pending_if_new`, ensure CI is configured with `fail_if_new: false` (see [Configuration Reference](configuration.md#quick-setup)): + +```ruby +Capybara::Screenshot::Diff.configure do |screenshot, diff| + diff.fail_if_new = false # Allow baselines to be added +end +``` + +--- + +## Upgrading to v1.12.0 + +### Overview + +Version 1.12.0 is a **minor release** with new features, performance improvements, and default behavior changes. This guide will help you upgrade smoothly. + +**Estimated upgrade time:** 5-15 minutes depending on your setup + +--- + +## Quick Upgrade Path (Most Users) + +For **most users**, upgrading is as simple as: + +```ruby +# In your Gemfile +gem 'capybara-screenshot-diff', '~> 1.12.0' +``` + +```bash +bundle update capybara-screenshot-diff +bundle exec rake test # Verify tests still pass +``` + +**That's it!** The zero-config setup still works out of the box. Your existing screenshot comparisons will continue to work with v1.12.0. + +--- + +## Breaking Changes & Migration Steps + +### 1. Default Behavior Changes (Most Important) + +Three settings now have different defaults. This is the most likely source of unexpected test failures. + +#### `blur_active_element` — Now defaults to `true` + +**Before (v1.11.x):** Cursor blinking could delay screenshots +**After (v1.12.0):** Cursor is automatically hidden + +**Action required:** Only if you want the old behavior + +```ruby +# To restore v1.x behavior: +Capybara::Screenshot.blur_active_element = false +``` + +#### `hide_caret` — Now defaults to `true` + +**Before (v1.11.x):** Input caret visible in screenshots +**After (v1.12.0):** Caret is transparent for stable screenshots + +**Action required:** Only if you want the old behavior + +```ruby +# To restore v1.x behavior: +Capybara::Screenshot.hide_caret = false +``` + +#### `fail_if_new` — Now defaults to `true` in CI + +**Before (v1.11.x):** New screenshots allowed in CI +**After (v1.12.0):** New screenshots fail tests in CI (when `ENV['CI']` is set) + +**Action required:** Only if you want to allow new screenshots in CI + +```ruby +# To allow new screenshots in CI: +Capybara::Screenshot::Diff.fail_if_new = false +``` + +**Why this changed:** This prevents accidental baseline additions in CI pipelines. Most teams want this behavior. + +--- + +### 2. SVN Support Removed + +**Before (v1.11.x):** Could use SVN for version control +**After (v1.12.0):** Git only + +**Action required:** If using SVN, migrate to Git + +```bash +# Check if you're using SVN for screenshots +git grep svn test/ # Look for svn commands in your tests +``` + +If you find SVN usage: +1. Export your SVN repository to Git +2. Update your CI/CD to use Git +3. Re-commit all screenshot baselines with Git + +**Why this changed:** SVN support was rarely used and added maintenance burden. + +--- + +### 3. ActiveSupport No Longer Required + +**Before (v1.11.x):** ActiveSupport was a runtime dependency +**After (v1.12.0):** Pure Ruby, no ActiveSupport required + +**Action required:** None (this is a positive change!) + +If your project only had ActiveSupport because of this gem, you can now remove it: + +```ruby +# In your Gemfile — can likely be removed if only used for this gem +# gem 'activesupport' # ← Remove if not used elsewhere +``` + +**Why this changed:** Lighter installations, faster boot times. + +--- + +### 4. Internal API Changes + +**Before (v1.11.x):** Could use internal classes like `CaptureStrategy`, `ComparisonLoader` +**After (v1.12.0):** These have been inlined/refactored + +**Action required:** Only if using internal APIs + +Check your codebase: + +```bash +# Search for internal API usage +grep -r "CaptureStrategy" test/ lib/ +grep -r "ComparisonLoader" test/ lib/ +grep -r "ScreenshotCoordinator" test/ lib/ +grep -r "ImagePreprocessor" test/ lib/ +``` + +If you find usage, these were never part of the public API and should be replaced with the documented public API. + +**Why this changed:** Simplified architecture, better performance, easier maintenance. + +--- + +## New Features to Try + +### HTML Reporter (Recommended) + +Get an interactive dashboard showing all screenshot differences: + +```ruby +# Add to test_helper.rb or spec_helper.rb +require 'capybara_screenshot_diff/reporters/html' +``` + +After running tests: + +```bash +open doc/screenshots/snap_diff_report.html +``` + +**Features:** +- Side-by-side comparison with diff toggle +- Thumbnail sidebar for navigation +- Search functionality +- Summary statistics + +--- + +### Standalone Image Comparison + +Compare any two images without Capybara or a browser: + +```ruby +result = Capybara::Screenshot::Diff.compare("baseline.png", "current.png") +result.quick_equal? # => true if byte-identical +result.different? # => true if visually different +``` + +**Use cases:** +- PDF regression testing +- Generated image validation +- CI artifact verification + +--- + +### Perceptual Color Distance (Anti-aliasing Fix) + +Eliminate false positives from font rendering differences: + +```ruby +# Global configuration +Capybara::Screenshot::Diff.perceptual_threshold = 2.0 + +# Or per-screenshot +screenshot 'dashboard', perceptual_threshold: 2.0 +``` + +**dE00 Scale Reference:** +- `< 1.0` — Not perceptible by human eyes +- `1-2` — Perceptible through close observation (anti-aliasing, font hinting) +- `2-10` — Perceptible at a glance (color shifts, layout changes) +- `> 10` — Clearly different colors + +**Why use this:** If you see false positives from font rendering differences across CI environments. + +--- + +### `assert_no_screenshot_changes` + +Assert that an action produces no visual change: + +```ruby +test "clicking cancel doesn't change page" do + visit '/edit' + screenshot 'before_cancel' + + click_button 'Cancel' + + assert_no_screenshot_changes 'after_cancel' +end +``` + +--- + +### Simplified Configuration + +Use the new `Diff.configure` block: + +```ruby +# In test_helper.rb — one line, that's it +Capybara::Screenshot::Diff.configure do |screenshot, diff| + screenshot.window_size = [1280, 1024] + screenshot.stability_time_limit = 1 + diff.driver = :vips + diff.tolerance = 0.0005 +end +``` + +--- + +## Performance Improvements + +Enjoy faster screenshot comparisons: + +- **ChunkyPNG:** Eliminated array allocations in shift-detection (~30% faster for large images) +- **VIPS:** Cached computations at construction (~15% faster) +- **General:** Memoized region area size, replaced closures with blocks + +**No action required** — these are automatic improvements. + +--- + +## Ruby & Rails Compatibility + +### Supported Versions + +- **Ruby:** 3.2, 3.3, 3.4, 3.5 (new!), 4.0 (new!) +- **Rails:** 7.1, 7.2, 8.0 + +### Upgrade Notes + +**Ruby 4.0:** Fully compatible! If you see DSLStub ordering issues, they're fixed in v1.12.0. + +**Rails 8.0:** Works out of the box with updated dependencies. + +--- + +## Testing Your Upgrade + +### Step 1: Update Gemfile + +```ruby +gem 'capybara-screenshot-diff', '~> 1.12.0' +``` + +### Step 2: Bundle Update + +```bash +bundle update capybara-screenshot-diff +``` + +### Step 3: Run Tests + +```bash +bundle exec rake test +``` + +### Step 4: Check for New Screenshot Failures + +If tests fail with new screenshot errors in CI: + +1. **Option A:** Commit the new baselines (recommended if changes are intentional) +2. **Option B:** Set `fail_if_new = false` temporarily (not recommended long-term) + +### Step 5: Enable HTML Reporter (Optional) + +```ruby +require 'capybara_screenshot_diff/reporters/html' +``` + +Run tests and open `doc/screenshots/snap_diff_report.html` to review differences. + +--- + +## Troubleshooting + +### "Tests fail with new screenshots in CI" + +**Cause:** `fail_if_new` now defaults to `true` in CI + +**Solution:** + +```bash +# Commit the new baselines +git add doc/screenshots/ +git commit -m "Add screenshot baselines for v1.12.0 upgrade" +``` + +Or temporarily allow them: + +```ruby +Capybara::Screenshot::Diff.fail_if_new = false +``` + +### "Screenshots look different after upgrade" + +**Cause:** `blur_active_element` and `hide_caret` now default to `true` + +**Solution:** Restore v1.x behavior temporarily: + +```ruby +Capybara::Screenshot.blur_active_element = false +Capybara::Screenshot.hide_caret = false +``` + +Then re-record baselines with the new defaults (recommended): + +```bash +# Delete old baselines +rm doc/screenshots/*.png + +# Run tests to generate new baselines +bundle exec rake test + +# Commit new baselines +git add doc/screenshots/ +git commit -m "Re-record baselines with v1.12.0 defaults" +``` + +### "NoMethodError on internal class" + +**Cause:** Using internal APIs that were refactored + +**Solution:** Use the public API instead. Check the documentation for the correct interface. + +--- + +## Rollback Plan + +If you need to rollback: + +```ruby +# Pin to previous version +gem 'capybara-screenshot-diff', '~> 1.12.0' +``` + +```bash +bundle update capybara-screenshot-diff +``` + +All screenshot baselines are compatible — no data loss. + +--- + +## Need Help? + +- **Documentation:** [README.md](../README.md) +- **Changelog:** [CHANGELOG.md](CHANGELOG.md) +- **Issues:** [GitHub Issues](https://github.com/snap-diff/snap_diff-capybara/issues) +- **DeepWiki:** [Code Documentation](https://deepwiki.com/snap-diff/snap_diff-capybara) + +--- + +## Summary Checklist + +- [ ] Update gem version to `~> 1.12.0` +- [ ] Run `bundle update capybara-screenshot-diff` +- [ ] Run test suite +- [ ] Check for new screenshot failures in CI +- [ ] Decide on `fail_if_new` behavior +- [ ] Decide on `blur_active_element` and `hide_caret` defaults +- [ ] Enable HTML reporter (optional) +- [ ] Re-record baselines if needed +- [ ] Commit changes +- [ ] Review upgrade issues in [GitHub Issues](https://github.com/snap-diff/snap_diff-capybara/issues) + +**Congratulations!** You're now running v1.12.0 🎉 diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..a2a9466a --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,295 @@ +# Architecture + +This document describes the internal architecture of `capybara-screenshot-diff` — how screenshots are captured, compared, and reported, and how the components fit together. + +Since the v2 namespace move (ADR-004), the implementation lives in `lib/snap_diff/` under the `SnapDiff` namespace. The old file paths (`lib/capybara/screenshot/diff/`, `lib/capybara_screenshot_diff/`) remain as thin forwarders, and the old constants resolve to the same objects via `lib/snap_diff/legacy_shims.rb` with a one-time deprecation warning. Class names below use the canonical `SnapDiff::` names, with legacy names noted where they differ. For the user-facing view of the same surface, see [SnapDiff — the canonical API](snapdiff.md). + +## Overview + +``` +┌─────────────────────────────────────────────────────────┐ +│ Test Framework │ +│ (Minitest / RSpec / Cucumber / Custom) │ +└──────────────┬──────────────────────────────┬────────────┘ + │ │ + ▼ ▼ +┌──────────────────────────┐ ┌──────────────────────────┐ +│ SnapDiff::DSL │ │ SnapDiff::AssertionRegistry│ +│ (screenshot, etc) │ │ (SnapDiff.session) │ +└──────────────┬───────────┘ └──────────────┬───────────┘ + │ │ + ▼ ▼ +┌──────────────────────────┐ ┌──────────────────────────┐ +│ ScreenshotMatcher │ │ ScreenshotAssertion │ +│ (capture + compare) │ │ (validate results) │ +└──────┬───────────┬────────┘ └──────────────┬───────────┘ + │ │ │ + ▼ ▼ ▼ +┌──────────┐ ┌──────────┐ ┌──────────────────────┐ +│Screenshot│ │Stable │ │ Reporters │ +│er │ │Screenshot│ │ (Default / HTML / │ +│ │ │er │ │ Custom) │ +└─────┬────┘ └──────────┘ └──────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────┐ +│ SnapDiff::Comparison (layered compare) │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │1. Byte │ │2. Pixel │ │3. Region │ │ +│ │ compare │ │ compare │ │ analyze │ │ +│ └──────────┘ └──────────┘ └──────────┘ │ +└─────────────────────┬────────────────────┘ + │ + ▼ +┌──────────────────────────────────────────┐ +│ VipsDriver (the image backend) │ +│ ┌────────────────────────────────────┐ │ +│ │ libvips via ruby-vips │ │ +│ └────────────────────────────────────┘ │ +└──────────────────────────────────────────┘ +``` + +## Component Breakdown + +### 1. DSL Layer (`lib/snap_diff/dsl.rb`) + +`SnapDiff::DSL` — `CapybaraScreenshotDiff::DSL` remains an eager same-object alias. + +The entry point for test code. `assert_matches_screenshot` is the primary assertion method (captures and compares). `screenshot` is a convenience wrapper with a `compare:` option — `compare: true` (default) delegates to `assert_matches_screenshot`, while `compare: false` delegates to the new `capture_screenshot` method. `capture_screenshot` takes screenshots without assertions. `assert_no_screenshot_changes` keeps its behavior but now delegates to `assert_matches_screenshot` (that redirect is part of the #191 fix). Users can safely override `screenshot` in their test classes without affecting internal gem flow. + +**Flow:** +1. Checks if screenshots are `active?` (returns `false` if disabled) +2. Builds a full screenshot name via `ScreenshotNamer` (handles sections, groups, counters) +3. Delegates to `ScreenshotMatcher` to capture and prepare comparison +4. Creates a `ScreenshotAssertion` — either adds it to the thread-local registry (delayed validation) or validates immediately + +### 2. ScreenshotMatcher (`lib/snap_diff/screenshot_matcher.rb`) + +The orchestrator that coordinates capture and comparison: + +1. **Viewport preparation** — `SnapDiff::Capture::Viewport.prepare!` (`lib/snap_diff/capture/viewport.rb`) verifies the browser window is the expected size (raise-only, never resizes); runs once per capture, outside any stability retry loop +2. **Area calculation** — resolves crop regions and skip areas (supports CSS selectors and coordinates) +3. **Base screenshot checkout** — retrieves the committed baseline from git via `Vcs.checkout_vcs` +4. **Capture** — delegates to `Screenshoter` or `StableScreenshoter` depending on `stability_time_limit` +5. **Comparison** — creates a `SnapDiff::Comparison` object (lazy — actual comparison happens on first access) +6. **Assertion** — returns a `ScreenshotAssertion` with the comparison attached + +**Key design decision:** The "new" screenshot is taken *first*, then compared against the baseline. This means if no baseline exists (first run), we skip comparison entirely and the test passes. + +### 3. Screenshoter & StableScreenshoter (`lib/snap_diff/screenshoter.rb`, `lib/snap_diff/stable_screenshoter.rb`) + +**Screenshoter:** The basic capture flow: +1. Prepares the page (blur active element, hide caret, disable animations, wait for images) +2. Takes a browser screenshot via `Capybara.current_session.save_screenshot` +3. Processes the screenshot (resize for retina, apply crop) + +**StableScreenshoter:** Wraps `Screenshoter` with stability detection: +1. Takes sequential screenshots at `stability_time_limit` intervals +2. Compares consecutive attempts for byte-level equality +3. Returns once two consecutive screenshots are identical +4. Fails with `UnstableImage` if timeout (`wait`) is reached, generating annotated attempt images for debugging + +### 4. Comparison (`lib/snap_diff/comparison.rb`) + +`SnapDiff::Comparison` (legacy name: `ImageCompare`); its result value object is `SnapDiff::ComparisonResult` (`lib/snap_diff/comparison_result.rb`, legacy name: `Difference`). + +The comparison engine uses a **layered optimization strategy** to balance speed and accuracy: + +| Layer | Method | What it checks | Speed | When it returns | +|-------|--------|----------------|-------|-----------------| +| 1 | `quick_equal?` | File size + byte-level comparison | Fastest | Equal → `true`, Different → Layer 2 | +| 2 | `quick_equal?` continues | Dimension check + pixel comparison | Fast | Different dimensions → `false`, Same pixels → `true` | +| 3 | `processed` / `different?` | Full region analysis with tolerance | Slower | Detailed diff with region, heatmap, annotations | + +**Key details:** +- `quick_equal?` is designed for fast rejection — it early-returns as soon as a difference is found +- `different?` triggers the full comparison if not already processed +- `processed` guarantees the comparison is complete and returns the result with all metadata +- `Comparison#analyze_difference` handles the actual pixel analysis, delegating to the driver + +### 5. The image backend (`lib/snap_diff/drivers/vips_driver.rb`) + +`SnapDiff::Drivers::VipsDriver` does the image work. 2.1 removed the abstraction that used to sit around it — the `SnapDiff::Driver` mixin, the `SnapDiff::Drivers` registry (`.loaded` / `.available` / `.for` / `.detect_available`), the `driver:` setting and `driver: :auto`. `ruby-vips` is a gemspec runtime dependency, so there is nothing to detect and nothing to select; `Comparison` and `Screenshoter` each construct a `VipsDriver` directly (it is stateless). `Drivers` survives only as the namespace the class is published under. + +| Operation | VipsDriver | +|-----------|-----------| +| `load_images` | `Vips::Image` from file | +| `same_dimension?` | Compare width × height | +| `same_pixels?` | Pixel-level equality | +| `find_difference_region` | Difference mask → Region | +| `crop` | Vips image crop | +| `save_image_to` | Vips `write_to_file` | +| `filter_image_with_median` | Vips median filter | +| `add_black_box` | Draw filled rect | +| `merge` | Composite images | +| `highlight_mask` | Conditional color overlay | + +**Loader cache:** `#from_file` passes `revalidate: true`. libvips caches loader operations on filename + mtime, and mtime has one-second resolution — without this, rewriting a screenshot path and re-reading it within the same second serves the PREVIOUS image. See the regression test in `test/unit/drivers/vips_driver_test.rb`. + +There is no custom-driver path; see [SnapDiff — the canonical API](snapdiff.md) and [Image Processing](drivers.md). + +### 6. Difference Region Detection + +`VipsDriver` uses a **difference mask** approach: +1. Compute absolute difference between images: `(new - base).abs` +2. Optional: apply perceptual color distance (CIE dE00) instead of raw RGB +3. Project the mask to find the bounding region of non-zero pixels +4. Return the tight bounding box of all differences + +### 7. SnapManager & Snap (`lib/snap_diff/snap_manager.rb`, `lib/snap_diff/snap.rb`) + +**Snap** represents a single screenshot file with path management: +- `path` — the actual screenshot file +- `base_path` — the `.base.ext` VCS checkout +- `attempt_path` — stability attempt files (`.attempt_00.png`, etc.) +- Provides cleanup of diff artifacts (`.diff.png`, `.heatmap.diff.png`) + +**SnapManager** is the factory and path manager: +- Creates `Snap` instances +- Handles VCS checkout of baselines +- Manages file operations (copy, move, cleanup) + +### 8. VCS (`lib/snap_diff/vcs.rb`) + +Handles baseline retrieval from git. Uses `git show HEAD:` to extract the committed version. Supports Git LFS via `git lfs smudge`. Returns `false` if the file doesn't exist in VCS (first-run scenario). + +### 9. Reporters (`lib/snap_diff/reporters/default.rb`, `lib/snap_diff/reporters/html.rb`) + +**Default reporter:** Generates annotated diff images: +- `image.diff.png` — new screenshot with diff region outlined in red +- `image.base.diff.png` — baseline with diff region outlined in red +- `image.heatmap.diff.png` — heatmap overlay of pixel differences + +**HTML reporter:** Generates an interactive dashboard (`snap_diff_report.html`) with: +- Sidebar with thumbnails and search +- 4 view modes (both/base/new/heatmap) +- Annotated toggle for showing diff outlines +- Per-image zoom with synchronized panning across side-by-side views +- Keyboard navigation and shortcuts +- Responsive layout for mobile + +**Custom reporters:** Implement `record(assertions)`, `finalize` and `summary`, then register via `SnapDiff::Reporting.register(reporter)` — the canonical way in, because the append happens under the mutex. The process-global reporter lifecycle (registration, notification, finalization) is owned by `SnapDiff::Reporting` (`lib/snap_diff/reporting.rb`); `CapybaraScreenshotDiff.reporters` / `.finalize_reporters!` are thin public shims over it, and `reporters` stays a mutable array for compatibility (appending directly still works, it just skips the lock). See [Custom reporters](snapdiff.md#custom-reporters). + +### 10. Assertion Lifecycle + +``` +Test begins + │ + ├─ setup: resize_window_if_needed + │ + ├─ test body: + │ └─ screenshot("name") + │ ├─ ScreenshotMatcher builds assertion + │ ├─ delayed=true → add to Thread-local registry + │ └─ delayed=false → validate immediately + │ + ├─ teardown: + │ ├─ SnapDiff.session.verify + │ │ ├─ iterates the fiber-local assertions + │ │ ├─ calls validate on each + │ │ └─ raises SnapDiff::ExpectationNotMet if any differed + │ │ + │ └─ SnapDiff.reset (always, in an ensure) + │ ├─ SnapDiff::Reporting.notify — mutex-protected reporter snapshot + │ └─ clears the session + │ + └─ end of suite (Minitest.after_run / RSpec after(:suite) / Cucumber AfterAll): + └─ SnapDiff::Reporting.finalize! + ├─ Generates HTML report (if reporter registered) + └─ Prints each reporter's summary +``` + +### 11. Thread Safety + +| Concern | Mechanism | +|---------|-----------| +| Assertion registry | Thread-local storage (`Thread.current[:capybara_screenshot_diff_registry]`) | +| Reporter notification | Mutex-protected snapshot of reporter list before iteration | +| HTML reporter internals | Mutex protecting `@failures`, `@total`, `@finalized` | +| Screenshot naming | Per-thread `ScreenshotNamer` instance | +| Global configuration | One process-wide `SnapDiff::Config` instance — must be set before tests run, not mutated during parallel execution | +| File system | Atomic `FileUtils.mv`, unique paths per screenshot name + counter, thread-safe `mkpath` | + +### 12. Configuration System + +Since ADR-008 step 1 the storage ownership is inverted from the original v2 consolidation: **`SnapDiff::Config` (`lib/snap_diff/config.rb`) IS the storage** — one eagerly-created instance, reachable as `SnapDiff.config`, holding every setting as a plain `attr_accessor`. It is the leaf of the config require graph and requires nothing that leads back to either entry point. + +The legacy `Capybara::Screenshot.*` / `Capybara::Screenshot::Diff.*` accessors are thin delegators generated from `SnapDiff::LegacyShims::CONFIG_MAPPING` (both singleton and instance methods, matching what `mattr_accessor` used to define) that forward to that one object. One storage, two views — a write through either surface is visible through the other structurally, not by synchronization. + +Since the 3.0-readiness pass, `lib/snap_diff/legacy_shims.rb` is the single file that holds the v1 surface as code: the `const_missing` forwarders, `CONFIG_MAPPING` and its generator, the derived forwarders (`Screenshot.active?`, `Diff.configure`, `Diff.default_options`, …) and `SnapDiff.start`. `Config` itself names nothing from the v1 namespaces — it declares its settings in `Config::SETTINGS`, and `LegacyShims::CONFIG_MAPPING` says which legacy holder each one is exposed on (an invariant pinned by `snap_diff_config_test.rb`). `lib/capybara/screenshot/diff/config_legacy.rb` remains at the old path as a pair of requires. + +The two legacy views are organized into two namespaces: + +**`Capybara::Screenshot`** — capture settings: +- `window_size`, `stability_time_limit`, `blur_active_element`, `hide_caret`, `disable_animations` +- `save_path`, `root`, `screenshot_format`, `add_driver_path`, `add_os_path` +- `enabled`, `capybara_screenshot_options` + +**`Capybara::Screenshot::Diff`** — comparison settings: +- `driver`, `tolerance`, `color_distance_limit`, `perceptual_threshold`, `shift_distance_limit` +- `area_size_limit`, `skip_area`, `fail_if_new`, `fail_on_difference`, `delayed` + +The canonical way in is `SnapDiff.configure { |config| ... }` (all 25 settings flat on one object). `SnapDiff.start` and `Capybara::Screenshot::Diff.configure` are the two-holder block shape over the same storage — since ADR-008 step 7b, `Diff.configure` forwards to `SnapDiff.start` rather than the other way round. + +`Config` also owns the derived values that used to live on the legacy modules: `active?` (ex `Capybara::Screenshot.active?`), `screenshot_area` / `screenshot_area_abs`, and `default_options` (ex `Capybara::Screenshot::Diff.default_options`, the option hash handed to `SnapDiff::Comparison`). The legacy module methods one-line forward here. + +**Default timing contract:** every default is evaluated once, in `Config#initialize`, which runs at require time of `config.rb` — the same load moment the old `mattr_accessor` default blocks evaluated at. `fail_if_new` (from `ENV["CI"]`) and `root` (from `Rails.root`) must never become lazy read-time defaults. The one deliberately live value is `default_options[:wait]`, a method-body read of `Capybara.default_max_wait_time`. + +## File Layout + +``` +lib/ + snap_diff.rb # SnapDiff module: compare/start/configure/config + snap_diff/ # Canonical implementation (v2) + dsl.rb # screenshot(), screenshot_group(), etc. + config.rb # SnapDiff::Config — THE storage for all 25 settings + errors.rb # Error / ExpectationNotMet / UnstableImage / WindowSizeMismatchError + region.rb # SnapDiff::Region — bounding box (+ eager top-level ::Region alias) + comparison.rb # Layered comparison engine (ex-ImageCompare) + comparison_result.rb # Comparison result value object (ex-Difference) + drivers/ + vips_driver.rb # THE image backend (libvips) + capture/ + viewport.rb # Per-capture viewport preparation seam + screenshoter.rb # Basic browser screenshot capture + stable_screenshoter.rb # Stability detection wrapper + screenshot_matcher.rb # Orchestrator for capture + compare + screenshot_assertion.rb # Assertion + registry objects + screenshot_namer.rb # Name/path generation with sections/groups + snap_manager.rb # Screenshot file management + snap.rb # Single screenshot file abstraction + reporting.rb # Process-global reporter lifecycle + reporters/ + html.rb # Interactive HTML report reporter + templates/report.html.erb # HTML report template + annotation_service.rb # Diff-image annotation (RED_RGBA / ORANGE_RGBA) + image_preprocessor.rb # Pre-processing (skip areas, median filter) + area_calculator.rb # Crop/skip area coordinate resolution + browser_helpers.rb # DOM manipulation helpers + attempts_reporter.rb # Debug reporting for unstable captures + error_with_filtered_backtrace.rb # Error with filtered stack + vcs.rb # Git baseline checkout + utils.rb # Driver detection + os.rb # OS detection + static.rb # Non-Rails static site serving + version.rb # Gem version + integrations/ + minitest.rb # Minitest assertions integration + rspec.rb # RSpec matcher integration + cucumber.rb # Cucumber World integration + capybara_screenshot_diff.rb # Umbrella entry point + eager error-class aliases + capybara_screenshot_diff/ # Legacy paths — mostly thin forwarders + minitest.rb / rspec.rb / cucumber.rb # Legacy entry points (load the full gem) + screenshot_assertion.rb # CapybaraScreenshotDiff session/reporter shims + ... # Everything else forwards to snap_diff/ + capybara/screenshot/diff.rb # Convenience require (loads minitest) + capybara/screenshot/diff/ + config_legacy.rb # Legacy accessor surface, delegating to SnapDiff::Config + region.rb # Forwarder to snap_diff/region.rb + version.rb # Capybara::Screenshot::Diff::VERSION (gemspec reads it) + ... # Everything else forwards to snap_diff/ +``` + +Most legacy `Capybara::Screenshot::Diff::*` and `CapybaraScreenshotDiff::*` constants resolve lazily via `snap_diff/legacy_shims.rb` (`const_missing`), pointing at the same objects with a one-time deprecation warning. The error classes are the deliberate exception: they are **eager** same-object aliases, because `rescue` clauses and `defined?` / `const_defined?` feature detection in adopter code must keep behaving exactly as before (`const_defined?` never triggers `const_missing`). Same for `LOADED_DRIVERS`, pinned as an eager alias of `SnapDiff::Drivers.loaded` so user registrations through the old constant are not silently dropped. + +See [SnapDiff — the canonical API](snapdiff.md) for the canonical surface, and [UPGRADING.md](UPGRADING.md) for the migration guide. diff --git a/docs/ci-integration.md b/docs/ci-integration.md new file mode 100644 index 00000000..65d0806f --- /dev/null +++ b/docs/ci-integration.md @@ -0,0 +1,248 @@ +# CI & Non-Rails Integration + +## Non-Rails Projects (Hugo, Jekyll, Static Sites) + +```ruby +# test/test_helper.rb +require 'snap_diff/static' + +SnapDiff.serve("_site") # or "public", "build", "dist" +``` + +
+Legacy names (still supported) + +```ruby +require 'capybara_screenshot_diff/static' +CapybaraScreenshotDiff.serve("_site") +``` +
+ +This sets up Capybara to serve static files and configures screenshot paths automatically. + +## .gitignore Setup + +See the [Quick Start section](../README.md#quick-start-5-minutes) in the README for recommended `.gitignore` patterns. + +Only commit the baseline screenshots (e.g., `homepage.png`). The `.base.png`, `.diff.png`, `.heatmap.diff.png`, and report files are regenerated on every test run. + +## GitHub Actions Integration + +### 1. Enable the HTML report + +Add to your test helper: + +```ruby +require 'snap_diff/reporters/html' # canonical +# require 'capybara_screenshot_diff/reporters/html' # legacy, same thing +``` + +### 2. Reusable composite action (recommended) + +The simplest way — one step handles artifact upload, job summary, and PR comments: + +```yaml +# .github/workflows/test.yml +jobs: + test: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write # Required for PR comments + + steps: + - uses: actions/checkout@v6 + + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + + - name: Run tests + run: bundle exec rake test + + - name: Upload screenshot reports + if: failure() + uses: snap-diff/snap_diff-capybara/.github/actions/upload-screenshots@master + with: + name: screenshots + pr-comment: 'true' +``` + +That's it. On failure, this will: +- Upload diff images + HTML report as artifacts +- Post a PR comment with links to the inline report and full artifact download +- Add a job summary with report links (visible in the Actions UI) + +#### Inputs + +| Input | Default | Description | +|-------|---------|-------------| +| `name` | (required) | Artifact name prefix | +| `report-path` | `doc/screenshots` | Path to HTML report directory | +| `retention-days` | `2` | Days to retain artifacts | +| `pr-comment` | `false` | Post PR comment with report link (requires `pull-requests: write`) | + +#### Outputs + +| Output | Description | +|--------|-------------| +| `report-url` | Direct URL to the inline HTML report artifact | +| `report-full-url` | Direct URL to the full report artifact (with images) | + +### 3. Ruby + libvips setup action + +For consistent CI environments (libvips, font antialiasing disabled), use the setup action: + +```yaml + - uses: snap-diff/snap_diff-capybara/.github/actions/setup-ruby-and-dependencies@master + with: + ruby-version: '4.0' + cache-apt-packages: true +``` + +This installs Ruby, libvips (with apt caching), and disables font antialiasing for consistent rendering across CI runs. + +#### Inputs + +| Input | Default | Description | +|-------|---------|-------------| +| `ruby-version` | (required) | Ruby version to install | +| `cache-apt-packages` | `false` | Cache libvips apt packages for faster runs | +| `ruby-cache-version` | — | Bundler cache version key | + +### 4. Full example with both actions + +```yaml +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + pull-requests: write + + steps: + - uses: actions/checkout@v6 + + - uses: snap-diff/snap_diff-capybara/.github/actions/setup-ruby-and-dependencies@master + with: + ruby-version: '4.0' + cache-apt-packages: true + + - run: bundle exec rake test + + - uses: snap-diff/snap_diff-capybara/.github/actions/upload-screenshots@master + if: failure() + with: + name: screenshots + pr-comment: 'true' +``` + +### 5. Manual setup (without composite actions) + +If you prefer full control, here's the expanded YAML: + +
+Expand manual setup + +```yaml +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + + - name: Install libvips + run: sudo apt-get install -y libvips-dev + + - name: Run tests + run: bundle exec rake test + + - name: Upload screenshot report + if: failure() + uses: actions/upload-artifact@v7 + with: + name: screenshot-report + path: doc/screenshots/snap_diff_report.html + archive: false + retention-days: 2 + + - name: Upload full report with images + if: failure() + uses: actions/upload-artifact@v7 + with: + name: screenshot-report-full + path: doc/screenshots/ + retention-days: 2 +``` + +
+ +## Update Baselines in CI + +When intentional UI changes are made, baselines need to be re-recorded. You can do this locally: + +```bash +RECORD_SCREENSHOTS=1 bundle exec rake test +git add test/fixtures/screenshots/ +git commit -m "chore: update screenshot baselines" +``` + +Or add a workflow that maintainers can trigger manually: + +
+Expand update-baselines workflow + +```yaml +# .github/workflows/update-baselines.yml +name: Update Screenshot Baselines + +on: + workflow_dispatch: + inputs: + branch: + description: 'Branch to update baselines on' + required: true + default: 'main' + +permissions: + contents: write + +jobs: + update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.branch }} + + - uses: snap-diff/snap_diff-capybara/.github/actions/setup-ruby-and-dependencies@master + with: + ruby-version: '4.0' + cache-apt-packages: true + + - name: Record new baselines + run: RECORD_SCREENSHOTS=1 bundle exec rake test + continue-on-error: true + + - name: Commit updated baselines + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add test/fixtures/ doc/screenshots/ + git diff --staged --quiet || git commit -m "chore: update screenshot baselines" + git push +``` + +
+ +**How it works:** +1. Go to Actions → "Update Screenshot Baselines" → "Run workflow" +2. Enter the branch name (e.g. your PR branch) +3. The workflow records new baselines, commits, and pushes + +[← Back to README](../README.md) diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..2fc1b5c6 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,409 @@ +# Configuration Reference + +## Quick Setup + +**Canonical (v2):** every setting lives on one flat object, `SnapDiff.config`. + +```ruby +# In test_helper.rb or rails_helper.rb +SnapDiff.configure do |config| + config.window_size = [1280, 1024] + config.stability_time_limit = 1 + config.blur_active_element = true + config.hide_caret = true + config.tolerance = 0.0005 + config.color_distance_limit = 15 +end +``` + +**Legacy (still supported):** the two-holder block, split across `Capybara::Screenshot` and +`Capybara::Screenshot::Diff`. + +```ruby +Capybara::Screenshot::Diff.configure do |screenshot, diff| + screenshot.window_size = [1280, 1024] + screenshot.stability_time_limit = 1 + screenshot.blur_active_element = true + screenshot.hide_caret = true + diff.driver = :vips + diff.tolerance = 0.0005 + diff.color_distance_limit = 15 +end +``` + +`SnapDiff::Config` **is** the storage; the legacy accessors are thin delegators onto it. There is +one source of truth, so a write through either surface is visible through the other — mixing them +is safe, and you can migrate a suite one line at a time: + +```ruby +SnapDiff.config.window_size = [1280, 1024] +Capybara::Screenshot.window_size # => [1280, 1024] +``` + +Every option name below is identical on both surfaces — only the receiver changes. The one +exception: `Capybara::Screenshot.enabled` is `SnapDiff.config.screenshot_enabled`, because +`SnapDiff.config.enabled` is taken by `Capybara::Screenshot::Diff.enabled`. See +[SnapDiff — the canonical API](snapdiff.md) for the full SnapDiff-native surface. + +**Note:** `fail_if_new` defaults to `true` in CI environments (when `ENV['CI']` is set). New screenshots are allowed locally but rejected in CI — no configuration needed. + +**Note:** Setting `Capybara::Screenshot.enabled = false` is sufficient to disable all screenshots. There is no need to define no-op modules or monkey-patch the gem. + +## Recommended tolerance values + +| Use Case | `tolerance` | `color_distance_limit` | `stability_time_limit` | +|----------|-------------|------------------------|----------------------| +| Animated/complex pages | 0.01 | 30 | 2s | +| Standard Rails apps | 0.001 (default) | 15 | 1s | +| Pixel-perfect design tests | 0.0001 | 5 | 1s | + +**Note:** `tolerance` defaults to 0.001 (allows 0.1% pixel difference). + +## Choosing the Right Color Comparison Method + +**Important:** `perceptual_threshold`, `color_distance_limit`, and `tolerance` serve different purposes. Use this decision tree: + +### Step 1: Choose color comparison method (pick ONE) + +| Method | Scale | Best for | +|--------|-------|----------| +| `perceptual_threshold` | 0-100+ (dE00) | Cross-OS/browser font rendering, anti-aliasing | +| `color_distance_limit` | 0-510 (RGBA Euclidean) | Fine-grained RGB control | + +**Recommendation:** Use `perceptual_threshold: 2.0` for most cases. It matches human perception and needs less tuning. + +**⚠️ Color comparison methods are exclusive:** `perceptual_threshold` and `color_distance_limit` cannot both be active — if you set both, `perceptual_threshold` wins and `color_distance_limit` is ignored. However, `tolerance` works with **both** methods and is applied by default (0.001). This means even with `perceptual_threshold: 2.0`, the `tolerance: 0.001` default still filters results. + +### Step 2: Set tolerance (optional, independent) + +| Setting | What it does | Scale | +|---------|--------------|-------| +| `tolerance` | Maximum allowed *ratio* of different pixels | 0.0-1.0 | + +**Example:** `tolerance: 0.001` allows 0.1% of the image to differ (e.g., 125 pixels in a 1280×1024 screenshot). + +**Key difference:** +- `perceptual_threshold` / `color_distance_limit` → **"how different can a pixel be?"** +- `tolerance` → **"how many pixels can differ?"** + +### Quick start + +```ruby +# Modern approach (recommended) +screenshot 'dashboard', perceptual_threshold: 2.0 + +# Allow small noise regions +screenshot 'dashboard', perceptual_threshold: 2.0, tolerance: 0.001 + +# Raw RGB distance instead of perceptual +screenshot 'dashboard', color_distance_limit: 15 +``` + +## Configuration Tiers + +**Tier 1 — Zero config (works immediately):** +`blur_active_element`, `hide_caret`, and `fail_if_new` (in CI) are enabled by default. +Just `require 'snap_diff/integrations/minitest'` (legacy: `capybara_screenshot_diff/minitest`) and call `screenshot`. + +**Tier 2 — Set when tests are flaky:** + +| Setting | When to use | +|---------|-------------| +| `window_size` | Screenshots differ between machines due to different browser sizes | +| `tolerance` | Sub-pixel rendering differences cause false positives | +| `skip_area` | Dynamic content (timestamps, ads) changes between runs | +| `stability_time_limit` | Animations or loading states cause inconsistent captures | + +**Tier 3 — Advanced tuning:** + +| Setting | When to use | +|---------|-------------| +| `perceptual_threshold` | Anti-aliasing false positives across OS/browser versions | +| `area_size_limit` | Allow small diff regions below a pixel count | +| `color_distance_limit` | Fine-tune raw RGB channel tolerance | +| `median_filter_window_size` | Smooth noise before comparison (VIPS only) | + +--- + +## Common Options + +### Screen size + +You can specify the desired screen size using + +```ruby +Capybara::Screenshot.window_size = [1024, 768] +``` + +This will force the screen shots to the given size, and skip taking screen shots +unless the desired window size can be achieved. + +### Disabling screen shots + +If you want to skip taking screen shots, set + +```ruby +Capybara::Screenshot.enabled = false +``` + +You can of course set this by an environment variable + +```ruby +Capybara::Screenshot.enabled = ENV['TAKE_SCREENSHOTS'] +``` + +### Disabling diff + +If you want to skip the assertion for change in the screen shot, set + +```ruby +Capybara::Screenshot::Diff.enabled = false +``` + +Using an environment variable + +```ruby +Capybara::Screenshot::Diff.enabled = ENV['COMPARE_SCREENSHOTS'] +``` + +### Tolerate screenshot differences + +To allow screenshot differences, but still fail on functional errors, you can set the following option: + +```ruby +Capybara::Screenshot::Diff.fail_on_difference = false +``` + +It defaults to `true`. This can be useful in continuous integration to a generate a screenshot difference +report while still reporting functional errors. + +### Does not tolerate new screenshots + +To fail the test if a new screenshot is taken, set the following option: + +```ruby +Capybara::Screenshot::Diff.fail_if_new = true +``` + +If `fail_if_new` is set to `true`, the test will fail if a new screenshot is taken +that does not have a corresponding previous image to compare against. +This can be useful in situations where you want to ensure +that every screenshot taken by your tests corresponds to an expected state of your application. + +### Marks new screenshots as pending + +To mark tests as pending (skipped) if a new screenshot is taken without a baseline, set: + +```ruby +Capybara::Screenshot::Diff.pending_if_new = true +# Required in CI, because fail_if_new defaults to true there and raises before +# the pending marker is applied. +Capybara::Screenshot::Diff.fail_if_new = false +``` + +If `pending_if_new` is set to `true`, the test will be marked as skipped in teardown +when a new screenshot has no committed baseline to compare against. +This is complementary to `fail_if_new` (which raises immediately); `fail_if_new` takes precedence since it raises first. +This option is useful when you want to record new screenshots without blocking CI, but still track them as needing review. + +### Screen shot save path + +By default, `Capybara::Screenshot::Diff` saves screenshots to a +`doc/screenshots` folder, relative to either `Rails.root` (if you're in Rails), +or your current directory otherwise. + +If you want to change where screenshots are saved to, then there are two +configuration options that that are relevant. + +The most likely one you'll want to modify is ... + +```ruby +Capybara::Screenshot.save_path = "other/path" +``` + +The `save_path` option is relative to `Capybara::Screenshot.root`. + +`Capybara::Screenshot.root` defaults to either `Rails.root` (if you're in +Rails) or your current directory. You can change it to something entirely +different if necessary, such as when using an alternative web framework. + +```ruby +Capybara::Screenshot.root = Hanami.root +``` + +### Screen shot stability + +To ensure that animations are finished before saving a screen shot, you can add +a stability time limit. If the stability time limit is set, a second screen +shot will be taken and compared to the first. This is repeated until two +subsequent screen shots are identical. + +```ruby +Capybara::Screenshot.stability_time_limit = 0.1 +``` + +This can be overridden on a single screenshot: + +```ruby +test 'stability_time_limit' do + visit '/' + screenshot 'index', stability_time_limit: 0.5 +end +``` + +### Maximum wait limit + +When the `stability_time_limit` is set, but no stable screenshot can be taken, a timeout occurs. +The timeout occurs after `Capybara.default_max_wait_time`, but can be overridden by an option. + +```ruby +test 'max wait time' do + visit '/' + screenshot 'index', wait: 20.seconds +end +``` + +### Hiding the caret for active input elements + +In Chrome the screenshot includes the blinking input cursor. This can make it impossible to get a +stable screenshot. To get around this you can set the `hide caret` option: + +```ruby +Capybara::Screenshot.hide_caret = true +``` + +This will make the cursor (caret) transparent (invisible), so the blinking does not delay the screen shot. + + +### Removing focus from the active element + +Another way to avoid the cursor blinking is to set the `blur_active_element` option: + +```ruby +Capybara::Screenshot.blur_active_element = true +``` + +This will remove the focus from the active element, removing the blinking cursor. + + + +### Allowed color distance + +Sometimes you want to allow small differences in the images. For example, Chrome renders the same +page slightly differently sometimes. You can set set the color difference threshold for the +comparison using the `color_distance_limit` option to the `screenshot` method: + +```ruby +test 'color threshold' do + visit '/' + screenshot 'index', color_distance_limit: 30 +end +``` + +The difference is calculated as the euclidean distance. You can also set this globally: + +```ruby +Capybara::Screenshot::Diff.color_distance_limit = 42 +``` + + +### Allowed shift distance — removed in 2.1 + +The `shift_distance_limit` option let you tolerate small movements in the image (for example, +jquery-tablesorter rendering the same table slightly differently each run). It was implemented +only by the ChunkyPNG driver, and 2.1 removed that driver — libvips is the only backend now, +and it has no shift-distance comparison. + +Setting it anywhere is a `NoMethodError` on the config object and an ignored key per +screenshot. Use one of these instead: + +| Instead of `shift_distance_limit` | Why | +|---|---| +| `median_filter_window_size` | The same idea, far faster — smooths the image before comparing. See [Image Processing](drivers.md#median-filter-size) | +| `tolerance` | Allows a ratio of the pixels to differ, wherever they are | +| `color_distance_limit` | Allows each pixel to differ by a colour distance | + +### Allowed difference size + +You can set set a threshold for the differing area size for the comparison +using the `area_size_limit` option to the `screenshot` method: + +```ruby +test 'area threshold' do + visit '/' + screenshot 'index', area_size_limit: 17 +end +``` + +The difference is calculated as `width * height`. You can also set this globally: + +```ruby +Capybara::Screenshot::Diff.area_size_limit = 42 +``` + + +### Skipping an area + +Sometimes you have expected change that you want to ignore. +You can use the `skip_area` option with `[left, top, right, bottom]` +or css selector like `'#footer'` or `'.container .skipped_element'` to the `screenshot` method to ignore an area. +Be aware that if the selector is not in the page then the library will wait the default wait time for it to appear. +Therefore, it is best to only use css selectors for skip_areas you know will be in the page: + +```ruby +test 'unstable area' do + visit '/' + screenshot 'index', skip_area: [[17, 6, 27, 16], '.container .skipped_element', '#footer'] +end +``` + +The arguments are `[left, top, right, bottom]` for the area you want to ignore. You can also set this globally: + +```ruby +Capybara::Screenshot::Diff.skip_area = [0, 0, 64, 48] +``` + +If you need to ignore multiple areas: + +```ruby +screenshot 'index', skip_area: [[0, 0, 64, 48], [17, 6, 27, 16], 'css_selector .element'] +``` + +### Skipping stack frames in the error output + +If you would like to override the `screenshot` method or for some other reason would like to skip stack +frames when reporting image differences, you can use the `skip_stack_frames` option: + +```ruby +test 'test visiting the index' do + visit root_path + screenshot :index +end + +private + +def screenshot(name, **options) + super(name, skip_stack_frames: 1, **options) +end +``` + +### Screenshot Format + +You can specify the format of the screenshots taken by setting the `screenshot_format` option. By default, the format is set to `"png"`. However, you can change this to any format supported by your image processing driver. For example, to set the format to `"webp"`, you can do the following: + +```ruby +Capybara::Screenshot.screenshot_format = "webp" +``` + +### Customize Capybara#screenshot options + +Allow to bypass screenshot options to Capybara driver. + +```ruby +# To create full page screenshots for Selenium +Capybara::Screenshot.capybara_screenshot_options[:full_page] = true + +screenshot('index', median_filter_window_size: 2, capybara_screenshot_options: {full_page: false}) +``` + +[← Back to README](../README.md) diff --git a/docs/docker-testing.md b/docs/docker-testing.md new file mode 100644 index 00000000..5dd861fd --- /dev/null +++ b/docs/docker-testing.md @@ -0,0 +1,24 @@ +# Docker Testing + +## Running tests in Docker + +Screenshot tests depend on exact browser rendering, which varies across OS and browser versions. Use `bin/dtest` to run tests inside Docker for consistent, reproducible results matching CI: + +```bash +bin/dtest # Run all tests with all drivers +bin/dtest test/integration/ # Run specific test directory +``` + +This builds a Docker image with Chrome and runs the test suite against three Capybara drivers: `cuprite`, `selenium_chrome_headless`, and `selenium_headless`. + +## Recording baseline screenshots + +Screenshot baselines are committed to the repo and compared against during tests. When you set up the project for the first time, or after upgrading the browser/driver, you need to re-record them: + +```bash +RECORD_SCREENSHOTS=1 bin/dtest +``` + +This skips screenshot comparisons and saves new baselines instead. Without this step, tests will fail because your local browser renders pixels differently from the previously committed baselines. + +[← Back to README](../README.md) diff --git a/docs/drivers.md b/docs/drivers.md new file mode 100644 index 00000000..282d74a4 --- /dev/null +++ b/docs/drivers.md @@ -0,0 +1,112 @@ +# Image Processing + +Comparison runs on [libvips](https://www.libvips.org/) through the +[`ruby-vips`](https://www.rubydoc.info/gems/ruby-vips/Vips/Image) gem. There is nothing to +configure and nothing to choose: `ruby-vips` is a runtime dependency of this gem, so Bundler +installs it for you. + +**libvips itself is a system library** and is not installed by Bundler. Add it with your +package manager: + +```sh +brew install vips # macOS +apt-get install libvips # Debian/Ubuntu +``` + +## Removed in 2.1: the driver abstraction + +2.0 shipped two backends and a way to pick between them. 2.1 removed the choice. + +| Removed in 2.1 | What to do instead | +|---|---| +| the `:chunky_png` driver | install libvips (above); comparisons run on it automatically | +| the `driver:` setting and `driver: :auto` | delete the line — there is one backend | +| `shift_distance_limit` | ChunkyPNG-only, with no libvips equivalent. Use `median_filter_window_size`, `tolerance` or `color_distance_limit` | +| `SnapDiff::Driver` (the custom-driver mixin) | nothing — see below | +| `SnapDiff::Drivers.loaded` (the registry) | nothing — see below | +| `SnapDiff::Drivers.available` / `SnapDiff::Utils.detect_available_drivers` | nothing to detect; a missing `ruby-vips` is now a Bundler resolution error | + +`SnapDiff::Drivers::VipsDriver` is the one name from this area that survives, and it is +internal: nothing in normal use has to mention it. + +**Custom drivers: there is no migration path.** The abstraction is removed whole — the +mixin, the registry, and selection by name. Third-party drivers stop working and nothing +replaces them. This was deliberate: one backend is what keeps the comparison engine honest. +If you maintain one, say so on +[the issue tracker](https://github.com/snap-diff/snap_diff-capybara/issues). + +## Perceptual color comparison + +By default, color differences are measured using raw RGB channel distance. This can produce +false positives from anti-aliasing and sub-pixel font rendering — the same page rendered on +different OS versions or browsers will have slightly different pixel values at text edges. + +The `perceptual_threshold` option uses the CIE dE00 formula instead, which measures color +difference the way human eyes perceive it. Anti-aliasing artifacts typically score below 2.0 +on the dE00 scale and are automatically ignored. + +```ruby +# Per-screenshot: ignore anti-aliasing, catch real visual changes +screenshot 'dashboard', perceptual_threshold: 2.0 + +# Global: apply to all screenshots +SnapDiff.config.perceptual_threshold = 2.0 + +# dE00 scale reference: +# < 1.0 — not perceptible by human eyes +# 1-2 — perceptible through close observation (anti-aliasing, font hinting) +# 2-10 — perceptible at a glance (color shifts, layout changes) +# > 10 — clearly different colors +``` + +Use `perceptual_threshold` when you see false positives from font rendering differences across +CI environments, or when `color_distance_limit` with raw RGB requires frequent tuning. + +**⚠️ Important:** `perceptual_threshold` and `color_distance_limit` are **mutually exclusive**. +If you set both, `perceptual_threshold` takes priority and `color_distance_limit` is silently ignored. + +These options use different scales and algorithms: +- `perceptual_threshold` → CIE dE00 perceptual distance (0-100+) +- `color_distance_limit` → Euclidean RGBA distance (0-510) + +## Tolerance level + +You can set a "tolerance" anywhere from 0% to 100%. This is the amount of change that's allowable. +If the screenshot has changed by more than that amount, it'll flag it as a failure. + +This is an alternative to "Allowed difference size", where the difference area is calculated +including valid pixels. "Tolerance" compares only different pixels. + +You can use the `tolerance` option to the `screenshot` method to set level: + +```ruby +test 'unstable area' do + visit '/' + # tolerance: 0.01 allows 1% of pixels to differ (use for noisy pages) + screenshot 'index', tolerance: 0.01 +end +``` + +You can also set this globally: + +```ruby +# Default is 0.001 (0.1% pixel difference allowed) +SnapDiff.config.tolerance = 0.001 +``` + +## Median filter size + +This is an alternative to "Allowed shift distance", but much faster. +You can find more about this strategy on [Median Filter](https://en.wikipedia.org/wiki/Median_filter). +Think about this like smoothing of the image, before comparison. + +You can use the `median_filter_window_size` option to the `screenshot` method to set level: + +```ruby +test 'unstable area' do + visit '/' + screenshot 'index', median_filter_window_size: 2 +end +``` + +[← Back to README](../README.md) diff --git a/docs/framework-setup.md b/docs/framework-setup.md new file mode 100644 index 00000000..392f1f8b --- /dev/null +++ b/docs/framework-setup.md @@ -0,0 +1,105 @@ +# Framework Setup + +> **Canonical equivalents.** This page uses the legacy `CapybaraScreenshotDiff` names, which keep +> working. Each has a `SnapDiff` home: +> +> | This page | Canonical | +> |-----------|-----------| +> | `require "capybara_screenshot_diff/minitest"` | `require "snap_diff/integrations/minitest"` | +> | `require "capybara_screenshot_diff/rspec"` | `require "snap_diff/integrations/rspec"` | +> | `require "capybara_screenshot_diff/cucumber"` | `require "snap_diff/integrations/cucumber"` | +> | `CapybaraScreenshotDiff::DSL` | `SnapDiff::DSL` | +> | `CapybaraScreenshotDiff::Minitest::Assertions` | `SnapDiff::Minitest::Assertions` | +> | `CapybaraScreenshotDiff.finalize_reporters!` | `SnapDiff::Reporting.finalize!` | +> +> The canonical setup, written out in full, is in [SnapDiff — the canonical API](snapdiff.md). + +## Including DSL + +To use the screenshot capturing and change detection features in your tests, include the `CapybaraScreenshotDiff::DSL` in your test classes. It provides the `screenshot` method to capture and compare screenshots. + +There are different modules for different testing frameworks integrations. + +## Minitest + +For Minitest, need to require `capybara_screenshot_diff/minitest`. +In your test class, include the `CapybaraScreenshotDiff::Minitest::Assertions` module: + +```ruby +require 'capybara_screenshot_diff/minitest' + +class ApplicationSystemTestCase < ActionDispatch::SystemTestCase + # Make the Capybara & Capybara Screenshot Diff DSLs available in tests + include CapybaraScreenshotDiff::DSL + # Make `assert_*` methods behave like Minitest assertions + include CapybaraScreenshotDiff::Minitest::Assertions + + def test_my_feature + visit '/' + assert_matches_screenshot 'index' + end +end +``` + +## RSpec + +To use the screenshot capturing and change detection features in your tests, +include the `CapybaraScreenshotDiff::DSL` in your test classes. +It adds `match_screenshot` matcher to RSpec. + +> **Important**: +> The `CapybaraScreenshotDiff::DSL` is automatically included in all feature and system tests by default. + + +```ruby +require 'capybara_screenshot_diff/rspec' + +describe 'Permissions admin', type: :feature do + it 'works with permissions' do + visit('/') + expect(page).to match_screenshot('home_page') + end +end + + +describe 'Permissions admin', type: :non_feature do + include CapybaraScreenshotDiff::DSL + + it 'works with permissions' do + visit('/') + expect(page).to match_screenshot('home_page') + end +end +``` + +## Cucumber + +Load Cucumber support by adding the following line (typically to your `features/support/env.rb` file): + +```ruby +require 'capybara_screenshot_diff/cucumber' +``` + +And in the steps you can use: + +```ruby +Then('I should not see any visual difference') do + screenshot 'homepage' +end +``` + +## Custom Test Frameworks + +Minitest, RSpec, and Cucumber are supported out of the box. For other frameworks, call the +end-of-suite hook yourself: + +```ruby +SnapDiff::Reporting.finalize! # canonical +CapybaraScreenshotDiff.finalize_reporters! # legacy, same thing +``` + +This generates the HTML report and prints the summary. A framework also needs the per-test +lifecycle wired up — see +[Frameworks other than Minitest/RSpec/Cucumber](snapdiff.md#frameworks-other-than-minitestrspeccucumber). + +[← Back to README](../README.md) diff --git a/docs/images/snap_diff_annotated.png b/docs/images/snap_diff_annotated.png new file mode 100644 index 00000000..b593be7c Binary files /dev/null and b/docs/images/snap_diff_annotated.png differ diff --git a/docs/images/snap_diff_web_ui.png b/docs/images/snap_diff_web_ui.png new file mode 100644 index 00000000..997698e6 Binary files /dev/null and b/docs/images/snap_diff_web_ui.png differ diff --git a/docs/migration-guide.md b/docs/migration-guide.md new file mode 100644 index 00000000..2859aedc --- /dev/null +++ b/docs/migration-guide.md @@ -0,0 +1,286 @@ +# Migration Guide + +Migrate your visual regression testing from other tools to `capybara-screenshot-diff`. This guide covers the most common migration paths. + +## Why Switch? + +| Factor | Percy / Chromatic | BackstopJS | `capybara-screenshot-diff` | +|--------|-------------------|------------|--------------------------| +| Pricing | Paid SaaS (snapshot limits) | Free | Free (MIT) | +| Infrastructure | Cloud service, API tokens | Node + Puppeteer | Ruby gem, no external services | +| Baselines | Hosted on their servers | Local files | Git (committed to repo) | +| Review | Web dashboard | HTML report | HTML report + GitHub PR comments | +| PR integration | GitHub app | Manual CI steps | Reusable GitHub Action | +| Offline | ❌ Requires internet | ✅ | ✅ | +| Diff in PRs | Screenshot in comment | Manual | Upload artifact + PR comment | + +## From Percy + +### Setup changes + +**Before (Percy):** +```ruby +# Gemfile +gem 'percy-capybara' + +# test helper +require 'percy/capybara' + +# test +def test_homepage + visit '/' + Percy::Capybara.screenshot('homepage') +end +``` + +**After (capybara-screenshot-diff):** +```ruby +# Gemfile +gem 'capybara-screenshot-diff' + +# test helper +require 'capybara_screenshot_diff/minitest' + +# test class +class ApplicationSystemTestCase < ActionDispatch::SystemTestCase + include CapybaraScreenshotDiff::Minitest::Assertions + + test "homepage" do + visit '/' + screenshot 'homepage' + end +end +``` + +### What changes + +| Concept | Percy | capybara-screenshot-diff | +|---------|-------|-------------------------| +| Baseline storage | Percy cloud | Committed to git (`doc/screenshots/`) | +| First run | Uploads to Percy | Saves locally, passes automatically | +| CI setup | `PERCY_TOKEN` env var | GitHub Action (3 lines) | +| Diff review | Percy dashboard | `snap_diff_report.html` or PR artifacts | +| Update baselines | Percy's "Approve" button | Delete file, re-run tests, commit | +| Snapshot limits | Paid plan dependent | Unlimited | +| Parallel builds | Built-in | Thread-safe with t-locals + mutex | + +### CI migration + +**Before (Percy GitHub Action):** +```yaml +- name: Percy Test + run: PERCY_TOKEN=${{ secrets.PERCY_TOKEN }} bundle exec rake test +``` + +**After (capybara-screenshot-diff):** +```yaml +- uses: snap-diff/snap_diff-capybara/.github/actions/setup-ruby-and-dependencies@master + with: + ruby-version: '4.0' +- run: bundle exec rake test +- uses: snap-diff/snap_diff-capybara/.github/actions/upload-screenshots@master + if: failure() + with: + name: screenshots + pr-comment: 'true' +``` + +### Migration steps + +1. **Remove Percy gem and configuration** +2. **Add `capybara-screenshot-diff`** to your Gemfile +3. **Replace `Percy::Capybara.screenshot` calls** with `screenshot` (or `match_screenshot` for RSpec) +4. **Run tests once** to generate baselines +5. **Commit baselines** (`git add doc/screenshots/`) +6. **Set up CI** with the GitHub Actions upload step +7. **Remove Percy integration** from CI + +--- + +## From Chromatic + +### Setup changes + +**Before (Chromatic + Storybook):** +```js +// .storybook/preview.js +import { withScreenshot } from 'chromatic'; + +export const decorators = [withScreenshot]; +``` + +**After (capybara-screenshot-diff):** +```ruby +# test/system/stories_test.rb +class StoriesTest < ApplicationSystemTestCase + test "landing page story" do + visit '/iframe.html?id=pages-landing--default' + screenshot 'stories/landing-page' + end +end +``` + +### Key differences + +| Concept | Chromatic | capybara-screenshot-diff | +|---------|-----------|-------------------------| +| Focus | Storybook components | Full-page system tests | +| Baseline | Chromatic cloud | Git-committed | +| Review | Chromatic web UI | HTML report + PR artifacts | +| CI integration | Chromatic GitHub App | GitHub Actions + PR comments | +| Thresholds | Visual catch (AI) | Configurable tolerance (numeric) | + +### Migration approach + +Chromatic is primarily for Storybook component testing. If you want to continue testing individual components: + +1. **Replace with Capybara system tests** that visit each component's rendered page +2. **Use `crop:` option** to isolate specific elements: `screenshot 'button', crop: '.my-button'` +3. **Use `skip_area:` option** to ignore dynamic regions: `screenshot 'dashboard', skip_area: ['.timestamp']` + +--- + +## From BackstopJS + +### Setup changes + +**Before (BackstopJS):** +```json +// backstop.json +{ + "id": "homepage", + "viewports": [{"width": 1280, "height": 1024}], + "scenarios": [{ + "label": "Homepage", + "url": "http://localhost:3000", + "referenceUrl": "http://localhost:3000", + "selectors": ["document"] + }], + "paths": { + "bitmaps_reference": "backstop_data/bitmaps_reference", + "bitmaps_test": "backstop_data/bitmaps_test", + "html_report": "backstop_data/html_report" + } +} +``` + +**After (capybara-screenshot-diff):** +```ruby +class HomepageTest < ApplicationSystemTestCase + test "homepage" do + visit '/' + screenshot 'homepage' + end +end +``` + +### Key differences + +| Concept | BackstopJS | capybara-screenshot-diff | +|---------|-----------|-------------------------| +| Language | JavaScript + Node | Ruby (runs in test suite) | +| Dependencies | Node, Puppeteer/Chromium | Ruby gems + optional libvips | +| Test runner | Standalone CLI | Minitest, RSpec, Cucumber | +| Selectors | CSS selectors for scenarios | CSS selectors for crop/skip_area | +| Viewports | Per-scenario config | Global `window_size` setting | +| CI report | HTML report | HTML report + GitHub Actions | +| Stability | `misMatchThreshold` + `delay` | `tolerance` + `stability_time_limit` | + +### Configuration mapping + +| BackstopJS option | capybara-screenshot-diff equivalent | +|-------------------|-------------------------------------| +| `misMatchThreshold` | `tolerance` (0.0-1.0 scale, e.g. `0.01` = 1%) | +| `delay` | `stability_time_limit` (seconds) | +| `selectors` | `crop:` option with CSS selector | +| `hideSelectors` | `skip_area:` option with CSS selectors | +| `removeSelectors` | N/A — use `skip_area` or modify DOM before screenshot | +| `waitTimeout` | `wait:` option (defaults to `Capybara.default_max_wait_time`) | +| `viewports` | `window_size: [width, height]` | +| `onReadyScript` | Custom setup in your test's `setup` block | + +### CI migration + +**Before (BackstopJS in CI):** +```yaml +- run: npx backstop test --config=backstop.json +``` + +**After (capybara-screenshot-diff in CI):** +```yaml +- uses: snap-diff/snap_diff-capybara/.github/actions/setup-ruby-and-dependencies@master +- run: bundle exec rake test +- uses: snap-diff/snap_diff-capybara/.github/actions/upload-screenshots@master + if: failure() + with: + name: screenshots + pr-comment: 'true' +``` + +### Migration steps + +1. **Remove BackstopJS configuration** (`backstop.json`, npm dependencies) +2. **Convert scenarios to Capybara tests** — each scenario becomes a `screenshot` call +3. **Map threshold and delay settings** to `tolerance` and `stability_time_limit` +4. **Run tests** to generate baselines +5. **Commit baselines** (`git add doc/screenshots/`) +6. **Update CI** to use the GitHub Actions setup + +--- + +## General Migration Checklist + +- [ ] Remove old gem/npm dependencies +- [ ] Add `capybara-screenshot-diff` to Gemfile +- [ ] Require the appropriate adapter (`minitest`, `rspec`, or `cucumber`) +- [ ] Replace screenshot calls with `screenshot` / `match_screenshot` +- [ ] Configure `window_size` for consistent viewport dimensions +- [ ] Set `tolerance` or `perceptual_threshold` if your previous tool had a mismatch threshold +- [ ] Add `.gitignore` patterns for diff artifacts +- [ ] Run tests to generate baseline screenshots +- [ ] Commit baselines to git +- [ ] Set up CI with artifact upload +- [ ] Optional: add HTML reporter and PR commenting + +## Common Gotchas + +### "My baselines are on Percy/Chromatic servers" + +You'll need to take fresh screenshots. Either: +- Visit each page and capture manually +- Run tests with `RECORD_SCREENSHOTS=1` to generate all baselines at once + +### "I had hundreds of BackstopJS scenarios" + +Start small. Migrate one test file at a time. The `screenshot_group` feature helps organize related screenshots: +```ruby +screenshot_group 'checkout' +screenshot 'step1' +screenshot 'step2' +# Produces: doc/screenshots/checkout/00_step1.png, 01_step2.png +``` + +### "My tests are slow now" + +Use the VIPS driver for ~50ms comparisons per image: +```ruby +gem 'ruby-vips' +Capybara::Screenshot::Diff.driver = :vips +``` + +### "The diffs look different from what I'm used to" + +Each tool uses different comparison algorithms: +- **Percy:** Proprietary pixel-level comparison with AI smoothing +- **Chromatic:** Visual catch algorithm (structure-aware) +- **BackstopJS:** Resemble.js pixel comparison +- **capybara-screenshot-diff:** Raw pixel difference with configurable tolerance + +Start with default settings, then adjust `tolerance` or `perceptual_threshold` based on your needs. + +## Need Help? + +- [Architecture Overview](docs/architecture.md) — understanding how comparisons work +- [Configuration Reference](docs/configuration.md) — all available options +- [CI Integration](docs/ci-integration.md) — setting up in CI +- [GitHub Issues](https://github.com/snap-diff/snap_diff-capybara/issues) — ask questions diff --git a/docs/organization.md b/docs/organization.md new file mode 100644 index 00000000..541f6f27 --- /dev/null +++ b/docs/organization.md @@ -0,0 +1,204 @@ +# Screenshot Organization + +## Taking screenshots + +Add `screenshot ''` to your tests. The screenshot will be saved in +the `doc/screenshots` directory. + +Change your existing `save_screenshot` calls to `screenshot` + +```ruby +test 'my useful feature' do + visit '/' + screenshot 'welcome_index' + click_button 'Useful feature' + screenshot 'feature_index' + click_button 'Perform action' + screenshot 'action_performed' +end +``` + +This will produce a sequence of images like this + +``` +doc + screenshots + action_performed + feature_index + welcome_index +``` + +To store the screenshot history, add the `doc/screenshots` directory to your +version control system (git). + +Screenshots are compared to the previously COMMITTED version of the same screenshot. + +**Note:** When a screenshot differs, diff artifacts (`.diff.png`, `.heatmap.diff.png`, etc.) are generated alongside the baseline. Add `*.diff.png`, `*.base.png`, `*.diff.webp`, `*.base.webp`, and `snap_diff_report.html` to your `.gitignore`. Clean up artifacts with `rake snap_diff:clean`. + +## Screenshot groups + +Commonly it is useful to group screenshots around a feature, and record them as +a sequence. To do this, add a `screenshot_group` call to the start of your +test. + +```ruby +test 'my useful feature' do + screenshot_group 'useful_feature' + visit '/' + screenshot 'welcome_index' + click_button 'Useful feature' + screenshot 'feature_index' + click_button 'Perform action' + screenshot 'action_performed' +end +``` + +This will produce a sequence of images like this + +``` +doc + screenshots + useful_feature + 00_welcome_index + 01_feature_index + 02_action_performed +``` + +**Note:** `screenshot_group` sets the group name for organizing screenshots. It does not delete existing files. + + +## Screenshot sections + +You can introduce another level above the screenshot group called a +`screenshot_section`. The section name is inserted just before the group name +in the save path. If called in the setup of the test, all screenshots in +that test will get the same prefix: + +```ruby +setup do + screenshot_section 'my_feature' +end + +test 'my subfeature' do + screenshot_group 'subfeature' + visit '/feature' + click_button 'Interesting button' + screenshot 'subfeature_index' + click_button 'Perform action' + screenshot 'action_performed' +end +``` + +This will produce a sequence of images like this + +``` +doc + screenshots + my_feature + subfeature + 00_subfeature_index + 01_action_performed +``` + + +## Setting `screenshot_section` and/or `screenshot_group` for all tests + +Setting the `screenshot_section` and/or `screenshot_group` for all tests can be +done in the super class setup: + +```ruby +class ApplicationSystemTestCase < ActionDispatch::SystemTestCase + setup do + screenshot_section class_name.underscore.sub(/(_feature|_system)?_test$/, '') + screenshot_group name[5..-1] + end +end +``` + +`screenshot_section` and/or `screenshot_group` can still be overridden in each +test. + + +## Capturing one area instead of the whole page + +You can crop images before comparison to be run, by providing region to crop as `[left, top, right, bottom]` or by css selector like `body .tag` + +```ruby +test 'the cool' do + visit '/feature' + screenshot 'cool_element', crop: '#my_element' +end +``` + +**Note:** When using a retina device screenshots dimensions might be off. If +you are using (headless) chrome you can prevent this by setting the +`force-device-scale-factor` argument to `1`. + +For Rails system specs using selenium you can do so for example by using the +following snippet: + +```ruby +driven_by :selenium, using: :chrome_headless do |options| + options.args << '--force-device-scale-factor=1' +end +``` + +## Multiple Capybara drivers + +Often it is useful to test your app using different browsers. To avoid the +screenshots for different Capybara drivers to overwrite each other, set + +```ruby +Capybara::Screenshot.add_driver_path = true +``` + +The example above will then save your screenshots like this +(for poltergeist and selenium): + +``` +doc + screenshots + poltergeist + useful_feature + 00_welcome_index + 01_feature_index + 02_action_performed + selenium + useful_feature + 00_welcome_index + 01_feature_index + 02_action_performed +``` + +## Multiple OSs + +If you run your tests on multiple operating systems, you will most likely find +the screen shots differ. To avoid the screenshots for different OSs to +overwrite each other, set + +```ruby +Capybara::Screenshot.add_os_path = true +``` + +The example above will then save your screenshots like this +(for Linux and Windows): + +``` +doc + screenshots + linux + useful_feature + 00_welcome_index + 01_feature_index + 02_action_performed + windows + useful_feature + 00_welcome_index + 01_feature_index + 02_action_performed +``` + +If you combine this config with the `add_driver_path` config, the driver will be +put in front of the OS name. + +[← Back to README](../README.md) diff --git a/docs/reporters.md b/docs/reporters.md new file mode 100644 index 00000000..ab4c1c5b --- /dev/null +++ b/docs/reporters.md @@ -0,0 +1,61 @@ +# Reporters + +## Web UI for Reviewing Screenshot Changes + +Generate an interactive Web UI report of screenshot differences: + +```ruby +# Add to test_helper.rb — one line, that's it +require 'snap_diff/reporters/html' # canonical +# require 'capybara_screenshot_diff/reporters/html' # legacy, same thing +``` + +After running tests, open the report (generated only when there are failures): + +```bash +open doc/screenshots/snap_diff_report.html +``` + +The report includes a sidebar with thumbnails, side-by-side comparison with diff toggle, search, and summary stats. No configuration needed — just require it. + +**Note:** The report is not generated when all screenshots match. In parallel test environments, each worker writes to the same file — the last worker's results will be in the report. + +## Custom Reporters + +Build your own reporter by implementing `record`, `finalize` and `summary`: + +```ruby +class MyReporter + def record(assertions) + assertions.each do |assertion| + next unless assertion.compare&.difference&.different? + # process the failure — send to Slack, write JSON, etc. + end + end + + def finalize + # called once at end of suite — write summary, upload report, etc. + end + + def summary + # printed to stdout after finalize; return nil to print nothing + nil + end +end + +# Register in test_helper.rb +SnapDiff::Reporting.register(MyReporter.new) # canonical — appends under the mutex +# CapybaraScreenshotDiff.reporters << MyReporter.new # legacy, same list, skips the lock +``` + +Reporters are notified before assertions are cleared on each test teardown. `finalize` runs from +the framework's end-of-suite hook (`Minitest.after_run`, RSpec `after(:suite)`, Cucumber +`AfterAll`), which calls `SnapDiff::Reporting.finalize!`. + +**Do implement `summary`.** `finalize!` calls it unconditionally, so a reporter without it is +finalized and then warned about. A reporter that raises is warned about and skipped — the others +still run. + +Full details in [Custom reporters](snapdiff.md#custom-reporters). + +[← Back to README](../README.md) diff --git a/docs/snapdiff.md b/docs/snapdiff.md new file mode 100644 index 00000000..64a5f654 --- /dev/null +++ b/docs/snapdiff.md @@ -0,0 +1,297 @@ +# SnapDiff — the canonical API + +Everything in this gem lives under `SnapDiff` since v2. This page is the SnapDiff-native +reference: setup, configuration, the object map, and the extension points — all using canonical +names only. + +The legacy `Capybara::Screenshot::Diff` / `CapybaraScreenshotDiff` names still work — they resolve +to the same objects — and the rest of the docs still teach them. The first legacy API a process +touches prints one migration notice; on top of that, *lazily shimmed* constants warn once each. +Some legacy names are silent by design. [UPGRADING.md](UPGRADING.md#deprecation-warnings) lists +exactly which is which. Nothing here replaces a working setup — it is what you write for **new** code. +For migrating an existing suite, see [UPGRADING.md](UPGRADING.md). + +## Quick start + +### Minitest + +```ruby +# test/test_helper.rb +require "snap_diff/integrations/minitest" +``` + +```ruby +# test/application_system_test_case.rb +class ApplicationSystemTestCase < ActionDispatch::SystemTestCase + include SnapDiff::Minitest::Assertions # brings in SnapDiff::DSL too +end +``` + +```ruby +class HomepageTest < ApplicationSystemTestCase + test "homepage" do + visit "/" + assert_matches_screenshot "homepage" + end +end +``` + +`SnapDiff::Minitest::Assertions` already includes `SnapDiff::DSL`, so a separate +`include SnapDiff::DSL` is not needed (it is harmless if you have it). + +> **The require path is `snap_diff/integrations/…`, not `snap_diff/…`.** +> `require "snap_diff/minitest"` raises `LoadError` — there is no such file. The integrations +> live one level down, mirroring `lib/snap_diff/integrations/`. + +### RSpec + +```ruby +# spec/rails_helper.rb +require "snap_diff/integrations/rspec" +``` + +This registers the `match_screenshot` matcher and includes `SnapDiff::DSL` into +`type: :feature` and `type: :system` examples automatically: + +```ruby +RSpec.describe "Homepage", type: :system do + it "looks right" do + visit "/" + expect(page).to match_screenshot("homepage") + end +end +``` + +For other example types, include the DSL yourself: + +```ruby +RSpec.describe "Admin", type: :request do + include SnapDiff::DSL +end +``` + +### Cucumber + +```ruby +# features/support/env.rb +require "snap_diff/integrations/cucumber" +``` + +The DSL is added to the Cucumber `World`, so steps can call `screenshot` / +`assert_matches_screenshot` directly. This file must be loaded from inside a Cucumber run — it +calls `World`, `Before`, `After` and `AfterAll` at load time and raises `NoMethodError` if +required outside one. + +### Static sites (Hugo, Jekyll, plain HTML) + +```ruby +require "snap_diff/static" + +SnapDiff.serve("_site") # or "public", "build", "dist" +SnapDiff.serve("_site", root: Dir.pwd) # root defaults to Dir.pwd +``` + +`SnapDiff.serve` points Capybara at the built directory and sets the screenshot root. It also +loads the Minitest integration. See +[CI & Non-Rails Integration](ci-integration.md#non-rails-projects-hugo-jekyll-static-sites). + +## Configuration + +All 25 settings live on one flat object, `SnapDiff.config` (a `SnapDiff::Config`). + +```ruby +# test_helper.rb / rails_helper.rb +SnapDiff.configure do |config| + config.window_size = [1280, 1024] + config.tolerance = 0.0005 + config.save_path = "doc/screenshots" +end + +# Or set them one at a time +SnapDiff.config.hide_caret = true +SnapDiff.config.tolerance # => 0.0005 +``` + +`SnapDiff::Config` **is** the storage. The legacy `Capybara::Screenshot.*` and +`Capybara::Screenshot::Diff.*` accessors are thin delegators onto it — one storage, two views — +so a write through either surface is immediately visible through the other: + +```ruby +SnapDiff.config.window_size = [1280, 1024] +Capybara::Screenshot.window_size # => [1280, 1024] +``` + +`SnapDiff.start` is the same call shape as the old `Capybara::Screenshot::Diff.configure`, if +you prefer the two-holder form: + +```ruby +SnapDiff.start do |screenshot, diff| + screenshot.window_size = [1280, 1024] + diff.tolerance = 0.0005 +end +``` + +Three derived, read-only values are computed from the settings above: + +| Method | What it returns | +|--------|-----------------| +| `SnapDiff.config.active?` | Whether screenshots are taken at all (ex `Capybara::Screenshot.active?`) | +| `SnapDiff.config.screenshot_area` | `save_path`, optionally segmented per OS and per Capybara driver | +| `SnapDiff.config.default_options` | The capture/compare defaults handed to `SnapDiff::Comparison` | + +Every option's meaning is documented in the +[Configuration Reference](configuration.md) — the names are identical, only the receiver differs. +The one rename: `Capybara::Screenshot.enabled` is `SnapDiff.config.screenshot_enabled`, because +`SnapDiff.config.enabled` is taken by `Capybara::Screenshot::Diff.enabled`. + +## Object map + +`require "snap_diff"` gives you the compare/configure core. The test-suite pieces come with the +integration require; a few objects need their own require, noted below. + +| Object | What it is for | +|--------|----------------| +| `SnapDiff.config`, `SnapDiff::Config` | Every setting, one flat object. The storage. | +| `SnapDiff.configure`, `SnapDiff.start` | Config block helpers (consolidated / v1 shape) | +| `SnapDiff.compare` | Compare two image files directly, no browser | +| `SnapDiff::Comparison` | The layered comparison engine (ex `ImageCompare`) | +| `SnapDiff::Comparison::Images` | Frozen bundle a comparison operates on: both images, their paths, the driver and the options | +| `SnapDiff::ComparisonResult` | Result value object: `different?`, region, metadata (ex `Difference`) | +| `SnapDiff::Region` | Bounding box value object — `from_edge_coordinates`, `to_edge_coordinates` | +| `SnapDiff::DSL` | `screenshot`, `assert_matches_screenshot`, `capture_screenshot`, groups/sections | +| `SnapDiff::Minitest::Assertions` | Minitest wiring (`snap_diff/integrations/minitest`) | +| `SnapDiff::Error` | Base class for every error this gem raises | +| `SnapDiff::ExpectationNotMet` | A screenshot did not match its baseline | +| `SnapDiff::UnstableImage` | No stable capture within `stability_time_limit` / `wait` | +| `SnapDiff::WindowSizeMismatchError` | Browser window is not the configured `window_size` | +| `SnapDiff::Driver` | Mixin with the shared driver defaults (`require "snap_diff/driver"`) — **removed in 2.1** | +| `SnapDiff::Drivers` | Driver factory and registry — `.for`, `.loaded`, `.available` — **removed in 2.1** | +| `SnapDiff::Reporting` | Process-global reporter lifecycle (`require "snap_diff/reporting"`) | +| `SnapDiff::Reporters::HTML` | The interactive HTML report (`require "snap_diff/reporters/html"`) | +| `SnapDiff::Reporters::Default` | Builds the annotated diff images and the failure message | +| `SnapDiff.session` | The per-test assertion registry (fiber-local) | +| `SnapDiff.reset` | Ends a test: notifies reporters, clears the session | +| `SnapDiff.pending_screenshots_message` | Skip message when a new screenshot has no baseline | +| `SnapDiff::Capture::Viewport` | Per-capture viewport check seam (`require "snap_diff/capture/viewport"`) | +| `SnapDiff.serve` | Point Capybara at a static site directory (`require "snap_diff/static"`) | + +## Compare two images without a browser + +Works on anything on disk — rendered PDFs, generated charts, CI artifacts: + +```ruby +require "snap_diff" + +result = SnapDiff.compare("baseline.png", "current.png") +result.quick_equal? # => true when byte-identical / pixel-identical +result.different? # => true when the difference exceeds the configured thresholds +result.difference # => SnapDiff::ComparisonResult with region and metadata + +# Per-call option overrides, merged over SnapDiff.config.default_options +SnapDiff.compare("baseline.png", "current.png", tolerance: 0.5) +``` + +Note the argument order: baseline first, current second ("compare baseline against current"). + +## Custom reporters + +A reporter is any object answering `record(assertions)`, `finalize`, and `summary`. Register it +once, for the rest of the process: + +```ruby +require "snap_diff/reporting" + +class SlackReporter + def initialize = @failures = [] + + # Called once per finished test, with that test's assertions. + def record(assertions) + assertions.each do |assertion| + next unless assertion.compare&.difference&.different? + @failures << assertion.name + end + end + + # Called once at end of suite. + def finalize + post_to_slack(@failures) unless @failures.empty? + end + + # Printed to stdout after finalize. Return nil to print nothing. + def summary + @failures.empty? ? nil : "#{@failures.size} screenshot(s) changed" + end +end + +SnapDiff::Reporting.register(SlackReporter.new) +``` + +`register` appends under a mutex, so concurrent registrations cannot lose one — prefer it over +mutating `SnapDiff::Reporting.reporters` directly. + +Do implement `summary`. `finalize!` calls it unconditionally; a reporter without it is finalized +and then warned about (`[snap_diff] Reporter … failed (NoMethodError: undefined method 'summary')`). + +A reporter that raises is warned about and skipped — the other reporters still run. + +The bundled HTML report is just a pre-registered reporter of this kind: + +```ruby +require "snap_diff/reporters/html" # registers SnapDiff::Reporters::HTML itself +``` + +### Frameworks other than Minitest/RSpec/Cucumber + +The three bundled integrations call the lifecycle for you. Wiring another framework means calling +three things: + +```ruby +require "snap_diff/dsl" # SnapDiff::DSL, BrowserHelpers, the session accessors +require "snap_diff/reporting" # SnapDiff::Reporting + +# before each test +SnapDiff::BrowserHelpers.resize_window_if_needed + +# after each test +SnapDiff.session.verify # raises SnapDiff::ExpectationNotMet on a mismatch +msg = SnapDiff.pending_screenshots_message # non-nil => skip the test with this message +SnapDiff.reset # always: notifies reporters, clears the session + +# after the suite +SnapDiff::Reporting.finalize! +``` + +## Custom drivers — removed in 2.1 + +**There is no migration path, and that is deliberate.** 2.1 removed the driver abstraction +whole: the `SnapDiff::Driver` mixin, the `SnapDiff::Drivers.loaded` registry, +`SnapDiff::Drivers.available` detection, the `driver:` setting, and `driver: :auto`. libvips +is the only backend, `ruby-vips` is a runtime dependency of this gem, and +`SnapDiff::Drivers::VipsDriver` is wired in directly. + +A third-party driver stops working on 2.1 and nothing replaces it. One backend is what keeps +the comparison engine honest — every option means one thing, and the reported figures come +from one implementation. If you maintain a driver, say so on +[the issue tracker](https://github.com/snap-diff/snap_diff-capybara/issues); that is the only +thing that can reopen this. + +Everything the abstraction was used for from the outside has a direct answer: + +| You used | Now | +|---|---| +| `SnapDiff.config.driver = :vips` | delete the line | +| `screenshot "index", driver: :vips` | delete the option | +| `driver: :auto` | delete it — there is one backend | +| `SnapDiff::Drivers.available` to branch on what is installed | nothing to branch on; a missing `ruby-vips` is a Bundler resolution error | +| `SnapDiff::Drivers.loaded[:mine] = MyDriver` | no replacement | + +## Related + +- [Framework Setup](framework-setup.md) — the same three integrations under their legacy names +- [Configuration Reference](configuration.md) — what every option does +- [Image Processing](drivers.md) — libvips, perceptual threshold, tolerance +- [Web UI & Custom Reporters](reporters.md) — the HTML report in detail +- [Architecture](architecture.md) — how the pieces fit together internally +- [UPGRADING.md](UPGRADING.md) — migrating an existing suite off the legacy names + +[← Back to README](../README.md) diff --git a/docs/thread_safety.md b/docs/thread_safety.md new file mode 100644 index 00000000..db007c16 --- /dev/null +++ b/docs/thread_safety.md @@ -0,0 +1,113 @@ +# Thread Safety Guide for Parallel Testing + +This document explains how `snap_diff` behaves under Rails parallel tests with the `:thread` strategy. + +## Overview + +`snap_diff` is thread safe for parallel test execution as long as global configuration is set before tests run. Per-thread state is isolated, and shared state is protected where it matters. + +## Architecture Summary + +### Per-thread Assertion Registry + +Each thread gets its own `AssertionRegistry` stored in thread-local storage: + +```ruby +def registry + Thread.current[:capybara_screenshot_diff_registry] ||= AssertionRegistry.new +end +``` + +This prevents cross-thread leakage for assertions and screenshot naming. + +### Reporters Snapshot on Notify + +Reporters are notified using a snapshot protected by an eagerly initialized mutex: + +```ruby +@reporters_mutex = Mutex.new + +def notify_reporters(assertions) + reporters_snapshot = reporters_mutex.synchronize { reporters.dup } + reporters_snapshot.each { |reporter| reporter.record(assertions) } +end +``` + +This ensures a stable list while notifying without forcing a global lock around reporter work. + +### HTML Reporter Internal Lock + +The HTML reporter protects `@failures`, `@total`, and `@finalized` with a mutex so `record` and `finalize` can run safely: + +```ruby +@mutex.synchronize do + return if @finalized + @total += total + @failures.concat(failures) +end +``` + +`@finalized` is set only after `write_report` succeeds, so a failed write can be retried. + +### Screenshot Naming Isolation + +Each thread gets its own `ScreenshotNamer` via the per-thread registry, so counters, sections, and groups do not collide. + +### SnapManager Per Call + +`SnapManager` returns a new instance for each call, avoiding shared mutable state. + +## Global Configuration + +Configuration uses `mattr_accessor` and should be set once before tests run. Do not mutate config during parallel execution. + +## Parallel Test Lifecycle + +- Setup: per-thread registry is created, config is read +- Execution: assertions are added to the thread-local registry +- Teardown: `verify` and `reset` operate on the thread-local registry, reporters are notified +- Exit: reporters finalize once per process (using mutex-protected snapshot) + +## Usage Examples + +```ruby +parallelize(workers: :number_of_processors, with: :threads) + +Capybara::Screenshot::Diff.configure do |screenshot, diff| + screenshot.window_size = [1280, 1024] + screenshot.save_path = "doc/screenshots" + diff.tolerance = 0.001 +end +``` + +## Do and Do Not + +Do: +- Set config once in test helper +- Pass per-screenshot options in the call + +Do not: +- Change global config inside tests +- Manually mutate registry internals + +## File System Notes + +- Paths are unique per screenshot name and counter +- `FileUtils.mv` is atomic on most file systems +- Directory creation uses `mkpath` + +## Load-time thread safety + +Runtime state is thread-local (above), but *loading* the gem is a separate +concern. The require graph is deliberately acyclic: `lib/snap_diff/*` units +depend only on the config-storage leaf (`snap_diff/config`, which the legacy +view `capybara/screenshot/diff/config_legacy` requires) and specific sibling +units, the umbrella files depend on the units, and nothing requires back up +the chain. + +Eager mutual requires between entry points are forbidden, even guarded ones: +per-thread "loading" flags cannot serialize Ruby's process-global per-file +require locks, so two threads requiring opposite entry points first can +deadlock (lock-order inversion — observed deterministically before this +design). If two files ever need each other, extract the shared piece into a +leaf both can require instead. diff --git a/gemfiles/rails42.gemfile b/gemfiles/edge_gems.rb similarity index 55% rename from gemfiles/rails42.gemfile rename to gemfiles/edge_gems.rb index 2d2b5fa3..c4745b3d 100644 --- a/gemfiles/rails42.gemfile +++ b/gemfiles/edge_gems.rb @@ -3,5 +3,7 @@ gems = "#{File.dirname __dir__}/gems.rb" eval File.read(gems), binding, gems -gem "actionpack", "~>4.2.7" -gem "bigdecimal", "<2", platform: :mri +git "https://github.com/rails/rails.git" do + gem "activesupport" + gem "actionpack" +end diff --git a/gemfiles/rails60_gems.rb b/gemfiles/rails60_gems.rb deleted file mode 100644 index 265afa0f..00000000 --- a/gemfiles/rails60_gems.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -gems = "#{File.dirname __dir__}/gems.rb" -eval File.read(gems), binding, gems - -gem "actionpack", "~> 6.0.1", "< 6.1" -gem "capybara", ">= 2.15" -gem "selenium-webdriver" diff --git a/gemfiles/rails61_gems.rb b/gemfiles/rails61_gems.rb deleted file mode 100644 index 200877de..00000000 --- a/gemfiles/rails61_gems.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -gems = "#{File.dirname __dir__}/gems.rb" -eval File.read(gems), binding, gems - -gem "actionpack", "~> 6.1.0", "< 6.2" -gem "capybara", ">= 3.26" diff --git a/gemfiles/rails71_gems.rb b/gemfiles/rails71_gems.rb new file mode 100644 index 00000000..db46028c --- /dev/null +++ b/gemfiles/rails71_gems.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +gems = "#{File.dirname __dir__}/gems.rb" +eval File.read(gems), binding, gems + +gem "activesupport", "~> 7.1.0", require: %w[logger active_support/deprecator active_support] +gem "actionpack", "~> 7.1.0", require: %w[action_controller action_dispatch] diff --git a/gemfiles/rails50.gemfile b/gemfiles/rails72_gems.rb similarity index 78% rename from gemfiles/rails50.gemfile rename to gemfiles/rails72_gems.rb index cb412ff2..5099876d 100644 --- a/gemfiles/rails50.gemfile +++ b/gemfiles/rails72_gems.rb @@ -3,4 +3,4 @@ gems = "#{File.dirname __dir__}/gems.rb" eval File.read(gems), binding, gems -gem "actionpack", "~>5.0.1" +gem "actionpack", "~> 7.2.0" diff --git a/gemfiles/rails51.gemfile b/gemfiles/rails80_gems.rb similarity index 64% rename from gemfiles/rails51.gemfile rename to gemfiles/rails80_gems.rb index b1db25dd..548e3714 100644 --- a/gemfiles/rails51.gemfile +++ b/gemfiles/rails80_gems.rb @@ -3,4 +3,5 @@ gems = "#{File.dirname __dir__}/gems.rb" eval File.read(gems), binding, gems -gem "actionpack", "~>5.1.2" +gem "activesupport", "~> 8.0.0" +gem "actionpack", "~> 8.0.0" diff --git a/gemfiles/rails52.gemfile b/gemfiles/rails81_gems.rb similarity index 64% rename from gemfiles/rails52.gemfile rename to gemfiles/rails81_gems.rb index 170b73db..086ea305 100644 --- a/gemfiles/rails52.gemfile +++ b/gemfiles/rails81_gems.rb @@ -3,4 +3,5 @@ gems = "#{File.dirname __dir__}/gems.rb" eval File.read(gems), binding, gems -gem "actionpack", "~>5.2.1" +gem "activesupport", "~> 8.1.0" +gem "actionpack", "~> 8.1.0" diff --git a/gems.rb b/gems.rb index 4f3ea7cf..ff700c13 100644 --- a/gems.rb +++ b/gems.rb @@ -7,22 +7,30 @@ gem "rake" -# Image processing libraries -gem "oily_png", platform: :ruby -gem "ruby-vips", require: false - -# Test -gem "minitest", require: false -gem "minitest-stub-const", require: false -gem "simplecov", require: false +# ruby-vips is a gemspec runtime dependency since 2.1 (the only backend), so +# it is not listed here. chunky_png/oily_png went with the chunky_png driver. + +group :test do + gem "capybara", ">= 3.26" + gem "mutex_m" # Needed for RubyMine debugging. Try removing it. + gem "minitest", "< 6", require: false + gem "minitest-mock", require: false + gem "minitest-stub-const", require: false + gem "simplecov", require: false + gem "rspec", require: false +end # Capybara Server gem "puma", require: false +gem "rackup", require: false # Capybara Drivers gem "cuprite", require: false -gem "selenium-webdriver", require: false -gem "webdrivers", require: false +gem "selenium-webdriver", ">= 4.11", require: false + +# Test Frameworks +# gem "cucumber", require: false +# gem "cucumber-rails", require: false group :tools do gem "standard", require: false diff --git a/lib/capybara/screenshot/diff.rb b/lib/capybara/screenshot/diff.rb deleted file mode 100644 index ee631421..00000000 --- a/lib/capybara/screenshot/diff.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -require "capybara/dsl" -require "capybara/screenshot/diff/version" -require "capybara/screenshot/diff/drivers/utils" -require "capybara/screenshot/diff/image_compare" -require "capybara/screenshot/diff/test_methods" - -module Capybara - module Screenshot - extend Os - mattr_accessor :add_driver_path - mattr_accessor :add_os_path - mattr_accessor :blur_active_element - mattr_accessor :enabled - mattr_accessor :hide_caret - mattr_reader(:root) { (defined?(Rails.root) && Rails.root) || Pathname(".").expand_path } - mattr_accessor :stability_time_limit - mattr_accessor :window_size - mattr_accessor(:save_path) { "doc/screenshots" } - mattr_accessor(:use_lfs) - - class << self - def root=(path) - @@root = Pathname(path).expand_path - end - - def active? - enabled || (enabled.nil? && Diff.enabled) - end - - def screenshot_area - parts = [Capybara::Screenshot.save_path] - parts << Capybara.current_driver.to_s if Capybara::Screenshot.add_driver_path - parts << os_name if Capybara::Screenshot.add_os_path - File.join parts - end - - def screenshot_area_abs - root / screenshot_area - end - end - - # Module to track screen shot changes - module Diff - include Capybara::DSL - include Capybara::Screenshot::Os - - mattr_accessor :area_size_limit - mattr_accessor :color_distance_limit - mattr_accessor(:enabled) { true } - mattr_accessor :shift_distance_limit - mattr_accessor :skip_area - mattr_accessor(:driver) { :auto } - mattr_accessor(:tolerance) { 0.001 } - - AVAILABLE_DRIVERS = Utils.detect_available_drivers.freeze - - def self.included(klass) - klass.include TestMethods - klass.setup do - if Capybara::Screenshot.window_size - if page.driver.respond_to?(:resize) - page.driver.resize(*Capybara::Screenshot.window_size) - elsif selenium? - page.driver.browser.manage.window.resize_to(*Capybara::Screenshot.window_size) - end - end - end - - klass.teardown do - if Capybara::Screenshot::Diff.enabled && @test_screenshots - test_screenshot_errors = @test_screenshots - .map { |caller, name, compare| assert_image_not_changed(caller, name, compare) } - test_screenshot_errors.compact! - fail(test_screenshot_errors.join("\n\n")) if test_screenshot_errors.any? - end - end - end - end - end -end diff --git a/lib/capybara/screenshot/diff/drivers/chunky_png_driver.rb b/lib/capybara/screenshot/diff/drivers/chunky_png_driver.rb deleted file mode 100644 index dad0c522..00000000 --- a/lib/capybara/screenshot/diff/drivers/chunky_png_driver.rb +++ /dev/null @@ -1,355 +0,0 @@ -# frozen_string_literal: true - -require "chunky_png" - -module Capybara - module Screenshot - module Diff - # Compare two images and determine if they are equal, different, or within some comparison - # range considering color values and difference area size. - module Drivers - class ChunkyPNGDriver - include ChunkyPNG::Color - - attr_reader :new_file_name, :old_file_name - - def initialize(new_file_name, old_file_name = nil, **options) - @new_file_name = new_file_name - @old_file_name = old_file_name || "#{new_file_name}~" - - @color_distance_limit = options[:color_distance_limit] - @shift_distance_limit = options[:shift_distance_limit] - @skip_area = options[:skip_area] - - reset - end - - # Resets the calculated data about the comparison with regard to the "new_image". - # Data about the original image is kept. - def reset - @max_color_distance = @color_distance_limit ? 0 : nil - @max_shift_distance = @shift_distance_limit ? 0 : nil - end - - def load_images(old_file_name, new_file_name) - old_bytes, new_bytes = load_image_files(old_file_name, new_file_name) - - _load_images(old_bytes, new_bytes) - end - - def filter_image_with_median(_image) - raise NotImplementedError - end - - def add_black_box(image, _region) - image - end - - def difference_level(_diff_mask, old_img, region) - size(region).to_f / image_area_size(old_img) - end - - def image_area_size(old_img) - width_for(old_img) * height_for(old_img) - end - - def shift_distance_equal? - # Stub - false - end - - def shift_distance_different? - # Stub - true - end - - def find_difference_region(new_image, old_image, color_distance_limit, shift_distance_limit, area_size_limit, fast_fail: false) - return nil, nil if new_image.pixels == old_image.pixels - - if fast_fail && !(color_distance_limit || shift_distance_limit || area_size_limit) - return [0, 0, width_for(new_image), height_for(new_image)], nil - end - - region = find_top(old_image, new_image) - region = if region.nil? || region[1].nil? - nil - else - find_diff_rectangle(old_image, new_image, region) - end - - [region, nil] - end - - def height_for(image) - image.height - end - - def width_for(image) - image.width - end - - def size(region) - return 0 unless region - - (region[2] - region[0] + 1) * (region[3] - region[1] + 1) - end - - def max_color_distance - calculate_metrics unless @max_color_distance - @max_color_distance - end - - def max_shift_distance - calculate_metrics unless @max_shift_distance || !@shift_distance_limit - @max_shift_distance - end - - def adds_error_details_to(log) - max_color_distance = self.max_color_distance.ceil(1) - max_shift_distance = self.max_shift_distance - - log[:max_color_distance] = max_color_distance - log.merge!(max_shift_distance: max_shift_distance) if max_shift_distance - end - - def crop(dimensions, i) - i.crop(0, 0, *dimensions) - end - - def from_file(filename) - ChunkyPNG::Image.from_file(filename) - end - - # private - - def calculate_metrics - old_file, new_file = load_image_files(@old_file_name, @new_file_name) - - if old_file == new_file - @max_color_distance = 0 - @max_shift_distance = 0 - return - end - - old_image, new_image = _load_images(old_file, new_file) - calculate_max_color_distance(new_image, old_image) - calculate_max_shift_limit(new_image, old_image) - end - - def calculate_max_color_distance(new_image, old_image) - pixel_pairs = old_image.pixels.zip(new_image.pixels) - @max_color_distance = pixel_pairs.inject(0) { |max, (p1, p2)| - next max unless p1 && p2 - - d = ChunkyPNG::Color.euclidean_distance_rgba(p1, p2) - [max, d].max - } - end - - def calculate_max_shift_limit(new_img, old_img) - (0...new_img.width).each do |x| - (0...new_img.height).each do |y| - shift_distance = - shift_distance_at(new_img, old_img, x, y, color_distance_limit: @color_distance_limit) - if shift_distance && (@max_shift_distance.nil? || shift_distance > @max_shift_distance) - @max_shift_distance = shift_distance - return if @max_shift_distance == Float::INFINITY # rubocop: disable Lint/NonLocalExitFromIterator - end - end - end - end - - def save_image_to(image, filename) - image.save(filename) - end - - def resize_image_to(image, new_width, new_height) - image.resample_bilinear(new_width, new_height) - end - - def load_image_files(old_file_name, file_name) - old_file = File.binread(old_file_name) - new_file = File.binread(file_name) - [old_file, new_file] - end - - def dimension_changed?(old_image, new_image) - return unless old_image.dimension != new_image.dimension - - change_msg = [old_image, new_image].map { |i| "#{i.width}x#{i.height}" }.join(" => ") - warn "Image size has changed for #{@new_file_name}: #{change_msg}" - true - end - - def draw_rectangles(images, (left, top, right, bottom), (r, g, b)) - images.map do |image| - new_img = image.dup - new_img.rect(left - 1, top - 1, right + 1, bottom + 1, ChunkyPNG::Color.rgb(r, g, b)) - new_img - end - end - - private - - def find_diff_rectangle(org_img, new_img, region) - left, top, right, bottom = find_left_right_and_top(org_img, new_img, region) - bottom = find_bottom(org_img, new_img, left, right, bottom) - [left, top, right, bottom] - end - - def find_top(old_img, new_img) - old_img.height.times do |y| - old_img.width.times do |x| - return [x, y, x, y] unless same_color?(old_img, new_img, x, y) - end - end - nil - end - - def find_left_right_and_top(old_img, new_img, region) - left = region[0] || old_img.width - 1 - top = region[1] - bottom = region[2] - right = region[3] || 0 - old_img.height.times do |y| - (0...left).find do |x| - next if same_color?(old_img, new_img, x, y) - - top ||= y - bottom = y - left = x - right = x if x > right - x - end - (old_img.width - 1).step(right + 1, -1).find do |x| - unless same_color?(old_img, new_img, x, y) - bottom = y - right = x - end - end - end - [left, top, right, bottom] - end - - def find_bottom(old_img, new_img, left, right, bottom) - if bottom - (old_img.height - 1).step(bottom + 1, -1).find do |y| - (left..right).find do |x| - bottom = y unless same_color?(old_img, new_img, x, y) - end - end - end - bottom - end - - def same_color?(old_img, new_img, x, y) - @skip_area&.each do |skip_start_x, skip_start_y, skip_end_x, skip_end_y| - return true if skip_start_x <= x && x <= skip_end_x && skip_start_y <= y && y <= skip_end_y - end - - color_distance = - color_distance_at(new_img, old_img, x, y, shift_distance_limit: @shift_distance_limit) - if !@max_color_distance || color_distance > @max_color_distance - @max_color_distance = color_distance - end - color_matches = color_distance == 0 || (@color_distance_limit && @color_distance_limit > 0 && - color_distance <= @color_distance_limit) - return color_matches if !@shift_distance_limit || @max_shift_distance == Float::INFINITY - - shift_distance = (color_matches && 0) || - shift_distance_at(new_img, old_img, x, y, color_distance_limit: @color_distance_limit) - if shift_distance && (@max_shift_distance.nil? || shift_distance > @max_shift_distance) - @max_shift_distance = shift_distance - end - color_matches - end - - def color_distance_at(new_img, old_img, x, y, shift_distance_limit:) - org_color = old_img[x, y] - if shift_distance_limit - start_x = [0, x - shift_distance_limit].max - end_x = [x + shift_distance_limit, new_img.width - 1].min - xs = (start_x..end_x).to_a - start_y = [0, y - shift_distance_limit].max - end_y = [y + shift_distance_limit, new_img.height - 1].min - ys = (start_y..end_y).to_a - new_pixels = xs.product(ys) - distances = new_pixels.map { |dx, dy| - new_color = new_img[dx, dy] - ChunkyPNG::Color.euclidean_distance_rgba(org_color, new_color) - } - distances.min - else - ChunkyPNG::Color.euclidean_distance_rgba(org_color, new_img[x, y]) - end - end - - def shift_distance_at(new_img, old_img, x, y, color_distance_limit:) - org_color = old_img[x, y] - shift_distance = 0 - loop do - bounds_breached = 0 - top_row = y - shift_distance - if top_row >= 0 # top - ([0, x - shift_distance].max..[x + shift_distance, new_img.width - 1].min).each do |dx| - if color_matches(new_img, org_color, dx, top_row, color_distance_limit) - return shift_distance - end - end - else - bounds_breached += 1 - end - if shift_distance > 0 - if (x - shift_distance) >= 0 # left - ([0, top_row + 1].max..[y + shift_distance, new_img.height - 2].min) - .each do |dy| - if color_matches(new_img, org_color, x - shift_distance, dy, color_distance_limit) - return shift_distance - end - end - else - bounds_breached += 1 - end - if (y + shift_distance) < new_img.height # bottom - ([0, x - shift_distance].max..[x + shift_distance, new_img.width - 1].min).each do |dx| - if color_matches(new_img, org_color, dx, y + shift_distance, color_distance_limit) - return shift_distance - end - end - else - bounds_breached += 1 - end - if (x + shift_distance) < new_img.width # right - ([0, top_row + 1].max..[y + shift_distance, new_img.height - 2].min) - .each do |dy| - if color_matches(new_img, org_color, x + shift_distance, dy, color_distance_limit) - return shift_distance - end - end - else - bounds_breached += 1 - end - end - break if bounds_breached == 4 - - shift_distance += 1 - end - Float::INFINITY - end - - def color_matches(new_img, org_color, x, y, color_distance_limit) - new_color = new_img[x, y] - return new_color == org_color unless color_distance_limit - - color_distance = ChunkyPNG::Color.euclidean_distance_rgba(org_color, new_color) - color_distance <= color_distance_limit - end - - def _load_images(old_file, new_file) - [ChunkyPNG::Image.from_blob(old_file), ChunkyPNG::Image.from_blob(new_file)] - end - end - end - end - end -end diff --git a/lib/capybara/screenshot/diff/drivers/utils.rb b/lib/capybara/screenshot/diff/drivers/utils.rb deleted file mode 100644 index 527f65cc..00000000 --- a/lib/capybara/screenshot/diff/drivers/utils.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Capybara - module Screenshot - module Diff - module Utils - def self.detect_available_drivers - result = [] - begin - result << :vips if defined?(Vips) || require("vips") - rescue LoadError - # vips not present - end - begin - result << :chunky_png if defined?(ChunkyPNG) || require("chunky_png") - rescue LoadError - # chunky_png not present - end - result - end - end - end - end -end diff --git a/lib/capybara/screenshot/diff/drivers/vips_driver.rb b/lib/capybara/screenshot/diff/drivers/vips_driver.rb deleted file mode 100644 index 37089376..00000000 --- a/lib/capybara/screenshot/diff/drivers/vips_driver.rb +++ /dev/null @@ -1,180 +0,0 @@ -# frozen_string_literal: true - -begin - require "vips" -rescue LoadError => e - warn 'Required ruby-vips gem is missing. Add `gem "ruby-vips"` to Gemfile' if e.message.include?("vips") - raise -end - -require_relative "./chunky_png_driver" - -module Capybara - module Screenshot - module Diff - # Compare two images and determine if they are equal, different, or within some comparison - # range considering color values and difference area size. - module Drivers - class VipsDriver - attr_reader :new_file_name, :old_file_name, :options - - def initialize(new_file_name, old_file_name = nil, **options) - @new_file_name = new_file_name - @old_file_name = old_file_name || "#{new_file_name}~" - - @options = options || {} - - reset - end - - # Resets the calculated data about the comparison with regard to the "new_image". - # Data about the original image is kept. - def reset - end - - def shift_distance_equal? - warn "[capybara-screenshot-diff] Instead of shift_distance_limit " \ - "please use median_filter_window_size and color_distance_limit options" - chunky_png_comparator.quick_equal? - end - - def shift_distance_different? - warn "[capybara-screenshot-diff] Instead of shift_distance_limit " \ - "please use median_filter_window_size and color_distance_limit options" - chunky_png_comparator.different? - end - - def find_difference_region(new_image, old_image, color_distance_limit, _shift_distance_limit, _area_size_limit, fast_fail: false) - diff_mask = VipsUtil.difference_mask(color_distance_limit, old_image, new_image) - region = VipsUtil.difference_region_by(diff_mask) - - [region, diff_mask] - end - - def size(region) - return 0 unless region - - (region[2] - region[0]) * (region[3] - region[1]) - end - - def adds_error_details_to(_log) - end - - # old private - - def inscribed?(dimensions, i) - dimension(i) == dimensions || i.width < dimensions[0] || i.height < dimensions[1] - end - - def crop(dimensions, i) - i.crop(0, 0, *dimensions) - end - - def filter_image_with_median(image, median_filter_window_size) - image.median(median_filter_window_size) - end - - def add_black_box(memo, region) - memo.draw_rect([0, 0, 0, 0], *region, fill: true) - end - - def chunky_png_comparator - @chunky_png_comparator ||= ImageCompare.new( - @new_file_name, - @old_file_name, - @options.merge(driver: :chunky_png, tolerance: nil, median_filter_window_size: nil) - ) - end - - def difference_level(diff_mask, old_img, _region = nil) - VipsUtil.difference_area_size_by(diff_mask).to_f / image_area_size(old_img) - end - - def image_area_size(old_img) - width_for(old_img) * height_for(old_img) - end - - def height_for(image) - image.height - end - - def width_for(image) - image.width - end - - def save_image_to(image, filename) - image.write_to_file(filename) - end - - def resize_image_to(image, new_width, new_height) - image.resize(1.* new_width / new_height) - end - - def load_images(old_file_name, new_file_name, driver = self) - [driver.from_file(old_file_name), driver.from_file(new_file_name)] - end - - def from_file(filename) - result = ::Vips::Image.new_from_file(filename) - - result = result.colourspace("srgb") if result.bands < 3 - result = result.bandjoin(255) if result.bands == 3 - - result - end - - def dimension_changed?(org_image, new_image) - return false if dimension(org_image) == dimension(new_image) - - change_msg = [org_image, new_image].map { |i| "#{i.width}x#{i.height}" }.join(" => ") - warn "Image size has changed for #{@new_file_name}: #{change_msg}" - - true - end - - def dimension(image) - [image.width, image.height] - end - - def draw_rectangles(images, (left, top, right, bottom), rgba) - images.map do |image| - image.draw_rect(rgba, left - 1, top - 1, right - left + 2, bottom - top + 2) - end - end - - class VipsUtil - def self.difference(old_image, new_image, color_distance: 0) - diff_mask = difference_mask(color_distance, new_image, old_image) - difference_region_by(diff_mask) - end - - def self.difference_area(old_image, new_image, color_distance: 0) - difference_mask = difference_mask(color_distance, new_image, old_image) - difference_area_size_by(difference_mask) - end - - def self.difference_area_size_by(difference_mask) - diff_mask = difference_mask == 0 - diff_mask.hist_find.to_a[0][0].max - end - - def self.difference_mask(color_distance, old_image, new_image) - (new_image - old_image).abs > color_distance - end - - def self.difference_region_by(diff_mask) - columns, rows = diff_mask.project - - left = columns.profile[1].min - right = columns.width - columns.flip("horizontal").profile[1].min - top = rows.profile[0].min - bottom = rows.height - rows.flip("vertical").profile[0].min - - [left, top, right, bottom] - end - end - end - end - end - end -end diff --git a/lib/capybara/screenshot/diff/image_compare.rb b/lib/capybara/screenshot/diff/image_compare.rb deleted file mode 100644 index bc71ac0a..00000000 --- a/lib/capybara/screenshot/diff/image_compare.rb +++ /dev/null @@ -1,240 +0,0 @@ -# frozen_string_literal: true - -module Capybara - module Screenshot - module Diff - LOADED_DRIVERS = {} - - # Compare two images and determine if they are equal, different, or within some comparison - # range considering color values and difference area size. - class ImageCompare < SimpleDelegator - attr_reader :driver, :driver_options - - attr_reader :annotated_new_file_name, :annotated_old_file_name, :area_size_limit, - :color_distance_limit, :new_file_name, :old_file_name, :shift_distance_limit, - :skip_area - - def initialize(new_file_name, old_file_name = nil, **driver_options) - @new_file_name = new_file_name - @old_file_name = old_file_name || "#{new_file_name}~" - @annotated_old_file_name = "#{new_file_name.chomp(".png")}.committed.png" - @annotated_new_file_name = "#{new_file_name.chomp(".png")}.latest.png" - - @driver_options = driver_options - - @color_distance_limit = driver_options[:color_distance_limit] || 0 - @area_size_limit = driver_options[:area_size_limit] - @shift_distance_limit = driver_options[:shift_distance_limit] - @dimensions = driver_options[:dimensions] - @skip_area = driver_options[:skip_area] - @tolerance = driver_options[:tolerance] - @median_filter_window_size = driver_options[:median_filter_window_size] - - driver_klass = find_driver_class_for(@driver_options.fetch(:driver, :chunky_png)) - @driver = driver_klass.new(@new_file_name, @old_file_name, **@driver_options) - - super(@driver) - end - - # Compare the two image files and return `true` or `false` as quickly as possible. - # Return falsish if the old file does not exist or the image dimensions do not match. - def quick_equal? - return false unless old_file_exists? - return true if new_file_size == old_file_size - - # old_bytes, new_bytes = load_image_files(@old_file_name, @new_file_name) - # return true if old_bytes == new_bytes - - images = driver.load_images(@old_file_name, @new_file_name) - old_image, new_image = preprocess_images(images, driver) - - return false if driver.dimension_changed?(old_image, new_image) - - region, meta = driver.find_difference_region( - new_image, - old_image, - @color_distance_limit, - @shift_distance_limit, - @area_size_limit, - fast_fail: true - ) - - self.difference_region = region - - return true if difference_region_empty?(new_image, region) - - return true if @area_size_limit && driver.size(region) <= @area_size_limit - - return true if @tolerance && @tolerance >= driver.difference_level(meta, old_image, region) - - # TODO: Remove this or find similar solution for vips - return true if @shift_distance_limit && driver.shift_distance_equal? - - false - end - - # Compare the two images referenced by this object, and return `true` if they are different, - # and `false` if they are the same. - # Return `nil` if the old file does not exist or if the image dimensions do not match. - def different? - return nil unless old_file_exists? - - images = driver.load_images(@old_file_name, @new_file_name) - - old_image, new_image = preprocess_images(images, driver) - - if driver.dimension_changed?(old_image, new_image) - save(new_image, old_image, @annotated_new_file_name, @annotated_old_file_name) - - self.difference_region = 0, 0, driver.width_for(old_image), driver.height_for(old_image) - - return true - end - - region, meta = driver.find_difference_region( - new_image, - old_image, - @color_distance_limit, - @shift_distance_limit, - @area_size_limit - ) - self.difference_region = region - - return not_different if difference_region_empty?(old_image, region) - return not_different if @area_size_limit && driver.size(region) <= @area_size_limit - return not_different if @tolerance && @tolerance > driver.difference_level(meta, old_image, region) - - # TODO: Remove this or find similar solution for vips - return not_different if @shift_distance_limit && !driver.shift_distance_different? - - annotate_and_save(images, region) - - true - end - - def clean_tmp_files - FileUtils.cp @old_file_name, @new_file_name if old_file_exists? - File.delete(@old_file_name) if old_file_exists? - File.delete(@annotated_old_file_name) if File.exist?(@annotated_old_file_name) - File.delete(@annotated_new_file_name) if File.exist?(@annotated_new_file_name) - end - - DIFF_COLOR = [255, 0, 0, 255].freeze - SKIP_COLOR = [255, 192, 0, 255].freeze - - def annotate_and_save(images, region = difference_region) - annotated_images = driver.draw_rectangles(images, region, DIFF_COLOR) - @skip_area.to_a.flatten.each_slice(4) do |region| - annotated_images = driver.draw_rectangles(annotated_images, region, SKIP_COLOR) - end - save(*annotated_images, @annotated_new_file_name, @annotated_old_file_name) - end - - def save(new_img, old_img, annotated_new_file_name, annotated_old_file_name) - driver.save_image_to(old_img, annotated_old_file_name) - driver.save_image_to(new_img, annotated_new_file_name) - end - - def old_file_exists? - @old_file_name && File.exist?(@old_file_name) - end - - def reset - self.difference_region = nil - driver.reset - end - - def error_message - result = { - area_size: driver.size(difference_region), - region: difference_region - } - - driver.adds_error_details_to(result) - - ["(#{result.to_json})", new_file_name, annotated_old_file_name, annotated_new_file_name].join("\n") - end - - def difference_region - return nil unless @left || @top || @right || @bottom - - [@left, @top, @right, @bottom] - end - - private - - def find_driver_class_for(driver) - driver = AVAILABLE_DRIVERS.first if driver == :auto - - LOADED_DRIVERS[driver] ||= - case driver - when :chunky_png - require "capybara/screenshot/diff/drivers/chunky_png_driver" - Drivers::ChunkyPNGDriver - when :vips - require "capybara/screenshot/diff/drivers/vips_driver" - Drivers::VipsDriver - else - fail "Wrong adapter #{driver.inspect}. Available adapters: #{AVAILABLE_DRIVERS.inspect}" - end - end - - def old_file_size - @old_file_size ||= old_file_exists? && File.size(@old_file_name) - end - - def new_file_size - File.size(@new_file_name) - end - - def not_different - clean_tmp_files - false - end - - def load_images(old_file_name, new_file_name, driver = self) - [driver.from_file(old_file_name), driver.from_file(new_file_name)] - end - - def preprocess_images(images, driver = self) - old_img = preprocess_image(images.first, driver) - new_img = preprocess_image(images.last, driver) - - [old_img, new_img] - end - - def preprocess_image(image, driver = self) - result = image - - if @dimensions && driver.inscribed?(@dimensions, result) - result = driver.crop(@dimensions, result) - end - - if @median_filter_window_size - result = driver.filter_image_with_median(image, @median_filter_window_size) - end - - if @skip_area - result = @skip_area.reduce(result) { |image, region| driver.add_black_box(image, region) } - end - - result - end - - def difference_region=(region) - @left, @top, @right, @bottom = region - end - - def difference_region_empty?(new_image, region) - region.nil? || - ( - region[1] == height_for(new_image) && - region[0] == width_for(new_image) && - region[2].zero? && - region[3].zero? - ) - end - end - end - end -end diff --git a/lib/capybara/screenshot/diff/os.rb b/lib/capybara/screenshot/diff/os.rb deleted file mode 100644 index d96d86bd..00000000 --- a/lib/capybara/screenshot/diff/os.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Capybara - module Screenshot - module Os - ON_WINDOWS = !!(RbConfig::CONFIG["host_os"] =~ /mswin|mingw|cygwin/) - ON_MAC = !!(RbConfig::CONFIG["host_os"] =~ /darwin/) - ON_LINUX = !!(RbConfig::CONFIG["host_os"] =~ /linux/) - - def os_name - return "windows" if ON_WINDOWS - return "macos" if ON_MAC - return "linux" if ON_LINUX - - "unknown" - end - end - end -end diff --git a/lib/capybara/screenshot/diff/stabilization.rb b/lib/capybara/screenshot/diff/stabilization.rb deleted file mode 100644 index dcfd21c7..00000000 --- a/lib/capybara/screenshot/diff/stabilization.rb +++ /dev/null @@ -1,191 +0,0 @@ -# frozen_string_literal: true - -require_relative "os" - -module Capybara - module Screenshot - module Diff - module Stabilization - include Os - - IMAGE_WAIT_SCRIPT = <<-JS.strip_heredoc.freeze - function pending_image() { - var images = document.images; - for (var i = 0; i < images.length; i++) { - if (!images[i].complete) { - return images[i].src; - } - } - return false; - }() - JS - - def take_stable_screenshot(comparison, stability_time_limit:, wait:) - previous_file_name = comparison.old_file_name - screenshot_started_at = last_image_change_at = Time.now - clean_stabilization_images(comparison.new_file_name) - - 1.step do |i| - take_right_size_screenshot(comparison) - if comparison.quick_equal? - clean_stabilization_images(comparison.new_file_name) - break - end - comparison.reset - - if previous_file_name - stabilization_comparison = make_stabilization_comparison_from( - comparison, - comparison.new_file_name, - previous_file_name - ) - if stabilization_comparison.quick_equal? - if (Time.now - last_image_change_at) > stability_time_limit - clean_stabilization_images(comparison.new_file_name) - break - end - next - else - last_image_change_at = Time.now - end - end - - previous_file_name = "#{comparison.new_file_name.chomp(".png")}" \ - "_x#{format("%02i", i)}_#{(Time.now - screenshot_started_at).round(1)}s" \ - "_#{stabilization_comparison.difference_region&.to_s&.gsub(", ", "_") || :initial}.png~" - FileUtils.mv comparison.new_file_name, previous_file_name - - check_max_wait_time( - comparison, - screenshot_started_at, - max_wait_time: max_wait_time(comparison.shift_distance_limit, wait) - ) - end - end - - def notice_how_to_avoid_this - unless @_csd_retina_warned - warn "Halving retina screenshot. " \ - 'You should add "force-device-scale-factor=1" to your Chrome chromeOptions args.' - @_csd_retina_warned = true - end - end - - private - - def make_stabilization_comparison_from(comparison, new_file_name, previous_file_name) - ImageCompare.new(new_file_name, previous_file_name, **comparison.driver_options) - end - - def reduce_retina_image_size(file_name, driver) - return if !ON_MAC || !selenium? || !Capybara::Screenshot.window_size - - expected_image_width = Capybara::Screenshot.window_size[0] - saved_image = driver.from_file(file_name) - return if driver.width_for(saved_image) < expected_image_width * 2 - - notice_how_to_avoid_this - - new_height = expected_image_width * driver.height_for(saved_image) / driver.width_for(saved_image) - resized_image = driver.resize_image_to(saved_image, expected_image_width, new_height) - - Dir.mktmpdir do |dir| - resized_image_file = "#{dir}/resized.png" - driver.save_image_to(resized_image, resized_image_file) - FileUtils.mv(resized_image_file, file_name) - end - end - - def stabilization_images(base_file) - Dir["#{base_file.chomp(".png")}_x*.png~"].sort - end - - def clean_stabilization_images(base_file) - FileUtils.rm stabilization_images(base_file) - end - - def prepare_page_for_screenshot(timeout:) - assert_images_loaded(timeout: timeout) - if Capybara::Screenshot.blur_active_element - active_element = execute_script(<<-JS) - ae = document.activeElement; - if (ae.nodeName == "INPUT" || ae.nodeName == "TEXTAREA") { - ae.blur(); - return ae; - } - return null; - JS - blurred_input = page.driver.send :unwrap_script_result, active_element - end - if Capybara::Screenshot.hide_caret && !@hid_caret - execute_script(<<~JS) - var style = document.createElement('style'); - document.head.appendChild(style); - var styleSheet = style.sheet; - styleSheet.insertRule("* { caret-color: transparent !important; }", 0); - JS - @hid_caret = true - end - blurred_input - end - - def take_right_size_screenshot(comparison) - save_screenshot(comparison.new_file_name) - - # TODO(uwe): Remove when chromedriver takes right size screenshots - reduce_retina_image_size(comparison.new_file_name, comparison.driver) - # ODOT - end - - def check_max_wait_time(comparison, screenshot_started_at, max_wait_time:) - return if (Time.now - screenshot_started_at) < max_wait_time - - annotate_stabilization_images(comparison) - # FIXME(uwe): Change to store the failure and only report if the test succeeds functionally. - fail("Could not get stable screenshot within #{max_wait_time}s\n" \ - "#{stabilization_images(comparison.new_file_name).join("\n")}") - end - - def annotate_stabilization_images(comparison) - previous_file = comparison.old_file_name - stabilization_images(comparison.new_file_name).each do |file_name| - if File.exist? previous_file - stabilization_comparison = make_stabilization_comparison_from( - comparison, - file_name, - previous_file - ) - if stabilization_comparison.different? - FileUtils.mv stabilization_comparison.annotated_new_file_name, file_name - end - FileUtils.rm stabilization_comparison.annotated_old_file_name - end - previous_file = file_name - end - end - - def max_wait_time(shift_distance_limit, wait) - shift_factor = shift_distance_limit ? (shift_distance_limit * 2 + 1) ^ 2 : 1 - wait * shift_factor - end - - def assert_images_loaded(timeout:) - return unless respond_to? :evaluate_script - - start = Time.now - loop do - pending_image = evaluate_script IMAGE_WAIT_SCRIPT - break unless pending_image - - assert( - (Time.now - start) < timeout, - "Images not loaded after #{timeout}s: #{pending_image.inspect}" - ) - - sleep 0.1 - end - end - end - end - end -end diff --git a/lib/capybara/screenshot/diff/test_methods.rb b/lib/capybara/screenshot/diff/test_methods.rb deleted file mode 100644 index 264fc015..00000000 --- a/lib/capybara/screenshot/diff/test_methods.rb +++ /dev/null @@ -1,130 +0,0 @@ -# frozen_string_literal: true - -require "English" -require "capybara" -require "action_controller" -require "action_dispatch" -require "active_support/core_ext/string/strip" -require_relative "image_compare" -require_relative "stabilization" -require_relative "vcs" - -# Add the `screenshot` method to ActionDispatch::IntegrationTest -module Capybara - module Screenshot - module Diff - module TestMethods - include Stabilization - include Vcs - - def initialize(*) - super - @screenshot_counter = nil - @screenshot_group = nil - @screenshot_section = nil - @test_screenshot_errors = nil - @test_screenshots = nil - end - - def group_parts - parts = [] - parts << @screenshot_section if @screenshot_section.present? - parts << @screenshot_group if @screenshot_group.present? - parts - end - - def full_name(name) - File.join group_parts.<<(name).map(&:to_s) - end - - def screenshot_dir - File.join [Screenshot.screenshot_area] + group_parts - end - - def current_capybara_driver_class - Capybara.current_session.driver.class - end - - def selenium? - current_capybara_driver_class <= Capybara::Selenium::Driver - end - - def screenshot_section(name) - @screenshot_section = name.to_s - end - - def screenshot_group(name) - @screenshot_group = name.to_s - @screenshot_counter = 0 - return unless Screenshot.active? && name.present? - - FileUtils.rm_rf screenshot_dir - end - - # @return [Boolean] wether a screenshot was taken - def screenshot( - name, - stability_time_limit: Screenshot.stability_time_limit, - wait: Capybara.default_max_wait_time, - **driver_options - ) - return false unless Screenshot.active? - return false if window_size_is_wrong? - - driver_options = { - area_size_limit: Diff.area_size_limit, - color_distance_limit: Diff.color_distance_limit, - driver: Diff.driver, - shift_distance_limit: Diff.shift_distance_limit, - skip_area: Diff.skip_area, - tolerance: Diff.tolerance - }.merge(driver_options) - - # Allow nil or single or multiple areas - if driver_options[:skip_area] - driver_options[:skip_area] = driver_options[:skip_area].compact.flatten&.each_cons(4)&.to_a - end - - if @screenshot_counter - name = "#{format("%02i", @screenshot_counter)}_#{name}" - @screenshot_counter += 1 - end - name = full_name(name) - file_name = "#{Screenshot.screenshot_area_abs}/#{name}.png" - - FileUtils.mkdir_p File.dirname(file_name) - comparison = ImageCompare.new(file_name, **driver_options) - checkout_vcs(name, comparison) - begin - blurred_input = prepare_page_for_screenshot(timeout: wait) - if stability_time_limit - take_stable_screenshot(comparison, stability_time_limit: stability_time_limit, wait: wait) - else - take_right_size_screenshot(comparison) - end - ensure - blurred_input&.click - end - - return false unless comparison.old_file_exists? - - (@test_screenshots ||= []) << [caller(1..1).first, name, comparison] - - true - end - - def window_size_is_wrong? - selenium? && Screenshot.window_size && - page.driver.browser.manage.window.size != - ::Selenium::WebDriver::Dimension.new(*Screenshot.window_size) - end - - def assert_image_not_changed(caller, name, comparison) - return unless comparison.different? - - "Screenshot does not match for '#{name}' #{comparison.error_message}\nat #{caller}" - end - end - end - end -end diff --git a/lib/capybara/screenshot/diff/vcs.rb b/lib/capybara/screenshot/diff/vcs.rb deleted file mode 100644 index 2e12de49..00000000 --- a/lib/capybara/screenshot/diff/vcs.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -require_relative "os" -module Capybara - module Screenshot - module Diff - module Vcs - SILENCE_ERRORS = Os::ON_WINDOWS ? "2>nul" : "2>/dev/null" - - def restore_git_revision(name, target_file_name) - redirect_target = "#{target_file_name} #{SILENCE_ERRORS}" - show_command = "git show HEAD~0:./#{Capybara::Screenshot.screenshot_area}/#{name}.png" - if Capybara::Screenshot.use_lfs - `#{show_command} | git lfs smudge > #{redirect_target}` - else - `#{show_command} > #{redirect_target}` - end - FileUtils.rm_f(target_file_name) unless $CHILD_STATUS == 0 - end - - def checkout_vcs(name, comparison) - svn_file_name = "#{Capybara::Screenshot.screenshot_area_abs}/.svn/text-base/#{name}.png.svn-base" - if File.exist?(svn_file_name) - committed_file_name = svn_file_name - FileUtils.cp committed_file_name, comparison.old_file_name - else - svn_info = `svn info #{comparison.new_file_name} #{SILENCE_ERRORS}` - if svn_info.present? - wc_root = svn_info.slice(/(?<=Working Copy Root Path: ).*$/) - checksum = svn_info.slice(/(?<=Checksum: ).*$/) - if checksum - committed_file_name = "#{wc_root}/.svn/pristine/#{checksum[0..1]}/#{checksum}.svn-base" - FileUtils.cp committed_file_name, comparison.old_file_name - end - else - restore_git_revision(name, comparison.old_file_name) - end - end - end - end - end - end -end diff --git a/lib/capybara/screenshot/diff/version.rb b/lib/capybara/screenshot/diff/version.rb deleted file mode 100644 index 039a12bf..00000000 --- a/lib/capybara/screenshot/diff/version.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -module Capybara - module Screenshot - module Diff - VERSION = "1.4.0" - end - end -end diff --git a/lib/snap_diff-capybara.rb b/lib/snap_diff-capybara.rb new file mode 100644 index 00000000..c3a5a4db --- /dev/null +++ b/lib/snap_diff-capybara.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +# Bundler.require entry point for `gem "snap_diff-capybara"`: Bundler +# requires the gem's own name, and its dash->slash fallback ("snap_diff/ +# capybara") misses too, so without this file a Rails user gets a silent +# no-op and a confusing NameError later. +require "snap_diff/integrations/minitest" diff --git a/lib/snap_diff.rb b/lib/snap_diff.rb new file mode 100644 index 00000000..a9535755 --- /dev/null +++ b/lib/snap_diff.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +# Dual-install guard: capybara-screenshot-diff and snap_diff-capybara ship +# identical files. With BOTH activated, every require silently resolves +# from whichever gem activated first, so version skew between the two is +# undetectable. Refuse that setup at the entry point. Local dev from +# source loads neither spec, so the guard fires only when both are +# genuinely installed as gems. +# +# snap_diff/errors is pulled in first (two dependency-free files) so +# DualInstallError can sit under SnapDiff::Error like every other error the +# gem raises -- docs/snapdiff.md promises Error is the catch-all. +require "snap_diff/errors" + +module SnapDiff + DualInstallError = Class.new(Error) + + # @api private + def self.assert_single_gem!(loaded_specs = Gem.loaded_specs) + return unless loaded_specs.key?("capybara-screenshot-diff") && loaded_specs.key?("snap_diff-capybara") + + raise DualInstallError, + "Both `capybara-screenshot-diff` and `snap_diff-capybara` gems are installed. " \ + "They ship identical files, so files load from whichever gem activated first " \ + "and versions can silently diverge. Remove one of them from your Gemfile." + end +end +SnapDiff.assert_single_gem! + +# "capybara/dsl" is needed directly (not just transitively) so +# `Capybara.default_max_wait_time` in Config#default_options resolves even +# when "snap_diff" is required standalone (SnapDiffTest's +# "standalone-loadable in a fresh process" regression test). +require "capybara/dsl" +require "snap_diff/config" +require "snap_diff/comparison" +require "snap_diff/version" +# SnapDiff.session/.reset/.pending_screenshots_message are part of the +# documented core surface (docs/snapdiff.md object map lists them with no +# extra require), so the entry point owns them rather than leaving them to +# whichever integration happens to be loaded. +require "snap_diff/screenshot_assertion" + +# The namespace for the gem. 2.1 deleted the v1 +Capybara::Screenshot+ / +# +CapybaraScreenshotDiff+ trees, so this is the only one. +module SnapDiff + # Compare two images on disk with the configured defaults. + # + # Note the argument order swap: callers pass baseline first (reading + # "compare baseline against current"), Comparison takes current first. + def self.compare(baseline_path, current_path, **options) + Comparison.new(current_path, baseline_path, config.default_options.merge(options)) + end + + # THE config entry point (ADR-008): yields the single consolidated + # {SnapDiff::Config} object. +SnapDiff.start+ yielded the two v1 config + # holders, so it could not outlive them -- 2.1 removed it rather than + # renaming it. + # + # SnapDiff.configure do |config| + # config.window_size = [1280, 1024] + # config.tolerance = 0.0005 + # end + def self.configure + yield config + end + + # SnapDiff.config itself is defined in snap_diff/config.rb (the storage + # leaf), where the Config instance is created eagerly at require time so + # the ENV["CI"] / Rails.root defaults evaluate at load, not first call. +end diff --git a/lib/snap_diff/annotation_service.rb b/lib/snap_diff/annotation_service.rb new file mode 100644 index 00000000..592115fa --- /dev/null +++ b/lib/snap_diff/annotation_service.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module SnapDiff + # Annotation colors, defined here (not in the capybara_screenshot_diff + # umbrella) so they resolve in processes that only `require "snap_diff"`. + RED_RGBA = [255, 0, 0, 255].freeze + ORANGE_RGBA = [255, 192, 0, 255].freeze + + # Draws diff/skip-area rectangles and the heatmap overlay for a Difference, + # and saves the resulting images to their `.diff.*` / `.heatmap.diff.*` paths. + # + # Extracted from Reporters::Default (ADR-004 PR 7) so the reporter only + # builds error messages; this class owns all image annotation work. + class AnnotationService + attr_reader :annotated_image_path, :annotated_base_image_path, :heatmap_diff_path + + def initialize(difference) + @difference = difference + + comparison = difference.comparison + ext = comparison.new_image_path.extname.delete_prefix(".") + screenshot_format = comparison.options[:screenshot_format] || (ext unless ext.empty?) || "png" + @annotated_image_path = comparison.new_image_path.sub_ext(".diff.#{screenshot_format}") + @annotated_base_image_path = comparison.base_image_path.sub_ext(".diff.#{screenshot_format}") + @heatmap_diff_path = comparison.new_image_path.sub_ext(".heatmap.diff.#{screenshot_format}") + end + + def clean_tmp_files + annotated_base_image_path.unlink if annotated_base_image_path.exist? + annotated_image_path.unlink if annotated_image_path.exist? + heatmap_diff_path.unlink if heatmap_diff_path.exist? + end + + def annotate_and_save_images + save_annotation_for(new_image, annotated_image_path) + save_annotation_for(base_image, annotated_base_image_path) + save_heatmap_diff if difference.diff_mask + end + + def save_annotation_for(image, image_path) + image = annotate_difference(image, difference.region) + image = annotate_skip_areas(image, difference.comparison.skip_area) if difference.comparison.skip_area + + save(image, image_path.to_path) + end + + def annotate_difference(image, region) + driver.draw_rectangles([image], region, RED_RGBA, offset: 1).first + end + + def annotate_skip_areas(image, skip_areas) + skip_areas.reduce(image) do |memo, region| + driver.draw_rectangles([memo], region, ORANGE_RGBA).first + end + end + + def save(image, image_path) + driver.save_image_to(image, image_path.to_s) + end + + private + + attr_reader :difference + + def save_heatmap_diff + merged_image = driver.merge(new_image, base_image) + highlighted_mask = driver.highlight_mask(difference.diff_mask, merged_image, color: RED_RGBA) + + save(highlighted_mask, heatmap_diff_path.to_path) + end + + def base_image + difference.comparison.base_image + end + + def new_image + difference.comparison.new_image + end + + def driver + @_driver ||= difference.comparison.driver + end + end +end diff --git a/lib/snap_diff/area_calculator.rb b/lib/snap_diff/area_calculator.rb new file mode 100644 index 00000000..0e90127d --- /dev/null +++ b/lib/snap_diff/area_calculator.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require "snap_diff/region" + +module SnapDiff + class AreaCalculator + def initialize(crop_coordinates, skip_area) + @crop_coordinates = crop_coordinates + @skip_area = skip_area + end + + def calculate_crop + return @_calculated_crop if defined?(@_calculated_crop) + return @_calculated_crop = nil unless @crop_coordinates + + # TODO: Move out from this class, this should be done on before screenshot and should not depend on Browser + @crop_coordinates = BrowserHelpers.bounds_for_css(@crop_coordinates).first if @crop_coordinates.is_a?(String) + @_calculated_crop = Region.from_edge_coordinates(*@crop_coordinates) + end + + # Cast skip areas params into Region + # and if there is crop then makes absolute coordinates to eb relative to crop top left corner + def calculate_skip_area + return nil unless @skip_area + + crop_region = calculate_crop + skip_area = Array(@skip_area) + + css_selectors, coords_list = skip_area.compact.partition { |region| region.is_a? String } + regions, coords_list = coords_list.partition { |region| region.is_a? Region } + + regions.concat(build_regions_for(BrowserHelpers.bounds_for_css(*css_selectors))) unless css_selectors.empty? + regions.concat(build_regions_for(coords_list.flatten.each_slice(4))) unless coords_list.empty? + + regions.compact! + + if crop_region + regions + .map! { |region| crop_region.find_relative_intersect(region) } + .filter! { |region| region&.present? } + end + + regions + end + + private + + def build_regions_for(coordinates) + coordinates + .map { |entry| Region.from_edge_coordinates(*entry) } + .tap { |region| region.compact! } + end + end +end diff --git a/lib/snap_diff/attempts_reporter.rb b/lib/snap_diff/attempts_reporter.rb new file mode 100644 index 00000000..c2eec960 --- /dev/null +++ b/lib/snap_diff/attempts_reporter.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require "fileutils" + +require "snap_diff/comparison" + +module SnapDiff + class AttemptsReporter + def initialize(snapshot, comparison_options, stability_options = {}) + @snapshot = snapshot + @comparison_options = comparison_options + @wait = stability_options[:wait] + end + + def generate + attempts_screenshot_paths = @snapshot.find_attempts_paths + + annotate_attempts(attempts_screenshot_paths) + + "Could not get stable screenshot within #{@wait}s:\n#{attempts_screenshot_paths.join("\n")}" + end + + def build_comparison_for(attempt_path, previous_attempt_path) + Comparison.new(attempt_path, previous_attempt_path, @comparison_options) + end + + private + + def annotate_attempts(attempts_screenshot_paths) + previous_file = nil + attempts_screenshot_paths.reverse_each do |file_name| + if previous_file && File.exist?(previous_file) + attempts_comparison = build_comparison_for(file_name, previous_file) + + if attempts_comparison.different? + FileUtils.mv(attempts_comparison.reporter.annotated_base_image_path, previous_file, force: true) + else + warn "[capybara-screenshot-diff] Some attempts was stable, but mistakenly marked as not: " \ + "#{previous_file} and #{file_name} are equal" + end + + FileUtils.rm(attempts_comparison.reporter.annotated_image_path, force: true) + end + + previous_file = file_name + end + + previous_file + end + end +end diff --git a/lib/snap_diff/browser_helpers.rb b/lib/snap_diff/browser_helpers.rb new file mode 100644 index 00000000..ee129bb4 --- /dev/null +++ b/lib/snap_diff/browser_helpers.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +require "snap_diff/region" + +module SnapDiff + module BrowserHelpers + def self.resize_window_if_needed + # The respond_to? guard this replaced existed because the legacy + # mattr_accessor might not be installed yet; Config always has the + # attribute, so only the value matters now. + window_size = SnapDiff.config.window_size + resize_to(window_size) if window_size + end + + def self.resize_to(window_size) + if session.driver.respond_to?(:resize) + session.driver.resize(*window_size) + elsif BrowserHelpers.selenium? + session.driver.browser.manage.window.resize_to(*window_size) + end + end + + def self.selenium? + current_capybara_driver_class <= Capybara::Selenium::Driver + end + + def self.window_size_is_wrong?(expected_window_size = nil) + selenium? && expected_window_size && + session.driver.browser.manage.window.size != ::Selenium::WebDriver::Dimension.new(*expected_window_size) + end + + def self.bounds_for_css(*css_selectors) + css_selectors.reduce([]) do |regions, selector| + regions.concat(all_visible_regions_for(selector)) + end + end + + IMAGE_WAIT_SCRIPT = <<~JS + function pending_image() { + const images = document.images + for (var i = 0; i < images.length; i++) { + if (!images[i].complete && images[i].loading !== "lazy") { + return images[i].src + } + } + return false + }(window) + JS + + HIDE_CARET_SCRIPT = <<~JS + if (!document.getElementById('csdHideCaretStyle')) { + let style = document.createElement('style'); + style.setAttribute('id', 'csdHideCaretStyle'); + document.head.appendChild(style); + let styleSheet = style.sheet; + styleSheet.insertRule("* { caret-color: transparent !important; }", 0); + } + JS + + def self.hide_caret + session.execute_script(HIDE_CARET_SCRIPT) + end + + DISABLE_ANIMATIONS_SCRIPT = <<~JS + if (!document.getElementById('csdDisableAnimationsStyle')) { + let style = document.createElement('style'); + style.setAttribute('id', 'csdDisableAnimationsStyle'); + style.textContent = '*, *::before, *::after { animation-duration: 0s !important; animation-delay: 0s !important; transition-duration: 0s !important; transition-delay: 0s !important; }'; + document.head.appendChild(style); + } + JS + + def self.disable_animations + session.execute_script(DISABLE_ANIMATIONS_SCRIPT) + end + + FIND_ACTIVE_ELEMENT_SCRIPT = <<~JS + function activeElement(){ + const ae = document.activeElement; + if (ae.nodeName === "INPUT" || ae.nodeName === "TEXTAREA") { + ae.blur(); + return ae; + } + return null; + }(window); + JS + + def self.blur_from_focused_element + session.evaluate_script(FIND_ACTIVE_ELEMENT_SCRIPT) + end + + GET_BOUNDING_CLIENT_RECT_SCRIPT = <<~JS + [ + this.getBoundingClientRect().left, + this.getBoundingClientRect().top, + this.getBoundingClientRect().right, + this.getBoundingClientRect().bottom + ] + JS + + def self.all_visible_regions_for(selector) + BrowserHelpers.session.all(selector, visible: true).map { |el| region_for(el) } + end + + def self.region_for(element) + element.evaluate_script(GET_BOUNDING_CLIENT_RECT_SCRIPT).map { |point| point.negative? ? 0 : point.ceil.to_i } + end + + def self.session + Capybara.current_session + end + + def self.pending_image_to_load + BrowserHelpers.session.evaluate_script(IMAGE_WAIT_SCRIPT) + end + + def self.current_capybara_driver_class + session.driver.class + end + end +end diff --git a/lib/snap_diff/capture/viewport.rb b/lib/snap_diff/capture/viewport.rb new file mode 100644 index 00000000..e9c9a6dd --- /dev/null +++ b/lib/snap_diff/capture/viewport.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require_relative "../browser_helpers" + +module SnapDiff + module Capture + # Per-capture viewport preparation seam. + # + # Called exactly once per capture, before the screenshoter runs and outside + # any stability retry loop. It only validates the window size + # (raise-only, never resizes). + module Viewport + module_function + + # @param expected_window_size [Array(Integer, Integer), nil] the configured window size + # @raise [SnapDiff::WindowSizeMismatchError] if the browser + # window does not match the expected size. + def prepare!(expected_window_size) + return unless BrowserHelpers.window_size_is_wrong?(expected_window_size) + + current_size = BrowserHelpers.selenium? ? + BrowserHelpers.session.driver.browser.manage.window.size.to_s : + "unknown" + + raise SnapDiff::WindowSizeMismatchError.new(<<~ERROR.chomp, caller) + Window size mismatch detected! + Expected: #{expected_window_size.inspect} + Actual: #{current_size} + + Screenshots cannot be compared when window sizes don't match. + Please ensure the browser window is properly sized before taking screenshots. + ERROR + end + end + end +end diff --git a/lib/snap_diff/comparison.rb b/lib/snap_diff/comparison.rb new file mode 100644 index 00000000..5c778a52 --- /dev/null +++ b/lib/snap_diff/comparison.rb @@ -0,0 +1,228 @@ +# frozen_string_literal: true + +require "pathname" +require "fileutils" + +require "snap_diff/comparison_result" +require "snap_diff/drivers/vips_driver" +require "snap_diff/image_preprocessor" +require "snap_diff/reporters/default" + +module SnapDiff + # Handles comparison of two images with a focus on performance and accuracy. + # + # This class implements a multi-layered optimization strategy for image comparison: + # + # 1. Early File-based Checks (Fastest): + # - Verifies both images exist (raises ArgumentError if not) + # - Compares file sizes (different sizes → different images) + # - Performs byte-by-byte comparison for identical files (exact match) + # + # 2. Quick Comparison (Fast): + # - Compares image dimensions (different dimensions → different images) + # - Performs pixel-by-pixel comparison if dimensions match + # + # 3. Detailed Analysis (Slower): + # - Only performed if quick comparison finds differences + # - Handles anti-aliasing, color tolerance, and shift detection + # - Respects skip_area and other comparison parameters + # + # This layered approach ensures optimal performance by: + # - Using the fastest possible method for early rejection + # - Only performing expensive operations when absolutely necessary + # - Maintaining high accuracy for complex comparisons + class Comparison + # Holds the two images (and their paths/options/driver) being compared + # (ADR-008 step 5: ex-Capybara::Screenshot::Diff::Comparison struct). + Images = Struct.new(:new_image, :base_image, :options, :driver, :new_image_path, :base_image_path) do + def skip_area + options[:skip_area] + end + end + + TOLERABLE_OPTIONS = [:tolerance, :color_distance_limit, :area_size_limit].freeze + + attr_reader :driver, :driver_options + attr_reader :image_path, :base_image_path + attr_reader :difference, :error_message + + def initialize(image_path, base_image_path, options = {}) + @image_path = Pathname.new(image_path) + @base_image_path = Pathname.new(base_image_path) + + ensure_files_exist! + + @driver_options = options.freeze + # One backend since 2.1, constructed rather than looked up: the driver + # is stateless, so nothing is gained by threading one instance through + # the options hash the way the registry used to. + @driver = Drivers::VipsDriver.new + @without_tolerable_options = (driver_options.keys & TOLERABLE_OPTIONS).empty? + end + + # Performs a quick comparison of two image files. + # + # This method is optimized for speed and will return as soon as a difference is found. + # It's used for fast rejection before performing more expensive comparisons. + # + # @return [Boolean] + # - `true` if images are exactly identical (byte-for-byte match) + # - `false` if images are different or if a quick difference is detected + # + # @note This method will raise ArgumentError if either image file is missing. + def quick_equal? + if base_image_path.size == image_path.size + return true if files_identical?(base_image_path, image_path) + end + + result, difference = find_difference(quick_mode: true) + self.difference = difference + result + end + + def ensure_files_exist! + raise ArgumentError, "There is no original (base) screenshot located at #{@base_image_path}" unless @base_image_path.exist? + raise ArgumentError, "There is no new screenshot located at #{@image_path}" unless @image_path.exist? + end + + # Determines if the images are different according to the comparison rules. + # + # This method performs a full comparison if not already done, including any + # configured tolerances for color differences and shift distances. + # + # @return [Boolean] + # - `true` if the images are different beyond configured tolerances + # - `false` if the images are considered identical + # + # @see #processed + # @see #analyze_difference + def different? + processed.difference.different? + end + + def dimensions_changed? + difference.failed_by&.[](:different_dimensions) + end + + def reporter + @reporter ||= build_reporter + end + + def processed? + !!difference + end + + def processed + self.difference = find_difference(quick_mode: false) unless processed? + @error_message ||= reporter.generate + self + end + + private + + def without_tolerable_options? + @without_tolerable_options + end + + def load_images_and_build_comparison(base_path, new_path, options) + base_img, new_img = driver.load_images(base_path, new_path) + Images.new(new_img, base_img, options, driver, new_path, base_path) + end + + def image_preprocessor + @image_preprocessor ||= ImagePreprocessor.new(driver, driver_options) + end + + def find_difference(quick_mode: false) + # Validate images exist + return build_null_difference("missing_image") unless images_exist? + + # Create comparison with preprocessed images + comparison = load_comparison(base_image_path, image_path, driver_options) + + analyze_difference(comparison, quick_mode: quick_mode) + end + + # Analyzes the comparison and determines if images are different. + # + # @param comparison [Comparison::Images] The comparison object containing images to analyze. + # @param quick_mode [Boolean] When true, performs minimal checks and returns early. + # In quick mode, returns [is_equal, difference] where: + # - is_equal is true if images are considered equal + # - difference is a ComparisonResult object or nil + # When false, returns a ComparisonResult object directly. + # @return [Array, ComparisonResult] Result format depends on quick_mode parameter. + def analyze_difference(comparison, quick_mode: true) + # Handle dimension differences + unless driver.same_dimension?(comparison) + result = ComparisonResult.build_null(comparison, comparison.base_image_path, comparison.new_image_path, {different_dimensions: true}) + return quick_mode ? [false, result] : result + end + + # Handle identical pixels + if driver.same_pixels?(comparison) + result = ComparisonResult.build_null(comparison, comparison.base_image_path, comparison.new_image_path) + return quick_mode ? [true, result] : result + end + + # Handle early return for non-tolerable options + if quick_mode && without_tolerable_options? + return [false, nil] + end + + # Process difference region + region = driver.find_difference_region(comparison) + + # Only create a proper difference object if we've completed the comparison + quick_mode ? [!region.different?, region] : region + end + + def difference=(new_difference) + @error_message = nil + @reporter = nil + @difference = new_difference + end + + def build_reporter + current_difference = difference || build_null_difference + Reporters::Default.new(current_difference) + end + + # Loads and preprocesses images for detailed comparison. + # + # This method is responsible for: + # 1. Loading both images using the configured driver + # 2. Applying any necessary preprocessing (cropping, normalization) + # 3. Creating a Comparison::Images object that holds the image data + # + # @param base_path [String,Pathname] Path to the baseline/reference image + # @param new_path [String,Pathname] Path to the new/candidate image + # @param options [Hash] Comparison options including: + # - :crop [Array] Optional crop area [x, y, width, height] + # - :skip_area [Array] Areas to exclude from comparison + # - :tolerance [Numeric] Color tolerance threshold + # @return [Comparison::Images] Prepared comparison object ready for analysis + # @raise [ArgumentError] If image files are invalid or unreadable + def load_comparison(base_path, new_path, options) + comparison = load_images_and_build_comparison(base_path, new_path, options) + image_preprocessor.process_comparison(comparison) + end + + def build_null_difference(failed_by = nil) + comparison = Images.new(nil, nil, driver_options, driver, image_path, base_image_path).freeze + ComparisonResult.build_null(comparison, base_image_path, image_path, failed_by) + end + + # Check if both images exist + def images_exist? + base_image_path.exist? && image_path.exist? + end + + # Check if files are identical by content + def files_identical?(file1, file2) + FileUtils.compare_file(file1, file2) + rescue SystemCallError, IOError + false + end + end +end diff --git a/lib/snap_diff/comparison_result.rb b/lib/snap_diff/comparison_result.rb new file mode 100644 index 00000000..b01010a9 --- /dev/null +++ b/lib/snap_diff/comparison_result.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require "json" + +module SnapDiff + # Represents the result of comparing two images + # + # This value object encapsulates the result of an image comparison operation. + # It follows the Single Responsibility Principle by focusing solely on representing + # the difference state, including: + # - Whether images are different or equal + # - Why they differ (dimensions, pixels, etc.) + # - The specific region of difference + # - Whether differences are tolerable based on configured thresholds + # + # As part of the layered comparison architecture, this class represents the final + # output of the comparison process, containing all data needed for reporting. + class ComparisonResult < Struct.new(:region, :meta, :comparison, :failed_by, :base_image_path, :image_path, keyword_init: nil) + def self.build_null(comparison, base_image_path, new_image_path, failed_by = nil) + ComparisonResult.new( + nil, + {difference_level: nil, max_color_distance: 0}, + comparison, + failed_by, + base_image_path, + new_image_path + ).freeze + end + + def different? + failed? || !(blank? || tolerable?) + end + + def equal? + !different? + end + + def failed? + !!failed_by + end + + def options + comparison.options + end + + def tolerance + options[:tolerance] + end + + def skip_area + comparison.skip_area + end + + def area_size_limit + options[:area_size_limit] + end + + def blank? + region.nil? || region_area_size.zero? + end + + def region_area_size + @region_area_size ||= region&.size || 0 + end + + def ratio + meta[:difference_level] + end + + # Serializable difference metrics. The raw diff mask image is excluded — + # it is an image object, not a metric, and is reachable via #diff_mask. + def to_h + {area_size: region_area_size, region: coordinates}.merge!(meta.except(:diff_mask)) + end + + def coordinates + region&.to_edge_coordinates + end + + # One-line debugging summary with the difference metrics. + # (Error messages use #to_h — see Reporters::Default#build_error_message.) + def inspect + "#<#{self.class.name} different=#{different?} failed_by=#{failed_by.inspect} " \ + "area_size=#{region_area_size} region=#{coordinates.inspect} " \ + "difference_level=#{ratio.inspect} " \ + "base=#{original_image_path} new=#{new_image_path}>" + end + + def tolerable? + !!((area_size_limit && area_size_limit >= region_area_size) || (tolerance && tolerance >= ratio)) + end + + # Path accessors for backward compatibility + def new_image_path + image_path || comparison&.new_image_path + end + + def original_image_path + base_image_path || comparison&.base_image_path + end + + def diff_mask + meta[:diff_mask] + end + end +end diff --git a/lib/snap_diff/config.rb b/lib/snap_diff/config.rb new file mode 100644 index 00000000..3b3a9291 --- /dev/null +++ b/lib/snap_diff/config.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +require "pathname" + +# This file is the LEAF of the config require graph (ADR-008 step 1): it must +# never require anything that leads back to an entry point. + +# Referenced by Config#initialize (screenshoter/manager defaults), which +# runs at the eager Config.new at the bottom of this file, so they must be +# real, already-loaded classes first. Neither requires back here. +require "snap_diff/screenshoter" +require "snap_diff/snap_manager" + +module SnapDiff + # Every setting the gem has, behind one object: + # SnapDiff.config.attr. Since 2.1 deleted the v1 + # namespaces this is not just the single storage but the single surface -- + # {SnapDiff.configure} is the one config entry point (ADR-008). + # + # Default timing contract (pinned by config_default_timing_test.rb): + # every default below is evaluated ONCE, in #initialize, which runs at + # require time of this file (the eager +Config.new+ at the bottom) -- the + # same load moment the old +mattr_accessor+ default blocks evaluated at. + # In particular +fail_if_new+ (from ENV["CI"]) and +root+ (from + # +Rails.root+ / pwd) must never become lazy read-time defaults, memoized + # or not. The one deliberately LIVE value, +default_options[:wait]+, is + # not storage at all: it stays a method-body read of + # +Capybara.default_max_wait_time+ in +#default_options+. + class Config + # Every setting this object stores, in the order the two legacy holders + # used to declare them. + # + # +screenshot_enabled+ is the one name that differs from its legacy + # spelling: +Capybara::Screenshot.enabled+ and + # +Capybara::Screenshot::Diff.enabled+ are independent settings (see + # {#active?}, which reads both) that happened to share a bare name in + # their own modules. A flat Config can't expose two attributes both + # called +enabled+, so the Screenshot-side one is renamed here; Diff's + # keeps the bare +enabled+ name since it's the one most existing + # configuration touches directly. + SETTINGS = %i[ + add_driver_path + add_os_path + blur_active_element + screenshot_enabled + hide_caret + disable_animations + root + stability_time_limit + window_size + save_path + use_lfs + screenshot_format + capybara_screenshot_options + delayed + area_size_limit + fail_if_new + pending_if_new + fail_on_difference + color_distance_limit + enabled + skip_area + tolerance + perceptual_threshold + screenshoter + manager + ].freeze + + attr_accessor(*(SETTINGS - %i[root])) + attr_reader :root + + def initialize + # Every setting gets its ivar up front (nil-defaulted ones included) + # so the full set always exists -- test_helper's per-test isolation + # snapshots/restores config by instance variable, and an ivar that + # only appears on first write would escape that snapshot and leak + # between tests. + SETTINGS.each { |key| instance_variable_set(:"@#{key}", nil) } + # Capybara::Screenshot side. + @blur_active_element = true + @hide_caret = true + # Raw Rails.root (no coercion), matching the old mattr_reader default; + # only the writer below coerces. + @root = (defined?(Rails) && defined?(Rails.root) && Rails.root) || Pathname(".").expand_path + @save_path = "doc/screenshots" + @screenshot_format = "png" + @capybara_screenshot_options = {} + # Capybara::Screenshot::Diff side. + @delayed = true + @fail_if_new = !ENV["CI"].nil? && !ENV["CI"].empty? + @pending_if_new = false + @fail_on_difference = true + @enabled = true + @screenshoter = SnapDiff::Screenshoter + @manager = SnapDiff::SnapManager + end + + def root=(path) + @root = Pathname(path).expand_path + end + + # --- Derived config (ADR-008 step 7b) ------------------------------- + # Read-only values computed from the storage above. They used to live + # on the legacy modules; those now one-line forward here. + + # ex +Capybara::Screenshot.active?+. The two +enabled+ settings are + # independent (see {SETTINGS}): the Screenshot-side one wins whenever it + # was set at all, and only a nil there falls through to the Diff-side + # one. + def active? + screenshot_enabled || (screenshot_enabled.nil? && enabled) + end + + # ex +Capybara::Screenshot.screenshot_area+: the save_path, optionally + # segmented per OS and per Capybara driver. + def screenshot_area + parts = [save_path] + parts << Os.name if add_os_path + parts << Capybara.current_driver.to_s if add_driver_path + File.join(*parts) + end + + # ex +Capybara::Screenshot.screenshot_area_abs+. + def screenshot_area_abs + root / screenshot_area + end + + # The capture/compare defaults handed to {SnapDiff::Comparison}. Carries + # the one literal that is not a stored setting -- the vips tolerance + # floor, now unconditional: 2.1 made libvips the only backend, so the + # driver == :vips guard this used to carry was always true. + def default_options + { + area_size_limit: area_size_limit, + color_distance_limit: color_distance_limit, + screenshot_format: screenshot_format, + capybara_screenshot_options: capybara_screenshot_options, + perceptual_threshold: perceptual_threshold, + skip_area: skip_area, + stability_time_limit: stability_time_limit, + tolerance: tolerance || 0.001, + # Deliberately LIVE (pinned by config_default_timing_test.rb): + # read at call time, never frozen into storage. + wait: Capybara.default_max_wait_time + } + end + end + + # Instantiated eagerly so the require-time defaults above are evaluated + # NOW, at load, not at the first SnapDiff.config call. + @config = Config.new + + # The single consolidated settings object -- and the single storage. + # See {SnapDiff::Config}. + def self.config + @config + end +end diff --git a/lib/snap_diff/drivers/vips_driver.rb b/lib/snap_diff/drivers/vips_driver.rb new file mode 100644 index 00000000..62648a79 --- /dev/null +++ b/lib/snap_diff/drivers/vips_driver.rb @@ -0,0 +1,208 @@ +# frozen_string_literal: true + +begin + require "vips" +rescue LoadError => e + raise 'Required ruby-vips gem is missing. Add `gem "ruby-vips"` to Gemfile' if e.message.match?(/vips/i) + raise +end + +require "snap_diff/comparison_result" +# Defines SnapDiff::RED_RGBA, the highlight_mask default color. +require "snap_diff/annotation_service" + +module SnapDiff + module Drivers + # THE image backend. 2.1 removed the driver abstraction (the + # +SnapDiff::Driver+ mixin, the +SnapDiff::Drivers+ registry and + # driver: :auto selection) along with the chunky_png driver, so + # this class is no longer one of several -- +ruby-vips+ is a runtime + # dependency and every comparison runs through here. The dimension + # helpers below came from the mixin; with one includer they live here. + # + # +Drivers+ survives only as the namespace this class has always been + # published under (docs/drivers.md), not as a registry. + class VipsDriver + PNG_EXTENSION = ".png" + + # libvips caches loader operations keyed on filename + mtime, and mtime + # has ONE-SECOND resolution -- so overwriting a path and re-reading it + # within the same second hands back the PREVIOUS image. This gem does + # exactly that: the screenshoter writes `.png`, `checkout_base_screenshot` + # writes `.base.png` from VCS, and the comparison then reads both. + # + # `revalidate: true` tells the loader to skip the cached result (libvips + # 8.15+). It was latent until 2.1 because comparisons built without an + # explicit driver defaulted to chunky_png, which always re-read the file. + REVALIDATE = Vips.at_least_libvips?(8, 15) ? {revalidate: true}.freeze : {}.freeze + + def find_difference_region(comparison) + new_image, base_image, options = comparison.new_image, comparison.base_image, comparison.options + + diff_mask = if options[:perceptual_threshold] + self.class.perceptual_difference_mask(base_image, new_image, options[:perceptual_threshold]) + else + self.class.difference_mask(base_image, new_image, options[:color_distance_limit]) + end + region = self.class.difference_region_by(diff_mask) + # TODO: schedule research when we got this case for VIPs + # region = nil if region && region_covers_entire_image?(region, base_image) + + result = ComparisonResult.new(region, {}, comparison) + + unless result.blank? + result.meta[:difference_level] = difference_level(diff_mask, base_image) if comparison.options[:tolerance] + result.meta[:diff_mask] = diff_mask + end + + result + end + + def crop(region, i) + i.crop(*region.to_top_left_corner_coordinates) + rescue Vips::Error => e + warn( + "[capybara-screenshot-diff] Crop has been failed for " \ + "{ region: #{region.to_top_left_corner_coordinates.inspect}, image: #{dimension(i).join("x")} }" + ) + raise e + end + + def filter_image_with_median(image, median_filter_window_size) + image.median(median_filter_window_size) + end + + def add_black_box(memo, region) + return memo unless region + + memo.draw_rect([0, 0, 0, 0], *region.to_top_left_corner_coordinates, fill: true) + end + + def difference_level(diff_mask, old_img, _region = nil) + self.class.difference_area_size_by(diff_mask).to_f / image_area_size(old_img) + end + + MAX_FILENAME_LENGTH = 200 + + # Vips could not work with the same file. Per each process we require to create new file + def save_image_to(image, filename) + # Dir::Tmpname will happily produce tempfile names that are too long for most unix filesystems, + # which leads to "unix error: File name too long". Apply a limit to avoid this. + limited_filename = filename.to_s[-MAX_FILENAME_LENGTH..] || filename.to_s + ::Dir::Tmpname.create([limited_filename, PNG_EXTENSION]) do |tmp_image_filename| + image.write_to_file(tmp_image_filename) + FileUtils.mv(tmp_image_filename, filename) + end + end + + def resize_image_to(image, new_width, new_height) + image.resize(new_width.to_f / image.width, vscale: new_height.to_f / image.height) + end + + def load_images(old_file_name, new_file_name) + [from_file(old_file_name), from_file(new_file_name)] + end + + def from_file(filename) + result = ::Vips::Image.new_from_file(filename.to_s, **REVALIDATE) + + result = result.colourspace(:srgb) if result.bands < 3 + result = result.bandjoin(255) if result.bands == 3 + + result + end + + def draw_rectangles(images, region, rgba, offset: 0) + images.map do |image| + image.draw_rect(rgba, region.left - offset, region.top - offset, region.width + (offset * 2), region.height + (offset * 2)) + end + end + + def same_pixels?(comparison) + (comparison.new_image == comparison.base_image).min == 255 + end + + # --- ex-SnapDiff::Driver mixin -------------------------------------- + # Dimension helpers, unchanged. They were shared because there were two + # drivers; there is one. + + def same_dimension?(comparison) + dimension(comparison.base_image) == dimension(comparison.new_image) + end + + def height_for(image) + image.height + end + + def width_for(image) + image.width + end + + def image_area_size(image) + width_for(image) * height_for(image) + end + + def dimension(image) + [width_for(image), height_for(image)] + end + + def merge(new_image, base_image) + base_image.composite2(new_image, :over) + end + + def highlight_mask(diff_mask, merged_image, color: SnapDiff::RED_RGBA) + diff_mask.ifthenelse(color, merged_image * 0.75) + end + + private + + class << self + def difference_area_size_by(difference_mask) + diff_mask = difference_mask == 0 + diff_mask.hist_find.to_a[0][0].max + end + + def difference_mask(base_image, new_image, color_distance = nil) + result = (new_image - base_image).abs + color_distance ? result > color_distance : result + end + + def perceptual_difference_mask(base_image, new_image, threshold = 2.0) + color_diff = perceptual_color_diff(base_image, new_image) > threshold + alpha_diff = alpha_channel_diff(base_image, new_image) + alpha_diff ? (color_diff | alpha_diff) : color_diff + end + + def perceptual_color_diff(base_image, new_image) + base_rgb = (base_image.bands > 3) ? base_image.extract_band(0, n: 3) : base_image + new_rgb = (new_image.bands > 3) ? new_image.extract_band(0, n: 3) : new_image + base_lab = base_rgb.colourspace(:lab) + new_lab = new_rgb.colourspace(:lab) + base_lab.dE00(new_lab) + rescue Vips::Error + base_lab.dE76(new_lab) + end + + def alpha_channel_diff(base_image, new_image) + return unless base_image.bands > 3 && new_image.bands > 3 + + (base_image.extract_band(3) - new_image.extract_band(3)).abs > 0 + end + + def difference_region_by(diff_mask) + columns, rows = diff_mask.bandor.project + + left = columns.profile[1].min + right = columns.width - columns.flip(:horizontal).profile[1].min + + top = rows.profile[0].min + bottom = rows.height - rows.flip(:vertical).profile[0].min + + return nil if right < left || bottom < top + + Region.from_edge_coordinates(left, top, right, bottom) + end + end + end + end +end diff --git a/lib/snap_diff/dsl.rb b/lib/snap_diff/dsl.rb new file mode 100644 index 00000000..2580e89b --- /dev/null +++ b/lib/snap_diff/dsl.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +# The canonical entry point. Everything a user is told to require under +# snap_diff/ (this file, snap_diff/integrations/*, snap_diff/static) routes +# through here, so this one line is what makes SnapDiff.configure/.start/ +# .compare/::VERSION -- and the dual-install guard -- present no matter +# which of those paths the user picked. +require "snap_diff" + +# DSL includes Capybara::DSL directly below, so it needs the base gem +# loaded regardless of what pulled this file in. +require "capybara/dsl" +require "snap_diff/config" +require "snap_diff/comparison" +require "snap_diff/screenshot_matcher" +require_relative "screenshot_namer" +require_relative "screenshot_assertion" + +module SnapDiff + # DSL for taking screenshots and making assertions in Capybara tests. + # This module provides methods for taking screenshots, comparing them against baselines, + # and managing the comparison process with various configuration options. + # + # The DSL is designed to be included in your test context (e.g., RSpec, Minitest) + # to provide screenshot comparison capabilities. + module DSL + include Capybara::DSL + + def screenshot_section(name) + SnapDiff.session.screenshot_namer.section = name + end + + def screenshot_group(name) + SnapDiff.session.screenshot_namer.group = name + end + + # Takes a screenshot and compares it against a baseline image. + # + # The method follows a layered optimization strategy for comparison: + # 1. First checks if screenshot functionality is active + # 2. Builds a full screenshot name using the current context + # 3. Creates a screenshot assertion object + # 4. Either validates immediately or defers validation based on options + # + # @param name [String] The base name of the screenshot, used to generate the filename. + # @param skip_stack_frames [Integer] The number of stack frames to skip when reporting errors. + # @param options [Hash] Additional options for taking the screenshot and comparison. + # @option options [Boolean] :delayed (SnapDiff.config.delayed) + # Whether to validate the screenshot immediately or delay validation. + # @option options [Array] :crop [left, top, right, bottom] Edge coordinates to crop the screenshot to. + # @option options [Array>] :skip_area Array of [left, top, right, bottom] edge coordinates to ignore. + # @option options [Numeric] :tolerance (0.001) Color tolerance for comparison. + # Represents the maximum allowed ratio of different pixels (0.0-1.0 scale). + # @option options [Numeric] :color_distance_limit Maximum allowed color distance between pixels. + # Uses Euclidean RGBA distance (0-510 scale). Mutually exclusive with :perceptual_threshold. + # @option options [Numeric] :perceptual_threshold Maximum perceptual color difference (CIE dE00). + # Uses human perception-based scale (0-100+). Takes priority over :color_distance_limit if both set. + # @option options [Numeric] :area_size_limit Maximum allowed difference area size in pixels. + # @return [Boolean] True if the screenshot was successfully captured and processed. + # @raise [SnapDiff::ExpectationNotMet] If comparison fails and immediate validation is enabled. + # @raise [SnapDiff::UnstableImage] If the image comparison is unstable. + # @raise [SnapDiff::WindowSizeMismatchError] If the window size doesn't match expectations. + def assert_matches_screenshot(name, skip_stack_frames: 0, **options) + return false unless SnapDiff.config.active? + + # Get the full name with section and group information + full_name = SnapDiff.session.screenshot_namer.full_name(name) + + # Build the screenshot assertion; the actual comparison is deferred + # until ScreenshotAssertion#validate! runs. + assertion = SnapDiff::ScreenshotMatcher + .new(full_name, options) + .build_screenshot_assertion(skip_stack_frames: skip_stack_frames + 1) + + return false unless assertion + + # Determine if validation should be delayed or immediate + delayed = options.fetch(:delayed, SnapDiff.config.delayed) + + if delayed + SnapDiff.session.add_assertion(assertion) + else + assertion.validate! + end + + true + end + + # Convenience wrapper around {#assert_matches_screenshot} and {#capture_screenshot}. + # @param compare [Boolean] When false, only captures the screenshot without comparing it to a baseline. + # @see #assert_matches_screenshot + # @see #capture_screenshot + def screenshot(name, skip_stack_frames: 0, compare: true, **options) + if compare + assert_matches_screenshot(name, skip_stack_frames: skip_stack_frames + 1, **options) + else + capture_screenshot(name, **options) + end + end + + # Captures a screenshot without comparing it to a baseline. + # @param name [String] The base name of the screenshot, used to generate the filename. + # @param options [Hash] Additional options for taking the screenshot. See {#assert_matches_screenshot}. + # @return [Boolean] True if the screenshot was successfully captured. + def capture_screenshot(name, **options) + return false unless SnapDiff.config.active? + + full_name = SnapDiff.session.screenshot_namer.full_name(name) + SnapDiff::ScreenshotMatcher.new(full_name, options).capture + + true + end + + # Asserts the current page has no visual changes from the baseline. + # Override in your base test class to add project-specific behavior + # (e.g., waiting for Turbo, default skip areas). + def assert_no_screenshot_changes(name, skip_stack_frames: 0, **opts) + assert_matches_screenshot(name, skip_stack_frames: skip_stack_frames + 1, **opts) + end + end +end diff --git a/lib/snap_diff/error_with_filtered_backtrace.rb b/lib/snap_diff/error_with_filtered_backtrace.rb new file mode 100644 index 00000000..6404cdd3 --- /dev/null +++ b/lib/snap_diff/error_with_filtered_backtrace.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module SnapDiff + class BacktraceFilter + LIB_DIRECTORY = File.expand_path(File.join(File.dirname(__FILE__), "..")) + File::SEPARATOR + + def initialize(lib_directory = LIB_DIRECTORY) + # Trailing separator keeps the prefix match on a directory boundary, + # so "/app/lib" does not also reject "/app/library". + @lib_directory = lib_directory.end_with?(File::SEPARATOR) ? lib_directory : lib_directory + File::SEPARATOR + end + + # Filters out any backtrace lines originating from the library directory or from gems such as ActiveSupport, Minitest, and Railties + # @param backtrace [Array] + # @return [Array] + def filtered(backtrace) + backtrace + .reject { |location| File.expand_path(location).start_with?(@lib_directory) } + .reject { |l| l =~ /gems\/(activesupport|minitest|railties)/ } + end + end + + # @private + class ErrorWithFilteredBacktrace < StandardError + # @private + def initialize(message = nil, backtrace = []) + super(message) + filter = BacktraceFilter.new + set_backtrace(filter.filtered(backtrace)) + end + end +end diff --git a/lib/snap_diff/errors.rb b/lib/snap_diff/errors.rb new file mode 100644 index 00000000..56edfd12 --- /dev/null +++ b/lib/snap_diff/errors.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require "snap_diff/error_with_filtered_backtrace" + +# ADR-008 step 2: the gem's error classes live under SnapDiff. The old +# CapybaraScreenshotDiff names (capybara_screenshot_diff.rb) are EAGER +# same-object aliases of these classes -- deliberately not const_missing +# shims, because rescue clauses and defined?/const_defined? feature +# detection in adopter code must keep behaving exactly as before +# (const_defined? never triggers const_missing). +# +# Error is the catch-all docs/snapdiff.md advertises: EVERY error this gem +# raises inherits it, so `rescue SnapDiff::Error` really does catch them all +# (pinned by test/unit/errors_alias_test.rb, which discovers the classes +# rather than listing them). ErrorWithFilteredBacktrace is plumbing, not a +# second root. +module SnapDiff + class Error < ErrorWithFilteredBacktrace; end + + class ExpectationNotMet < Error; end + + class UnstableImage < Error; end + + class WindowSizeMismatchError < Error; end +end diff --git a/lib/snap_diff/image_preprocessor.rb b/lib/snap_diff/image_preprocessor.rb new file mode 100644 index 00000000..73496fd5 --- /dev/null +++ b/lib/snap_diff/image_preprocessor.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module SnapDiff + # Handles image preprocessing operations (skip_area and median filtering) + # + # This class applies preprocessing filters to images before comparison, + # such as masking specific regions (skip_area) or applying noise reduction. + # It's designed to work with either direct image objects or with options. + class ImagePreprocessor + attr_reader :driver, :options + + def initialize(driver, options = {}) + @driver = driver + @options = options + end + + # Process a comparison object directly + # This allows reusing the comparison's existing options + # @param [Comparison] comparison the comparison object + # @return [Comparison] the comparison object + def process_comparison(comparison) + # Process both images + comparison.base_image = process_image(comparison.base_image, comparison.base_image_path) + comparison.new_image = process_image(comparison.new_image, comparison.new_image_path) + + comparison + end + + private + + def process_image(image, _path) + result = image + result = apply_skip_area(result) if skip_area + result = apply_median_filter(result) if median_filter_window_size + result + end + + def apply_skip_area(image) + skip_area.reduce(image) do |result, region| + driver.add_black_box(result, region) + end + end + + # Unconditional since 2.1: libvips is the only backend and it implements + # the filter. The `driver.supports?` guard (and the warning it fell back + # to) existed for chunky_png, which did not. + def apply_median_filter(image) + driver.filter_image_with_median(image, median_filter_window_size) + end + + def skip_area + options[:skip_area] + end + + def median_filter_window_size + options[:median_filter_window_size] + end + end +end diff --git a/lib/snap_diff/integrations/cucumber.rb b/lib/snap_diff/integrations/cucumber.rb new file mode 100644 index 00000000..b64f279e --- /dev/null +++ b/lib/snap_diff/integrations/cucumber.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# See the matching comment in integrations/minitest.rb. +require_relative "../dsl" +require "snap_diff/screenshot_assertion" +require "snap_diff/reporting" + +World(::SnapDiff::DSL) + +Before do + SnapDiff.config.delayed = false + SnapDiff::BrowserHelpers.resize_window_if_needed +end + +After do |scenario| + if !scenario.failed? && (msg = SnapDiff.pending_screenshots_message) + skip_this_scenario(msg) + end +ensure + SnapDiff.reset +end + +AfterAll { SnapDiff::Reporting.finalize! } diff --git a/lib/snap_diff/integrations/minitest.rb b/lib/snap_diff/integrations/minitest.rb new file mode 100644 index 00000000..baadf30c --- /dev/null +++ b/lib/snap_diff/integrations/minitest.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require "minitest" +require_relative "../dsl" +# SnapDiff.session/.reset/.pending_screenshots_message live in +# snap_diff/screenshot_assertion, SnapDiff::Reporting.finalize! in +# snap_diff/reporting -- neither is pulled in by requiring dsl.rb alone. +require "snap_diff/screenshot_assertion" +require "snap_diff/reporting" + +used_deprecated_entrypoint = caller.any? do |path| + path.include?("capybara-screenshot-diff.rb") || path.include?("capybara/screenshot/diff.rb") +end + +if used_deprecated_entrypoint + warn <<~MSG + [DEPRECATION] The default activation of `capybara_screenshot_diff/minitest` will be removed. + Please `require "capybara_screenshot_diff/minitest"` explicitly. + MSG +end + +module SnapDiff + module Minitest + module Assertions + include ::SnapDiff::DSL + + def assert_matches_screenshot(*args, skip_stack_frames: 0, **opts) + self.assertions += 1 + + super(*args, skip_stack_frames: skip_stack_frames + 1, **opts) + rescue ::SnapDiff::ExpectationNotMet => e + raise ::Minitest::Assertion, e.message + end + + def setup + super + ::SnapDiff::BrowserHelpers.resize_window_if_needed + end + + def before_teardown + super + SnapDiff.session.verify + + # Computed here (before teardown/reset), but the actual `skip` is + # deferred to `after_teardown` so a real error raised by the user's + # `teardown` isn't masked by a pending skip recorded before it ran. + @capybara_screenshot_diff_pending_message = SnapDiff.pending_screenshots_message + rescue SnapDiff::ExpectationNotMet => e + assertion = ::Minitest::Assertion.new(e) + assertion.set_backtrace(e.backtrace) + failures << assertion + ensure + SnapDiff.reset + end + + def after_teardown + super + + # Never mask a real failure (from `verify` above or from the user's + # own `teardown`) with a pending marker. + if failures.empty? && (msg = @capybara_screenshot_diff_pending_message) + skip(msg) + end + end + end + end +end + +::Minitest.after_run { SnapDiff::Reporting.finalize! } if ::Minitest.respond_to?(:after_run) diff --git a/lib/snap_diff/integrations/rspec.rb b/lib/snap_diff/integrations/rspec.rb new file mode 100644 index 00000000..178ae802 --- /dev/null +++ b/lib/snap_diff/integrations/rspec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require "rspec/core" +require_relative "../dsl" +# See the matching comment in integrations/minitest.rb. +require "snap_diff/screenshot_assertion" +require "snap_diff/reporting" + +RSpec::Matchers.define :match_screenshot do |name, **options| + description { "match screenshot '#{name}'" } + + match do |_page| + assert_matches_screenshot(name, **options) + true + end + + failure_message do + "Expected page to match screenshot '#{name}'" + end + + failure_message_when_negated do + "Expected page not to match screenshot '#{name}'" + end +end + +RSpec.configure do |config| + config.include SnapDiff::DSL, type: :feature + config.include SnapDiff::DSL, type: :system + + config.before do + if self.class.include?(SnapDiff::DSL) + SnapDiff::BrowserHelpers.resize_window_if_needed + end + end + + # `append_after` (as opposed to the default `after`, which prepends) adds + # this hook to the *end* of the after-hook chain regardless of when it's + # registered relative to the user's own `after`/`config.after` hooks. RSpec + # runs `after(:each)` hooks in reverse registration order, so a plain + # `config.after` here would run BEFORE a user hook registered earlier in + # their own spec_helper (before this file was required) -- and if that + # later-running user hook raises, its exception gets folded into + # `pending_exception` instead of `example.exception`, silently masking the + # failure behind our pending skip. `append_after` runs after the full user + # after-chain no matter the registration order, closing that gap. + config.append_after do |example| + if self.class.include?(SnapDiff::DSL) + begin + SnapDiff.session.verify + + # Never mask a real failure with a pending marker. Kept as + # defense-in-depth: `append_after` observes failures from plain + # `after`/`prepend_after` user hooks, but appended hooks run FIFO, + # so a user `append_after` registered after this gem still runs + # later than us — RSpec has no "run absolutely last" construct. + # Mitigation for such consumers: require this gem last. + if example.exception.nil? && (msg = SnapDiff.pending_screenshots_message) + skip(msg) + end + rescue SnapDiff::ExpectationNotMet => e + raise RSpec::Expectations::ExpectationNotMetError.new(e.message).tap { |ex| ex.set_backtrace(e.backtrace) } + ensure + SnapDiff.reset + end + end + end + + config.after(:suite) { SnapDiff::Reporting.finalize! } +end diff --git a/lib/snap_diff/os.rb b/lib/snap_diff/os.rb new file mode 100644 index 00000000..5eb2a0bd --- /dev/null +++ b/lib/snap_diff/os.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module SnapDiff + module Os + ON_WINDOWS = !!(RbConfig::CONFIG["host_os"] =~ /mswin|mingw|cygwin/) + ON_MAC = !!(RbConfig::CONFIG["host_os"] =~ /darwin/) + ON_LINUX = !!(RbConfig::CONFIG["host_os"] =~ /linux/) + + def self.name + return "windows" if ON_WINDOWS + return "macos" if ON_MAC + return "linux" if ON_LINUX + + "unknown" + end + end +end diff --git a/lib/snap_diff/region.rb b/lib/snap_diff/region.rb new file mode 100644 index 00000000..9fc422dc --- /dev/null +++ b/lib/snap_diff/region.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +module SnapDiff + class Region + attr_accessor :x, :y, :width, :height + + def initialize(x, y, width, height) + @x, @y, @width, @height = x, y, width, height + end + + def self.from_edge_coordinates(left, top, right, bottom) + return nil unless left && top && right && bottom + return nil if right < left || bottom < top + + Region.new(left, top, right - left, bottom - top) + end + + def to_edge_coordinates + [left, top, right, bottom] + end + + def to_top_left_corner_coordinates + [x, y, width, height] + end + + def top + y + end + + def bottom + y + height + end + + def left + x + end + + def right + x + width + end + + def size + return 0 if width < 0 || height < 0 + + result = width * height + result.zero? ? 1 : result + end + + def to_a + [@x, @y, @width, @height] + end + + def find_intersect_with(region) + return nil unless intersect?(region) + + new_left = [x, region.x].max + new_top = [y, region.y].max + + Region.new(new_left, new_top, [right, region.right].min - new_left, [bottom, region.bottom].min - new_top) + end + + def intersect?(region) + left <= region.right && right >= region.left && top <= region.bottom && bottom >= region.top + end + + def move_by(right_by, down_by) + Region.new(x + right_by, y + down_by, width, height) + end + + def find_relative_intersect(region) + intersect = find_intersect_with(region) + return nil unless intersect + + intersect.move_by(-x, -y) + end + + def cover?(x, y) + x.between?(left, right) && y.between?(top, bottom) + end + + def empty? + width.zero? || height.zero? + end + + def blank? + empty? + end + + def present? + !empty? + end + + def inspect + "Region(x: #{x}, y: #{y}, width: #{width}, height: #{height})" + end + + # need to add this method to make it work with assert_equal + def ==(other) + case other + when Region + x == other.x && y == other.y && width == other.width && height == other.height + when Array + to_a == other + else + false + end + end + end +end + +# Compatibility alias -- EAGER and same-object on purpose, mirroring the +# ADR-008 step 2 rationale: user configs construct skip_area entries with +# top-level `Region.new(...)` and SnapDiff internals check +# `is_a?(SnapDiff::Region)`, so both names must be the exact same class; +# and `defined?(Region)` feature detection must be truthy as soon as this +# file loads (a lazy const_missing shim would return nil there). +Region = SnapDiff::Region diff --git a/lib/snap_diff/reporters/default.rb b/lib/snap_diff/reporters/default.rb new file mode 100644 index 00000000..5dc3e8bf --- /dev/null +++ b/lib/snap_diff/reporters/default.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require "snap_diff/annotation_service" + +module SnapDiff + module Reporters + class Default + attr_reader :difference + + def initialize(difference) + @difference = difference + @annotation_service = SnapDiff::AnnotationService.new(difference) + end + + def annotated_image_path + annotation_service.annotated_image_path + end + + def annotated_base_image_path + annotation_service.annotated_base_image_path + end + + def heatmap_diff_path + annotation_service.heatmap_diff_path + end + + def generate + if difference.equal? + # NOTE: Delete previous run runtime files + clean_tmp_files + return nil + end + + if difference.failed? && difference.failed_by[:different_dimensions] + return build_error_for_different_dimensions + end + + annotate_and_save_images + build_error_message + end + + def clean_tmp_files + annotation_service.clean_tmp_files + end + + def annotate_and_save_images + annotation_service.annotate_and_save_images + end + + def build_error_for_different_dimensions + change_msg = [comparison.base_image, comparison.new_image] + .map { |image| driver.dimension(image).join("x") } + .join(" => ") + + "Dimensions have changed: #{change_msg}\n#{base_image_path.to_path}\n#{image_path.to_path}" + end + + NEW_LINE = "\n" + + def build_error_message + [ + "(#{difference.to_h.to_json})", + image_path.to_path, + annotated_base_image_path.to_path, + annotated_image_path.to_path, + heatmap_diff_path.to_path + ].join(NEW_LINE) + end + + private + + attr_reader :annotation_service + + def base_image_path + comparison.base_image_path + end + + def image_path + comparison.new_image_path + end + + def driver + @_driver ||= comparison.driver + end + + def comparison + @_comparison ||= difference.comparison + end + end + end +end diff --git a/lib/snap_diff/reporters/html.rb b/lib/snap_diff/reporters/html.rb new file mode 100644 index 00000000..b24c4e9e --- /dev/null +++ b/lib/snap_diff/reporters/html.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require "base64" +require "erb" +require "fileutils" +require "pathname" +require "json" +# The auto-registration block at the bottom of this file registers with +# SnapDiff::Reporting. +require "snap_diff/reporting" +require "snap_diff/config" + +module SnapDiff + module Reporters + class HTML + attr_reader :failures, :total + + def initialize(output_path: nil, embed_images: false) + @explicit_output_path = output_path + @embed_images = embed_images + @failures = [] + @total = 0 + @finalized = false + @mutex = Mutex.new + end + + def record(assertions) + return if @finalized + + failures = [] + total = 0 + + assertions.each do |assertion| + compare = assertion.compare + next unless compare + + total += 1 + next unless compare.difference&.different? + + failures << failure_entry_for(assertion.name, compare) + rescue => e + warn "[snap_diff] Reporter skipped '#{assertion.name}': #{e.message}" if ENV["DEBUG"] + end + + @mutex.synchronize do + return if @finalized + @total += total + @failures.concat(failures) + end + end + + def finalize + @mutex.synchronize do + return if @finalized + return if failures.empty? + + write_report + @finalized = true + output_path + end + end + + def output_path + @output_path ||= Pathname.new(@explicit_output_path || self.class.default_output_path) + end + + def passed = total - failures.size + def failed = failures.size + + def summary + return if total.zero? + + screenshots_label = (total == 1) ? "1 screenshot" : "#{total} screenshots" + + if failures.empty? + "[snap_diff] #{screenshots_label} compared, no failures." + else + failures_label = (failures.size == 1) ? "1 failure" : "#{failures.size} failures" + "[snap_diff] #{screenshots_label} compared, #{failures_label}. Report: #{output_path}" + end + end + + def render + ERB.new(File.read(self.class.template_path)).result(binding) + end + + def self.template_path + File.expand_path("templates/report.html.erb", __dir__) + end + + def self.default_output_path + root = SnapDiff.config.root || Pathname.pwd + root / SnapDiff.config.save_path / "snap_diff_report.html" + end + + private + + def failure_entry_for(name, compare) + difference = compare.difference + { + name: name, + original: resolve_image(compare.base_image_path), + new: resolve_image(compare.image_path), + base_diff: resolve_image(compare.reporter.annotated_base_image_path), + diff: resolve_image(compare.reporter.annotated_image_path), + heatmap: resolve_image(compare.reporter.heatmap_diff_path), + diff_level: difference.ratio && (difference.ratio * 100).round(2), + area_size: difference.region_area_size, + max_color_distance: difference.meta[:max_color_distance]&.round(1) + } + end + + def resolve_image(path) + return unless path + + pathname = Pathname.new(path).expand_path + return unless pathname.exist? + + @embed_images ? data_uri(pathname) : pathname.relative_path_from(output_path.dirname.expand_path).to_s + end + + def data_uri(pathname) + ext = pathname.extname.delete_prefix(".") + mime = (ext == "webp") ? "image/webp" : "image/png" + "data:#{mime};base64,#{Base64.strict_encode64(pathname.binread)}" + end + + def write_report + FileUtils.mkdir_p(output_path.dirname) + File.write(output_path, render) + end + end + end +end + +# Auto-register reporter. +# Framework adapters (Minitest, RSpec, Cucumber) call SnapDiff::Reporting.finalize! via native hooks. +# For custom frameworks, call SnapDiff::Reporting.finalize! manually. +unless SnapDiff::Reporting.reporters.any?(SnapDiff::Reporters::HTML) + SnapDiff::Reporting.register(SnapDiff::Reporters::HTML.new(embed_images: !!ENV["CI"])) +end diff --git a/lib/snap_diff/reporters/templates/report.html.erb b/lib/snap_diff/reporters/templates/report.html.erb new file mode 100644 index 00000000..8c756e5a --- /dev/null +++ b/lib/snap_diff/reporters/templates/report.html.erb @@ -0,0 +1,463 @@ + + + + + + SnapDiff Report — <%= failed %> failures + + + + +
+ +
+
+ + 1/<%= failed %> + +
+ + +
+
+ <%= failed %> failed + <%= passed %> passed + <%= total %> total +
+
+ +
+ + +
+
+ View +
+ + + + +
+
+ +
+ +
+
+
+ +
+ + 100% + + +
+
+
+ + + + diff --git a/lib/snap_diff/reporting.rb b/lib/snap_diff/reporting.rb new file mode 100644 index 00000000..3f3096a4 --- /dev/null +++ b/lib/snap_diff/reporting.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module SnapDiff + # Process-global reporter lifecycle: registration, per-test notification, + # end-of-suite finalization. One list of reporters for the whole process, + # guarded by one mutex. + # + # Deliberately separate from the per-test session lifecycle + # (SnapDiff.session): reporters outlive any single test, the session does + # not. CapybaraScreenshotDiff keeps its public + # reporters/reporters_mutex/finalize_reporters! methods as thin shims + # over this module. + module Reporting + @reporters = [] + @mutex = Mutex.new + + class << self + attr_reader :reporters, :mutex + + # Registers a reporter for the rest of the process. The canonical way + # in: the append happens under the mutex, so concurrent registrations + # cannot lose one (issue #217 item 2). `reporters` stays public and + # mutable for compatibility -- appending to it directly still works, + # it just skips the lock. + def register(reporter) + @mutex.synchronize { @reporters << reporter } + reporter + end + + # Delivers a finished test's assertions to every registered reporter. + # Iterates over a snapshot so a reporter mutating the list mid-notify + # cannot affect the current round. A raising reporter is warned about + # and skipped; the rest are still notified. + def notify(assertions) + return if assertions.nil? || assertions.empty? + + reporters_snapshot = @mutex.synchronize { @reporters.dup } + return if reporters_snapshot.empty? + + reporters_snapshot.each do |reporter| + reporter.record(assertions) + rescue => e + warn "[snap_diff] Reporter #{reporter.class} failed (#{e.class}: #{e.message})" + end + end + + # End-of-suite hook: finalizes each reporter and prints its summary. + # A raising reporter is warned about and skipped; the rest are still + # finalized. + def finalize! + @mutex.synchronize { @reporters.dup }.each do |reporter| + reporter.finalize + if (msg = reporter.summary) + $stdout.puts msg + end + rescue => e + warn "[snap_diff] Reporter #{reporter.class} failed (#{e.class}: #{e.message})" + end + end + end + end +end diff --git a/lib/snap_diff/screenshot_assertion.rb b/lib/snap_diff/screenshot_assertion.rb new file mode 100644 index 00000000..574aa0ed --- /dev/null +++ b/lib/snap_diff/screenshot_assertion.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +require "fileutils" +require_relative "screenshot_namer" +require_relative "reporting" + +module SnapDiff + # --- Session lifecycle (per-test) --- + # + # The canonical home of the per-test session: the AssertionRegistry + # holding the assertions and new-screenshot names of the test running + # here. CapybaraScreenshotDiff.registry / .reset / + # .pending_screenshots_message are thin forwarders over these. + # + # Note: Thread.current[] is *fiber*-local, so a session is per fiber, not + # per thread. That is pre-existing, documented behaviour (issue #217); + # ADR-008 step 6 relocates the accessor without touching the semantics. + def self.session + Thread.current[:capybara_screenshot_diff_registry] ||= AssertionRegistry.new + end + + # Ends a test: hands the finished session's assertions to the reporters, + # then clears it. The one deliberate bridge between the process-global + # reporter lifecycle (SnapDiff::Reporting) and the per-test session. + def self.reset + Reporting.notify(session.assertions) + session.reset + end + + # Message to skip the test with when a new screenshot has no baseline yet + # and `pending_if_new` is enabled. Adapters call this after verifying + # screenshots, and skip the test with the returned message when present. + # + # @return [String, nil] the pending message, or nil when there is nothing to report + def self.pending_screenshots_message + return unless SnapDiff.config.pending_if_new && session.new_screenshots_present? + + "No baseline for: #{session.new_screenshots.join(", ")}. Commit the captured screenshots to record them." + end + + class ScreenshotAssertion + attr_reader :name, :args + attr_accessor :compare, :caller + + def initialize(name, **args) + @name = name + @args = args + end + + # One-line debugging summary. Never triggers the (expensive, file-touching) + # comparison itself: an unprocessed comparison shows as "pending". + def inspect + return "#<#{self.class.name} #{name.inspect} (no comparison)>" unless compare + + state = if compare.processed? + compare.difference.different? ? "different" : "matches" + else + "pending" + end + "#<#{self.class.name} #{name.inspect} #{state} new=#{compare.image_path} base=#{compare.base_image_path}>" + end + + def validate + return unless compare + + if compare.different? + "Screenshot does not match for '#{name}': #{compare.error_message}\n#{caller.join("\n")}" + else + archive_baseline! + nil + end + end + + # Commits the baseline after a passing comparison: the base image is + # moved over the actual image, so the captured screenshot becomes the + # recorded baseline again. This is the file-mutating half of the verify + # flow, kept explicit and separate from the pure "are they different?" + # question. Idempotent: a second call is a no-op. + def archive_baseline! + return unless compare && !compare.different? && compare.base_image_path.exist? + + FileUtils.mv(compare.base_image_path, compare.image_path, force: true) + end + + def validate! + error_msg = validate + + if error_msg + raise SnapDiff::ExpectationNotMet.new(error_msg, caller) + end + end + + # Verifies that all scheduled screenshots do not show any unintended differences. + # + # @param screenshots [Array(Array(Array(String), String, Comparison))] The list of match screenshots jobs. Defaults to all screenshots taken during the test. + # @return [Array, nil] Returns an array of error messages if there are screenshot differences, otherwise nil. + # @note This method is typically called at the end of a test to assert all screenshots are as expected. + def self.verify_screenshots!(screenshots) + return unless SnapDiff.config.active? && SnapDiff.config.fail_on_difference + + test_screenshot_errors = screenshots.map do |assertion| + assertion.validate + end + + test_screenshot_errors.compact! + + test_screenshot_errors.empty? ? nil : test_screenshot_errors + end + end + + class AssertionRegistry + attr_reader :assertions, :screenshot_namer, :new_screenshots + + def initialize + @assertions = [] + @new_screenshots = [] + @screenshot_namer = SnapDiff::ScreenshotNamer.new + end + + def add_assertion(assertion) + return unless assertion&.compare + + @assertions.push(assertion) + + assertion + end + + def assertions_present? + !@assertions.empty? + end + + def record_new_screenshot(name) + @new_screenshots.push(name) + end + + def new_screenshots_present? + !@new_screenshots.empty? + end + + def verify(screenshots = assertions) + result = ScreenshotAssertion.verify_screenshots!(screenshots) + return unless result + + raise SnapDiff::ExpectationNotMet.new(result.join("\n\n"), failed_assertions.first.caller) + end + + def failed_assertions + assertions.filter { |screenshot_assert| screenshot_assert.compare&.different? } + end + + def reset + @assertions.clear + @new_screenshots.clear + @screenshot_namer = SnapDiff::ScreenshotNamer.new + end + end +end diff --git a/lib/snap_diff/screenshot_matcher.rb b/lib/snap_diff/screenshot_matcher.rb new file mode 100644 index 00000000..09760446 --- /dev/null +++ b/lib/snap_diff/screenshot_matcher.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +require "snap_diff/snap_manager" +require_relative "screenshoter" +require_relative "stable_screenshoter" +require_relative "browser_helpers" +require_relative "capture/viewport" +require_relative "vcs" +require_relative "area_calculator" + +module SnapDiff + class ScreenshotMatcher + attr_reader :screenshot_full_name, :driver_options, :screenshot_format + + def initialize(screenshot_full_name, options = {}) + @screenshot_full_name = screenshot_full_name + @driver_options = SnapDiff.config.default_options.merge(options) + + @screenshot_format = @driver_options[:screenshot_format] + @snapshot = SnapDiff::SnapManager.snapshot(screenshot_full_name, @screenshot_format) + end + + def build_screenshot_assertion(skip_stack_frames: 0) + Capture::Viewport.prepare!(SnapDiff.config.window_size) + prepare_screenshot_options + check_base_screenshot + + capture_options, comparison_options = extract_capture_and_comparison_options(driver_options) + + capture_screenshot(capture_options, comparison_options) + + # Pre-computation: No need to compare without base screenshot + # NOTE: Consider to return PreValid Assertion Value Object with hard coded valid result + unless need_to_compare? + SnapDiff.session.record_new_screenshot(screenshot_full_name) + return + end + + create_screenshot_assertion(skip_stack_frames + 1, comparison_options) + end + + # Captures a screenshot without comparing it to a baseline. + def capture + Capture::Viewport.prepare!(SnapDiff.config.window_size) + prepare_screenshot_options + + capture_options, comparison_options = extract_capture_and_comparison_options(driver_options) + + @snapshot.manager.create_output_directory_for(@snapshot.path) + capture_screenshot(capture_options, comparison_options) + end + + private + + def need_to_compare? + @snapshot.base_path.exist? + end + + def prepare_screenshot_options + area_calculator = AreaCalculator.new(driver_options.delete(:crop), driver_options[:skip_area]) + + driver_options[:crop] = area_calculator.calculate_crop + driver_options[:skip_area] = area_calculator.calculate_skip_area + end + + def check_base_screenshot + @snapshot.checkout_base_screenshot + + if SnapDiff.config.fail_if_new && !@snapshot.base_path.exist? + raise SnapDiff::ExpectationNotMet.new(<<~ERROR.chomp, caller) + No existing screenshot found for #{@snapshot.base_path}! + To record baselines: RECORD_SCREENSHOTS=1 bundle exec rake test + To allow new screenshots: SnapDiff.config.fail_if_new = false + ERROR + end + end + + def capture_screenshot(capture_options, comparison_options) + screenshoter = if capture_options[:stability_time_limit] + StableScreenshoter.new(capture_options, comparison_options) + else + SnapDiff.config.screenshoter.new(capture_options, comparison_options) + end + screenshoter.take_comparison_screenshot(@snapshot) + end + + def create_screenshot_assertion(skip_stack_frames, comparison_options) + assertion = SnapDiff::ScreenshotAssertion.new(screenshot_full_name) + assertion.caller = caller(skip_stack_frames + 1) + assertion.compare = Comparison.new(@snapshot.path, @snapshot.base_path, comparison_options) + assertion + end + + # Pure partition of one options hash into [capture_options, + # comparison_options]. Unlike the previous delete-based carve, this + # method itself does not mutate its input hash. + def extract_capture_and_comparison_options(driver_options = {}) + capture_options = { + # screenshot options + capybara_screenshot_options: driver_options[:capybara_screenshot_options], + crop: driver_options[:crop], + # delivery options + screenshot_format: driver_options[:screenshot_format], + # stability options + stability_time_limit: driver_options[:stability_time_limit], + wait: driver_options[:wait] + } + + [capture_options, driver_options.except(:crop, :stability_time_limit, :wait)] + end + end +end diff --git a/lib/snap_diff/screenshot_namer.rb b/lib/snap_diff/screenshot_namer.rb new file mode 100644 index 00000000..697446d1 --- /dev/null +++ b/lib/snap_diff/screenshot_namer.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require "fileutils" +require "pathname" + +module SnapDiff + # Handles the naming, path generation, and organization of screenshots. + # This class encapsulates logic related to screenshot sections, groups, + # and counters, providing a centralized way to determine screenshot filenames + # and directories. + class ScreenshotNamer + attr_reader :section, :group + + def initialize + @section = nil + @group = nil + @counter = nil + end + + # Sets the current section for screenshots. + # @param name [String, nil] The name of the section. + def section=(name) + @section = name&.to_s + reset_group_counter + end + + # Sets the current group for screenshots and resets the counter. + # @param name [String, nil] The name of the group. + def group=(name) + @group = name&.to_s + reset_group_counter + end + + # Builds the full, unique name for a screenshot, including any counter. + # @param base_name [String] The base name for the screenshot. + # @return [String] The full screenshot name. + def full_name(base_name) + name = base_name.to_s + + if @counter + name = format("%02i_%s", @counter, name) + @counter += 1 + end + + File.join(*directory_parts.push(name.to_s)) + end + + # Returns the directory parts (section and group) for constructing paths. + # @return [Array] An array of directory names. + def directory_parts + parts = [] + parts << @section unless @section.nil? || @section.empty? + parts << @group unless @group.nil? || @group.empty? + parts + end + + private + + def reset_group_counter + @counter = (@group.nil? || @group.empty?) ? nil : 0 + end + end +end diff --git a/lib/snap_diff/screenshoter.rb b/lib/snap_diff/screenshoter.rb new file mode 100644 index 00000000..c61374e5 --- /dev/null +++ b/lib/snap_diff/screenshoter.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +require "tempfile" + +require_relative "os" +require_relative "browser_helpers" +require_relative "drivers/vips_driver" + +module SnapDiff + class Screenshoter + attr_reader :capture_options, :driver + + # @param capture_options [Hash] Options for capturing (window_size, wait, etc.) + # @param _comparison_options [Hash] Ignored since 2.1 removed driver + # selection; kept so the two-argument call sites stay unchanged. + def initialize(capture_options, _comparison_options = {}) + @capture_options = capture_options + @driver = SnapDiff::Drivers::VipsDriver.new + end + + def crop + @capture_options[:crop] + end + + def wait + @capture_options[:wait] + end + + def capybara_screenshot_options + @capture_options[:capybara_screenshot_options] || {} + end + + # Try to get screenshot from browser. + # On `stability_time_limit` it checks that page stop updating by comparison several screenshot attempts + # On reaching `wait` limit then it has been failed. On failing we annotate screenshot attempts to help to debug + def take_comparison_screenshot(snapshot) + capture_screenshot_at(snapshot) + snapshot.cleanup_attempts! + end + + PNG_EXTENSION = ".png" + + def take_screenshot(screenshot_path) + blurred_input = prepare_page_for_screenshot(timeout: wait) + + # Take browser screenshot and save + save_and_process_screenshot(screenshot_path) + + blurred_input&.click + end + + def process_screenshot(stored_path, screenshot_path) + screenshot_image = driver.from_file(stored_path) + + # TODO(uwe): Remove when chromedriver takes right size screenshots + screenshot_image = resize_if_needed(screenshot_image) if selenium_with_retina_screen? + + screenshot_image = driver.crop(crop, screenshot_image) if crop + + driver.save_image_to(screenshot_image, screenshot_path) + end + + def notice_how_to_avoid_this + unless defined?(@_csd_retina_warned) + warn "Halving retina screenshot. " \ + 'You should add "force-device-scale-factor=1" to your Chrome chromeOptions args.' + @_csd_retina_warned = true + end + end + + def prepare_page_for_screenshot(timeout:) + wait_images_loaded(timeout: timeout) if timeout + + blurred_input = BrowserHelpers.blur_from_focused_element if SnapDiff.config.blur_active_element + + BrowserHelpers.hide_caret if SnapDiff.config.hide_caret + BrowserHelpers.disable_animations if SnapDiff.config.disable_animations + + blurred_input + end + + def wait_images_loaded(timeout:) + return unless timeout + + deadline_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout + loop do + pending_image = BrowserHelpers.pending_image_to_load + break unless pending_image + + if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline_at + raise SnapDiff::ExpectationNotMet.new("Images have not been loaded after #{timeout}s: #{pending_image.inspect}", caller) + end + + sleep 0.025 + end + end + + private + + def save_and_process_screenshot(screenshot_path) + tmpfile = Tempfile.new([screenshot_path.basename.to_s, PNG_EXTENSION]) + BrowserHelpers.session.save_screenshot(tmpfile.path, **capybara_screenshot_options) + # Load saved screenshot and pre-process it + process_screenshot(tmpfile.path, screenshot_path) + ensure + tmpfile&.close! + end + + def capture_screenshot_at(snapshot) + take_screenshot(snapshot.next_attempt_path!) + + snapshot.commit_last_attempt + end + + def resize_if_needed(saved_image) + expected_image_width = SnapDiff.config.window_size[0] + return saved_image if driver.width_for(saved_image) < expected_image_width * 2 + + notice_how_to_avoid_this + + new_height = expected_image_width * driver.height_for(saved_image) / driver.width_for(saved_image) + driver.resize_image_to(saved_image, expected_image_width, new_height) + end + + def selenium_with_retina_screen? + Os::ON_MAC && BrowserHelpers.selenium? && SnapDiff.config.window_size + end + end +end diff --git a/lib/snap_diff/snap.rb b/lib/snap_diff/snap.rb new file mode 100644 index 00000000..34e5fa27 --- /dev/null +++ b/lib/snap_diff/snap.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module SnapDiff + class Snap + attr_reader :full_name, :format, :path, :base_path, :manager, :attempt_path, :prev_attempt_path, :attempts_count + + def initialize(full_name, format, manager: SnapManager.instance) + @full_name = full_name + @format = format + @path = manager.abs_path_for(Pathname.new(@full_name).sub_ext(".#{@format}")) + @base_path = @path.sub_ext(".base.#{@format}") + @manager = manager + @attempts_count = 0 + end + + def delete! + path.delete if path.exist? + base_path.delete if base_path.exist? + cleanup_diff_artifacts! + cleanup_attempts! + end + + def checkout_base_screenshot + @manager.checkout_file(path, base_path) + end + + def path_for(version = :actual) + case version + when :base + base_path + else + path + end + end + + def next_attempt_path! + @prev_attempt_path = @attempt_path + @attempt_path = path.sub_ext(sprintf(".attempt_%02i.#{format}", @attempts_count)) + ensure + @attempts_count += 1 + end + + def commit_last_attempt + @manager.move(attempt_path, path) + end + + def cleanup_attempts! + @manager.cleanup_attempts!(self) + @attempts_count = 0 + end + + def find_attempts_paths + Dir[@manager.abs_path_for("**/#{full_name}.attempt_[0-9][0-9].#{format}")] + end + + private + + def cleanup_diff_artifacts! + [ + path.sub_ext(".diff.#{format}"), + path.sub_ext(".heatmap.diff.#{format}"), + base_path.sub_ext(".diff.#{format}") + ].each { |f| f.unlink if f.exist? } + end + end +end diff --git a/lib/snap_diff/snap_manager.rb b/lib/snap_diff/snap_manager.rb new file mode 100644 index 00000000..3611fc88 --- /dev/null +++ b/lib/snap_diff/snap_manager.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +require "fileutils" + +require_relative "vcs" + +require_relative "snap" + +module SnapDiff + class SnapManager + attr_reader :root + + def initialize(root) + @root = Pathname.new(root) + @snapshots = Set.new + end + + def snapshot(screenshot_full_name, screenshot_format = "png") + Snap.new(screenshot_full_name, screenshot_format, manager: self).tap do |snapshot| + @snapshots << snapshot + end + end + + def self.snapshot(screenshot_full_name, screenshot_format = "png") + instance.snapshot(screenshot_full_name, screenshot_format) + end + + # Pure path lookup: builds the Snap (path bundle) for a name WITHOUT + # registering it for cleanup. Use this when you only need to know where + # a screenshot lives; use #snapshot when taking one that cleanup! owns. + def path_for(screenshot_full_name, screenshot_format = "png") + Snap.new(screenshot_full_name, screenshot_format, manager: self) + end + + def self.path_for(screenshot_full_name, screenshot_format = "png") + instance.path_for(screenshot_full_name, screenshot_format) + end + + def abs_path_for(relative_path) + @root / relative_path + end + + def checkout_file(path, as_path) + create_output_directory_for(as_path) unless as_path.exist? + SnapDiff::Vcs.checkout_vcs(root, path, as_path) + end + + def provision_snap_with(snap, path, version: :actual) + managed_path = snap.path_for(version) + create_output_directory_for(managed_path) unless managed_path.exist? + FileUtils.cp(path, managed_path) + end + + def create_output_directory_for(path = nil) + path ? path.dirname.mkpath : root.mkpath + end + + def cleanup! + snapshots.each do |snapshot| + cleanup_attempts!(snapshot) + snapshot.delete! + end + # With the manager memoized per thread the set would otherwise grow for + # the whole run, and a later cleanup! could delete a path an earlier + # test tracked but the current test re-provisioned. + snapshots.clear + end + + def self.cleanup! + instance.cleanup! + end + + def cleanup_attempts!(snapshot) + FileUtils.rm_rf snapshot.find_attempts_paths, secure: true + end + + def move(new_screenshot_path, screenshot_path) + FileUtils.mv(new_screenshot_path, screenshot_path, force: true) + end + + def screenshots + root.children.map { |f| f.basename.to_s } + end + + attr_reader :snapshots + + def self.screenshots + instance.screenshots + end + + def self.root + instance.root + end + + # Thread-local memoized manager (D7 fix): snapshots tracked via the + # class-method path and class-method cleanup! now share one instance, so + # cleanup! actually deletes what was tracked. Rebuilt when the configured + # manager class or screenshot root changes (tests re-root per example). + # + # Thread.current[] is fiber-local — same construct as + # SnapDiff.session; both split per fiber and must migrate + # together if async support ever lands. + # + # Scope of the fix: default-root unit tests and before_setup re-rooters + # get working cleanup. System tests do NOT — system_test_case.rb re-roots + # in `setup do`, and LIFO teardown restores the root BEFORE test_helper's + # cleanup! runs, so the root-mismatch check below rebuilds an empty + # manager there. That miss is protective: firing cleanup on the restored + # instance would delete committed baselines under + # test/fixtures/app/doc/screenshots/ that VCS rollback just restored. + # Do not reorder those teardowns without revisiting this. + def self.instance + manager_class = SnapDiff.config.manager + root = Pathname.new(SnapDiff.config.screenshot_area_abs) + + current = Thread.current[:snap_diff_manager] + unless current&.instance_of?(manager_class) && current.root == root + current = Thread.current[:snap_diff_manager] = manager_class.new(root) + end + current + end + end +end diff --git a/lib/snap_diff/stable_screenshoter.rb b/lib/snap_diff/stable_screenshoter.rb new file mode 100644 index 00000000..06f7e196 --- /dev/null +++ b/lib/snap_diff/stable_screenshoter.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module SnapDiff + class StableScreenshoter + STABILITY_OPTIONS = [:stability_time_limit, :wait] + + attr_reader :stability_time_limit, :wait + + # Initializes a new instance of StableScreenshoter + # + # This method sets up a new screenshoter with specific capture and comparison options. It validates the presence of + # `:stability_time_limit` and `:wait` in capture options and ensures that `:stability_time_limit` is less than or equal to `:wait`. + # + # @param capture_options [Hash] The options for capturing screenshots, must include `:stability_time_limit` and `:wait`. + # @param comparison_options [Hash] The options for comparing screenshots, defaults to `{}`. Same signature as {SnapDiff::Screenshoter#initialize}. + # @raise [ArgumentError] If `:wait` or `:stability_time_limit` are not provided, or if `:stability_time_limit` is greater than `:wait`. + def initialize(capture_options, comparison_options = {}) + @stability_time_limit, @wait = capture_options.fetch_values(*STABILITY_OPTIONS) + + raise ArgumentError, "wait should be provided for stable screenshots" unless wait + raise ArgumentError, "stability_time_limit should be provided for stable screenshots" unless stability_time_limit + raise ArgumentError, "stability_time_limit (#{stability_time_limit}) should be less or equal than wait (#{wait}) for stable screenshots" unless stability_time_limit <= wait + + @comparison_options = comparison_options + + @screenshoter = SnapDiff.config.screenshoter.new(capture_options.except(:stability_time_limit), @comparison_options) + end + + # Takes a comparison screenshot ensuring page stability + # + # Attempts to take a stable screenshot of the page by comparing several screenshot attempts until the page stops updating + # or the `:wait` limit is reached. If unable to achieve a stable state within the time limit, it annotates the attempts + # to aid debugging. + # + # @param snapshot Snap The snapshot details to take a stable screenshot of. + # @return [void] + # @raise [SnapDiff::UnstableImage] If a stable screenshot cannot be obtained within the specified `:wait` time. + def take_comparison_screenshot(snapshot) + result = take_stable_screenshot(snapshot) + + # We failed to get stable browser state! Generate difference between attempts to overview moving parts! + unless result + # The failed-stability outcome is data (the annotated attempts report); raising is this + # boundary's decision, not the reporter's. + # FIXME(uwe): Hand the failure to the verify path so it only reports after the test's own assertions run. + raise SnapDiff::UnstableImage.new(stability_failure_report(snapshot), caller) + end + + # store success attempt as actual screenshot + snapshot.commit_last_attempt + + # cleanup all previous attempts + snapshot.cleanup_attempts! + end + + def take_stable_screenshot(snapshot) + # We try to compare first attempt with checkout version, in order to not run next screenshots + deadline_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) + wait + + # Cleanup all previous attempts for sure + snapshot.cleanup_attempts! + + loop do + attempt_next_screenshot(snapshot) + + return true if attempt_successful?(snapshot) + return false if timeout?(deadline_at) + + sleep(stability_time_limit) + end + end + + private + + def attempt_successful?(snapshot) + return false unless snapshot.prev_attempt_path + + build_last_attempts_comparison_for(snapshot).quick_equal? + rescue ArgumentError + false + end + + def attempt_next_screenshot(snapshot) + @screenshoter.take_screenshot(snapshot.next_attempt_path!) + end + + def timeout?(deadline_at) + Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline_at + end + + def build_last_attempts_comparison_for(snapshot) + Comparison.new(snapshot.attempt_path, snapshot.prev_attempt_path, @comparison_options) + end + + # Builds the failed-stability outcome as data: annotates the attempt artifacts on disk + # and returns the report message. Does not raise. + # @return [String] the annotated attempts report + def stability_failure_report(snapshot) + require_relative "attempts_reporter" + SnapDiff::AttemptsReporter.new(snapshot, @comparison_options, {wait: wait, stability_time_limit: stability_time_limit}).generate + end + end +end diff --git a/lib/snap_diff/static.rb b/lib/snap_diff/static.rb new file mode 100644 index 00000000..18769d97 --- /dev/null +++ b/lib/snap_diff/static.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require "rack/files" +require_relative "integrations/minitest" + +module SnapDiff + def self.serve(directory, root: Dir.pwd) + Capybara.app = Rack::Files.new(directory) + SnapDiff.config.root = root + end +end diff --git a/lib/snap_diff/vcs.rb b/lib/snap_diff/vcs.rb new file mode 100644 index 00000000..49454be7 --- /dev/null +++ b/lib/snap_diff/vcs.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require "open3" +require_relative "os" + +module SnapDiff + module Vcs + def self.checkout_vcs(root, screenshot_path, checkout_path) + root_path = root.to_s + git_root, _, status = Open3.capture3("git", "-C", root_path, "rev-parse", "--show-toplevel") + return false unless status.success? + + git_root = git_root.chomp + vcs_file_path = Pathname.new(screenshot_path).expand_path.relative_path_from(Pathname.new(git_root)).to_s + + if SnapDiff.config.use_lfs + tmp_path = "#{checkout_path}.tmp" + success = system("git", "-C", root_path, "show", "HEAD:#{vcs_file_path}", out: tmp_path, err: File::NULL) + if success + system("git", "-C", root_path, "lfs", "smudge", in: tmp_path, out: checkout_path.to_s, err: File::NULL) + end + File.delete(tmp_path) if File.exist?(tmp_path) + else + success = system("git", "-C", root_path, "show", "HEAD:#{vcs_file_path}", out: checkout_path.to_s, err: File::NULL) + end + + unless success + checkout_path.delete if checkout_path.exist? + return false + end + + true + end + end +end diff --git a/lib/snap_diff/version.rb b/lib/snap_diff/version.rb new file mode 100644 index 00000000..328a025e --- /dev/null +++ b/lib/snap_diff/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module SnapDiff + VERSION = "2.0.0.beta3" +end diff --git a/matrix_test.rb b/matrix_test.rb deleted file mode 100755 index 6416eba7..00000000 --- a/matrix_test.rb +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env ruby -w -# frozen_string_literal: true - -update_gemfiles = ARGV.delete("--update") - -require "yaml" -travis = YAML.safe_load(File.read(".travis.yml")) - -def run_script(ruby, env, gemfile) - env.scan(/\b(?[A-Z_]+)="(?.+?)"/) do |key, value| - ENV[key] = value - end - puts "*" * 80 - puts "Testing #{ruby} #{gemfile} #{env}" - puts - system("chruby-exec #{ruby} -- bundle exec rake") || exit(1) - puts "Testing #{ruby} #{gemfile} OK" - puts "*" * 80 -end - -def use_gemfile(ruby, gemfile, update_gemfiles) - puts "$" * 80 - ENV["BUNDLE_GEMFILE"] = gemfile - - bundler_version = `grep -A1 "BUNDLED WITH" #{gemfile}.lock | tail -n 1` - bundler_version = "~> 2.0" if bundler_version.strip.empty? - - version_arg = "-v '#{bundler_version}'" - bundler_gem_check_cmd = "chruby-exec #{ruby} -- gem query -i -n bundler #{version_arg} >/dev/null" - system "#{bundler_gem_check_cmd} || chruby-exec #{ruby} -- gem install #{version_arg} bundler" || exit(1) - - if update_gemfiles - system "chruby-exec #{ruby} -- bundle update" - else - system "chruby-exec #{ruby} -- bundle check >/dev/null || chruby-exec #{ruby} -- bundle install" - end || exit(1) - yield - puts "$" * 80 -end - -travis["rvm"].each do |ruby| - next if /head/.match?(ruby) # ruby-install does not support HEAD installation - - puts "#" * 80 - puts "Testing #{ruby}" - puts - system "ruby-install --no-reinstall #{ruby}" || exit(1) - travis["gemfile"].each do |gemfile| - if travis["matrix"] && - (travis["matrix"]["exclude"].to_a + travis["matrix"]["allow_failures"].to_a) - .any? { |f| f["rvm"] == ruby && (f["gemfile"].nil? || f["gemfile"] == gemfile) } - puts "Skipping known failure." - next - end - use_gemfile(ruby, gemfile, update_gemfiles) do - travis["env"].each do |env| - run_script(ruby, env, gemfile) - end - end - end - puts "Testing #{ruby} OK" - puts "#" * 80 -end - -print "\033[0;32m" -print " TESTS PASSED OK!" -puts "\033[0m" diff --git a/scripts/generate_sample_report.rb b/scripts/generate_sample_report.rb new file mode 100644 index 00000000..5d77126d --- /dev/null +++ b/scripts/generate_sample_report.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +# Generate a sample HTML report for manual testing. +# Uses absolute file:// paths so images load when opened directly in a browser. + +require "bundler/setup" +require "snap_diff/integrations/minitest" +require "snap_diff/reporters/html" + +output_path = SnapDiff::Reporters::HTML.default_output_path + +# Build real comparisons using the gem's own Comparison. +# Each pair gets a unique copy of the base image to avoid annotation file conflicts. +pairs = [ + {name: "islands-map", base: "a", new: "b"}, + {name: "islands-variant", base: "a", new: "c"}, + {name: "portrait-layout", base: "portrait", new: "portrait_b"} +] + +embed = ARGV.include?("--embed") || !!ENV["CI"] +reporter = SnapDiff::Reporters::HTML.new(output_path: output_path, embed_images: embed) +fixtures = File.expand_path("../test/fixtures/images", __dir__) +tmp_dir = File.expand_path("../tmp/sample_images", __dir__) +FileUtils.mkdir_p(tmp_dir) + +assertions = pairs.map do |pair| + # Copy to tmp so each comparison has its own base/new files for annotations + base_copy = "#{tmp_dir}/#{pair[:name]}_base.png" + new_copy = "#{tmp_dir}/#{pair[:name]}_new.png" + FileUtils.cp("#{fixtures}/#{pair[:base]}.png", base_copy) + FileUtils.cp("#{fixtures}/#{pair[:new]}.png", new_copy) + + compare = SnapDiff::Comparison.new(new_copy, base_copy) + compare.processed + + SnapDiff::ScreenshotAssertion.new(pair[:name]).tap { |a| a.compare = compare } +end + +# Add passing assertions (identical images = no difference) +passing = %w[dashboard settings profile users].map do |name| + compare = SnapDiff::Comparison.new("#{fixtures}/a.png", "#{fixtures}/a.png") + compare.processed + SnapDiff::ScreenshotAssertion.new(name).tap { |a| a.compare = compare } +end + +reporter.record(assertions + passing) +reporter.finalize + +puts "Screenshots: #{reporter.failed} failed, #{reporter.passed} passed, #{reporter.total} total" diff --git a/test/capybara/screenshot/diff/drivers/chunky_png_driver_test.rb b/test/capybara/screenshot/diff/drivers/chunky_png_driver_test.rb deleted file mode 100644 index e5670557..00000000 --- a/test/capybara/screenshot/diff/drivers/chunky_png_driver_test.rb +++ /dev/null @@ -1,147 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" -require "capybara/screenshot/diff/image_compare" -require "capybara/screenshot/diff/drivers/chunky_png_driver" - -module Capybara - module Screenshot - module Diff - module Drivers - class ChunkyPNGDriverTest < ActionDispatch::IntegrationTest - include TestHelper - - test "it can be instantiated" do - assert ChunkyPNGDriver.new("images/b.png") - end - - test "it can be instantiated with dimensions" do - assert ChunkyPNGDriver.new("images/b.png", dimensions: [80, 80]) - end - - test "#different? for equal is negative" do - comp = make_comparison(:a, :a) - assert_not comp.different? - end - - test "#quick_equal? for equal is positive" do - comp = make_comparison(:a, :a) - assert comp.quick_equal? - end - - test "compare then dimensions and cleanup" do - comp = make_comparison(:a, :c) - assert comp.different? - assert_equal [11, 3, 48, 20], comp.difference_region - assert File.exist?(comp.old_file_name) - assert File.exist?(comp.annotated_old_file_name) - assert File.exist?(comp.annotated_new_file_name) - comp = make_comparison(:c, :c) - assert !comp.different? - assert !File.exist?(comp.old_file_name) - assert !File.exist?(comp.annotated_old_file_name) - assert !File.exist?(comp.annotated_new_file_name) - end - - test "compare of 1 pixel wide diff" do - comp = make_comparison(:a, :d) - assert comp.different? - assert_equal [9, 6, 9, 13], comp.difference_region - end - - test "compare with color_distance_limit above difference" do - comp = make_comparison(:a, :b, color_distance_limit: 223) - assert_not comp.different? - assert_equal 223, comp.max_color_distance.ceil - end - - test "compare with color_distance_limit below difference" do - comp = make_comparison(:a, :b, color_distance_limit: 222) - assert comp.different? - assert_equal 223, comp.max_color_distance.ceil - end - - test "compare with shift_distance_limit above difference" do - comp = make_comparison(:a, :b, shift_distance_limit: 11) - assert_not comp.different? - assert_equal 0, comp.max_shift_distance.ceil - end - - test "compare with shift_distance_limit below difference" do - comp = make_comparison(:a, :b, shift_distance_limit: 9) - assert comp.different? - assert_equal 11, comp.max_shift_distance.ceil - end - - test "quick_equal with color distance limit above max color distance" do - comp = make_comparison(:a, :b, color_distance_limit: 224) - assert comp.quick_equal? - assert_equal 223, comp.max_color_distance.ceil - end - - test "quick_equal with color distance limit" do - comp = make_comparison(:a, :b, color_distance_limit: 222) - assert !comp.quick_equal? - assert_equal 223, comp.max_color_distance.ceil - end - - test "max_color_distance a vs b" do - comp = make_comparison(:a, :b) - assert_equal 223, comp.max_color_distance.ceil - end - - test "max_color_distance a vs c" do - comp = make_comparison(:a, :c) - assert_equal 318, comp.max_color_distance.ceil - end - - test "max_color_distance a vs d" do - comp = make_comparison(:a, :d) - assert_equal 271, comp.max_color_distance.ceil - end - - test "max_color_distance 1.0" do - a_img = ChunkyPNG::Image.from_blob(File.binread("#{TEST_IMAGES_DIR}/a.png")) - a_img[9, 6] += 0x010000 - - comp = make_comparison(:a, :b) - other_img_filename = comp.new_file_name - a_img.save(other_img_filename) - - assert_equal 1, comp.max_color_distance - end - - test "size a vs a_cropped" do - comp = make_comparison(:a, :a_cropped) - comp.different? - assert_equal 6561, comp.size(comp.difference_region) - end - - # Test Interface Contracts - - test "from_file loads image from path" do - driver = ChunkyPNGDriver.new("#{Rails.root}/screenshot.png") - assert driver.from_file("#{TEST_IMAGES_DIR}/a.png") - end - - test "size requires region" do - driver = ChunkyPNGDriver.new("#{Rails.root}/screenshot.png") - assert driver.size(sample_region) - end - - private - - def make_comparison(old_img, new_img, **options) - comp = ImageCompare.new("#{Rails.root}/screenshot.png", **options.merge(driver: :chunky_png)) - set_test_images(comp, old_img, new_img) - comp - end - - def sample_region - [0, 0, 0, 0] - end - end - end - end - end -end diff --git a/test/capybara/screenshot/diff/drivers/utils_test.rb b/test/capybara/screenshot/diff/drivers/utils_test.rb deleted file mode 100644 index e79ae9ef..00000000 --- a/test/capybara/screenshot/diff/drivers/utils_test.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" -require "capybara/screenshot/diff/drivers/utils" -require "minitest/stub_const" - -module Capybara - module Screenshot - module Diff - class UtilsTest < ActiveSupport::TestCase - test "detect_available_drivers add vips when ruby-vips is available" do - Object.stub :require, ->(gem) { gem == "vips" } do - assert_includes Utils.detect_available_drivers, :vips - end - end - - test "detect_available_drivers does not add :vips when ruby-vips is unavailable" do - Object.stub_remove_const(:Vips) do - Object.stub :require, ->(gem) { gem != "vips" } do - assert_not_includes Utils.detect_available_drivers, :vips - end - end - end - - test "detect_available_drivers does not add :vips when there is no system libvips installed" do - Object.stub_remove_const(:Vips) do - Object.stub :require, ->(gem) { gem == "vips" && raise(LoadError.new("Could not ... vips")) } do - assert_not_includes Utils.detect_available_drivers, :vips - end - end - end - - test "detect_available_drivers returns vips before chunky_png if both gems are available" do - Object.stub_consts(Vips: Class.new, ChunkyPNG: Class.new) do - Object.stub :require, true do - assert_equal %i[vips chunky_png], Utils.detect_available_drivers - end - end - end - - test "detect_available_drivers add chunky_png when chunky_png is available" do - Object.stub :require, ->(gem) { gem == "chunky_png" } do - assert_includes Utils.detect_available_drivers, :chunky_png - end - end - - test "detect_available_drivers does not add chunky_png when chunky_png is not available" do - Object.stub_remove_const(:ChunkyPNG) do - Object.stub :require, ->(gem) { gem != "chunky_png" } do - assert_not_includes Utils.detect_available_drivers, :chunky_png - end - end - end - end - end - end -end diff --git a/test/capybara/screenshot/diff/drivers/vips_driver_test.rb b/test/capybara/screenshot/diff/drivers/vips_driver_test.rb deleted file mode 100644 index c330325e..00000000 --- a/test/capybara/screenshot/diff/drivers/vips_driver_test.rb +++ /dev/null @@ -1,201 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" -require "capybara/screenshot/diff/drivers/vips_driver" - -module Capybara - module Screenshot - module Diff - module Drivers - class VipsDriverTest < ActionDispatch::IntegrationTest - include TestHelper - - setup do - @new_screenshot_result = Tempfile.new(%w[screenshot .png], Rails.root) - end - - teardown do - if @new_screenshot_result - @new_screenshot_result.close - @new_screenshot_result.unlink - end - end - - test "#different? for equal is negative" do - comp = make_comparison(:a, :a) - assert_not comp.different? - end - - test "#quick_equal? for equal is positive" do - comp = make_comparison(:a, :a) - - assert comp.quick_equal? - end - - test "it can be instantiated" do - assert VipsDriver.new("images/b.png") - end - - test "it can be instantiated with dimensions" do - assert VipsDriver.new("images/b.png", dimensions: [80, 80]) - end - - test "when different does not clean runtime files" do - comp = make_comparison(:a, :c) - assert comp.different? - assert_equal [11.0, 3.0, 49.0, 21.0], comp.difference_region - assert File.exist?(comp.old_file_name) - assert File.exist?(comp.annotated_old_file_name) - assert File.exist?(comp.annotated_new_file_name) - end - - test "when equal clean runtime files" do - comp = make_comparison(:c, :c) - assert_not comp.different? - assert_not File.exist?(comp.old_file_name) - assert_not File.exist?(comp.annotated_old_file_name) - assert_not File.exist?(comp.annotated_new_file_name) - end - - test "compare of 1 pixel wide diff" do - comp = make_comparison(:a, :d) - assert comp.different? - assert_equal [9.0, 6.0, 10.0, 14.0], comp.difference_region - end - - test "compare with color_distance_limit above difference" do - comp = make_comparison(:a, :b, color_distance_limit: 255) - assert_not comp.different? - end - - test "compare with color_distance_limit below difference" do - comp = make_comparison(:a, :b, color_distance_limit: 3) - assert comp.different? - end - - test "compare with tolerance level more then area of the difference" do - comp = make_comparison(:a, :b, tolerance: 0.01) - assert comp.quick_equal? - assert_not comp.different? - end - - test "compare with tolerance level less then area of the difference" do - comp = make_comparison(:a, :b, tolerance: 0.000001) - assert_not comp.quick_equal? - assert comp.different? - end - - test "compare with median_filter_window_size when images have 1px line difference" do - comp = make_comparison(:a, :d, median_filter_window_size: 3, color_distance_limit: 8) - assert comp.quick_equal? - assert_not comp.different? - end - - test "quick_equal compare with shift_distance_limit above difference" do - comp = make_comparison(:a, :d, shift_distance_limit: 11) - assert comp.quick_equal? - end - - test "different with shift_distance_limit above difference" do - comp = make_comparison(:a, :d, shift_distance_limit: 20) - assert_not comp.different? - end - - test "quick_equal? with shift_distance_limit below difference" do - comp = make_comparison(:a, :b, shift_distance_limit: 9) - assert_not comp.quick_equal? - end - - test "different? with shift_distance_limit below difference" do - comp = make_comparison(:a, :b, shift_distance_limit: 9) - assert comp.different? - end - - test "quick_equal" do - comp = make_comparison(:a, :b) - assert_not comp.quick_equal? - end - - test "quick_equal with color distance limit below current level" do - comp = make_comparison(:a, :b, color_distance_limit: 2) - assert_not comp.quick_equal? - end - - test "quick_equal with color distance limit above current level" do - comp = make_comparison(:a, :b, color_distance_limit: 200) - assert comp.quick_equal? - end - - test "size a vs a_cropped" do - comp = make_comparison(:a, :a_cropped) - comp.different? - assert_equal 6400, comp.size(comp.difference_region) - end - - test "quick_equal compare skips difference if skip_area covers it" do - comp = make_comparison(:a, :d, skip_area: [[9, 0, 11, 80], [79, 79, 80, 80]]) - assert comp.quick_equal? - assert_not comp.different? - end - - test "quick_equal compare skips difference if skip_area does not cover it" do - comp = make_comparison(:a, :d, skip_area: [[79, 79, 80, 80], [78, 78, 80, 80]]) - assert_not comp.quick_equal? - assert comp.different? - end - - # Test Interface Contracts - - test "from_file loads image from path" do - driver = VipsDriver.new("#{Rails.root}/screenshot.png") - assert driver.from_file("#{TEST_IMAGES_DIR}/a.png") - end - - test "size requires region" do - driver = VipsDriver.new("#{Rails.root}/screenshot.png") - assert driver.size(sample_region) - end - - private - - def make_comparison(old_img, new_img, **driver_args) - result = ImageCompare.new(@new_screenshot_result.path, **driver_args.merge(driver: :vips)) - set_test_images(result, old_img, new_img) - result - end - - def sample_region - [0, 0, 0, 0] - end - end - - class VipsUtilTest < ActiveSupport::TestCase - test "segment difference without min color difference" do - old_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/a.png") - new_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/b.png") - - left, top, right, bottom = VipsDriver::VipsUtil.difference(old_image, new_image) - - assert_equal [20.0, 15.0, 30.0, 25.0], [left, top, right, bottom] - end - - test "segment difference" do - old_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/a.png") - new_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/b.png") - - left, top, right, bottom = VipsDriver::VipsUtil.difference(old_image, new_image) - - assert_equal [20.0, 15.0, 30.0, 25.0], [left, top, right, bottom] - end - - test "area of the difference" do - old_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/a.png") - new_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/d.png").bandjoin(255) - - assert_equal 8, VipsDriver::VipsUtil.difference_area(old_image, new_image, color_distance: 10) - end - end - end - end - end -end diff --git a/test/capybara/screenshot/diff/image_compare_test.rb b/test/capybara/screenshot/diff/image_compare_test.rb deleted file mode 100644 index f0673921..00000000 --- a/test/capybara/screenshot/diff/image_compare_test.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" -require "minitest/stub_const" - -module Capybara - module Screenshot - module Diff - class ImageCompareTest < ActionDispatch::IntegrationTest - include TestHelper - - test "it can be instantiated with chunky_png driver" do - comparison = ImageCompare.new("images/b.png") - assert_kind_of Drivers::ChunkyPNGDriver, comparison.driver - end - - test "it can be instantiated with explicit chunky_png adapter" do - comparison = ImageCompare.new("images/b.png", driver: :chunky_png) - assert_kind_of Drivers::ChunkyPNGDriver, comparison.driver - end - - test "it can be instantiated with vips adapter" do - comparison = ImageCompare.new("images/b.png", driver: :vips) - assert_kind_of Drivers::VipsDriver, comparison.driver - end - - test "it can be instantiated with vips adapter and tolerance option" do - comp = make_comparison(:a, :b, driver: :vips, tolerance: 0.02) - assert comp.quick_equal? - assert_not comp.different? - end - - test "could pass use tolerance for chunky_png driver" do - ImageCompare.new("images/b.png", driver: :chunky_png, tolerance: 0.02) - end - - test "it can be instantiated with dimensions" do - assert ImageCompare.new("images/b.png", dimensions: [80, 80]) - end - - test "for driver: :auto returns first from available drivers" do - comparison = ImageCompare.new("images/b.png", driver: :auto) - assert_kind_of Drivers::VipsDriver, comparison.driver - end - - test "for driver: :auto raise error if no drivers are available" do - Capybara::Screenshot::Diff.stub_const(:AVAILABLE_DRIVERS, []) do - assert_raise(RuntimeError) do - ImageCompare.new("images/b.png", driver: :auto) - end - end - end - end - end - end -end diff --git a/test/capybara/screenshot/diff/test_methods_test.rb b/test/capybara/screenshot/diff/test_methods_test.rb deleted file mode 100644 index 18b3a074..00000000 --- a/test/capybara/screenshot/diff/test_methods_test.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" - -module Capybara - module Screenshot - module Diff - class TestMethodsTest < ActionDispatch::IntegrationTest - include TestMethods - include TestHelper - - def test_assert_image_not_changed - message = assert_image_not_changed("caller", "name", make_comparison(:a, :c)) - value = RUBY_VERSION >= "2.4" ? 187.4 : 188 - assert_equal <<-MSG.strip_heredoc.chomp, message - Screenshot does not match for 'name' ({"area_size":684,"region":[11,3,48,20],"max_color_distance":#{value}}) - #{Rails.root}/screenshot.png - #{Rails.root}/screenshot.committed.png - #{Rails.root}/screenshot.latest.png - at caller - MSG - end - - def test_assert_image_not_changed_with_shift_distance_limit - message = - assert_image_not_changed("caller", "name", make_comparison(:a, :c, shift_distance_limit: 1)) - value = RUBY_VERSION >= "2.4" ? 5.0 : 5 - assert_equal <<-MSG.strip_heredoc.chomp, message - Screenshot does not match for 'name' ({"area_size":684,"region":[11,3,48,20],"max_color_distance":#{value},"max_shift_distance":15}) - #{Rails.root}/screenshot.png - #{Rails.root}/screenshot.committed.png - #{Rails.root}/screenshot.latest.png - at caller - MSG - end - - def test_screenshot_support_drivers_options - screenshot("a", driver: :vips) - end - end - end - end -end diff --git a/test/capybara/screenshot/diff_test.rb b/test/capybara/screenshot/diff_test.rb deleted file mode 100644 index 29643317..00000000 --- a/test/capybara/screenshot/diff_test.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" - -module Capybara - module Screenshot - class DiffTest < ActionDispatch::IntegrationTest - setup do - @orig_add_os_path = Capybara::Screenshot.add_os_path - Capybara::Screenshot.add_os_path = true - - @orig_add_driver_path = Capybara::Screenshot.add_driver_path - Capybara::Screenshot.add_driver_path = true - - @orig_window_size = Capybara::Screenshot.window_size - Capybara::Screenshot.window_size = [80, 80] - end - - include Capybara::Screenshot::Diff - include Diff::TestHelper - - teardown do - Capybara::Screenshot.add_os_path = @orig_add_os_path - Capybara::Screenshot.add_driver_path = @orig_add_driver_path - Capybara::Screenshot.window_size = @orig_window_size - end - - def test_that_it_has_a_version_number - refute_nil ::Capybara::Screenshot::Diff::VERSION - end - - def test_screenshot_groups_are_replaced - assert_nil @screenshot_group - screenshot_group "a" - assert_equal "a", @screenshot_group - screenshot_group "b" - assert_equal "b", @screenshot_group - end - - def test_screenshot_section_is_prepended - assert_nil @screenshot_section - assert_nil @screenshot_group - screenshot_section "a" - assert_equal "a", @screenshot_section - assert_match %r{doc/screenshots/rack_test/(macos|linux)/a}, screenshot_dir - screenshot_group "b" - assert_equal "b", @screenshot_group - assert_match %r{doc/screenshots/rack_test/(macos|linux)/a/b}, screenshot_dir - screenshot_group "c" - assert_equal "c", @screenshot_group - assert_match %r{doc/screenshots/rack_test/(macos|linux)/a/c}, screenshot_dir - end - - test "screenshot" do - screenshot_group "screenshot" - screenshot "a" - end - - def test_screenshot_with_alternate_save_path - default_path = Capybara::Screenshot.save_path - Capybara::Screenshot.save_path = "foo/bar" - screenshot_section "a" - screenshot_group "b" - screenshot "a" - assert_match %r{foo/bar/rack_test/(macos|linux)/a/b}, screenshot_dir - ensure - Capybara::Screenshot.save_path = default_path - end - - def test_screenshot_with_stability_time_limit - Capybara::Screenshot.stability_time_limit = 0.001 - screenshot "a" - ensure - Capybara::Screenshot.stability_time_limit = nil - end - - def test_screenshot_with_color_threshold - a_img = ChunkyPNG::Image.from_blob(File.binread("#{TEST_IMAGES_DIR}/a.png")) - a_val = a_img[9, 14] - a_img[9, 14] = a_val + 0x010000 + 0x000100 + 0x000001 - rev_filename = "#{Rails.root}/#{screenshot_dir}/a_0.png~" - FileUtils.mkdir_p(File.dirname(rev_filename)) - a_img.save(rev_filename) - - screenshot "a", color_distance_limit: 3 - ensure - File.delete(rev_filename) if File.exist?(rev_filename) - end - - test "full_name" do - assert_equal "a", full_name("a") - screenshot_group "b" - assert_equal "b/a", full_name("a") - screenshot_section "c" - assert_equal "c/b/a", full_name("a") - screenshot_group nil - assert_equal "c/a", full_name("a") - end - - test "full_name allows symbol" do - screenshot_group :b - assert_equal "b/a", full_name(:a) - end - - test "detect available diff drivers on the loading" do - # NOTE for tests we are loading both drivers, so we expect that all of them are available - assert_equal %i[vips chunky_png], Capybara::Screenshot::Diff::AVAILABLE_DRIVERS - end - end - end -end diff --git a/test/capybara/screenshot/screenshot_test.rb b/test/capybara/screenshot/screenshot_test.rb deleted file mode 100644 index cc242037..00000000 --- a/test/capybara/screenshot/screenshot_test.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" -require "minitest/mock" - -module Capybara - class ScreenshotTest < ActionDispatch::IntegrationTest - def test_screenshot_area_abs_is_absolute - assert Capybara::Screenshot.screenshot_area_abs.absolute? - end - - def test_root_is_a_pathname - # NOTE: We test that Rails.root is Pathname, which is true. - assert_kind_of Pathname, Capybara::Screenshot.root - assert Capybara::Screenshot.root.absolute? - end - - def test_root_could_be_assigned_relative_path - @orig_root = Capybara::Screenshot.root - - Capybara::Screenshot.root = "." - assert_kind_of Pathname, Capybara::Screenshot.root - assert Capybara::Screenshot.root.absolute? - ensure - Capybara::Screenshot.root = @orig_root - end - end -end diff --git a/test/fixtures/app/doc/screenshots/.keep b/test/fixtures/app/doc/screenshots/.keep new file mode 100644 index 00000000..e69de29b diff --git a/test/fixtures/app/doc/screenshots/index-blur_active_element-disabled.png b/test/fixtures/app/doc/screenshots/index-blur_active_element-disabled.png deleted file mode 100644 index 091e8c60..00000000 Binary files a/test/fixtures/app/doc/screenshots/index-blur_active_element-disabled.png and /dev/null differ diff --git a/test/fixtures/app/doc/screenshots/index-blur_active_element-enabled.png b/test/fixtures/app/doc/screenshots/index-blur_active_element-enabled.png deleted file mode 100644 index f8d4fbe4..00000000 Binary files a/test/fixtures/app/doc/screenshots/index-blur_active_element-enabled.png and /dev/null differ diff --git a/test/fixtures/app/doc/screenshots/index-hide_caret-disabled.png b/test/fixtures/app/doc/screenshots/index-hide_caret-disabled.png deleted file mode 100644 index 091e8c60..00000000 Binary files a/test/fixtures/app/doc/screenshots/index-hide_caret-disabled.png and /dev/null differ diff --git a/test/fixtures/app/doc/screenshots/index-hide_caret-enabled.png b/test/fixtures/app/doc/screenshots/index-hide_caret-enabled.png deleted file mode 100644 index 8c2bfb8b..00000000 Binary files a/test/fixtures/app/doc/screenshots/index-hide_caret-enabled.png and /dev/null differ diff --git a/test/fixtures/app/doc/screenshots/index.png b/test/fixtures/app/doc/screenshots/index.png deleted file mode 100644 index 46bec19d..00000000 Binary files a/test/fixtures/app/doc/screenshots/index.png and /dev/null differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/cropped_screenshot.png b/test/fixtures/app/doc/screenshots/linux/cuprite/cropped_screenshot.png new file mode 100644 index 00000000..dee2803e Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/cropped_screenshot.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/index-blur_active_element-disabled.png b/test/fixtures/app/doc/screenshots/linux/cuprite/index-blur_active_element-disabled.png new file mode 100644 index 00000000..825526c3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/index-blur_active_element-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/index-blur_active_element-enabled.png b/test/fixtures/app/doc/screenshots/linux/cuprite/index-blur_active_element-enabled.png new file mode 100644 index 00000000..a799e1fc Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/index-blur_active_element-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/index-cropped.png b/test/fixtures/app/doc/screenshots/linux/cuprite/index-cropped.png new file mode 100644 index 00000000..fe9296b3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/index-hide_caret-disabled.png b/test/fixtures/app/doc/screenshots/linux/cuprite/index-hide_caret-disabled.png new file mode 100644 index 00000000..825526c3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/index-hide_caret-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/index-hide_caret-enabled.png b/test/fixtures/app/doc/screenshots/linux/cuprite/index-hide_caret-enabled.png new file mode 100644 index 00000000..e69de29b diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/index-vips.webp b/test/fixtures/app/doc/screenshots/linux/cuprite/index-vips.webp new file mode 100644 index 00000000..47752ff3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/index-without-img-cropped.png b/test/fixtures/app/doc/screenshots/linux/cuprite/index-without-img-cropped.png new file mode 100644 index 00000000..fe9296b3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/index-without-img-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/index.png b/test/fixtures/app/doc/screenshots/linux/cuprite/index.png new file mode 100644 index 00000000..d2afd411 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/index.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/index_with_skip_area_as_array_of_css.png b/test/fixtures/app/doc/screenshots/linux/cuprite/index_with_skip_area_as_array_of_css.png new file mode 100644 index 00000000..d2afd411 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/index_with_skip_area_as_array_of_css.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/index_with_skip_area_as_array_of_css_and_p.png b/test/fixtures/app/doc/screenshots/linux/cuprite/index_with_skip_area_as_array_of_css_and_p.png new file mode 100644 index 00000000..d2afd411 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/index_with_skip_area_as_array_of_css_and_p.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index/00_index.png b/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index/00_index.png new file mode 100644 index 00000000..d2afd411 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index_as_webp/00_index-vips.webp b/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index_as_webp/00_index-vips.webp new file mode 100644 index 00000000..d2afd411 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index_as_webp/00_index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index_cropped/00_index-cropped.png b/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index_cropped/00_index-cropped.png new file mode 100644 index 00000000..fe9296b3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index_cropped/00_index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index_with_stability/00_index.png b/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index_with_stability/00_index.png new file mode 100644 index 00000000..d2afd411 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/cuprite/record_screenshot/record_index_with_stability/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/cropped_screenshot.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/cropped_screenshot.png new file mode 100644 index 00000000..cac46f2b Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/cropped_screenshot.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-blur_active_element-disabled.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-blur_active_element-disabled.png new file mode 100644 index 00000000..380a19b4 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-blur_active_element-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-blur_active_element-enabled.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-blur_active_element-enabled.png new file mode 100644 index 00000000..a0d3b82d Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-blur_active_element-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-cropped.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-cropped.png new file mode 100644 index 00000000..68cff241 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-hide_caret-disabled.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-hide_caret-disabled.png new file mode 100644 index 00000000..852fa522 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-hide_caret-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-hide_caret-enabled.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-hide_caret-enabled.png new file mode 100644 index 00000000..a0d3b82d Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-hide_caret-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-vips.webp b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-vips.webp new file mode 100644 index 00000000..47752ff3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-without-img-cropped.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-without-img-cropped.png new file mode 100644 index 00000000..68cff241 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index-without-img-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index.png new file mode 100644 index 00000000..b48868cd Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index_with_skip_area_as_array_of_css.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index_with_skip_area_as_array_of_css.png new file mode 100644 index 00000000..b48868cd Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index_with_skip_area_as_array_of_css.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index_with_skip_area_as_array_of_css_and_p.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index_with_skip_area_as_array_of_css_and_p.png new file mode 100644 index 00000000..b48868cd Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/index_with_skip_area_as_array_of_css_and_p.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index/00_index.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index/00_index.png new file mode 100644 index 00000000..d2afd411 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index_as_webp/00_index-vips.webp b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index_as_webp/00_index-vips.webp new file mode 100644 index 00000000..d2afd411 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index_as_webp/00_index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index_cropped/00_index-cropped.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index_cropped/00_index-cropped.png new file mode 100644 index 00000000..fe9296b3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index_cropped/00_index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index_with_stability/00_index.png b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index_with_stability/00_index.png new file mode 100644 index 00000000..d2afd411 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_chrome_headless/record_screenshot/record_index_with_stability/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/cropped_screenshot.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/cropped_screenshot.png new file mode 100644 index 00000000..4f8fd026 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/cropped_screenshot.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-blur_active_element-disabled.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-blur_active_element-disabled.png new file mode 100644 index 00000000..e17850a9 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-blur_active_element-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-blur_active_element-enabled.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-blur_active_element-enabled.png new file mode 100644 index 00000000..6fcff839 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-blur_active_element-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-cropped.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-cropped.png new file mode 100644 index 00000000..bda68f47 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-hide_caret-disabled.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-hide_caret-disabled.png new file mode 100644 index 00000000..6af9de06 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-hide_caret-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-hide_caret-enabled.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-hide_caret-enabled.png new file mode 100644 index 00000000..6fcff839 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-hide_caret-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-vips.webp b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-vips.webp new file mode 100644 index 00000000..d246b6c5 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-without-img-cropped.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-without-img-cropped.png new file mode 100644 index 00000000..bda68f47 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index-without-img-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/index.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index.png new file mode 100644 index 00000000..ad12eba3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/index_with_skip_area_as_array_of_css.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index_with_skip_area_as_array_of_css.png new file mode 100644 index 00000000..ad12eba3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index_with_skip_area_as_array_of_css.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/index_with_skip_area_as_array_of_css_and_p.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index_with_skip_area_as_array_of_css_and_p.png new file mode 100644 index 00000000..ad12eba3 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/index_with_skip_area_as_array_of_css_and_p.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index/00_index.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index/00_index.png new file mode 100644 index 00000000..36e78545 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index_as_webp/00_index-vips.webp b/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index_as_webp/00_index-vips.webp new file mode 100644 index 00000000..36e78545 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index_as_webp/00_index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index_cropped/00_index-cropped.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index_cropped/00_index-cropped.png new file mode 100644 index 00000000..bda68f47 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index_cropped/00_index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index_with_stability/00_index.png b/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index_with_stability/00_index.png new file mode 100644 index 00000000..36e78545 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/linux/selenium_headless/record_screenshot/record_index_with_stability/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/cropped_screenshot.png b/test/fixtures/app/doc/screenshots/macos/cuprite/cropped_screenshot.png new file mode 100644 index 00000000..a658b855 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/cropped_screenshot.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/index-blur_active_element-disabled.png b/test/fixtures/app/doc/screenshots/macos/cuprite/index-blur_active_element-disabled.png new file mode 100644 index 00000000..a9b42256 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/index-blur_active_element-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/index-blur_active_element-enabled.png b/test/fixtures/app/doc/screenshots/macos/cuprite/index-blur_active_element-enabled.png new file mode 100644 index 00000000..f8885f1a Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/index-blur_active_element-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/index-cropped.png b/test/fixtures/app/doc/screenshots/macos/cuprite/index-cropped.png new file mode 100644 index 00000000..615653f6 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/index-hide_caret-disabled.png b/test/fixtures/app/doc/screenshots/macos/cuprite/index-hide_caret-disabled.png new file mode 100644 index 00000000..a9b42256 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/index-hide_caret-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/index-hide_caret-enabled.png b/test/fixtures/app/doc/screenshots/macos/cuprite/index-hide_caret-enabled.png new file mode 100644 index 00000000..f8885f1a Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/index-hide_caret-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/index-vips.webp b/test/fixtures/app/doc/screenshots/macos/cuprite/index-vips.webp new file mode 100644 index 00000000..d5da2f3f Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/index-without-img-cropped.png b/test/fixtures/app/doc/screenshots/macos/cuprite/index-without-img-cropped.png new file mode 100644 index 00000000..615653f6 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/index-without-img-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/index.png b/test/fixtures/app/doc/screenshots/macos/cuprite/index.png new file mode 100644 index 00000000..d5d38baf Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/index.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/index_with_skip_area_as_array_of_css.png b/test/fixtures/app/doc/screenshots/macos/cuprite/index_with_skip_area_as_array_of_css.png new file mode 100644 index 00000000..99568acb Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/index_with_skip_area_as_array_of_css.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/index_with_skip_area_as_array_of_css_and_p.png b/test/fixtures/app/doc/screenshots/macos/cuprite/index_with_skip_area_as_array_of_css_and_p.png new file mode 100644 index 00000000..99568acb Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/index_with_skip_area_as_array_of_css_and_p.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index/00_index.png b/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index/00_index.png new file mode 100644 index 00000000..d5d38baf Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index_as_webp/00_index-vips.webp b/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index_as_webp/00_index-vips.webp new file mode 100644 index 00000000..d5d38baf Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index_as_webp/00_index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index_cropped/00_index-cropped.png b/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index_cropped/00_index-cropped.png new file mode 100644 index 00000000..d1fa4f38 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index_cropped/00_index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index_with_stability/00_index.png b/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index_with_stability/00_index.png new file mode 100644 index 00000000..d5d38baf Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/cuprite/record_screenshot/record_index_with_stability/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/cropped_screenshot.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/cropped_screenshot.png new file mode 100644 index 00000000..97299a30 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/cropped_screenshot.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-blur_active_element-disabled.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-blur_active_element-disabled.png new file mode 100644 index 00000000..936dcc55 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-blur_active_element-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-blur_active_element-enabled.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-blur_active_element-enabled.png new file mode 100644 index 00000000..e1fa7be1 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-blur_active_element-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-cropped.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-cropped.png new file mode 100644 index 00000000..d1fa4f38 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-hide_caret-disabled.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-hide_caret-disabled.png new file mode 100644 index 00000000..c6d04302 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-hide_caret-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-hide_caret-enabled.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-hide_caret-enabled.png new file mode 100644 index 00000000..e1fa7be1 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-hide_caret-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-vips.webp b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-vips.webp new file mode 100644 index 00000000..2c77ba1f Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-without-img-cropped.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-without-img-cropped.png new file mode 100644 index 00000000..d1fa4f38 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index-without-img-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index.png new file mode 100644 index 00000000..ab6a97cd Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index_with_skip_area_as_array_of_css.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index_with_skip_area_as_array_of_css.png new file mode 100644 index 00000000..53d0fa90 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index_with_skip_area_as_array_of_css.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index_with_skip_area_as_array_of_css_and_p.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index_with_skip_area_as_array_of_css_and_p.png new file mode 100644 index 00000000..53d0fa90 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/index_with_skip_area_as_array_of_css_and_p.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index/00_index.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index/00_index.png new file mode 100644 index 00000000..ab6a97cd Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index_as_webp/00_index-vips.webp b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index_as_webp/00_index-vips.webp new file mode 100644 index 00000000..ab6a97cd Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index_as_webp/00_index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index_cropped/00_index-cropped.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index_cropped/00_index-cropped.png new file mode 100644 index 00000000..d1fa4f38 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index_cropped/00_index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index_with_stability/00_index.png b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index_with_stability/00_index.png new file mode 100644 index 00000000..ab6a97cd Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_chrome_headless/record_screenshot/record_index_with_stability/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/cropped_screenshot.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/cropped_screenshot.png new file mode 100644 index 00000000..e2ad0111 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/cropped_screenshot.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-blur_active_element-disabled.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-blur_active_element-disabled.png new file mode 100644 index 00000000..a3607dca Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-blur_active_element-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-blur_active_element-enabled.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-blur_active_element-enabled.png new file mode 100644 index 00000000..0775227a Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-blur_active_element-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-cropped.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-cropped.png new file mode 100644 index 00000000..556025a7 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-hide_caret-disabled.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-hide_caret-disabled.png new file mode 100644 index 00000000..200f0f65 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-hide_caret-disabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-hide_caret-enabled.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-hide_caret-enabled.png new file mode 100644 index 00000000..0775227a Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-hide_caret-enabled.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-vips.webp b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-vips.webp new file mode 100644 index 00000000..8d3f7e23 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-without-img-cropped.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-without-img-cropped.png new file mode 100644 index 00000000..556025a7 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index-without-img-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/index.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index.png new file mode 100644 index 00000000..cf6c0f64 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/index_with_skip_area_as_array_of_css.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index_with_skip_area_as_array_of_css.png new file mode 100644 index 00000000..cf6c0f64 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index_with_skip_area_as_array_of_css.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/index_with_skip_area_as_array_of_css_and_p.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index_with_skip_area_as_array_of_css_and_p.png new file mode 100644 index 00000000..cf6c0f64 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/index_with_skip_area_as_array_of_css_and_p.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index/00_index.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index/00_index.png new file mode 100644 index 00000000..cf6c0f64 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index/00_index.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index_as_webp/00_index-vips.webp b/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index_as_webp/00_index-vips.webp new file mode 100644 index 00000000..cf6c0f64 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index_as_webp/00_index-vips.webp differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index_cropped/00_index-cropped.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index_cropped/00_index-cropped.png new file mode 100644 index 00000000..556025a7 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index_cropped/00_index-cropped.png differ diff --git a/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index_with_stability/00_index.png b/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index_with_stability/00_index.png new file mode 100644 index 00000000..cf6c0f64 Binary files /dev/null and b/test/fixtures/app/doc/screenshots/macos/selenium_headless/record_screenshot/record_index_with_stability/00_index.png differ diff --git a/test/fixtures/app/index-changed.html b/test/fixtures/app/index-changed.html deleted file mode 100644 index 9d02a3b1..00000000 --- a/test/fixtures/app/index-changed.html +++ /dev/null @@ -1,5 +0,0 @@ - - -

DIFFERENCE: No Image

- - diff --git a/test/fixtures/app/index-with-anim.html b/test/fixtures/app/index-with-anim.html new file mode 100644 index 00000000..a9af96d4 --- /dev/null +++ b/test/fixtures/app/index-with-anim.html @@ -0,0 +1,41 @@ + + + + + + + +

Animation

+ +
+ + + + + diff --git a/test/fixtures/app/index-without-img.html b/test/fixtures/app/index-without-img.html new file mode 100644 index 00000000..0f24812e --- /dev/null +++ b/test/fixtures/app/index-without-img.html @@ -0,0 +1,23 @@ + + + + + + +
+
+ + +
+ +
+ + +
+
+ + diff --git a/test/fixtures/app/index.html b/test/fixtures/app/index.html index 6bc13454..98f41784 100644 --- a/test/fixtures/app/index.html +++ b/test/fixtures/app/index.html @@ -1,15 +1,23 @@ + + + + -
+
- +
- +
diff --git a/test/fixtures/comparisons/a-and-b.diff.png b/test/fixtures/comparisons/a-and-b.diff.png new file mode 100644 index 00000000..123a3d97 Binary files /dev/null and b/test/fixtures/comparisons/a-and-b.diff.png differ diff --git a/test/fixtures/comparisons/a-and-b.heatmap.diff.png b/test/fixtures/comparisons/a-and-b.heatmap.diff.png new file mode 100644 index 00000000..6e7a09fa Binary files /dev/null and b/test/fixtures/comparisons/a-and-b.heatmap.diff.png differ diff --git a/test/fixtures/comparisons/a-and-c.diff.png b/test/fixtures/comparisons/a-and-c.diff.png new file mode 100644 index 00000000..af30d2c9 Binary files /dev/null and b/test/fixtures/comparisons/a-and-c.diff.png differ diff --git a/test/fixtures/comparisons/b-and-a.diff.png b/test/fixtures/comparisons/b-and-a.diff.png new file mode 100644 index 00000000..ae3fa8cb Binary files /dev/null and b/test/fixtures/comparisons/b-and-a.diff.png differ diff --git a/test/fixtures/comparisons/c-and-a.diff.png b/test/fixtures/comparisons/c-and-a.diff.png new file mode 100644 index 00000000..2b5c00d3 Binary files /dev/null and b/test/fixtures/comparisons/c-and-a.diff.png differ diff --git a/test/fixtures/images/README.md b/test/fixtures/images/README.md new file mode 100644 index 00000000..3a611dc9 --- /dev/null +++ b/test/fixtures/images/README.md @@ -0,0 +1,25 @@ +# Test Fixture Images + +These images are used by the unit test suite for image comparison testing. + +| File | Description | +|------|-------------| +| `a.png` | Base island map (80x80) | +| `b.png` | Modified island map — small region differs from `a.png` | +| `c.png` | Modified island map — different region differs from `a.png` | +| `d.png` | Modified island map — another variation | +| `a_cropped.png` | Cropped version of `a.png` | +| `portrait.png` | Portrait orientation image (3x6) | +| `portrait_b.png` | Modified portrait — differs from `portrait.png` | +| `a.webp` | WebP version of `a.png` | + +## Generated artifacts (gitignored) + +These files are generated during test runs and should NOT be committed: + +| Pattern | Description | +|---------|-------------| +| `*.base.png` | VCS checkout of committed baseline | +| `*.diff.png` | Annotated diff overlay | +| `*.base.diff.png` | Annotated base image with diff region | +| `*.heatmap.diff.png` | Heatmap of pixel differences | diff --git a/test/images/a.png b/test/fixtures/images/a.png similarity index 100% rename from test/images/a.png rename to test/fixtures/images/a.png diff --git a/test/fixtures/images/a.webp b/test/fixtures/images/a.webp new file mode 100644 index 00000000..a77ee5d0 Binary files /dev/null and b/test/fixtures/images/a.webp differ diff --git a/test/images/a_cropped.png b/test/fixtures/images/a_cropped.png similarity index 100% rename from test/images/a_cropped.png rename to test/fixtures/images/a_cropped.png diff --git a/test/images/b.png b/test/fixtures/images/b.png similarity index 100% rename from test/images/b.png rename to test/fixtures/images/b.png diff --git a/test/images/c.png b/test/fixtures/images/c.png similarity index 100% rename from test/images/c.png rename to test/fixtures/images/c.png diff --git a/test/images/d.png b/test/fixtures/images/d.png similarity index 100% rename from test/images/d.png rename to test/fixtures/images/d.png diff --git a/test/fixtures/images/portrait.png b/test/fixtures/images/portrait.png new file mode 100644 index 00000000..aa7f0ae7 Binary files /dev/null and b/test/fixtures/images/portrait.png differ diff --git a/test/fixtures/images/portrait_b.png b/test/fixtures/images/portrait_b.png new file mode 100644 index 00000000..b6089e5d Binary files /dev/null and b/test/fixtures/images/portrait_b.png differ diff --git a/test/fixtures/rspec_after_hook_order_masking_spec.rb b/test/fixtures/rspec_after_hook_order_masking_spec.rb new file mode 100644 index 00000000..21e5f378 --- /dev/null +++ b/test/fixtures/rspec_after_hook_order_masking_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require "capybara/rspec" + +# Registered BEFORE `capybara_screenshot_diff/rspec` is required below. +# RSpec's `after(:each)` hooks run in REVERSE registration order (the last +# hook registered runs first), so this user hook -- registered first, here +# -- runs AFTER the gem's own `config.after` hook, which is registered next +# when the require below loads. That reproduces the ordering hazard: by the +# time the gem's hook runs (first) and marks the example pending, this hook +# hasn't raised yet; by the time this hook raises (last), the gem has +# already committed to "pending". +RSpec.configure do |config| + config.after do + raise "deliberate failure: a config.after hook registered before the gem loaded" + end +end + +require "snap_diff/integrations/rspec" +require "support/stub_test_methods" + +unless defined?(SCREEN_SIZE) + # Deliberately does NOT require "test_helper": that pulls in + # `minitest/autorun`, whose `at_exit` hook hijacks this process's exit + # code when run standalone (as it is here, from a subprocess). Require + # only what an RSpec-only run actually needs. + require "support/setup_rails_app" + require "support/setup_capybara" + require "support/setup_capybara_drivers" +end + +# Regression fixture for the "gem's pending hook must run after the full +# user after-chain" guard in capybara_screenshot_diff/rspec.rb. +# +# Run via test/integration/rspec_after_hook_order_masking_test.rb, which +# executes this file in a fresh subprocess and asserts the run reports a +# failure (not a pending/skip), even though the example also records a new +# screenshot under `pending_if_new` and never fails on its own. +RSpec.describe "capybara_screenshot_diff/rspec after-hook ordering masking guard", type: :feature do + before do + Capybara.current_driver = Capybara.javascript_driver + Capybara.page.current_window.resize_to(*SCREEN_SIZE) + SnapDiff.config.window_size = SCREEN_SIZE + + SnapDiff.config.save_path = "doc/screenshots" + SnapDiff.config.root = Rails.root / "../test/fixtures/app" + SnapDiff.config.add_os_path = true + SnapDiff.config.add_driver_path = true + SnapDiff.config.tolerance = 0.5 + # This fixture runs standalone in its own subprocess (no + # ActiveSupport::TestCase setup forcing this off), and CI sets $CI, + # which flips the default on and would raise before we ever get here. + SnapDiff.config.fail_if_new = false + end + + it "keeps a real after-hook failure failing even when a new screenshot is pending" do + name = "pending-masking-after-hook-order" + allow(SnapDiff.config).to receive(:pending_if_new).and_return(true) + visit "/" + screenshot name + ensure + FileUtils.rm_f(SnapDiff::SnapManager.snapshot(name).path) + end +end diff --git a/test/fixtures/rspec_pending_masking_spec.rb b/test/fixtures/rspec_pending_masking_spec.rb new file mode 100644 index 00000000..c44fd190 --- /dev/null +++ b/test/fixtures/rspec_pending_masking_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require "capybara/rspec" + +require "snap_diff/integrations/rspec" +require "support/stub_test_methods" + +unless defined?(SCREEN_SIZE) + # Deliberately does NOT require "test_helper": that pulls in + # `minitest/autorun`, whose `at_exit` hook hijacks this process's exit + # code when run standalone (as it is here, from a subprocess). Require + # only what an RSpec-only run actually needs. + require "support/setup_rails_app" + require "support/setup_capybara" + require "support/setup_capybara_drivers" +end + +# Regression fixture for the "never mask a real failure with a pending +# marker" guard in capybara_screenshot_diff/rspec.rb's `config.after` hook +# (`example.exception.nil? && ...`). +# +# Run via test/integration/rspec_pending_masking_test.rb, which executes +# this file in a fresh subprocess and asserts the run reports a failure +# (not a pending/skip), even though the example also records a new +# screenshot under `pending_if_new`. +RSpec.describe "capybara_screenshot_diff/rspec pending_if_new masking guard", type: :feature do + before do + Capybara.current_driver = Capybara.javascript_driver + Capybara.page.current_window.resize_to(*SCREEN_SIZE) + SnapDiff.config.window_size = SCREEN_SIZE + + SnapDiff.config.save_path = "doc/screenshots" + SnapDiff.config.root = Rails.root / "../test/fixtures/app" + SnapDiff.config.add_os_path = true + SnapDiff.config.add_driver_path = true + SnapDiff.config.tolerance = 0.5 + # This fixture runs standalone in its own subprocess (no + # ActiveSupport::TestCase setup forcing this off), and CI sets $CI, + # which flips the default on and would raise before we ever get here. + SnapDiff.config.fail_if_new = false + end + + it "keeps a genuine failure failing even when a new screenshot is pending" do + name = "pending-masking-real-failure" + allow(SnapDiff.config).to receive(:pending_if_new).and_return(true) + visit "/" + screenshot name + + raise "deliberate failure: pending_if_new must never mask this" + ensure + FileUtils.rm_f(SnapDiff::SnapManager.snapshot(name).path) + end +end diff --git a/test/fixtures/rspec_spec.rb b/test/fixtures/rspec_spec.rb new file mode 100644 index 00000000..14fa65f7 --- /dev/null +++ b/test/fixtures/rspec_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require "capybara/rspec" + +require "snap_diff/integrations/rspec" +require "support/stub_test_methods" + +unless defined?(SCREEN_SIZE) + require "test_helper" + require "support/setup_capybara_drivers" +end + +RSpec.describe "capybara_screenshot_diff/rspec", type: :feature do + before do + Capybara.current_driver = Capybara.javascript_driver + Capybara.page.current_window.resize_to(*SCREEN_SIZE) + SnapDiff.config.window_size = SCREEN_SIZE + + SnapDiff.config.save_path = "doc/screenshots" + SnapDiff.config.root = Rails.root / "../test/fixtures/app" + SnapDiff.config.add_os_path = true + SnapDiff.config.add_driver_path = true + SnapDiff.config.tolerance = 0.5 + end + + it "should include SnapDiff::DSL in rspec" do + expect(self.class.ancestors).to include SnapDiff::DSL + end + + it "visits and compare screenshot on teardown" do + visit "/" + screenshot "index" + end + + it "use custom matcher" do + visit "/" + + expect(page).to match_screenshot("index", skip_stack_frames: 1) + end + + it "does not conflicts with rspec methods" do + expect { raise StandardError }.to raise_error(StandardError) + end + + it "marks the example pending when a new screenshot has no baseline and pending_if_new is enabled" do + name = "pending-if-new-example" + allow(SnapDiff.config).to receive(:pending_if_new).and_return(true) + visit "/" + screenshot name + ensure + FileUtils.rm_f(SnapDiff::SnapManager.snapshot(name).path) + end +end diff --git a/test/integration/browser_screenshot_test.rb b/test/integration/browser_screenshot_test.rb index a5e1572d..ae67b0ae 100644 --- a/test/integration/browser_screenshot_test.rb +++ b/test/integration/browser_screenshot_test.rb @@ -1,78 +1,281 @@ +# frozen_string_literal: true + require "system_test_case" class BrowserScreenshotTest < SystemTestCase + setup do + SnapDiff.config.blur_active_element = true + @original_tolerance = SnapDiff.config.tolerance + # Was branched on the driver (0.035 vips / 0.13 chunky_png); 2.1 left + # one backend, so only the vips figure survives. + SnapDiff.config.tolerance = 0.035 + end + + teardown do + SnapDiff.config.blur_active_element = nil + SnapDiff.config.tolerance = @original_tolerance + end + + def before_teardown + if SnapDiff.session.assertions_present? + # NOTE: We rollback new screenshots in order to remain their original state + # and only for debug mode we keep them + unless persist_comparisons? + SnapDiff.session.assertions.each(&method(:rollback_comparison_runtime_files)) + end + # NOTE: We clear tracked different errors in order to not raise error + SnapDiff.reset + end + super + end + def test_screenshot_without_changes visit "/" - screenshot "index" + assert_matches_screenshot "index" end def test_screenshot_with_changes - visit "/index-changed.html" - screenshot "index" + if ENV["RECORD_SCREENSHOTS"] + skip "we record screenshots only in" + end + visit "/" + + fill_in "First Field:", with: "Some changes in the field" + + assert_matches_screenshot("index", tolerance: nil) assert_screenshot_error_for("index") end def test_window_size_should_resize_browser_window_in_setup - assert_equal [800, 600], window_size - end - - def window_size - if page.driver.respond_to?(:window_size) - return page.driver.window_size(page.driver.current_window_handle) - end - - page.driver.browser.manage.window.size.to_a + assert_equal SCREEN_SIZE, window_size end def test_screenshot_with_hide_caret_enabled - Capybara::Screenshot.hide_caret = true - + SnapDiff.config.hide_caret = true visit "/" + fill_in "First Field:", with: "Test Input With Hide Caret" - screenshot "index-hide_caret-enabled" + + assert_matches_screenshot("index-hide_caret-enabled") ensure - Capybara::Screenshot.hide_caret = nil + SnapDiff.config.hide_caret = nil end def test_screenshot_with_hide_caret_disabled - Capybara::Screenshot.hide_caret = false + SnapDiff.config.hide_caret = false visit "/" fill_in "First Field:", with: "Test Input Without Hide Caret" # Hide caret is flaky issue, let's give more tries to take stable screenshot - screenshot "index-hide_caret-disabled", wait: Capybara.default_max_wait_time * 5 + assert_matches_screenshot "index-hide_caret-disabled", wait: Capybara.default_max_wait_time * 5 ensure - Capybara::Screenshot.hide_caret = nil + SnapDiff.config.hide_caret = nil end def test_screenshot_with_blur_active_element_enabled - Capybara::Screenshot.blur_active_element = true - + SnapDiff.config.blur_active_element = true visit "/" fill_in "First Field:", with: "Test Input With Hide Caret" - screenshot "index-blur_active_element-enabled" + + assert_matches_screenshot "index-blur_active_element-enabled" ensure - Capybara::Screenshot.blur_active_element = nil + SnapDiff.config.blur_active_element = nil end def test_screenshot_with_blur_active_element_disabled - Capybara::Screenshot.blur_active_element = false - + SnapDiff.config.blur_active_element = false visit "/" fill_in "First Field:", with: "Test Input Without Hide Caret" - screenshot "index-blur_active_element-disabled" + + assert_matches_screenshot "index-blur_active_element-disabled" ensure - Capybara::Screenshot.blur_active_element = nil + SnapDiff.config.blur_active_element = nil + end + + def test_screenshot_selected_element + visit "/" + + assert_matches_screenshot "cropped_screenshot", crop: [0, 100, 100, 200] + end + + test "skip_area accepts passing multiple coordinates as one array" do + if ENV["RECORD_SCREENSHOTS"] + skip "we record screenshots only in" + end + + visit "/" + fill_in "First Field:", with: "Changed" + fill_in "Second Field:", with: "Changed" + + assert_matches_screenshot("index", skip_area: [8, 100, 218, 140, 8, 140, 218, 180]) + + assert_no_screenshot_errors + end + + test "compare crops only when other part is not working" do + visit "/index-without-img.html" + + assert_matches_screenshot("index-without-img-cropped", crop: "form", color_distance_limit: 40) + + assert_no_screenshot_errors + end + + test "crop accepts css selector" do + visit "/index-without-img.html" + + if ENV["RECORD_SCREENSHOTS"] + skip "we record screenshots only in" + end + + assert_matches_screenshot("index-without-img-cropped", crop: "form") + + assert_no_screenshot_errors + end + + test "skip_area accepts css selector" do + visit "/" + + assert_matches_screenshot("index_with_skip_area_as_array_of_css", skip_area: ["form"]) + assert_matches_screenshot("index_with_skip_area_as_array_of_css_and_p", skip_area: [[90, 950, 180, 1000], "form"]) + end + + test "skip_area accepts css selector and ignores changes" do + if ENV["RECORD_SCREENSHOTS"] + skip "we record screenshots only in" + end + + visit "/" + + fill_in "First Field:", with: "Changed" + fill_in "Second Field:", with: "Changed" + + assert_matches_screenshot("index", skip_area: "form") + + assert_no_screenshot_errors + end + + test "cropped screenshot" do + visit "/index.html" + + assert_matches_screenshot("index-cropped", skip_area: "#first-field", crop: "form") + end + + test "skip_area converts coordinates to be relative to cropped region" do + if ENV["RECORD_SCREENSHOTS"] + skip "we record screenshots only in" + end + + visit "/index.html" + fill_in "First Field:", with: "New Change" + fill_in "Second Field:", with: "New Change" + + assert_matches_screenshot("index-cropped", skip_area: "#first-field", crop: "form", tolerance: 0.001) + + assert_not_predicate( + SnapDiff.session.failed_assertions, + :empty?, + "differences have not been found when they should have been" + ) + end + + test "skip_area by css selectors" do + if ENV["RECORD_SCREENSHOTS"] + skip "we record screenshots only in" + end + + visit "/" + fill_in "First Field:", with: "Test Input With Hide Caret" + + assert_matches_screenshot("index", skip_area: "form") + assert_no_screenshot_errors + end + + test "crop and skip_area by css selectors" do + if ENV["RECORD_SCREENSHOTS"] + skip "we record screenshots only in" + end + + visit "/index-without-img.html" + fill_in "First Field:", with: "Test Input With Hide Caret" + + assert_matches_screenshot("index-without-img-cropped", skip_area: "input", crop: "form") + + assert_no_screenshot_errors + end + + test "bounds_for_css for multiple elements returns all areas" do + visit "/" + + label_bounds = SnapDiff::BrowserHelpers.bounds_for_css("label") + + assert_equal 2, label_bounds.size + end + + test "rect_for for multiple elements returns first visible element" do + visit "/index.html" + + label_bound = rect_for("label") + + assert_equal 4, label_bound.size + end + + test "animated example" do + optional_test + + visit "/index-with-anim.html" + + assert_raises SnapDiff::UnstableImage, "Could not get stable screenshot within 0.5s:" do + # We need to run several times, + # because quick_equal could produce incorrect result, + # because of the same size screenshots + 10.times do + assert_matches_screenshot "index-with-anim", stability_time_limit: 0.33, wait: 0.5, tolerance: nil + end + end + ensure + SnapDiff::SnapManager.snapshot("index-with-anim").delete! + end + + def test_await_all_images_are_loaded + visit "/index.html" + assert_raises ::Minitest::Assertion do + SnapDiff::BrowserHelpers.stub(:pending_image_to_load, "http://127.0.0.1:62815/image.png") do + assert_matches_screenshot :index + end + end + assert_no_screenshot_errors end private + def rect_for(css_selector) + SnapDiff::BrowserHelpers.all_visible_regions_for(css_selector).first + end + + def window_size + if page.driver.respond_to?(:window_size) + return page.driver.window_size(page.driver.current_window_handle) + end + + page.driver.browser.manage.window.size.to_a + end + def assert_screenshot_error_for(screenshot_name) - assert_equal 1, @test_screenshots.length, "expecting to have just one difference" - assert_equal screenshot_name, @test_screenshots[0][1], "index screenshot should have difference for changed page" + assertions = SnapDiff.session.failed_assertions + + assert_equal 1, assertions&.length, "expecting to have just one difference" + assert_equal screenshot_name, assertions[0].name, "index screenshot should have difference for changed page" end - # TODO: Add test for stability to await while image are loading - # TODO: Allow to run tests for browser which was selected by ENV: chrome, firefox, cuprite and other + def assert_no_screenshot_errors + screenshots = SnapDiff.session.failed_assertions + + error_messages = screenshots.map { |assertion| assertion.compare.error_message } + + assert( + screenshots.empty?, + "expecting not to have any difference. But got next:\n\n#{error_messages.join(";\n")}" + ) + end end diff --git a/test/integration/record_screenshot_test.rb b/test/integration/record_screenshot_test.rb new file mode 100644 index 00000000..03b78580 --- /dev/null +++ b/test/integration/record_screenshot_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "system_test_case" + +class RecordScreenshotTest < SystemTestCase + setup do + screenshot_section class_name.underscore.sub(/(_feature|_system)?_test$/, "") unless SnapDiff.session.screenshot_namer.section + screenshot_group name[5..] unless SnapDiff.session.screenshot_namer.group + + @original_tolerance = SnapDiff.config.tolerance + # Was branched on the driver (0.035 vips / 0.7 chunky_png); 2.1 left + # one backend, so only the vips figure survives. + SnapDiff.config.tolerance = 0.035 + end + + teardown do + SnapDiff.config.blur_active_element = nil + SnapDiff.config.tolerance = @original_tolerance + end + + def test_record_index + visit "/" + + screenshot "index" + end + + def test_record_index_cropped + visit "/" + + screenshot "index-cropped", crop: "form" + end + + def test_record_index_as_webp + skip "VIPS not present. Skipping VIPS driver tests." unless defined?(Vips) + + visit "/" + + screenshot "index-vips", screenshot_format: "webp" + end + + def test_record_index_with_stability + visit "/" + + screenshot "index", stability_time_limit: 0.1, wait: (RUBY_ENGINE == "jruby") ? 10 : 1 + end +end diff --git a/test/integration/rspec_after_hook_order_masking_test.rb b/test/integration/rspec_after_hook_order_masking_test.rb new file mode 100644 index 00000000..2a8a8f61 --- /dev/null +++ b/test/integration/rspec_after_hook_order_masking_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "test_helper" +require "open3" +require "json" +require "tmpdir" + +# Regression test for the "gem's pending hook must run after the full user +# after-chain" guard in capybara_screenshot_diff/rspec.rb. +# +# This must run the fixture in a genuinely separate process: RSpec's +# `after(:each)` hooks run in REVERSE registration order, so a +# `config.after` hook registered BEFORE `capybara_screenshot_diff/rspec` +# is required runs AFTER the gem's own `config.after` hook. If the gem's +# hook isn't guaranteed to run last, it commits to "pending" before the +# user's hook has had a chance to raise, and that later raise gets folded +# into `pending_exception` (via `Example#set_exception`) rather than +# `example.exception`, silently masking the real failure. That only +# reproduces end-to-end via a real `RSpec::Core::Runner.run` exit status + +# report, not via an in-process assertion on some Ruby object. +class RspecAfterHookOrderMaskingTest < ActiveSupport::TestCase + test "a real failure from an after hook registered before the gem loaded is never masked as pending" do + spec_file = file_fixture("rspec_after_hook_order_masking_spec.rb").to_s + + Dir.mktmpdir do |dir| + json_path = File.join(dir, "result.json") + + script = <<~RUBY + require "rspec/core" + exit RSpec::Core::Runner.run([#{spec_file.inspect}, "--format", "json", "--out", #{json_path.inspect}], $stderr, $stdout) + RUBY + + out, status = Open3.capture2e(RbConfig.ruby, "-Ilib", "-Itest", "-e", script) + + refute status.success?, "expected the after-hook-order-masking fixture to fail the process, got:\n#{out}" + + summary = JSON.parse(File.read(json_path))["summary"] + + assert_equal 1, summary["example_count"], out + assert_equal 1, summary["failure_count"], + "expected the after hook's real failure to be reported as a failure, not masked:\n#{out}" + assert_equal 0, summary["pending_count"], + "expected pending_if_new to NOT mask the after hook's failure as pending:\n#{out}" + end + end +end diff --git a/test/integration/rspec_pending_masking_test.rb b/test/integration/rspec_pending_masking_test.rb new file mode 100644 index 00000000..32369363 --- /dev/null +++ b/test/integration/rspec_pending_masking_test.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require "test_helper" +require "open3" +require "json" +require "tmpdir" + +# Regression test for the "never mask a real failure with a pending +# marker" guard in capybara_screenshot_diff/rspec.rb's `config.after` hook. +# +# This must run the fixture in a genuinely separate process: RSpec's own +# after-hook exception handling folds a `skip` raised from an after-hook +# into the example's pending state (it resets `example.exception` because +# `Pending.mark_skipped!` already flipped `example.pending?` to true before +# the skip exception is caught). That only reproduces end-to-end via a +# real `RSpec::Core::Runner.run` exit status + report, not via an +# in-process assertion on some Ruby object. +class RspecPendingMaskingTest < ActiveSupport::TestCase + test "a genuine example failure is never masked as pending by pending_if_new" do + spec_file = file_fixture("rspec_pending_masking_spec.rb").to_s + + Dir.mktmpdir do |dir| + json_path = File.join(dir, "result.json") + + script = <<~RUBY + require "rspec/core" + exit RSpec::Core::Runner.run([#{spec_file.inspect}, "--format", "json", "--out", #{json_path.inspect}], $stderr, $stdout) + RUBY + + out, status = Open3.capture2e(RbConfig.ruby, "-Ilib", "-Itest", "-e", script) + + refute status.success?, "expected the pending-masking fixture to fail the process, got:\n#{out}" + + summary = JSON.parse(File.read(json_path))["summary"] + + assert_equal 1, summary["example_count"], out + assert_equal 1, summary["failure_count"], + "expected the deliberate failure to be reported as a failure, not masked:\n#{out}" + assert_equal 0, summary["pending_count"], + "expected pending_if_new to NOT mask the failure as pending:\n#{out}" + end + end +end diff --git a/test/integration/rspec_test.rb b/test/integration/rspec_test.rb new file mode 100644 index 00000000..c51e2f56 --- /dev/null +++ b/test/integration/rspec_test.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require "system_test_case" + +class RspecTest < SystemTestCase + test "RSpec integration runs successfully with capybara-screenshot-diff" do + # Ensure that the RSpec module is loaded + require "rspec/core" + + # Run the RSpec spec file + capture_output = StringIO.new + spec_file = file_fixture("rspec_spec.rb").to_s + rspec_status = RSpec::Core::Runner.run([spec_file], capture_output, capture_output) + + assert_equal 0, rspec_status, "RSpec tests failed:\n#{capture_output.string}" + end +end diff --git a/test/integration/test_methods_system_test.rb b/test/integration/test_methods_system_test.rb index ffafbff4..2f652999 100644 --- a/test/integration/test_methods_system_test.rb +++ b/test/integration/test_methods_system_test.rb @@ -1,38 +1,31 @@ # frozen_string_literal: true -require "system_test_case" - # NOTE: For this test we need only chrome browser, # because we can spot problem by counting running chrome driver processes -require "action_pack/version" -unless [ActionPack::VERSION::MAJOR, ActionPack::VERSION::MINOR].join >= "60" && ENV["CAPYBARA_DRIVER"].include?("selenium_chrome") - warn "Regression only for 6x Rails and driven_by construction" +unless ENV["CAPYBARA_DRIVER"].include?("selenium_chrome") return end +warn "Regression test for `driven_by :selenium_chrome` construction." +require "system_test_case" +require "action_pack/version" require "objspace" -module Capybara - module Screenshot - module Diff - class TestMethodsSystemTest < ActionDispatch::SystemTestCase - include TestMethods - include TestHelper +class TestMethodsSystemTest < ActionDispatch::SystemTestCase + include SnapDiff::DSL + include DSLStub - driven_by :selenium, using: :headless_chrome + driven_by :selenium, using: :headless_chrome - def test_current_capybara_driver_class_do_not_spawn_new_process_when_we_use_system_test_cases - # NOTE: There is possible that we have several drivers usage in the one suite, - # so each of them will have separate instance - other_activated_drivers = ObjectSpace.each_object(Capybara::Selenium::Driver).count + def test_current_capybara_driver_class_do_not_spawn_new_process_when_we_use_system_test_cases + # NOTE: There is possible that we have several drivers usage in the one suite, + # so each of them will have separate instance + other_activated_drivers = ObjectSpace.each_object(Capybara::Selenium::Driver).count - 3.times { current_capybara_driver_class } + 3.times { SnapDiff::BrowserHelpers.current_capybara_driver_class } - run_chrome_drivers = ObjectSpace.each_object(Capybara::Selenium::Driver).count - assert run_chrome_drivers.positive? - assert run_chrome_drivers - other_activated_drivers <= 1 - end - end - end + run_chrome_drivers = ObjectSpace.each_object(Capybara::Selenium::Driver).count + assert run_chrome_drivers.positive? + assert run_chrome_drivers - other_activated_drivers <= 1 end end diff --git a/test/support/driver_contract_tests.rb b/test/support/driver_contract_tests.rb new file mode 100644 index 00000000..3028ba4d --- /dev/null +++ b/test/support/driver_contract_tests.rb @@ -0,0 +1,234 @@ +# frozen_string_literal: true + +require "active_support/concern" + +# The image-backend contract, exercised against SnapDiff::Drivers::VipsDriver. +# +# 2.1 removed the driver abstraction, so this is no longer a SHARED contract -- +# there is one includer. It is kept, not folded into vips_driver_test.rb, +# because what it pins is still worth pinning and is a different KIND of claim +# from the tests there: those cover vips-specific mechanics (annotation files, +# class-level mask math, difference regions), while these are the behavioural +# obligations the rest of the gem relies on -- Comparison, ImagePreprocessor, +# Screenshoter and AnnotationService all call these methods and would break +# silently if a signature or a slot order drifted. +# +# What did NOT survive the collapse: the `supports?(feature)` capability probe +# (it existed so ImagePreprocessor could ask whether a driver implemented +# median filtering -- chunky_png did not, vips does, so the question is gone), +# and the value of running the same expectations against two implementations. +# Include in any driver test class that uses DSLStub (provides make_comparison). +module DriverContractTests + extend ActiveSupport::Concern + + included do + test "[contract] quick_equal? returns true for identical images" do + comp = make_comparison(:a, :a) + assert comp.quick_equal? + end + + test "[contract] different? returns false for identical images" do + comp = make_comparison(:a, :a) + assert_not comp.different? + end + + test "[contract] different? returns true for different images" do + comp = make_comparison(:a, :c) + assert comp.different? + end + + test "[contract] different? generates annotated images for different images" do + comp = make_comparison(:a, :c) + assert comp.different? + + assert File.exist?(comp.reporter.annotated_base_image_path) + assert File.exist?(comp.reporter.annotated_image_path) + end + + test "[contract] different? does not create annotated images for identical images" do + comp = make_comparison(:c, :c) + assert_not comp.different? + + assert_not File.exist?(comp.reporter.annotated_base_image_path) + assert_not File.exist?(comp.reporter.annotated_image_path) + end + + # Method presence / signature -------------------------------------------- + # Pins the interface the rest of the gem calls into. It survived the + # inheritance -> mixin -> single-class collapse unchanged, which is the + # point: these names are what Comparison, ImagePreprocessor, Screenshoter + # and AnnotationService depend on. + + test "[contract] driver implements the image-backend interface" do + driver = make_comparison(:a, :a).driver + + %i[ + load_images add_black_box find_difference_region crop from_file + save_image_to resize_image_to draw_rectangles same_pixels? + same_dimension? height_for width_for image_area_size dimension + filter_image_with_median + ].each do |method_name| + assert_respond_to driver, method_name, "driver should implement ##{method_name}" + end + end + + test "[contract] #find_difference_region, #same_pixels?, and #same_dimension? each take a single comparison argument" do + driver = make_comparison(:a, :a).driver + + assert_equal 1, driver.method(:find_difference_region).arity + assert_equal 1, driver.method(:same_pixels?).arity + assert_equal 1, driver.method(:same_dimension?).arity + end + + test "[contract] #load_images takes exactly the old and new file paths" do + driver = make_comparison(:a, :a).driver + assert_equal 2, driver.method(:load_images).arity + end + + # load_images ------------------------------------------------------------- + # Strengthened (vs. #204's original): a.png/b.png share dimensions, so a + # swapped return would go undetected. Uses fixtures with DIFFERENT + # dimensions so slot identity is verifiable by content. + + test "[contract] #load_images returns [old_image, new_image] without swapping slots" do + driver = make_comparison(:a, :a).driver + old_path = TEST_IMAGES_DIR / "a.png" # 80x80 + new_path = TEST_IMAGES_DIR / "a_cropped.png" # 80x60 + + old_image, new_image = driver.load_images(old_path, new_path) + + assert_equal [80, 80], driver.dimension(old_image) + assert_equal [80, 60], driver.dimension(new_image) + end + + # find_difference_region result shape -------------------------------------- + + test "[contract] different? exposes a Difference with a region, meta hash, and the comparison" do + comp = make_comparison(:a, :c) + assert comp.different? + + difference = comp.difference + assert_kind_of SnapDiff::ComparisonResult, difference + assert_not_nil difference.region + assert_kind_of Hash, difference.meta + assert_equal comp.driver, difference.comparison.driver + end + + # Dimension handling -------------------------------------------------------- + + test "[contract] #same_dimension? returns true when images share dimensions" do + driver = make_comparison(:a, :a).driver + old_image, new_image = driver.load_images(TEST_IMAGES_DIR / "a.png", TEST_IMAGES_DIR / "b.png") + comparison = SnapDiff::Comparison::Images.new(new_image, old_image, {}, driver) + + assert driver.same_dimension?(comparison) + end + + test "[contract] #same_dimension? returns false when images differ in dimensions" do + driver = make_comparison(:a, :a).driver + old_image, new_image = driver.load_images(TEST_IMAGES_DIR / "a.png", TEST_IMAGES_DIR / "a_cropped.png") + comparison = SnapDiff::Comparison::Images.new(new_image, old_image, {}, driver) + + assert_not driver.same_dimension?(comparison) + end + + test "[contract] #width_for, #height_for, #dimension, and #image_area_size agree with each other" do + driver = make_comparison(:a, :a).driver + image = driver.from_file(TEST_IMAGES_DIR / "a.png") + + assert_equal [driver.width_for(image), driver.height_for(image)], driver.dimension(image) + assert_equal driver.width_for(image) * driver.height_for(image), driver.image_area_size(image) + end + + # Same-image fast paths ------------------------------------------------- + + test "[contract] #same_pixels? returns true for pixel-identical images and false otherwise" do + driver = make_comparison(:a, :a).driver + + old_image, new_image = driver.load_images(TEST_IMAGES_DIR / "a.png", TEST_IMAGES_DIR / "a.png") + same_comparison = SnapDiff::Comparison::Images.new(new_image, old_image, {}, driver) + assert driver.same_pixels?(same_comparison) + + other_old_image, other_new_image = driver.load_images(TEST_IMAGES_DIR / "a.png", TEST_IMAGES_DIR / "c.png") + different_comparison = SnapDiff::Comparison::Images.new(other_new_image, other_old_image, {}, driver) + assert_not driver.same_pixels?(different_comparison) + end + + # Option handling ----------------------------------------------------------- + # Thresholds below are chosen with headroom on the driver's actual + # measurements for the a/b and a/d fixture pairs. + + test "[contract] tolerance option treats small differences as equal" do + comp = make_comparison(:a, :b, tolerance: 0.5) + assert_not comp.different? + end + + test "[contract] tolerance option still flags differences exceeding the ratio" do + comp = make_comparison(:a, :b, tolerance: 0.001) + assert comp.different? + end + + test "[contract] color_distance_limit option treats close colors as equal" do + comp = make_comparison(:a, :b, color_distance_limit: 255) + assert_not comp.different? + end + + test "[contract] color_distance_limit option still flags colors beyond the limit" do + comp = make_comparison(:a, :b, color_distance_limit: 1) + assert comp.different? + end + + test "[contract] skip_area option excludes covered regions from comparison" do + comp = make_comparison( + :a, + :d, + skip_area: [ + Region.from_edge_coordinates(9, 0, 11, 80), + Region.from_edge_coordinates(79, 79, 80, 80) + ] + ) + assert_not comp.different? + end + + test "[contract] skip_area option still detects differences outside the skipped regions" do + comp = make_comparison( + :a, + :d, + skip_area: [ + Region.from_edge_coordinates(79, 79, 80, 80), + Region.from_edge_coordinates(78, 78, 80, 80) + ] + ) + assert comp.different? + end + + # Error behavior on missing files -------------------------------------------- + + test "[contract] raises ArgumentError when the base (original) image is missing" do + error = assert_raises(ArgumentError) do + SnapDiff::Comparison.new(TEST_IMAGES_DIR / "a.png", TEST_IMAGES_DIR / "does_not_exist.png") + end + assert_match(/no original \(base\) screenshot/, error.message) + end + + test "[contract] raises ArgumentError when the new image is missing" do + error = assert_raises(ArgumentError) do + SnapDiff::Comparison.new(TEST_IMAGES_DIR / "does_not_exist.png", TEST_IMAGES_DIR / "a.png") + end + assert_match(/no new screenshot/, error.message) + end + + # Resize behavior ------------------------------------------------------- + # Behavioral (vs. #204's method-presence check above): asserts the actual + # output dimensions, not just that #resize_image_to responds. + + test "[contract] resize_image_to resizes a non-square source to the exact requested non-square dimensions" do + driver = make_comparison(:a, :a).driver + source = driver.from_file(TEST_IMAGES_DIR / "portrait.png") # 3x6, non-square + + resized = driver.resize_image_to(source, 40, 30) + + assert_equal [40, 30], driver.dimension(resized) + end + end +end diff --git a/test/support/dsl_stub.rb b/test/support/dsl_stub.rb new file mode 100644 index 00000000..5b7db45a --- /dev/null +++ b/test/support/dsl_stub.rb @@ -0,0 +1,65 @@ +require "active_support/concern" + +# Plain top-level module: test scaffolding has no business reopening a gem +# namespace -- least of all the v1 one 3.0 deletes. +module DSLStub + extend ActiveSupport::Concern + + def setup + super + @manager = SnapDiff::SnapManager.new(SnapDiff.config.root / "doc/screenshots") + SnapDiff.config.screenshoter = ScreenshoterStub + end + + def teardown + @manager.cleanup! + SnapDiff.config.screenshoter = SnapDiff::Screenshoter + SnapDiff.reset + super + end + + # Prepare comparison images and build a Comparison for them + def make_comparison(fixture_base_image, fixture_new_image = nil, destination: "screenshot", **options) + fixture_new_image ||= fixture_base_image + snap = create_snapshot_for(fixture_base_image, fixture_new_image, name: destination) + SnapDiff::Comparison.new(snap.path, snap.base_path, **options) + end + + # Prepare images for comparison in a test + # + # @param snap [SnapDiff::Snap] the snapshot to prepare + # @param expected [String] the base name of the original base image + # @param actual [String] the base name of the original new image + def set_test_images(snap, expected, actual) + @manager.provision_snap_with(snap, fixture_image_path_from(actual, snap.format), version: :actual) + @manager.provision_snap_with(snap, fixture_image_path_from(expected, snap.format), version: :base) + end + + ImageCompareStub = Struct.new( + :driver, :driver_options, :shift_distance_limit, :quick_equal?, :different?, :reporter, keyword_init: true + ) + + def build_image_compare_stub(equal: true) + ImageCompareStub.new( + driver: ::Minitest::Mock.new, + reporter: ::Minitest::Mock.new, + driver_options: SnapDiff.config.default_options, + shift_distance_limit: nil, + quick_equal?: equal, + different?: !equal + ) + end + + def take_stable_screenshot_with(snap, stability_time_limit: 0.01, wait: 10) + screenshoter = SnapDiff::StableScreenshoter.new({stability_time_limit: stability_time_limit, wait: wait}) + screenshoter.take_stable_screenshot(snap) + end + + def create_snapshot_for(expected, actual = nil, name: nil) + actual ||= expected + name ||= "#{actual}_#{Time.now.nsec}" + @manager.snapshot(name).tap do |snap| + set_test_images(snap, expected, actual) + end + end +end diff --git a/test/support/non_minitest_assertions.rb b/test/support/non_minitest_assertions.rb new file mode 100644 index 00000000..866faabe --- /dev/null +++ b/test/support/non_minitest_assertions.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "snap_diff/dsl" + +# Stands in for a non-minitest adapter: the raw session lifecycle a host +# framework's integration has to drive itself. +module NonMinitest + module Assertions + def self.included(klass) + klass.include SnapDiff::DSL + + klass.setup do + SnapDiff::BrowserHelpers.resize_window_if_needed + end + + klass.teardown do + SnapDiff.session.verify + ensure + SnapDiff.reset + end + end + end +end diff --git a/test/support/screenshoter_stub.rb b/test/support/screenshoter_stub.rb new file mode 100644 index 00000000..b0ce4333 --- /dev/null +++ b/test/support/screenshoter_stub.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require "snap_diff/screenshoter" + +class ScreenshoterStub < SnapDiff::Screenshoter + def pending_image_to_load + end + + # Stub of the Capybara's save_screenshot + def save_screenshot(path) + source_image = path.basename.to_path + source_image.slice!(/\.attempt_\d+/) + source_image.slice!(/^\d\d_/) + source_image.slice!(/_\d+(?=\.)/) + + FileUtils.mkdir_p(path.dirname) + FileUtils.cp(File.expand_path(source_image, TEST_IMAGES_DIR), path) + + path + end + + def evaluate_script(*) + # Do nothing + end + + def prepare_page_for_screenshot(**) + nil + end + + def take_screenshot(screenshot_path) + stored_path = save_screenshot(screenshot_path) # rubocop:disable Lint/Debugger + + process_screenshot(stored_path, screenshot_path) + end +end diff --git a/test/support/setup_capybara.rb b/test/support/setup_capybara.rb new file mode 100644 index 00000000..552ec802 --- /dev/null +++ b/test/support/setup_capybara.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "support/setup_rails_app" +require "capybara" +require "puma" + +# JRuby joins every live thread at interpreter teardown (`Ruby.tearDown` -> +# `ThreadService.teardown` -> `Thread#join`) where MRI simply kills them. +# Puma's reactor thread parks in a native kqueue/epoll poll that no +# interrupt can wake, so a Capybara-booted Puma keeps the process alive +# forever after an otherwise green run -- issue #244. Capybara's stock +# `:puma` block builds its `Puma::Server` in a block-local and joins it, +# leaving nothing anyone could stop; this equivalent block keeps the handle +# so the suite can shut the server down for real. +module StoppablePuma + BOOTED = [] + + def self.stop_all + BOOTED.each { |server| server.stop(true) } + BOOTED.clear + end +end + +Capybara.register_server :stoppable_puma do |app, port, host, **options| + options = {min_threads: 0, max_threads: 4, log_writer: Puma::LogWriter.strings}.merge(options) + Puma::Server.new(app, nil, options).tap do |server| + server.add_tcp_listener(host, port) + StoppablePuma::BOOTED << server + end.run.join +end + +Capybara.app = Rails.application +Capybara.default_max_wait_time = 1 +Capybara.disable_animation = true +Capybara.server = :stoppable_puma +Capybara.threadsafe = true + +# `minitest/autorun` owns this process's exit through its own `at_exit`, +# registered before this file loads -- a plain `at_exit` here would run +# BEFORE the suite does. The RSpec fixtures that load this file run +# standalone without minitest and need the plain hook. +if defined?(Minitest.after_run) + Minitest.after_run { StoppablePuma.stop_all } +else + at_exit { StoppablePuma.stop_all } +end diff --git a/test/support/setup_capybara_drivers.rb b/test/support/setup_capybara_drivers.rb index 2b1c4c50..46dfb17c 100644 --- a/test/support/setup_capybara_drivers.rb +++ b/test/support/setup_capybara_drivers.rb @@ -1,78 +1,95 @@ # frozen_string_literal: true -# Try to load other drivers if they are available +require "snap_diff/os" ENV["CAPYBARA_DRIVER"] ||= "cuprite" +SCREEN_SIZE = [800, 600] +if ENV["CAPYBARA_DRIVER"] == "selenium_chrome_headless" && SnapDiff::Os.name == "linux" + SCREEN_SIZE[1] += 87 # Add extra space for address field etc. +end +BROWSERS = {cuprite: "chrome", selenium_headless: "firefox", selenium_chrome_headless: "chrome"} + +CHROME_ARGS = { + "allow-running-insecure-content" => nil, + "autoplay-policy" => "user-gesture-required", + "disable-add-to-shelf" => nil, + "disable-background-networking" => nil, + "disable-background-timer-throttling" => nil, + "disable-backgrounding-occluded-windows" => nil, + "disable-breakpad" => nil, + "disable-checker-imaging" => nil, + "disable-client-side-phishing-detection" => nil, + "disable-component-extensions-with-background-pages" => nil, + "disable-datasaver-prompt" => nil, + "disable-default-apps" => nil, + "disable-desktop-notifications" => nil, + "disable-dev-shm-usage" => nil, + "disable-domain-reliability" => nil, + "disable-extensions" => nil, + "disable-features" => "TranslateUI,BlinkGenPropertyTrees", + "disable-gpu" => nil, + "disable-hang-monitor" => nil, + "disable-infobars" => nil, + "disable-ipc-flooding-protection" => nil, + "disable-notifications" => nil, + "disable-popup-blocking" => nil, + "disable-prompt-on-repost" => nil, + "disable-renderer-backgrounding" => nil, + "disable-setuid-sandbox" => nil, + "disable-site-isolation-trials" => nil, + "disable-sync" => nil, + "disable-web-security" => nil, + "enable-automation" => nil, + "enable-features" => "NetworkService,NetworkServiceInProcess", + "enable-logging" => "stderr", + "force-color-profile" => "srgb", + "force-device-scale-factor" => "1", + "hide-scrollbars" => nil, + "headless" => nil, + # "headless" => "new", + "ignore-certificate-errors" => nil, + "js-flags" => "--random-seed=1157259157", + "log-level" => "0", + "metrics-recording-only" => nil, + "mute-audio" => nil, + "no-default-browser-check" => nil, + "no-first-run" => nil, + "no-sandbox" => nil, + "password-store=basic" => nil, + "test-type" => nil, + "use-mock-keychain" => nil, + "window-size" => SCREEN_SIZE.join(",") +} + if ENV["CAPYBARA_DRIVER"] == "cuprite" + # NOTE: do not require cuprite by default require "capybara/cuprite" - CHROME_ARGS = { - "allow-running-insecure-content" => nil, - "autoplay-policy" => "user-gesture-required", - "disable-add-to-shelf" => nil, - "disable-background-networking" => nil, - "disable-background-timer-throttling" => nil, - "disable-backgrounding-occluded-windows" => nil, - "disable-breakpad" => nil, - "disable-checker-imaging" => nil, - "disable-client-side-phishing-detection" => nil, - "disable-component-extensions-with-background-pages" => nil, - "disable-datasaver-prompt" => nil, - "disable-default-apps" => nil, - "disable-desktop-notifications" => nil, - "disable-dev-shm-usage" => nil, - "disable-domain-reliability" => nil, - "disable-extensions" => nil, - "disable-features" => "TranslateUI,BlinkGenPropertyTrees", - "disable-gpu" => nil, - "disable-hang-monitor" => nil, - "disable-infobars" => nil, - "disable-ipc-flooding-protection" => nil, - "disable-notifications" => nil, - "disable-popup-blocking" => nil, - "disable-prompt-on-repost" => nil, - "disable-renderer-backgrounding" => nil, - "disable-setuid-sandbox" => nil, - "disable-site-isolation-trials" => nil, - "disable-sync" => nil, - "disable-web-security" => nil, - "enable-automation" => nil, - "enable-features" => "NetworkService,NetworkServiceInProcess", - "enable-logging" => "stderr", - "force-color-profile" => "srgb", - "force-device-scale-factor" => "1", - "hide-scrollbars" => nil, - "ignore-certificate-errors" => nil, - "js-flags" => "--random-seed=1157259157", - "log-level" => "0", - "metrics-recording-only" => nil, - "mute-audio" => nil, - "no-default-browser-check" => nil, - "no-first-run" => nil, - "no-sandbox" => nil, - "password-store=basic" => nil, - "test-type" => nil, - "use-mock-keychain" => nil - }.freeze + + cuprite_options = { + js_errors: true, + process_timeout: ENV["CI"] ? 40 : 5, + screen_size: SCREEN_SIZE, + timeout: ENV["CI"] ? 40 : 5, + window_size: SCREEN_SIZE, + browser_options: CHROME_ARGS + } Capybara.register_driver(:cuprite) do |app| - Capybara::Cuprite::Driver.new( - app, - browser_options: CHROME_ARGS, - js_errors: true, - process_timeout: 10, - screen_size: [800, 600], - timeout: ENV["CI"] ? 40 : 10, - window_size: [800, 600] - ) + Capybara::Cuprite::Driver.new(app, **cuprite_options) end +end -elsif ENV["CAPYBARA_DRIVER"].include?("chrome") - require "webdrivers/chromedriver" - Webdrivers::Chromedriver.update -else - require "webdrivers/geckodriver" - Webdrivers::Geckodriver.update +Capybara.register_driver :selenium_chrome_headless do |app| + version = Capybara::Selenium::Driver.load_selenium + options_key = Capybara::Selenium::Driver::CAPS_VERSION.satisfied_by?(version) ? :capabilities : :options + browser_options = Selenium::WebDriver::Chrome::Options.new + CHROME_ARGS.each { browser_options.add_argument("--#{_1}=#{_2}") } + Capybara::Selenium::Driver.new(app, :browser => :chrome, options_key => browser_options).tap do |driver| + driver.browser.manage.window.size = Selenium::WebDriver::Dimension.new(*SCREEN_SIZE) + end end +Capybara.save_path = Pathname.new("tmp/capybara").expand_path Capybara.javascript_driver = ENV.fetch("CAPYBARA_DRIVER", :cuprite).to_sym +Capybara.disable_animation = true diff --git a/test/support/setup_rails_app.rb b/test/support/setup_rails_app.rb new file mode 100644 index 00000000..bfee43da --- /dev/null +++ b/test/support/setup_rails_app.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require "rack" +require "rackup" if Rack::RELEASE >= "3" + +require "logger" # for Rails 7.0 +require "action_controller" + +# NOTE: Simulate Rails Environment +module Rails + def self.root + Pathname("../../tmp").expand_path(__dir__) + end + + def self.application + Rack::Builder.new { + use(Rack::Static, urls: [""], root: "test/fixtures/app", index: "index.html") + run ->(_env) { [200, {}, []] } + }.to_app + end +end diff --git a/test/support/stub_test_methods.rb b/test/support/stub_test_methods.rb new file mode 100644 index 00000000..bc4d79ac --- /dev/null +++ b/test/support/stub_test_methods.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative "screenshoter_stub" +require_relative "dsl_stub" diff --git a/test/support/test_doubles.rb b/test/support/test_doubles.rb new file mode 100644 index 00000000..3c60d4cc --- /dev/null +++ b/test/support/test_doubles.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +module TestDoubles + # Test double for file paths with configurable size and existence + class TestPath + attr_reader :size_value + + # Initialize a path with a size value and existence flag + # @param size_value [Integer] The size of the file + # @param exists [Boolean] Whether the file exists, defaults to true + def initialize(size_value, exists = true) + @size_value = size_value + @exists = exists + end + + def size + @size_value + end + + def exist? + @exists + end + end + + # Test double for image drivers with configurable behavior + class TestDriver + attr_reader :add_black_box_calls, :filter_calls, :dimension_check_calls, :pixel_check_calls, :difference_region_calls, :load_images_called, :load_images_args + attr_accessor :same_dimension_result, :same_pixels_result, :difference_region_result, :images_to_return + + # Initializes a new TestDriver + # @param is_vips_driver [Boolean] whether this driver should behave like a VipsDriver + # @param images_to_return [Array] images to return from load_images method + def initialize(is_vips_driver = false, images_to_return = nil) + @is_vips_driver = is_vips_driver + @images_to_return = images_to_return || [:base_image, :new_image] + @add_black_box_calls = [] + @filter_calls = [] + @dimension_check_calls = [] + @pixel_check_calls = [] + @difference_region_calls = [] + @load_images_called = false + @load_images_args = nil + @same_dimension_result = true + @same_pixels_result = true + @difference_region_result = nil + end + + def is_a?(klass) + return @is_vips_driver if klass == SnapDiff::Drivers::VipsDriver + super + end + + def add_black_box(image, region) + @add_black_box_calls << {image: image, region: region} + "processed_#{image}" + end + + def filter_image_with_median(image, size) + @filter_calls << {image: image, size: size} + # Return the filtered image, converting to the expected format + "filtered_#{image}" + end + + def same_dimension?(comparison) + @dimension_check_calls << comparison + @same_dimension_result + end + + def same_pixels?(comparison) + @pixel_check_calls << comparison + @same_pixels_result + end + + def find_difference_region(comparison) + @difference_region_calls << comparison + @difference_region_result + end + + def load_images(base_path, new_path) + @load_images_called = true + @load_images_args = [base_path, new_path] + @images_to_return + end + + def supports?(...) + @is_vips_driver + end + + # Returns Object so warning messages in ImagePreprocessor don't + # couple tests to the TestDriver class name. + def class + Object + end + end + + # Test double for difference results + class TestDifference + attr_reader :different_value + + def initialize(different_value) + @different_value = different_value + end + + def different? + @different_value + end + end + + # Simple test double for comparison objects + class TestComparison + attr_reader :new_image, :base_image, :options, :driver + attr_accessor :new_image_path, :base_image_path + + def initialize(options = {}) + @new_image = options[:new_image] + @base_image = options[:base_image] + @options = options[:options] || {} + @driver = options[:driver] + @new_image_path = options[:new_image_path] || options[:image_path] + @base_image_path = options[:base_image_path] + end + end +end diff --git a/test/support/test_helpers.rb b/test/support/test_helpers.rb new file mode 100644 index 00000000..03d47f8f --- /dev/null +++ b/test/support/test_helpers.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require "support/test_doubles" + +module TestHelpers + include TestDoubles + + # Common assertions for image comparison tests + module Assertions + # Asserts that a driver check was called a specific number of times + # @param driver [Object] The test driver object + # @param check_type [Symbol] :dimension_check, :pixel_check, or :difference_region + # @param times [Integer] The expected number of calls (default: 1) + def assert_driver_check_called(driver, check_type, times = 1) + calls = driver.public_send(:"#{check_type}_calls") + assert_equal times, calls.size, + "Expected #{check_type} to be called #{times} time(s), was called #{calls.size}" + end + + # Convenience aliases for backward compatibility + def assert_dimension_check_called(driver, times = 1) = assert_driver_check_called(driver, :dimension_check, times) + def assert_pixel_check_called(driver, times = 1) = assert_driver_check_called(driver, :pixel_check, times) + def assert_difference_region_called(driver, times = 1) = assert_driver_check_called(driver, :difference_region, times) + end + + # Common setup methods for test drivers + module DriverSetup + # Sets up driver results for testing + # @param driver [Object] The test driver object + # @param same_dimension [Boolean] Whether dimensions match (default: true) + # @param same_pixels [Boolean, nil] Whether pixels match (default: nil for no change) + # @param difference_region [Object, nil] The difference region result (default: nil) + def setup_driver_results(driver, same_dimension: true, same_pixels: nil, difference_region: nil) + driver.same_dimension_result = same_dimension + driver.same_pixels_result = same_pixels unless same_pixels.nil? + driver.difference_region_result = difference_region if difference_region + end + end + + # Common test data generators + module TestData + # Creates a test driver with the given options + # @param is_vips [Boolean] Whether to create a VIPS driver (default: false) + # @param images [Array, nil] Images to return from load_images (default: nil) + # @return [TestDoubles::TestDriver] A test driver object + def create_test_driver(is_vips: false, images: nil) + TestDoubles::TestDriver.new(is_vips, images) + end + end +end diff --git a/test/system_test_case.rb b/test/system_test_case.rb index 531955b2..bda94a9f 100644 --- a/test/system_test_case.rb +++ b/test/system_test_case.rb @@ -1,59 +1,95 @@ +# frozen_string_literal: true + require "test_helper" +require "snap_diff/integrations/minitest" +require "snap_diff/reporters/html" require "support/setup_capybara_drivers" -class SystemTestCase < ActionDispatch::IntegrationTest +class SystemTestCase < ActiveSupport::TestCase setup do Capybara.current_driver = Capybara.javascript_driver + Capybara.page.current_window.resize_to(*SCREEN_SIZE) + + SnapDiff.config.screenshot_enabled = true + SnapDiff.config.enabled = true # TODO: Reset original settings to previous values - @orig_root = Capybara::Screenshot.root - Capybara::Screenshot.root = "." - @orig_save_path = Capybara::Screenshot.save_path - Capybara::Screenshot.save_path = "test/fixtures/app/doc/screenshots" - Capybara::Screenshot.enabled = true - Capybara::Screenshot::Diff.enabled = true - Capybara::Screenshot::Diff.driver = ENV.fetch("SCREENSHOT_DRIVER", "vips").to_sym + @orig_root = SnapDiff.config.root + SnapDiff.config.root = Rails.root / "../test/fixtures/app" + + @orig_save_path = SnapDiff.config.save_path + SnapDiff.config.save_path = "./doc/screenshots" + + # `SnapDiff.config.driver = ENV.fetch("SCREENSHOT_DRIVER", ...)` is gone + # with the setting: 2.1 made libvips the only backend, so there is nothing + # for the env var to select. # TODO: Makes configurations copying and restoring much easier - @orig_add_os_path = Capybara::Screenshot.add_os_path - Capybara::Screenshot.add_os_path = false - @orig_add_driver_path = Capybara::Screenshot.add_driver_path - Capybara::Screenshot.add_driver_path = false - # NOTE: Only works before `include Capybara::Screenshot::Diff` line - @orig_window_size = Capybara::Screenshot.window_size - Capybara::Screenshot.window_size = [800, 600] + @orig_add_os_path = SnapDiff.config.add_os_path + SnapDiff.config.add_os_path = true + @orig_add_driver_path = SnapDiff.config.add_driver_path + SnapDiff.config.add_driver_path = true + # NOTE: Only works before the `include SnapDiff::DSL` line + @orig_window_size = SnapDiff.config.window_size + SnapDiff.config.window_size = SCREEN_SIZE # NOTE: For small screenshots we should have pixel perfect comparisons - @orig_tolerance = Capybara::Screenshot::Diff.tolerance - Capybara::Screenshot::Diff.tolerance = nil + @orig_tolerance = SnapDiff.config.tolerance + SnapDiff.config.tolerance = nil end - include Capybara::Screenshot::Diff + include SnapDiff::DSL + include SnapDiff::Minitest::Assertions teardown do - if @test_screenshots - @test_screenshots.each(&method(:rollback_comparison_runtime_files)) - # NOTE: We clear tracked different errors in order to not raise error - @test_screenshots.clear - end - # Restore to previous values - Capybara::Screenshot.root = @orig_root - Capybara::Screenshot.save_path = @orig_save_path - Capybara::Screenshot.add_os_path = @orig_add_os_path - Capybara::Screenshot.add_driver_path = @orig_add_driver_path - Capybara::Screenshot.window_size = @orig_window_size - Capybara::Screenshot::Diff.tolerance = @orig_tolerance + SnapDiff.config.root = @orig_root + SnapDiff.config.save_path = @orig_save_path + SnapDiff.config.add_os_path = @orig_add_os_path + SnapDiff.config.add_driver_path = @orig_add_driver_path + SnapDiff.config.window_size = @orig_window_size + SnapDiff.config.tolerance = @orig_tolerance Capybara.current_driver = Capybara.default_driver + + # The vips cache flush that used to live here (cache_set_max 0 then 1000) + # papered over libvips serving a stale image when a screenshot path was + # rewritten within the same second. VipsDriver#from_file now passes + # `revalidate: true`, so the workaround is gone -- see the regression test + # in test/unit/drivers/vips_driver_test.rb. end private - def rollback_comparison_runtime_files(screenshot_error) - screenshot_name, comparison = screenshot_error[1], screenshot_error[2] - restore_git_revision(screenshot_name, comparison.new_file_name) - comparison.clean_tmp_files + def rollback_comparison_runtime_files(screenshot_assert) + comparison = screenshot_assert.is_a?(SnapDiff::ScreenshotAssertion) ? screenshot_assert.compare : screenshot_assert[2] + return unless comparison + + save_annotations_for_debug(comparison) + + screenshot_path = comparison.image_path + SnapDiff::Vcs.checkout_vcs(SnapDiff.config.root, screenshot_path, screenshot_path) + + if comparison.difference + comparison.reporter.clean_tmp_files + end + end + + def save_annotations_for_debug(comparison) + debug_diffs_save_path = Pathname.new(Capybara.save_path) / "screenshots-diffs" / name + debug_diffs_save_path.mkpath unless debug_diffs_save_path.exist? + + if File.exist?(comparison.image_path) + FileUtils.cp(comparison.image_path, debug_diffs_save_path) + end + + if comparison.reporter.annotated_base_image_path.exist? + FileUtils.mv(comparison.reporter.annotated_base_image_path, debug_diffs_save_path, force: true) + end + + if comparison.reporter.annotated_image_path.exist? + FileUtils.mv(comparison.reporter.annotated_image_path, debug_diffs_save_path, force: true) + end end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 863bba98..1272b6a8 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -2,71 +2,111 @@ if ENV["COVERAGE"] require "simplecov" - SimpleCov.start - SimpleCov.minimum_coverage 92 + SimpleCov.start "test_frameworks" do + enable_coverage :branch + minimum_coverage line: 90, branch: 68 + + add_filter("gemfiles") + add_filter("test") + end end $LOAD_PATH.unshift File.expand_path("../lib", __dir__) -TEST_IMAGES_DIR = File.expand_path("images", __dir__) +require "pathname" +TEST_IMAGES_DIR = Pathname.new(File.expand_path("fixtures/images", __dir__)) + +require "support/setup_rails_app" +require "minitest/autorun" + +require "capybara/minitest" +require "support/setup_capybara" + +require "snap_diff/integrations/minitest" + +# The deprecation machinery this file used to configure -- the migration +# notice, the 2.1 removal warnings, and the `Warning` guard that raised on +# any `[snap_diff deprecation]` line -- went with 2.1. There is no channel +# left to suppress or to police: a legacy require now fails loudly by +# itself, and the two static gates (core_tree_has_no_legacy_deps_test, +# canonical_suite_has_no_legacy_refs_test) catch names that only appear in +# text. +# +# The DriverCoverage banner/abort went the same way. It guarded against a +# SILENT fallback to chunky_png when libvips was missing; with one backend a +# missing libvips is a `require "vips"` LoadError at boot, not a quiet +# downgrade. + +require "support/stub_test_methods" +require "support/setup_capybara_drivers" +require "support/test_helpers" + +SnapDiff.config.root = Rails.root +SnapDiff.config.save_path = "./doc/screenshots" + +class ActiveSupport::TestCase + include TestHelpers::Assertions + include TestHelpers::DriverSetup + include TestHelpers::TestData + + # Set up fixtures and test helpers + self.file_fixture_path = Pathname.new(File.expand_path("fixtures", __dir__)) + + # Snapshot ALL global config state before each test, restore after. + # Prevents one test from poisoning another via leaked config changes. + # Since ADR-008 step 1 the single storage is the SnapDiff.config instance + # (the legacy Capybara::Screenshot / ::Diff accessors delegate to it), so + # snapshotting its instance variables covers both surfaces. + setup do + config = SnapDiff.config + @_global_snapshots = config.instance_variables.map { |iv| + [iv, config.instance_variable_get(iv)] + } + @_orig_cwd = Dir.pwd + @_orig_capybara_app = Capybara.app -# NOTE: Simulate Rails Environment -module Rails - def self.root - Pathname("../tmp").expand_path(__dir__) + SnapDiff.config.fail_if_new = false + SnapDiff.config.blur_active_element = false + SnapDiff.config.hide_caret = false + SnapDiff.config.disable_animations = false end - def self.application - Rack::Builder.new { - use(Rack::Static, urls: [""], root: "test/fixtures/app", index: "index.html") - run ->(_env) { [200, {}, []] } - }.to_app + teardown do + # Restore all global config state + @_global_snapshots&.each do |iv, val| + SnapDiff.config.instance_variable_set(iv, val) + end + Dir.chdir(@_orig_cwd) if @_orig_cwd && Dir.pwd != @_orig_cwd + Capybara.app = @_orig_capybara_app if @_orig_capybara_app + SnapDiff::SnapManager.cleanup! unless persist_comparisons? end -end -require "capybara/screenshot/diff" -require "minitest/autorun" -require "capybara/minitest" + def persist_comparisons? + ENV["DEBUG"] || ENV["DISABLE_ROLLBACK_COMPARISON_RUNTIME_FILES"] || ENV["RECORD_SCREENSHOTS"] + end -require "capybara/dsl" -Capybara.disable_animation = true -Capybara.server = :puma, {Silent: true} -Capybara.threadsafe = true -Capybara.app = Rails.application - -# TODO(uwe): Remove when we stop support for Rails 4.2 -ActiveSupport.test_order = :random -# ODOT - -module Capybara - module Screenshot - module Diff - module TestHelper - private - - # Stub of the Capybara's save_screenshot - def save_screenshot(file_name) - source_image = File.basename(file_name) - source_image.slice!(/^\d\d_/) - FileUtils.cp File.expand_path("images/#{source_image}", __dir__), file_name - end - - def make_comparison(old_img, new_img, **options) - comp = ImageCompare.new("#{Rails.root}/screenshot.png", **options) - set_test_images(comp, old_img, new_img) - comp - end - - def set_test_images(comp, old_img, new_img) - FileUtils.mkdir_p File.dirname(comp.old_file_name) - FileUtils.cp "#{TEST_IMAGES_DIR}/#{old_img}.png", comp.old_file_name - FileUtils.cp "#{TEST_IMAGES_DIR}/#{new_img}.png", comp.new_file_name - end - - def evaluate_script(*) - # Do nothing - end - end + def optional_test + unless ENV["DISABLE_SKIP_TESTS"] + skip "This is optional test! To enable provide DISABLE_SKIP_TESTS=1" end end + + private + + def fixture_image_path_from(original_new_image, ext = "png") + file_fixture("images/#{original_new_image}.#{ext}") + end + + def assert_same_images(expected_image_name, image_path) + expected_image_path = file_fixture("comparisons/#{expected_image_name}") + assert_predicate(SnapDiff::Comparison.new(image_path, expected_image_path), :quick_equal?) + end + + def assert_stored_screenshot(filename) + assert_includes( + SnapDiff::SnapManager.screenshots, + filename, + "Screenshot #{filename} not found in #{SnapDiff::SnapManager.instance.root}" + ) + end end diff --git a/test/unit/annotation_service_test.rb b/test/unit/annotation_service_test.rb new file mode 100644 index 00000000..a25dabc2 --- /dev/null +++ b/test/unit/annotation_service_test.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff/annotation_service" + +unless defined?(Vips) + warn "VIPS not present. Skipping VIPS driver tests." + return +end +require "snap_diff/drivers/vips_driver" + +class AnnotationServiceTest < ActiveSupport::TestCase + setup do + @_tmpdir = Pathname.new(Dir.mktmpdir) + end + + teardown do + FileUtils.remove_entry @_tmpdir if @_tmpdir + end + + test "#annotate_and_save_images writes annotated and heatmap images" do + skip "VIPS not present. Skipping VIPS driver tests." unless defined?(Vips) + driver = SnapDiff::Drivers::VipsDriver.new + comparison = build_comparison_for(driver, "a.png", "b.png") + service = SnapDiff::AnnotationService.new(driver.find_difference_region(comparison)) + + service.annotate_and_save_images + + assert_same_images "a-and-b.heatmap.diff.png", service.heatmap_diff_path + end + + test "#clean_tmp_files removes annotated and heatmap images" do + skip "VIPS not present. Skipping VIPS driver tests." unless defined?(Vips) + driver = SnapDiff::Drivers::VipsDriver.new + comparison = build_comparison_for(driver, "a.png", "b.png") + service = SnapDiff::AnnotationService.new(driver.find_difference_region(comparison)) + service.annotate_and_save_images + + assert_predicate service.heatmap_diff_path, :exist? + + service.clean_tmp_files + + assert_not service.annotated_image_path.exist?, "diff should be cleaned" + assert_not service.annotated_base_image_path.exist?, "base diff should be cleaned" + assert_not service.heatmap_diff_path.exist?, "heatmap diff should be cleaned" + end + + test "#save_annotation_for bakes in a visibly different image when a skip_area is set" do + skip "VIPS not present. Skipping VIPS driver tests." unless defined?(Vips) + driver = SnapDiff::Drivers::VipsDriver.new + new_image = driver.from_file(TEST_IMAGES_DIR.join("a.png")) + base_image = driver.from_file(TEST_IMAGES_DIR.join("b.png")) + + with_skip_area = SnapDiff::Comparison::Images.new(new_image, base_image, {skip_area: [Region.new(0, 0, 10, 10)]}, driver, + @_tmpdir / "with_skip_area.png", @_tmpdir / "with_skip_area_base.png") + without_skip_area = SnapDiff::Comparison::Images.new(new_image, base_image, {}, driver, + @_tmpdir / "without_skip_area.png", @_tmpdir / "without_skip_area_base.png") + + service_with = SnapDiff::AnnotationService.new(driver.find_difference_region(with_skip_area)) + service_without = SnapDiff::AnnotationService.new(driver.find_difference_region(without_skip_area)) + service_with.annotate_and_save_images + service_without.annotate_and_save_images + + assert_not FileUtils.compare_file(service_with.annotated_base_image_path.to_s, service_without.annotated_base_image_path.to_s), + "annotated base image should differ once a skip_area rectangle is drawn onto it" + end + + private + + def build_comparison_for(driver, *images) + new_image = driver.from_file(TEST_IMAGES_DIR.join(images.first)) + base_image = driver.from_file(TEST_IMAGES_DIR.join(images.last)) + + SnapDiff::Comparison::Images.new(new_image, base_image, {}, driver, @_tmpdir / images.first, @_tmpdir / images.last) + end +end diff --git a/test/unit/area_calculator_test.rb b/test/unit/area_calculator_test.rb new file mode 100644 index 00000000..a223cf32 --- /dev/null +++ b/test/unit/area_calculator_test.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff/area_calculator" + +class AreaCalculatorTest < ActiveSupport::TestCase + class CalculateSkipAreaTest < self + test "#calculate_skip_area returns empty array when no skip areas overlap with crop area" do + skip_area = [[0, 0, 100, 100], [200, 200, 100, 100]] + crop_area = [100, 100, 100, 100] + calculator = SnapDiff::AreaCalculator.new(crop_area, skip_area) + + result = calculator.calculate_skip_area + + assert_empty result + end + + test "#calculate_skip_area returns intersecting regions when skip areas overlap with crop area" do + skip_area = [Region.new(50, 50, 150, 150)] + crop_area = Region.new(0, 0, 200, 200) + calculator = SnapDiff::AreaCalculator.new(crop_area, skip_area) + + result = calculator.calculate_skip_area + + assert_equal [Region.new(50, 50, 150, 150)], result + end + end + + class InitializationTest < self + test "#initialize handles Region objects for skip areas correctly" do + skip_area = [Region.new(0, 0, 100, 100)] + crop_area = Region.new(0, 0, 200, 200) + + calculator = SnapDiff::AreaCalculator.new(crop_area, skip_area) + + assert_equal [Region.new(0, 0, 100, 100)], calculator.calculate_skip_area + end + + test "#initialize converts array coordinates to Region objects" do + skip_area = [[0, 0, 100, 100]] + crop_area = [0, 0, 200, 200] + + calculator = SnapDiff::AreaCalculator.new(crop_area, skip_area) + result = calculator.calculate_skip_area + + assert_equal 1, result.size + assert_kind_of Region, result.first + assert_equal [0, 0, 100, 100], + [result.first.left, result.first.top, result.first.right, result.first.bottom] + end + end + + class EdgeCaseTest < self + test "#calculate_skip_area returns empty array when skip_areas is empty" do + calculator = SnapDiff::AreaCalculator.new([0, 0, 100, 100], []) + + result = calculator.calculate_skip_area + + assert_empty result + end + + test "#calculate_skip_area returns nil when skip_areas is not provided (nil)" do + calculator = SnapDiff::AreaCalculator.new([0, 0, 100, 100], nil) + + result = calculator.calculate_skip_area + + assert_nil result + end + end +end diff --git a/test/unit/attempts_reporter_test.rb b/test/unit/attempts_reporter_test.rb new file mode 100644 index 00000000..1333ef1d --- /dev/null +++ b/test/unit/attempts_reporter_test.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff" +# Not on any entry point's require path: stable_screenshoter pulls it in +# lazily, at the moment a capture actually goes unstable. +require "snap_diff/attempts_reporter" + +module SnapDiff + # Guard #2 from the v2 core-redesign acceptance contract (D8). + # + # The stability-failure path — unstable page -> UnstableImage raised -> + # annotated diff artifacts on disk — had zero direct coverage: no test file + # mentioned AttemptsReporter at all. These tests pin today's behavior + # (raise from inside capture) before the redesign turns the failure into + # data raised later at verify time. + class AttemptsReporterTest < ActiveSupport::TestCase + setup do + @manager = SnapDiff::SnapManager.new(SnapDiff.config.root / "attempts_reporter_test") + @manager.create_output_directory_for + end + + teardown do + @manager.cleanup! + end + + test "#generate returns the timeout message listing every attempt artifact" do + snap = attempt_snapshot("unstable_message", %i[a b]) + + message = AttemptsReporter.new(snap, {}, {wait: 2, stability_time_limit: 0.1}).generate + + assert_match(/Could not get stable screenshot within 2s/, message) + snap.find_attempts_paths.each do |attempt_path| + assert_includes message, attempt_path.to_s + end + end + + test "#generate overwrites later attempts with annotated diffs" do + snap = attempt_snapshot("unstable_annotated", %i[a b]) + newest_attempt = Pathname.new(snap.find_attempts_paths.max) + original_bytes = newest_attempt.binread + + AttemptsReporter.new(snap, {}, {wait: 2, stability_time_limit: 0.1}).generate + + assert_predicate newest_attempt, :exist? + assert_not_equal original_bytes, newest_attempt.binread, + "each attempt after the first should be replaced by its annotated diff for debugging" + end + + # End to end through StableScreenshoter: a page that never stabilizes + # (every attempt differs from the previous one) times out, raises + # UnstableImage from inside capture, and leaves annotated attempts behind. + test "StableScreenshoter raises UnstableImage with annotated attempts when the page never stabilizes" do + alternating_screenshoter = Class.new(SnapDiff::Screenshoter) do + def take_screenshot(screenshot_path) + @flip = !@flip + FileUtils.mkdir_p(screenshot_path.dirname) + FileUtils.cp(TEST_IMAGES_DIR / "#{@flip ? "a" : "b"}.png", screenshot_path) + end + end + + snap = @manager.snapshot("unstable_end_to_end") + + error = nil + SnapDiff.config.stub(:screenshoter, alternating_screenshoter) do + error = assert_raises(SnapDiff::UnstableImage) do + StableScreenshoter + .new({stability_time_limit: 0.05, wait: 0.2}, {}) + .take_comparison_screenshot(snap) + end + end + + assert_match(/Could not get stable screenshot within 0.2s/, error.message) + + attempts = snap.find_attempts_paths + assert_operator attempts.size, :>=, 2, "the unstable run must leave its attempt artifacts for debugging" + # Attempts after the first are overwritten with annotated diffs. + annotated_attempt = Pathname.new(attempts.max) + assert_not_equal (TEST_IMAGES_DIR / "a.png").binread, annotated_attempt.binread + assert_not_equal (TEST_IMAGES_DIR / "b.png").binread, annotated_attempt.binread + end + + private + + # Builds a snapshot with one attempt file per fixture, oldest first. + def attempt_snapshot(name, fixtures) + @manager.snapshot(name).tap do |snap| + fixtures.each do |fixture| + attempt_path = snap.next_attempt_path! + FileUtils.mkdir_p(attempt_path.dirname) + FileUtils.cp(fixture_image_path_from(fixture), attempt_path) + end + end + end + end +end diff --git a/test/unit/backtrace_filter_test.rb b/test/unit/backtrace_filter_test.rb new file mode 100644 index 00000000..ff6a022f --- /dev/null +++ b/test/unit/backtrace_filter_test.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff/error_with_filtered_backtrace" + +class BacktraceFilterTest < ActiveSupport::TestCase + test "#filtered removes lines originating from the given lib directory" do + filter = SnapDiff::BacktraceFilter.new("/app/lib/") + + result = filter.filtered([ + "/app/lib/snap_diff/foo.rb:1:in 'bar'", + "/app/test/some_test.rb:5:in 'test_thing'" + ]) + + assert_equal ["/app/test/some_test.rb:5:in 'test_thing'"], result + end + + test "#filtered removes lines from activesupport, minitest, and railties gems" do + filter = SnapDiff::BacktraceFilter.new("/app/lib/") + + result = filter.filtered([ + "/gems/activesupport-7.0.0/lib/foo.rb:1:in 'bar'", + "/gems/minitest-5.0.0/lib/minitest.rb:2:in 'run'", + "/gems/railties-7.0.0/lib/baz.rb:3:in 'call'", + "/app/test/some_test.rb:5:in 'test_thing'" + ]) + + assert_equal ["/app/test/some_test.rb:5:in 'test_thing'"], result + end + + test "#filtered keeps lines outside the lib directory and unrelated gems" do + filter = SnapDiff::BacktraceFilter.new("/app/lib/") + backtrace = [ + "/app/test/some_test.rb:5:in 'test_thing'", + "/gems/rack-3.0.0/lib/rack.rb:1:in 'call'" + ] + + assert_equal backtrace, filter.filtered(backtrace) + end + + test "#filtered does not treat a sibling directory sharing the prefix as inside lib" do + filter = SnapDiff::BacktraceFilter.new("/app/lib") + + backtrace = ["/app/library/foo.rb:1:in 'bar'"] + + assert_equal backtrace, filter.filtered(backtrace) + end + + test "#initialize defaults to the library's own lib directory" do + filter = SnapDiff::BacktraceFilter.new + lib_file = File.expand_path("../../lib/snap_diff/error_with_filtered_backtrace.rb", __dir__) + + result = filter.filtered([ + "#{lib_file}:1:in 'filtered'", + "/app/test/some_test.rb:5:in 'test_thing'" + ]) + + assert_equal ["/app/test/some_test.rb:5:in 'test_thing'"], result + end +end diff --git a/test/unit/canonical_suite_has_no_legacy_refs_test.rb b/test/unit/canonical_suite_has_no_legacy_refs_test.rb new file mode 100644 index 00000000..104945ae --- /dev/null +++ b/test/unit/canonical_suite_has_no_legacy_refs_test.rb @@ -0,0 +1,181 @@ +# frozen_string_literal: true + +require "test_helper" + +# The TEST-TREE half of the split that core_tree_has_no_legacy_deps_test.rb +# guards for lib/. +# +# Before 2.1 this gate protected a FUTURE deletion: a test asserting legacy +# behaviour passed then and would fail the day the v1 trees went. The deletion +# has landed, so a legacy `require` now fails by itself and this gate no longer +# has to predict anything. It stays for the half that still does not fail on +# its own -- a legacy NAME in a string, a heredoc-embedded subprocess script, +# an assertion message or a docstring. Those survive the deletion and start +# lying the moment it happens. +# +# Scope: test/unit/ and test/integration/ -- the whole suite now that +# test/legacy/ is gone. +# +# WHOLE-LINE comments are ignored, same as the twin gate: a comment +# explaining that a forwarder used to live under the old name is history, not +# a dependency. Everything else on a code line counts, strings included -- a +# subprocess script embedded in a heredoc runs, and a legacy require inside +# one dies with the trees just as loudly as one at the top of the file. +class CanonicalSuiteHasNoLegacyRefsTest < ActiveSupport::TestCase + TEST_ROOT = Pathname.new(__dir__).join("..").expand_path + + # This file cannot scan itself: a line-level allowlist has to quote the + # exact lines it blesses, and every such quote is itself a legacy + # reference. Scanning self would demand an allowlist entry for the + # allowlist, whose text is again an offence -- no fixed point exists. + SELF = Pathname.new(File.expand_path(__FILE__)) + + ALL_FILES = ( + Dir[TEST_ROOT.join("unit/**/*_test.rb")] + Dir[TEST_ROOT.join("integration/**/*_test.rb")] + ).map { |path| Pathname.new(path) }.sort.freeze + + CANONICAL_FILES = ALL_FILES.reject { |file| file == SELF }.freeze + + # A require of a doomed path: the v1 trees (`capybara/screenshot/...`, + # `capybara_screenshot_diff...`, the `capybara-screenshot-diff` gem-name + # entry) and the two core files that build the v1 surface and go with it. + # Plain `require "capybara"` / `"capybara/minitest"` is the base gem and + # must not match. + # + # Unanchored, unlike the lib-side twin: tests drive subprocesses, so a + # legacy require is as likely to sit inside a heredoc or a `-e` string as + # at the top of the file, and both really load it. The optional backslash + # covers the escaped-quote form those scripts use. + LEGACY_REQUIRE = %r{ + require(_relative)?\s+\\?["'](\.{1,2}/)* + (capybara(/screenshot|_screenshot_diff|-screenshot-diff)|snap_diff/(legacy_shims|deprecation)) + }x + + # A read or write of a v1 namespace constant. + LEGACY_CONSTANT = /(? [ + 'LEGACY_CONSTANT = /(? [ + "Capybara::Screenshot", + "CapybaraScreenshotDiff", + "SnapDiff::Deprecation", + "SnapDiff::Removal", + "SnapDiff::Driver", + "SnapDiff::Drivers::AVAILABLE_DRIVERS", + "REMOVED_METHODS = %w[start silence_deprecations silence_deprecations=].freeze" + ] + }.freeze + + test "no canonical test references the legacy namespaces or entry points" do + refute_empty CANONICAL_FILES, "canonical glob matched nothing -- the gate would pass vacuously" + assert_equal ALL_FILES.size - 1, CANONICAL_FILES.size, + "SELF no longer names a scanned file, so this gate is scanning itself or nothing" + + offenders = CANONICAL_FILES.flat_map { |file| offences(file) } + + assert_empty offenders, <<~MSG + Test(s) name a surface 2.1 removed. The v1 trees, the deprecation + channel and the driver abstraction are gone, so these names resolve to + nothing -- and where they sit in a string or a docstring, nothing else + will say so. + + Rewrite against what the gem actually has (`SnapDiff.config.*`, + `SnapDiff.configure`, `SnapDiff::Os`, `SnapDiff::Minitest::Assertions`, + `SnapDiff::Drivers::VipsDriver`, `require "snap_diff/..."`): + + #{offenders.join("\n")} + MSG + end + + test "the allowlist names only lines that still exist" do + stale = ALLOWED.flat_map do |path, lines| + file = TEST_ROOT.join(path) + # A deleted file is the most stale an entry can get; report it rather + # than letting Pathname#read blow up with Errno::ENOENT. + next ["#{path}: allowlisted file no longer exists"] unless file.exist? + + present = significant_lines(file).map(&:first) + (lines - present).map { |line| "#{path}: allowlisted line no longer present: `#{line}`" } + end + + assert_empty stale, <<~MSG + The allowlist is out of date -- these entries protect nothing and only + hide future regressions. Delete them: + + #{stale.join("\n")} + MSG + end + + private + + def offences(file) + rel = file.relative_path_from(TEST_ROOT).to_s + allowed = ALLOWED.fetch(rel, []) + + significant_lines(file).filter_map do |line, number| + next if allowed.include?(line) + + reason = + if LEGACY_REQUIRE.match?(line) then "requires a doomed path" + elsif LEGACY_CONSTANT.match?(line) then "references a v1 namespace constant" + elsif LEGACY_SHIM_SURFACE.match?(line) then "uses a shim-only name that the deletion removes" + end + "#{rel}:#{number}: #{reason} -- `#{line}`" if reason + end + end + + # [stripped line, 1-based line number] for every line that is not blank and + # not a whole-line comment. + def significant_lines(file) + file.read.lines.each_with_index.filter_map do |line, index| + stripped = line.strip + [stripped, index + 1] unless stripped.empty? || stripped.start_with?("#") + end + end +end diff --git a/test/unit/capture/viewport_test.rb b/test/unit/capture/viewport_test.rb new file mode 100644 index 00000000..e9a85111 --- /dev/null +++ b/test/unit/capture/viewport_test.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff" + +module SnapDiff + module Capture + # Direct coverage for the per-capture viewport preparation seam + # (5.5-lite item 6): a raise-only window-size guard. + class ViewportTest < ActiveSupport::TestCase + test "prepare! is a no-op when the window size matches" do + BrowserHelpers.stub(:window_size_is_wrong?, false) do + assert_nil Viewport.prepare!([800, 600]) + end + end + + test "prepare! raises WindowSizeMismatchError when the window size is wrong" do + BrowserHelpers.stub(:window_size_is_wrong?, true) do + BrowserHelpers.stub(:selenium?, false) do + error = assert_raises(SnapDiff::WindowSizeMismatchError) do + Viewport.prepare!([800, 600]) + end + assert_includes error.message, "[800, 600]" + assert_includes error.message, "Actual: unknown" + end + end + end + + # The selenium arm of the ternary was never taken -- the test above + # stubs selenium? to false, and nothing else calls prepare! -- so the + # whole `session.driver.browser.manage.window.size` chain (the only + # reason the message is ever useful) was unexecuted. Under selenium the + # message must name the size the browser ACTUALLY has. + test "prepare! reports the real window size when the driver is selenium" do + window = Struct.new(:size).new("(width: 1024, height: 768)") + manage = Struct.new(:window).new(window) + browser = Struct.new(:manage).new(manage) + driver = Struct.new(:browser).new(browser) + session = Struct.new(:driver).new(driver) + + BrowserHelpers.stub(:window_size_is_wrong?, true) do + BrowserHelpers.stub(:selenium?, true) do + BrowserHelpers.stub(:session, session) do + error = assert_raises(SnapDiff::WindowSizeMismatchError) do + Viewport.prepare!([800, 600]) + end + + assert_includes error.message, "Expected: [800, 600]" + assert_includes error.message, "Actual: (width: 1024, height: 768)" + end + end + end + end + end + end +end diff --git a/test/unit/compare_api_test.rb b/test/unit/compare_api_test.rb new file mode 100644 index 00000000..d768e273 --- /dev/null +++ b/test/unit/compare_api_test.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require "test_helper" + +# The canonical entry point for the file-to-file compare API. The v1 +# `Capybara::Screenshot::Diff.compare` forwarder over it is exercised in +# test/legacy/legacy_forwarders_test.rb. +class CompareApiTest < ActiveSupport::TestCase + test ".compare returns ImageCompare instance" do + result = SnapDiff.compare( + TEST_IMAGES_DIR / "a.png", + TEST_IMAGES_DIR / "a.png" + ) + assert_kind_of SnapDiff::Comparison, result + end + + test ".compare detects identical images" do + result = SnapDiff.compare( + TEST_IMAGES_DIR / "a.png", + TEST_IMAGES_DIR / "a.png" + ) + assert result.quick_equal? + assert_not result.different? + end + + test ".compare detects different images" do + result = SnapDiff.compare( + TEST_IMAGES_DIR / "a.png", + TEST_IMAGES_DIR / "b.png" + ) + assert_not result.quick_equal? + assert result.different? + end + + # ".compare accepts driver option" is gone with the option: 2.1 removed + # driver selection, so there is nothing left to accept. + test ".compare compares two images with the configured defaults" do + result = SnapDiff.compare(TEST_IMAGES_DIR / "a.png", TEST_IMAGES_DIR / "a.png") + assert result.quick_equal? + end + + test ".compare accepts tolerance options" do + result = SnapDiff.compare( + TEST_IMAGES_DIR / "a.png", + TEST_IMAGES_DIR / "b.png", + tolerance: 1.0 + ) + assert_not result.different? + end +end diff --git a/test/unit/config_default_timing_test.rb b/test/unit/config_default_timing_test.rb new file mode 100644 index 00000000..5735c38c --- /dev/null +++ b/test/unit/config_default_timing_test.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require "test_helper" +require "open3" + +# ADR-005 step 1 guard: pins WHEN each config default is evaluated, per +# documented entry point, in fresh subprocesses (test_helper preloads the +# whole gem, so only a subprocess can observe require-time behavior). +# +# Current behavior being pinned: +# +# - the ENV/pwd-derived defaults (fail_if_new from ENV["CI"], root from +# Rails.root/pwd) are evaluated ONCE, when snap_diff/config.rb is first +# required. Mutating ENV, cwd, or Rails.root after the require -- even +# before the first read -- must NOT change the value. A refactor that +# turns any of these into a lazy (read-time) default, memoized or not, +# goes red here. +# - default_options[:wait] is the opposite: it reads +# Capybara.default_max_wait_time at CALL time, live, every call. +# +# Canonical entry points only, read through SnapDiff.config only. The v1 +# entry points and the "both surfaces agree" half re-run these same scripts +# from test/legacy/legacy_config_default_timing_test.rb, which is deleted +# with the v1 trees in 2.1. +class ConfigDefaultTimingTest < ActiveSupport::TestCase + ENTRY_POINTS = %w[ + snap_diff + snap_diff/integrations/minitest + ].freeze + + def run_probe(script, env) + out, status = Open3.capture2e(env, RbConfig.ruby, "-Ilib", "-e", script) + + assert status.success?, "probe failed:\n#{out}" + end + + CHECK_HELPER = <<~'RUBY' + def check(name, expected, actual) + return if expected == actual + abort("#{name}: expected #{expected.inspect}, got #{actual.inspect}") + end + RUBY + + # Probe A: defaults snapshot without CI/Rails + require-time freezing of + # ENV- and pwd-derived defaults + call-time liveness of the Capybara- + # coupled wait. ENV/cwd are mutated after the require but BEFORE the + # first read, so both eager-at-require (current, expected) and any lazy + # read-time variant are distinguished. + SNAPSHOT_SCRIPT = CHECK_HELPER + <<~'RUBY' + require "pathname" + launch_pwd = Pathname(".").expand_path + + require ENV.fetch("PROBE_ENTRY") + + ENV["CI"] = "1" + require "tmpdir" + Dir.chdir(Dir.tmpdir) + + # Expected default values (CI unset, no Rails, at require time). + # fail_if_new false / root == launch pwd also pin require-time + # evaluation: ENV["CI"] and cwd were changed above, pre-first-read. + { + add_driver_path: nil, + add_os_path: nil, + blur_active_element: true, + screenshot_enabled: nil, + hide_caret: true, + disable_animations: nil, + root: launch_pwd, + stability_time_limit: nil, + window_size: nil, + save_path: "doc/screenshots", + use_lfs: nil, + screenshot_format: "png", + capybara_screenshot_options: {}, + delayed: true, + area_size_limit: nil, + fail_if_new: false, + pending_if_new: false, + fail_on_difference: true, + color_distance_limit: nil, + enabled: true, + skip_area: nil, + tolerance: nil, + perceptual_threshold: nil, + screenshoter: SnapDiff::Screenshoter, + manager: SnapDiff::SnapManager + }.each do |name, expected| + check("SnapDiff.config.#{name}", expected, SnapDiff.config.public_send(name)) + end + + # Capybara-coupled wait is read at CALL time (live), not frozen. + Capybara.default_max_wait_time = 42.5 + check("default_options[:wait] follows Capybara.default_max_wait_time set after require", + 42.5, SnapDiff.config.default_options[:wait]) + RUBY + + # Probe B: ENV["CI"] present (non-empty) BEFORE the require flips the + # fail_if_new default on -- and unsetting it after the require does not + # flip it back (frozen at require time). + CI_SET_SCRIPT = CHECK_HELPER + <<~RUBY + require ENV.fetch("PROBE_ENTRY") + ENV.delete("CI") + check(:fail_if_new, true, SnapDiff.config.fail_if_new) + RUBY + + # Probe C: a Rails module with .root defined BEFORE the require wins over + # the pwd fallback -- and reassigning Rails.root after the require (pre + # first read) is not seen (frozen at require time). + RAILS_ROOT_SCRIPT = CHECK_HELPER + <<~RUBY + require "pathname" + + module Rails + @root = Pathname("/fake-rails-root-at-require") + class << self + attr_accessor :root + end + end + + require ENV.fetch("PROBE_ENTRY") + Rails.root = Pathname("/fake-rails-root-after-require") + check(:root, Pathname("/fake-rails-root-at-require"), SnapDiff.config.root) + RUBY + + ENTRY_POINTS.each do |entry| + test "#{entry}: defaults snapshot matches; ENV/pwd frozen at require, wait live" do + run_probe(SNAPSHOT_SCRIPT, {"PROBE_ENTRY" => entry, "CI" => nil}) + end + + test "#{entry}: CI=1 before require turns fail_if_new on; unset after require does not turn it off" do + run_probe(CI_SET_SCRIPT, {"PROBE_ENTRY" => entry, "CI" => "1"}) + end + + test "#{entry}: Rails.root defined before require wins; reassigning it after require is not seen" do + run_probe(RAILS_ROOT_SCRIPT, {"PROBE_ENTRY" => entry, "CI" => nil}) + end + end +end diff --git a/test/unit/core_tree_has_no_legacy_deps_test.rb b/test/unit/core_tree_has_no_legacy_deps_test.rb new file mode 100644 index 00000000..4fb0bf4a --- /dev/null +++ b/test/unit/core_tree_has_no_legacy_deps_test.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require "test_helper" + +# Keeps the removed names from creeping back into lib/. +# +# Its twin (legacy_tree_is_alias_only_test.rb) proved the v1 trees held no +# logic; the trees are gone and so is that test. This one used to prove the +# core did not reach BACK into them, which is what made the deletion a +# `git rm`. The deletion has happened -- a legacy `require` in lib/ now fails +# loudly on its own -- so the surviving job is the quiet half: a NAME in a +# string, a user-facing message or a docstring that mentions +# `Capybara::Screenshot.*`, `SnapDiff::Drivers.available` or +# `shift_distance_limit` still parses fine and simply lies. +# +# Scope: every file under lib/. The exclusion list this gate used to carry +# (legacy_shims.rb, deprecation.rb -- files that existed to BUILD the v1 +# surface) is empty: they were deleted, not excluded. +# +# WHOLE-LINE comments are ignored: "ex +SnapDiff.config.active?+" on its +# own line is history, not a dependency. Everything else on a code line +# counts, strings and trailing comments included -- a user-facing message +# naming a legacy accessor is a legacy reference that survives the deletion +# and starts lying the day it happens, and a trailing note on a live line is +# close enough to the code to be worth keeping honest. Move such a note to +# its own line if the gate objects. +class CoreTreeHasNoLegacyDepsTest < ActiveSupport::TestCase + LIB = Pathname.new(__dir__).join("../../lib").expand_path + + # EMPTY, and it must stay that way. It named the two files that built the v1 + # surface (legacy_shims.rb, deprecation.rb); 2.1 deleted them rather than + # exempting them, so there is nothing left under lib/ this gate skips. + DELETED_WITH_LEGACY_TREES = [].freeze + + CORE_FILES = ( + [LIB.join("snap_diff.rb")] + Dir[LIB.join("snap_diff/**/*.rb")].map { |p| Pathname.new(p) } + ).sort.reject { |file| DELETED_WITH_LEGACY_TREES.include?(file.relative_path_from(LIB).to_s) }.freeze + + # A require of anything in the v1 trees: `capybara/screenshot/...`, + # `capybara_screenshot_diff...`, `capybara-screenshot-diff`. Plain + # `require "capybara"` / `"capybara/dsl"` is the base gem, not this gem's + # legacy tree, so it must not match. + # + # The `(\.{1,2}/)*` is load-bearing: every core file sits one directory + # below lib/, so `require_relative "../capybara/screenshot/diff/version"` + # reaches the v1 tree and really loads it. Anchoring straight on the quote + # let that through. + LEGACY_REQUIRE = %r{\Arequire(_relative)?\s+["'](\.{1,2}/)*capybara(/screenshot|_screenshot_diff|-screenshot-diff)} + + # A read or write of a v1 namespace constant. + LEGACY_CONSTANT = /(?legacy edges that existed the day + # the gate was written, and every one of them is gone. Keep it empty: an + # entry is a decision to keep a core->legacy edge across the 2.1 deletion, + # so it needs a written reason here AND an ADR-008 update -- never just a + # red build turned green. + ALLOWED = {}.freeze + + test "no core file requires or references the v1 namespaces" do + refute_empty CORE_FILES, "core glob matched nothing -- the gate would pass vacuously" + + offenders = CORE_FILES.flat_map { |file| offences(file) } + + assert_empty offenders, <<~MSG + lib/ names something 2.1 removed -- the v1 compatibility trees, the + driver abstraction, or shift_distance_limit. Repoint these at what the + gem actually has (`SnapDiff.config.*`, `SnapDiff::Drivers::VipsDriver`, + `require "snap_diff/..."`): + + #{offenders.join("\n")} + MSG + end + + test "the allowlist names only lines that still exist" do + stale = ALLOWED.flat_map do |path, lines| + file = LIB.join(path) + # A deleted file is the most stale an entry can get; report it rather + # than letting Pathname#read blow up with Errno::ENOENT. + next ["#{path}: allowlisted file no longer exists"] unless file.exist? + + present = significant_lines(file).map(&:first) + (lines - present).map { |line| "#{path}: allowlisted line no longer present: `#{line}`" } + end + + assert_empty stale, <<~MSG + The allowlist is out of date -- these entries protect nothing and only + hide future regressions. Delete them: + + #{stale.join("\n")} + MSG + end + + private + + def offences(file) + rel = file.relative_path_from(LIB).to_s + allowed = ALLOWED.fetch(rel, []) + + significant_lines(file).filter_map do |line, number| + next if allowed.include?(line) + + reason = + if LEGACY_REQUIRE.match?(line) then "requires a v1 tree path" + elsif LEGACY_CONSTANT.match?(line) then "references a v1 namespace constant" + elsif REMOVED_SURFACE.match?(line) then "names a surface 2.1 removed" + end + "#{rel}:#{number}: #{reason} -- `#{line}`" if reason + end + end + + # [stripped line, 1-based line number] for every line that is not blank and + # not a whole-line comment. + def significant_lines(file) + file.read.lines.each_with_index.filter_map do |line, index| + stripped = line.strip + [stripped, index + 1] unless stripped.empty? || stripped.start_with?("#") + end + end +end diff --git a/test/unit/diff_test.rb b/test/unit/diff_test.rb new file mode 100644 index 00000000..3aa1d694 --- /dev/null +++ b/test/unit/diff_test.rb @@ -0,0 +1,243 @@ +# frozen_string_literal: true + +require "test_helper" +require "minitest/stub_const" +require "support/non_minitest_assertions" + +class DiffTest < ActiveSupport::TestCase + setup do + Capybara.current_driver = Capybara.default_driver + + @orig_add_driver_path = SnapDiff.config.add_driver_path + SnapDiff.config.add_driver_path = true + + @orig_add_os_path = SnapDiff.config.add_os_path + SnapDiff.config.add_os_path = true + + @orig_screenshot_format = SnapDiff.config.screenshot_format + + @orig_window_size = SnapDiff.config.window_size + SnapDiff.config.window_size = [80, 80] + end + + include SnapDiff::Minitest::Assertions + include DSLStub + + teardown do + SnapDiff::SnapManager.cleanup! unless persist_comparisons? + SnapDiff.reset + + SnapDiff.config.add_driver_path = @orig_add_driver_path + SnapDiff.config.add_os_path = @orig_add_os_path + SnapDiff.config.screenshot_format = @orig_screenshot_format + SnapDiff.config.window_size = @orig_window_size + end + + test "has a version number" do + refute_nil SnapDiff::VERSION + end + + test "updates screenshot group name" do + assert_nil screenshot_namer.group + screenshot_group "a" + assert_equal "a", screenshot_namer.group + screenshot_group "b" + assert_equal "b", screenshot_namer.group + end + + test "screenshot_section prepends section to path" do + assert_nil screenshot_namer.section + assert_nil screenshot_namer.group + + screenshot_section "a" + assert_equal "a", screenshot_namer.section + assert_match %r{doc/screenshots/(macos|linux)/rack_test/a}, screenshot_dir + + screenshot_group "b" + assert_equal "b", screenshot_namer.group + assert_match %r{doc/screenshots/(macos|linux)/rack_test/a/b}, screenshot_dir + + screenshot_group "c" + assert_equal "c", screenshot_namer.group + assert_match %r{doc/screenshots/(macos|linux)/rack_test/a/c}, screenshot_dir + end + + test "stores screenshot with given name" do + screenshot_group "screenshot" + assert_matches_screenshot "a" + end + + test "does not fail when fail_on_difference is false and screenshots differ" do + SnapDiff.config.stub(:fail_on_difference, false) do + test_case = SampleMiniTestCase.new(:_test_sample_screenshot_error) + test_case.run + assert_equal 0, test_case.failures.size + end + end + + test "writes screenshot to alternate save path" do + default_path = SnapDiff.config.save_path + SnapDiff.config.save_path = "foo/bar" + + screenshot_section "a" + screenshot_group "b" + screenshot "a", delayed: false + + assert_match %r{foo/bar/(macos|linux)/rack_test/a/b}, screenshot_dir + ensure + FileUtils.remove_entry SnapDiff.config.screenshot_area_abs + SnapDiff.config.save_path = default_path + end + + test "does not error when using stability_time_limit" do + default_stability_time_limit = SnapDiff.config.stability_time_limit + SnapDiff.config.stability_time_limit = 0.001 + + screenshot "a" + ensure + SnapDiff.config.stability_time_limit = default_stability_time_limit + end + + test "builds full name from string" do + assert_equal "a", build_full_name("a") + screenshot_group "b" + assert_equal "b/00_a", build_full_name("a") + screenshot_section "c" + assert_equal "c/b/00_a", build_full_name("a") + screenshot_group nil + assert_equal "c/a", build_full_name("a") + end + + test "builds full name from symbol" do + screenshot_group :b + assert_equal "b/00_a", build_full_name(:a) + end + + # "detects available diff drivers" is gone with driver detection itself: + # 2.1 removed SnapDiff::Drivers::AVAILABLE_DRIVERS along with the rest of the + # abstraction. There is nothing to detect when there is one backend, and + # `ruby-vips` is a gemspec runtime dependency, so its absence is a resolver + # error rather than a list this gem has to compute. + + test "aggregates failures on teardown for Minitest" do + test_case = SampleMiniTestCase.new(:_test_sample_screenshot_error) + + test_case.run + + assert_equal 1, test_case.failures.size + assert_includes test_case.failures.first.message, "expected error message" + end + + test "raises error on teardown for non-Minitest" do + test_case = SampleNotMiniTestCase.new + test_case._test_sample_screenshot_error + + expected_message = + "Screenshot does not match for 'sample_screenshot' expected error message for non minitest" + assert_raises(SnapDiff::ExpectationNotMet, expected_message) { test_case.teardown } + assert_empty(SnapDiff.session.assertions) + end + + class SampleMiniTestCase < ActiveSupport::TestCase + include SnapDiff::Minitest::Assertions + + # NOTE: we need to add `_` as prefix to skip this test from auto-run + def _test_sample_screenshot_error + mock = ::Minitest::Mock.new + mock.expect(:different?, true) + mock.expect(:different?, true) + mock.expect(:dimensions_changed?, false) + mock.expect(:base_image_path, Pathname.new("screenshot.base.png")) + mock.expect(:error_message, "expected error message") + + assertion = SnapDiff::ScreenshotAssertion.new("sample_screenshot") + assertion.caller = ["my_test.rb:42"] + assertion.compare = mock + SnapDiff.session.add_assertion(assertion) + + assert true + end + end + + class SampleNotMiniTestCase + def self.setup + # noop + end + + def self.teardown(&block) + @@teardown_callback = block + end + + def teardown + instance_eval(&@@teardown_callback) if @@teardown_callback + ensure + @@teardown_callback = nil + SnapDiff.reset + end + + include NonMinitest::Assertions + + def _test_sample_screenshot_error + comparison = ::Minitest::Mock.new + comparison.expect(:different?, true) # to find backtrace + comparison.expect(:different?, true) # to find messages + comparison.expect(:dimensions_changed?, false) + comparison.expect(:base_image_path, Pathname.new("screenshot.base.png")) + comparison.expect(:error_message, "expected error message for non minitest") + + assertion = SnapDiff::ScreenshotAssertion.new("sample_screenshot") + assertion.caller = ["my_test.rb:42"] + assertion.compare = comparison + SnapDiff.session.add_assertion(assertion) + end + end + + class ScreenshotFormatTest < ActiveSupport::TestCase + setup do + @orig_screenshot_format = SnapDiff.config.screenshot_format + end + + include DSLStub + include SnapDiff::Minitest::Assertions + + teardown do + SnapDiff.config.screenshot_format = @orig_screenshot_format + end + + test "stores screenshot using default format extension" do + skip "VIPS not present. Skipping VIPS driver tests." unless defined?(Vips) + snap = SnapDiff::SnapManager.snapshot("a", "webp") + + set_test_images(snap, :a, :a) + + SnapDiff.config.stub(:screenshot_format, "webp") do + screenshot "a" + + assert_stored_screenshot("a.webp") + end + end + + test "stores screenshot using overridden format extension" do + snap = SnapDiff::SnapManager.snapshot("a", "png") + set_test_images(snap, :a, :a) + + SnapDiff.config.stub(:screenshot_format, "webp") do + screenshot "a", screenshot_format: "png" + + assert_stored_screenshot("a.png") + end + end + end + + def screenshot_dir + File.join(SnapDiff.config.screenshot_area, *screenshot_namer.directory_parts) + end + + def screenshot_namer + SnapDiff.session.screenshot_namer + end + + def build_full_name(name) + SnapDiff.session.screenshot_namer.full_name(name) + end +end diff --git a/test/unit/difference_test.rb b/test/unit/difference_test.rb new file mode 100644 index 00000000..3acde90f --- /dev/null +++ b/test/unit/difference_test.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff/comparison_result" + +class DifferenceTest < ActiveSupport::TestCase + setup do + @difference = SnapDiff::ComparisonResult.new(nil, {}, nil, {different_dimensions: []}) + end + + test "#different? returns true when images have different dimensions" do + assert_predicate @difference, :different? + end + + test "#failed? returns true when images have different dimensions" do + assert_predicate @difference, :failed? + end + + test "#inspect is a one-line summary with the difference metrics" do + line = @difference.inspect + + assert_includes line, "different=true" + assert_includes line, "failed_by=" + assert_includes line, "area_size=0" + assert_includes line, "difference_level=" + assert_not_includes line, "\n" + end +end diff --git a/test/unit/drivers/vips_driver_test.rb b/test/unit/drivers/vips_driver_test.rb new file mode 100644 index 00000000..1cfe90d6 --- /dev/null +++ b/test/unit/drivers/vips_driver_test.rb @@ -0,0 +1,292 @@ +# frozen_string_literal: true + +require "test_helper" +require "support/driver_contract_tests" + +require "snap_diff/drivers/vips_driver" + +module SnapDiff + module Drivers + class VipsDriverTest < ActiveSupport::TestCase + include DSLStub + include DriverContractTests + + setup do + @new_screenshot_result = Tempfile.new(%w[screenshot .png], Rails.root) + end + + teardown do + if @new_screenshot_result + @new_screenshot_result.close + @new_screenshot_result.unlink + end + end + + # REGRESSION. libvips caches loaders on filename + mtime, and mtime has + # one-second resolution, so rewriting a path and re-reading it within the + # same second used to hand back the PREVIOUS image. #from_file passes + # `revalidate: true` to defeat that. + # + # This was latent while chunky_png existed: a Comparison built without an + # explicit driver defaulted to chunky_png, which re-read the file every + # time. 2.1 made vips the only backend, so the stale read became the only + # behaviour -- it turned identical images into "different" and vice versa. + # + # The teardown above used to flush the whole vips cache + # (`Vips.cache_set_max(0); Vips.cache_set_max(1000)`) to paper over this; + # with the driver fixed, that workaround is gone. + test "#from_file re-reads a path that was overwritten within the same second" do + driver = VipsDriver.new + path = Rails.root / "revalidate_probe.png" + + FileUtils.cp(TEST_IMAGES_DIR / "b.png", path) + first = driver.from_file(path) + first_avg = first.avg # force evaluation BEFORE the overwrite + + FileUtils.cp(TEST_IMAGES_DIR / "a.png", path) + second_avg = driver.from_file(path).avg + + assert_not_equal first_avg, second_avg, + "vips served the cached b.png after the path was overwritten with a.png" + assert_equal driver.from_file(TEST_IMAGES_DIR / "a.png").avg, second_avg + ensure + FileUtils.rm_f(path) + end + + test "#different? returns false when comparing identical images" do + comp = make_comparison(:a, :a) + assert_not comp.different? + end + + test "#quick_equal? returns true when comparing identical images" do + comp = make_comparison(:a, :a) + + assert comp.quick_equal? + end + + test "can be instantiated with default constructor" do + assert VipsDriver.new + end + + test "#different? preserves runtime files when images are different" do + comp = make_comparison(:a, :c) + assert comp.different? + assert_includes comp.error_message, "[11.0,3.0,49.0,21.0]" + assert File.exist?(comp.base_image_path) + assert File.exist?(comp.reporter.annotated_base_image_path) + assert File.exist?(comp.reporter.annotated_image_path) + end + + test "#different? cleans up runtime files when images are identical" do + comp = make_comparison(:c, :c) + assert_not comp.different? + + assert comp.reporter.annotated_base_image_path + assert comp.reporter.annotated_image_path + + assert_not File.exist?(comp.reporter.annotated_base_image_path) + assert_not File.exist?(comp.reporter.annotated_image_path) + end + + test "#different? detects single-pixel wide differences between images" do + comp = make_comparison(:a, :d) + assert comp.different? + assert_includes comp.error_message, "[9.0,6.0,10.0,14.0]" + end + + test "#different? respects color_distance_limit when within allowed threshold" do + comp = make_comparison(:a, :b, color_distance_limit: 255) + assert_not comp.different? + end + + test "#different? enforces color_distance_limit when beyond allowed threshold" do + comp = make_comparison(:a, :b, color_distance_limit: 3) + assert comp.different? + end + + test "#different? returns equal when tolerance is greater than difference area" do + comp = make_comparison(:a, :b, tolerance: 0.01) + assert comp.quick_equal? + assert_not comp.different? + assert_not comp.error_message + end + + test "#different? detects difference when tolerance is less than difference area" do + comp = make_comparison(:a, :b, tolerance: 0.000001) + assert_not comp.quick_equal? + assert comp.different? + end + + test "#different? handles single-pixel line differences with median filter" do + comp = make_comparison(:a, :d, median_filter_window_size: 3, color_distance_limit: 8) + assert comp.quick_equal? + assert_not comp.different? + end + + test "#quick_equal? returns false when images are different" do + comp = make_comparison(:a, :b) + assert_not comp.quick_equal? + end + + test "#quick_equal? respects color_distance_limit when below difference threshold" do + comp = make_comparison(:a, :b, color_distance_limit: 2) + assert_not comp.quick_equal? + end + + test "#quick_equal? respects color_distance_limit when above difference threshold" do + comp = make_comparison(:a, :b, color_distance_limit: 200) + assert comp.quick_equal? + end + + test "#different? detects dimension changes between images" do + comp = make_comparison(:a, :a_cropped) + assert comp.different? + assert_includes comp.error_message, "Dimensions have changed: " + assert_includes comp.error_message, "80x60" + end + + test "#quick_equal? skips differences covered by skip_area configuration" do + comp = make_comparison( + :a, + :d, + skip_area: [ + Region.from_edge_coordinates(9, 0, 11, 80), + Region.from_edge_coordinates(79, 79, 80, 80) + ] + ) + assert comp.quick_equal? + assert_not comp.different? + end + + test "#quick_equal? detects differences not covered by skip_area" do + comp = make_comparison( + :a, + :d, + skip_area: [ + Region.from_edge_coordinates(79, 79, 80, 80), + Region.from_edge_coordinates(78, 78, 80, 80) + ] + ) + assert_not comp.quick_equal? + assert comp.different? + end + + test "#different? uses perceptual_threshold when set" do + comp = make_comparison(:a, :b, perceptual_threshold: 2.0) + assert comp.different? + end + + test "#different? with perceptual_threshold returns equal for identical images" do + comp = make_comparison(:a, :a, perceptual_threshold: 2.0) + assert_not comp.different? + end + + # Test Interface Contracts + + test "#from_file successfully loads an image from the specified path" do + assert VipsDriver.new.from_file(TEST_IMAGES_DIR / "a.png") + end + + private + + def make_comparison(old_img, new_img, options = {}) + destination = Pathname.new(@new_screenshot_result.path) + super(old_img, new_img, destination: destination, **options) + end + + def sample_region + [0, 0, 0, 0] + end + end + + class VipsDriverClassMethodsTest < ActiveSupport::TestCase + test "VipsDriver.difference_region_by detects difference regions without color threshold" do + old_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/a.png") + new_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/b.png") + + left, top, right, bottom = difference(old_image, new_image) + + assert_equal [20.0, 15.0, 30.0, 25.0], [left, top, right, bottom] + + left, top, right, bottom = difference(old_image, new_image, color_distance: 0) + + assert_equal [20.0, 15.0, 30.0, 25.0], [left, top, right, bottom] + end + + test "VipsDriver.difference_region_by respects color_distance threshold" do + old_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/a.png") + new_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/b.png") + + left, top, right, bottom = difference(old_image, new_image, color_distance: 150) + + assert_equal [26.0, 18.0, 27.0, 19.0], [left, top, right, bottom] + end + + test "VipsDriver.difference_region_by returns correct region coordinates" do + old_image = Vips::Image.new_from_file(TEST_IMAGES_DIR.join("a.png").to_path) + new_image = Vips::Image.new_from_file(TEST_IMAGES_DIR.join("b.png").to_path) + + left, top, right, bottom = difference(old_image, new_image) + + assert_equal [20.0, 15.0, 30.0, 25.0], [left, top, right, bottom] + end + + test "VipsDriver.perceptual_difference_mask returns nil region for identical images" do + old_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/a.png") + same_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/a.png") + + diff_mask = VipsDriver.perceptual_difference_mask(old_image, same_image) + region = VipsDriver.difference_region_by(diff_mask) + + assert_nil region + end + + test "VipsDriver.perceptual_difference_mask detects differences between images" do + old_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/a.png") + new_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/b.png") + + diff_mask = VipsDriver.perceptual_difference_mask(old_image, new_image, 2.0) + region = VipsDriver.difference_region_by(diff_mask) + + assert_not_nil region + end + + test "VipsDriver.perceptual_difference_mask detects alpha channel differences" do + old_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/a.png") + # Make a copy with different alpha + transparent = old_image.extract_band(0, n: 3).bandjoin(128) + + diff_mask = VipsDriver.perceptual_difference_mask(old_image, transparent) + region = VipsDriver.difference_region_by(diff_mask) + + assert_not_nil region, "Should detect alpha channel difference" + end + + test "VipsDriver.perceptual_difference_mask respects threshold" do + old_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/a.png") + new_image = Vips::Image.new_from_file("#{TEST_IMAGES_DIR}/b.png") + + # High threshold — fewer differences detected + high_mask = VipsDriver.perceptual_difference_mask(old_image, new_image, 50.0) + high_region = VipsDriver.difference_region_by(high_mask) + + # Low threshold — more differences detected + low_mask = VipsDriver.perceptual_difference_mask(old_image, new_image, 2.0) + low_region = VipsDriver.difference_region_by(low_mask) + + assert_not_nil low_region + # High threshold region should be smaller or nil compared to low threshold + if high_region + assert high_region.size <= low_region.size + end + end + + private + + def difference(old_image, new_image, color_distance: nil) + diff_mask = VipsDriver.difference_mask(new_image, old_image, color_distance) + VipsDriver.difference_region_by(diff_mask).to_edge_coordinates + end + end + end +end diff --git a/test/unit/dsl_test.rb b/test/unit/dsl_test.rb new file mode 100644 index 00000000..cdc49110 --- /dev/null +++ b/test/unit/dsl_test.rb @@ -0,0 +1,254 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff" +require "snap_diff/screenshot_assertion" + +class DSLTest < ActiveSupport::TestCase + include SnapDiff::DSL + include DSLStub + + def before_setup + @original_root = SnapDiff.config.root + @new_root = Dir.mktmpdir + SnapDiff.config.root = Pathname.new(@new_root) + super + end + + def after_teardown + super + SnapDiff.config.root = @original_root + FileUtils.remove_entry(@new_root) if @new_root + end + + test "#screenshot raises error when screenshot is missing and fail_if_new is true" do + SnapDiff::Vcs.stub(:checkout_vcs, false) do + SnapDiff.config.stub(:fail_if_new, true) do + assert_raises SnapDiff::ExpectationNotMet, match: /No existing screenshot found for/ do + screenshot "not_existing_screenshot-name" + end + end + end + end + + # The reported figures are libvips': area_size and region come out as floats + # and there is no max_color_distance. They used to be chunky_png's (629 / + # [11,3,48,20] / max_color_distance 187.4) because a Comparison built without + # an explicit `driver:` fell back to chunky_png -- 2.1 deleted that driver + # and the selection that reached it, so this is the deletion showing through, + # not a drift in the reporter. + test "#assert_image_not_changed generates correct error message for image mismatch" do + message = assert_image_not_changed(["my_test.rb:42"], "name", make_comparison(:a, :c, destination: "screenshot.png")) + assert_equal <<~MSG.chomp, message + Screenshot does not match for 'name': ({"area_size":684.0,"region":[11.0,3.0,49.0,21.0]}) + #{SnapDiff.config.root}/doc/screenshots/screenshot.png + #{SnapDiff.config.root}/doc/screenshots/screenshot.base.diff.png + #{SnapDiff.config.root}/doc/screenshots/screenshot.diff.png + #{SnapDiff.config.root}/doc/screenshots/screenshot.heatmap.diff.png + my_test.rb:42 + MSG + end + + # Two tests are deleted rather than repointed: + # + # - "includes shift distance in error message": `shift_distance_limit` is + # implemented only by chunky_png and dies with it. libvips has no + # shift-distance comparison, so there is no equivalent message. + # - "supports driver options for image comparison": there are no driver + # options left to support. + + test "#screenshot compares against the baseline and reports no difference" do + assert_not screenshot("a") + end + + def assert_no_screenshot_jobs_scheduled + assert_not_predicate SnapDiff.session, :assertions_present? + end + + test "#screenshot with skip_stack_frames: 0 includes our_screenshot in caller" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + assert_no_screenshot_jobs_scheduled + + snap = create_snapshot_for(:a, :c) + + our_screenshot(snap.full_name, 0) + assert_equal 1, SnapDiff.session.assertions.size + assert_match(/our_screenshot'/, SnapDiff.session.assertions[0].caller.first) + assert_equal snap.full_name, SnapDiff.session.assertions[0].name + end + end + + test "#screenshot with skip_stack_frames: 1 includes test method in caller" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + assert_no_screenshot_jobs_scheduled + + snap = create_snapshot_for(:a, :c) + + our_screenshot(snap.full_name, 1) + assert_equal 1, SnapDiff.session.assertions.size + assert_match( + %r{/dsl_test.rb}, + SnapDiff.session.assertions[0].caller.first + ) + assert_equal snap.full_name, SnapDiff.session.assertions[0].name + end + end + + test "#assert_no_screenshot_changes reports caller from test method" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + assert_no_screenshot_jobs_scheduled + + snap = create_snapshot_for(:a, :c) + + assert_no_screenshot_changes(snap.full_name) + assert_equal 1, SnapDiff.session.assertions.size + assert_match( + %r{/dsl_test.rb}, + SnapDiff.session.assertions[0].caller.first + ) + end + end + + test "#screenshot with delayed: false raises error when images differ" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + SnapDiff.config.stub(:delayed, false) do + assert_raises(SnapDiff::ExpectationNotMet) do + snap = create_snapshot_for(:c, :a) + screenshot(snap.full_name, delayed: false) + end + end + end + end + + test "#screenshot with delayed: false succeeds when images match" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + SnapDiff.config.stub(:delayed, false) do + snap = create_snapshot_for(:a) + assert_nothing_raised { screenshot(snap.full_name, delayed: false) } + end + end + end + + test "#screenshot accepts skip_area and stability_time_limit options" do + assert_not screenshot(:a, skip_area: [0, 0, 1, 1], stability_time_limit: 0.01) + end + + test "#screenshot creates new screenshot file when it doesn't exist" do + screenshot(:c) + + snap = SnapDiff::SnapManager.snapshot("c") + assert_predicate snap.path, :exist? + end + + # Regression for https://github.com/snap-diff/snap_diff-capybara/issues/191: + # a user-defined #screenshot in the test class must not hijack the gem's internals. + test "#assert_no_screenshot_changes ignores user-defined #screenshot" do + def self.screenshot(*, **) + @user_screenshot_called = true + end + + SnapDiff::Vcs.stub(:checkout_vcs, true) do + snap = create_snapshot_for(:a, :c) + + assert_no_screenshot_changes(snap.full_name) + assert_not @user_screenshot_called + assert_equal 1, SnapDiff.session.assertions.size + end + end + + test "#assert_matches_screenshot ignores user-defined #screenshot" do + def self.screenshot(*, **) + @user_screenshot_called = true + end + + SnapDiff::Vcs.stub(:checkout_vcs, true) do + snap = create_snapshot_for(:a, :c) + + assert_matches_screenshot(snap.full_name) + assert_not @user_screenshot_called + assert_equal 1, SnapDiff.session.assertions.size + end + end + + test "#screenshot records new screenshots that have no baseline in the registry" do + SnapDiff::Vcs.stub(:checkout_vcs, false) do + screenshot "a" + + assert_equal ["a"], SnapDiff.session.new_screenshots + end + end + + test "SnapDiff.reset clears new_screenshots" do + SnapDiff::Vcs.stub(:checkout_vcs, false) do + screenshot "a" + assert_predicate SnapDiff.session, :new_screenshots_present? + + SnapDiff.reset + + assert_not_predicate SnapDiff.session, :new_screenshots_present? + assert_empty SnapDiff.session.new_screenshots + end + end + + test "#capture_screenshot writes the file and registers no assertion" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + capture_screenshot(:c) + + snap = SnapDiff::SnapManager.snapshot("c") + assert_predicate snap.path, :exist? + assert_no_screenshot_jobs_scheduled + end + end + + test "#capture_screenshot creates the destination directory for nested names" do + naive_screenshoter = Class.new do + def initialize(_capture_options, _comparison_options) + end + + def take_comparison_screenshot(snapshot) + File.binwrite(snapshot.path, "png") + end + end + + SnapDiff.config.stub(:screenshoter, naive_screenshoter) do + capture_screenshot("nested/dir/example") + + assert_predicate SnapDiff::SnapManager.snapshot("nested/dir/example").path, :exist? + end + end + + test "#capture_screenshot does not raise even when a differing baseline exists" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + snap = create_snapshot_for(:a, :c) + + assert_nothing_raised { capture_screenshot(snap.full_name) } + assert_no_screenshot_jobs_scheduled + end + end + + test "#screenshot with compare: false captures without registering an assertion" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + snap = create_snapshot_for(:a, :c) + snap.path.delete + + screenshot(snap.full_name, compare: false) + + assert_predicate snap.path, :exist? + assert_no_screenshot_jobs_scheduled + end + end + + private + + def our_screenshot(name, skip_stack_frames) + screenshot(name, skip_stack_frames: skip_stack_frames) + end + + # Pins the user-facing error-message shape produced by #validate. + def assert_image_not_changed(backtrace, name, comparison) + assertion = SnapDiff::ScreenshotAssertion.new(name) + assertion.caller = backtrace + assertion.compare = comparison + assertion.validate + end +end diff --git a/test/unit/errors_test.rb b/test/unit/errors_test.rb new file mode 100644 index 00000000..4bfecec8 --- /dev/null +++ b/test/unit/errors_test.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require "test_helper" + +# The canonical half of what used to be errors_alias_test.rb: the shape of +# the SnapDiff error hierarchy itself, which outlives the v1 aliases. The +# `CapybaraScreenshotDiff::*` alias half stayed behind in +# test/legacy/errors_alias_test.rb and goes with the v1 trees in 2.1. +class ErrorsTest < ActiveSupport::TestCase + test "error hierarchy is preserved" do + assert_operator SnapDiff::ExpectationNotMet, :<, SnapDiff::Error + assert_operator SnapDiff::UnstableImage, :<, SnapDiff::Error + assert_operator SnapDiff::Error, :<, SnapDiff::ErrorWithFilteredBacktrace + assert_operator SnapDiff::WindowSizeMismatchError, :<, SnapDiff::ErrorWithFilteredBacktrace + end + + # docs/snapdiff.md calls SnapDiff::Error "Base class for every error this + # gem raises" -- so `rescue SnapDiff::Error` has to actually catch every + # one of them. Discovered rather than listed: a new error class added + # outside the hierarchy fails here instead of quietly breaking that claim + # for adopters (WindowSizeMismatchError and DualInstallError both did). + test "every error the gem defines inherits SnapDiff::Error" do + plumbing = [SnapDiff::Error, SnapDiff::ErrorWithFilteredBacktrace] + errors = SnapDiff.constants + .map { |name| SnapDiff.const_get(name) } + .select { |const| const.is_a?(Class) && const < StandardError } - plumbing + + assert_operator errors.size, :>=, 4, "probe should see the gem's error classes" + + errors.each do |error| + assert_operator error, :<, SnapDiff::Error, "#{error} must inherit SnapDiff::Error" + end + end +end diff --git a/test/unit/image_compare_test.rb b/test/unit/image_compare_test.rb new file mode 100644 index 00000000..382eb2fb --- /dev/null +++ b/test/unit/image_compare_test.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true + +require "test_helper" + +# No `if defined?(Vips)` guard and no driver-selection tests: 2.1 removed the +# driver abstraction, so `ruby-vips` is a gemspec runtime dependency and +# SnapDiff::Drivers::VipsDriver is the only backend there is. +class ImageCompareTest < ActiveSupport::TestCase + include DSLStub + + test "#initialize always builds the vips driver" do + assert_kind_of SnapDiff::Drivers::VipsDriver, make_comparison(:b).driver + end + + test "#different? generates annotated diff images" do + comparison = make_comparison(:a, :b) + + assert comparison.different? + + assert_same_images("a-and-b.diff.png", comparison.reporter.annotated_base_image_path) + assert_same_images("b-and-a.diff.png", comparison.reporter.annotated_image_path) + end + + test "#different? handles very long input filenames" do + filename = %w[this-0000000000000000000000000000000000000000000000000-path/is/extremely/ + long/and/if/the/directories/are/flattened/in/ + the_temporary_they_will_cause_the_filename_to_exceed_ + the_limit_on_most_unix_systems_which_nobody_wants.png].join + comparison = make_comparison(:a, :b, destination: (Rails.root / filename)) + + assert comparison.different? + end + + test "#initialize respects the tolerance option" do + comp = make_comparison(:a, :b, tolerance: 0.02) + assert comp.quick_equal? + assert_not comp.different? + assert_equal 0.02, comp.driver_options[:tolerance] + end + + test "#initialize with dimensions creates valid comparison" do + comp = make_comparison(:b, dimensions: [80, 80]) + assert comp.quick_equal? + assert_not comp.different? + end +end + +# Guards the regression killed twice during ADR-004 review (migration-plan PR 5): skip_area +# masking must run at *comparison* time, against whatever base image is on disk (including a +# baseline checked out from VCS), not baked in at *capture* time. Capture-time masking would +# only ever touch the freshly-taken screenshot; a VCS-checked-out baseline predates that +# capture and would never get masked, so a masked new image compared against an unmasked +# baseline would still report a difference in the skip area. +# +# `make_comparison(:a, :c)` stands in for that scenario: `:a` plays the already-on-disk +# baseline (as if checked out from VCS), `:c` plays the freshly captured screenshot. The two +# fixtures are known to differ only within [11,3,48,20]. +class SkipAreaMasksVcsBaselineTest < ActiveSupport::TestCase + include DSLStub + + test "#different? masks the VCS-checked-out baseline, not just the new screenshot" do + full_image_region = Region.from_edge_coordinates(0, 0, 80, 80) + comparison = make_comparison(:a, :c, destination: "skip_area_vcs_baseline", skip_area: [full_image_region]) + + refute_predicate comparison, :different? + end +end + +class IntegrationRegressionTest < ActiveSupport::TestCase + include DSLStub + + # Was a two-element driver matrix ({} and {driver: :chunky_png}); with one + # backend the outer loop had one iteration, so it is gone rather than left + # as a loop over a single element. + test "identical images are quick_equal and not different" do + images = all_fixtures_images_names + Dir.chdir File.expand_path("../fixtures/images", __dir__) do + images.each do |old_img| + new_img = old_img + comparison = make_comparison(old_img, new_img) + assert(comparison.quick_equal?, "compare #{old_img} with #{new_img} should be quick_equal") + assert_not(comparison.different?, "compare #{old_img} with #{new_img} should not be different") + end + end + end + + test "different images are not quick_equal and are marked as different" do + images = all_fixtures_images_names + + images.each do |image| + other_images = images - [image] + other_images.each do |different_image| + comparison = make_comparison(image, different_image) + assert_not( + comparison.quick_equal?, + "compare #{image.inspect} with #{different_image.inspect} should not be quick_equal" + ) + assert( + comparison.different?, + "compare #{image.inspect} with #{different_image.inspect} should be different" + ) + end + end + end + + def all_fixtures_images_names + %w[a a_cropped b c d portrait portrait_b] + end +end + +class ImageCompareRefactorTest < ActiveSupport::TestCase + include DSLStub + include TestHelpers + + # Test #quick_equal? method + test "#quick_equal? returns true when comparing identical images" do + comparison = make_comparison(:a, :a) + assert_predicate comparison, :quick_equal? + end + + test "#quick_equal? returns false when comparing different images" do + comparison = make_comparison(:a, :b) + refute_predicate comparison, :quick_equal? + end + + test "#quick_equal? skips the expensive region scan when pixels differ and no tolerance options are set" do + comparison = make_comparison(:a, :b) + region_scan_calls = 0 + comparison.driver.define_singleton_method(:find_difference_region) do |*args| + region_scan_calls += 1 + TestDoubles::TestDifference.new(true) + end + + comparison.quick_equal? + + assert_equal 0, region_scan_calls, "find_difference_region should not run when no tolerance options are configured" + end + + # Test #different? method + test "#different? returns false when comparing identical images" do + comparison = make_comparison(:a, :a) + refute_predicate comparison, :different? + end + + test "#different? returns true when comparing different images" do + comparison = make_comparison(:a, :b) + assert_predicate comparison, :different? + end + + # Test #dimensions_changed? method + test "#dimensions_changed? returns true when images have different dimensions" do + comparison = make_comparison(:portrait, :a) + comparison.processed + + assert_predicate comparison, :dimensions_changed? + assert_kind_of SnapDiff::Reporters::Default, comparison.reporter + end + + test "#dimensions_changed? returns false when images have same dimensions" do + comparison = make_comparison(:a, :a) + comparison.processed + + refute_predicate comparison, :dimensions_changed? + end + + # Test reporter configuration + test "#reporter returns Default reporter by default" do + comparison = make_comparison(:a, :a) + assert_kind_of SnapDiff::Reporters::Default, comparison.reporter + end +end diff --git a/test/unit/image_preprocessor_test.rb b/test/unit/image_preprocessor_test.rb new file mode 100644 index 00000000..1629ea2f --- /dev/null +++ b/test/unit/image_preprocessor_test.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require "test_helper" +require "support/test_doubles" +require "support/test_helpers" + +class ImagePreprocessorTest < ActiveSupport::TestCase + include DSLStub + include TestHelpers + + def setup + super + @driver = create_test_driver + end + + test "#process_comparison returns comparison unchanged when no preprocessing options are provided" do + preprocessor = SnapDiff::ImagePreprocessor.new(@driver, {}) + comparison = SnapDiff::Comparison::Images.new(:new_image, :base_image, {}, @driver) + + result = preprocessor.process_comparison(comparison) + + assert_equal comparison, result + assert_empty @driver.add_black_box_calls + assert_empty @driver.filter_calls + end + + test "#process_comparison applies black box to skip areas when skip_area option is provided" do + skip_area = [{x: 10, y: 20, width: 30, height: 40}] + preprocessor = SnapDiff::ImagePreprocessor.new(@driver, skip_area: skip_area) + comparison = SnapDiff::Comparison::Images.new(:new_image, :base_image, {}, @driver) + + result = preprocessor.process_comparison(comparison) + + assert_equal comparison, result + assert_equal 2, @driver.add_black_box_calls.size + + first_call = @driver.add_black_box_calls[0] + second_call = @driver.add_black_box_calls[1] + + assert_equal skip_area.first, first_call[:region] + assert_equal skip_area.first, second_call[:region] + assert_equal :base_image, first_call[:image] + assert_equal :new_image, second_call[:image] + end + + test "#process_comparison applies the median filter when median_filter_window_size is specified" do + @driver = create_test_driver(is_vips: true) + window_size = 3 + options = {median_filter_window_size: window_size} + preprocessor = SnapDiff::ImagePreprocessor.new(@driver, options) + comparison = SnapDiff::Comparison::Images.new(:new_image, :base_image, {}, @driver) + + result = preprocessor.process_comparison(comparison) + + assert_equal comparison, result + assert_equal 2, @driver.filter_calls.size + + first_call = @driver.filter_calls[0] + second_call = @driver.filter_calls[1] + + assert_equal window_size, first_call[:size] + assert_equal window_size, second_call[:size] + assert_equal :base_image, first_call[:image] + assert_equal :new_image, second_call[:image] + end + + # The "warns and skips the median filter when the driver does not support it" + # test is gone with the capability probe it exercised: chunky_png was the + # driver that lacked #filter_image_with_median, and 2.1 removed it. With + # libvips the only backend the fallback branch was unreachable. +end diff --git a/test/unit/minitest_assertions_test.rb b/test/unit/minitest_assertions_test.rb new file mode 100644 index 00000000..64963d4d --- /dev/null +++ b/test/unit/minitest_assertions_test.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require "test_helper" + +class MinitestAssertionsTest < ActiveSupport::TestCase + # Runs a throwaway ::Minitest::Test that takes a single screenshot, so we can + # inspect how before_teardown resolved (passed/skipped/failed) without polluting + # the outer test's own assertions/reporting. + # + # @param teardown [Proc, nil] optional replacement `teardown` method, to + # simulate a user teardown that runs after `before_teardown`. Calls + # `super()` first so DSLStub's own cleanup still happens. + def run_inner_test(teardown: nil, &block) + test_class = Class.new(::Minitest::Test) do + include SnapDiff::Minitest::Assertions + include DSLStub + + define_method(:test_it, &block) + define_method(:teardown, &teardown) if teardown + end + test_class.new(:test_it).run + end + + test "#before_teardown skips the test when pending_if_new is enabled and a screenshot has no baseline" do + SnapDiff::Vcs.stub(:checkout_vcs, false) do + SnapDiff.config.stub(:pending_if_new, true) do + result = run_inner_test { screenshot("a") } + + assert_predicate result, :skipped? + assert_equal( + "No baseline for: a. Commit the captured screenshots to record them.", + result.failures.first.message + ) + end + end + end + + test "#screenshot and #assert_no_screenshot_changes count Minitest assertions" do + SnapDiff::Vcs.stub(:checkout_vcs, false) do + result = run_inner_test do + screenshot("a") + assert_no_screenshot_changes("b") + end + + assert_predicate result, :passed? + assert_equal 2, result.assertions + end + end + + test "#before_teardown does not mask a real teardown error behind a pending skip" do + SnapDiff::Vcs.stub(:checkout_vcs, false) do + SnapDiff.config.stub(:pending_if_new, true) do + result = run_inner_test(teardown: proc { + super() + raise "boom from teardown" + }) { screenshot("a") } + + refute_predicate result, :skipped? + assert_predicate result, :error? + end + end + end + + test "#before_teardown does not skip the test when pending_if_new is disabled" do + SnapDiff::Vcs.stub(:checkout_vcs, false) do + SnapDiff.config.stub(:pending_if_new, false) do + result = run_inner_test { screenshot("a") } + + assert_predicate result, :passed? + end + end + end +end diff --git a/test/unit/pending_screenshots_message_test.rb b/test/unit/pending_screenshots_message_test.rb new file mode 100644 index 00000000..f4db3fbc --- /dev/null +++ b/test/unit/pending_screenshots_message_test.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require "test_helper" + +class PendingScreenshotsMessageTest < ActiveSupport::TestCase + teardown do + SnapDiff.reset + end + + test "returns nil when pending_if_new is disabled" do + SnapDiff.config.stub(:pending_if_new, false) do + SnapDiff.session.record_new_screenshot("a") + + assert_nil SnapDiff.pending_screenshots_message + end + end + + test "returns nil when pending_if_new is enabled but no new screenshots were recorded" do + SnapDiff.config.stub(:pending_if_new, true) do + assert_nil SnapDiff.pending_screenshots_message + end + end + + test "returns the baseline message listing recorded screenshot names" do + SnapDiff.config.stub(:pending_if_new, true) do + SnapDiff.session.record_new_screenshot("a") + SnapDiff.session.record_new_screenshot("b") + + assert_equal( + "No baseline for: a, b. Commit the captured screenshots to record them.", + SnapDiff.pending_screenshots_message + ) + end + end + + test "reads from the calling thread's own registry, not other threads'" do + SnapDiff.config.stub(:pending_if_new, true) do + other_thread_result = Thread.new { + SnapDiff.session.record_new_screenshot("other-thread") + SnapDiff.pending_screenshots_message + }.value + + assert_equal( + "No baseline for: other-thread. Commit the captured screenshots to record them.", + other_thread_result + ) + assert_nil SnapDiff.pending_screenshots_message + end + end +end diff --git a/test/unit/region_alias_guard_test.rb b/test/unit/region_alias_guard_test.rb new file mode 100644 index 00000000..5c30bbcd --- /dev/null +++ b/test/unit/region_alias_guard_test.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff/region" +require "snap_diff/area_calculator" + +# ADR-008 step 3 guards: Region's home is SnapDiff::Region, but user +# configs build skip_area entries with top-level `Region.new(...)` and +# feature-detect with `defined?(Region)`. The top-level name must stay an +# EAGER same-object alias -- not a copy, not a lazy shim. +class RegionAliasGuardTest < ActiveSupport::TestCase + test "top-level Region is the exact same object as SnapDiff::Region" do + assert_same SnapDiff::Region, ::Region, + "expected ::Region and SnapDiff::Region to be the same class object" + end + + test "defined?(Region) is truthy without any const_missing round-trip" do + assert defined?(::Region), "defined?(::Region) must be truthy (eager alias, not a lazy shim)" + end + + test "a top-level Region in skip_area survives SnapDiff internals' is_a? checks" do + user_region = ::Region.new(5, 5, 10, 10) + + result = SnapDiff::AreaCalculator.new(nil, [user_region]).calculate_skip_area + + assert_equal [user_region], result + assert_same user_region, result.first, + "AreaCalculator's is_a?(Region) partition must keep the user's object as-is" + assert result.first.is_a?(SnapDiff::Region) + end +end diff --git a/test/unit/region_test.rb b/test/unit/region_test.rb new file mode 100644 index 00000000..79935163 --- /dev/null +++ b/test/unit/region_test.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require "test_helper" + +class RegionTest < ActiveSupport::TestCase + test "#move_by updates region coordinates by specified deltas" do + region = Region.new(10, 10, 10, 10).move_by(-5, -5) + + assert_equal 5, region.x + assert_equal 5, region.y + assert_equal 10, region.width + assert_equal 10, region.height + end + + test "#find_intersect_with returns intersection with another region" do + crop = Region.new(5, 5, 10, 10) + region = Region.new(10, 10, 20, 20).find_intersect_with(crop) + + assert_equal 10, region.x + assert_equal 10, region.y + assert_equal 5, region.width + assert_equal 5, region.height + end + + test "#find_relative_intersect returns intersection with relative coordinates" do + crop = Region.new(5, 5, 10, 10) + + region = crop.find_relative_intersect(Region.new(0, 0, 20, 20)) + + assert_equal 0, region.x + assert_equal 0, region.y + assert_equal 10, region.width + assert_equal 10, region.height + + region = crop.find_relative_intersect(Region.new(10, 10, 20, 20)) + + assert_equal 5, region.x + assert_equal 5, region.y + assert_equal 5, region.width + assert_equal 5, region.height + end + + test ".from_edge_coordinates returns nil when right or bottom is nil" do + assert_nil Region.from_edge_coordinates(0, 0, nil, nil) + end + + test ".from_edge_coordinates returns nil when region has zero or negative dimensions" do + assert_nil Region.from_edge_coordinates(10, 10, 9, 11) + assert_nil Region.from_edge_coordinates(10, 10, 11, 9) + end + + test "#== returns true when comparing with an identical Region" do + assert_equal Region.new(10, 10, 10, 10), Region.new(10, 10, 10, 10) + assert_not_equal Region.new(10, 10, 10, 10), Region.new(10, 10, 10, 11) + end + + test "#== returns true when comparing with equivalent Array of coordinates" do + assert_equal Region.new(10, 10, 10, 10), [10, 10, 10, 10] + assert_not_equal Region.new(10, 10, 10, 10), [10, 10, 10, 11] + end +end diff --git a/test/unit/registry_concurrency_test.rb b/test/unit/registry_concurrency_test.rb new file mode 100644 index 00000000..44221ee8 --- /dev/null +++ b/test/unit/registry_concurrency_test.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff" + +# Guard #5 from the v2 core-redesign acceptance contract. +# +# Extends the single thread-isolation test (pending_screenshots_message) to +# the whole registry surface: add_assertion / verify / reset racing across +# threads must never leak assertions between threads. This is the regression +# net for parallel test runners and for D7's thread-local store fix. +class RegistryConcurrencyTest < ActiveSupport::TestCase + PassingCompare = Struct.new(:name) do + def different? = false + + def base_image_path = Pathname.new("/nonexistent/#{name}.base.png") + end + + def build_assertion(name) + assertion = SnapDiff::ScreenshotAssertion.new(name) + assertion.caller = ["#{name}:1"] + assertion.compare = PassingCompare.new(name) + assertion + end + + # Memoized per fiber: repeated reads hand back the one registry the test + # is accumulating into, not a fresh empty one. + # (The identity of the v1 CapybaraScreenshotDiff.registry forwarder with + # this accessor is pinned in test/legacy/legacy_forwarders_test.rb.) + test "SnapDiff.session returns the same registry within a fiber" do + assert_same SnapDiff.session, SnapDiff.session + + SnapDiff.session.record_new_screenshot("shared_object_probe") + assert_equal ["shared_object_probe"], SnapDiff.session.new_screenshots + ensure + SnapDiff.session.reset + end + + test "each thread gets its own registry instance" do + here = SnapDiff.session + there = Thread.new { SnapDiff.session }.value + + assert_not_same here, there + end + + test "add_assertion, verify and reset in concurrent threads never leak across threads" do + threads = 2.times.map do |i| + Thread.new do + names = 25.times.map { |j| "thread_#{i}_shot_#{j}" } + observed = [] + + names.each do |name| + SnapDiff.session.add_assertion(build_assertion(name)) + SnapDiff.session.record_new_screenshot(name) + observed << SnapDiff.session.assertions.map(&:name) + end + + SnapDiff.session.verify # all compares pass -> must not raise + + final_names = SnapDiff.session.assertions.map(&:name) + new_screenshots = SnapDiff.session.new_screenshots.dup + + SnapDiff.reset + after_reset = SnapDiff.session.assertions.size + SnapDiff.session.new_screenshots.size + + {names: names, observed: observed, final_names: final_names, + new_screenshots: new_screenshots, after_reset: after_reset} + end + end + + threads.map(&:value).each do |result| + # At every point each thread saw only its own assertions, in order. + result[:observed].each_with_index do |snapshot_of_names, index| + assert_equal result[:names].first(index + 1), snapshot_of_names + end + assert_equal result[:names], result[:final_names] + assert_equal result[:names], result[:new_screenshots] + assert_equal 0, result[:after_reset], "reset must clear only the calling thread's registry" + end + + # The main thread's registry stayed untouched by the worker threads. + assert_not_predicate SnapDiff.session, :assertions_present? + assert_empty SnapDiff.session.new_screenshots + end + + test "a failing assertion in one thread does not fail verify in another" do + failing_compare = Struct.new(:name) { + def different? = true + + def error_message = "boom" + } + + failing_thread = Thread.new do + assertion = SnapDiff::ScreenshotAssertion.new("failing_shot") + assertion.caller = ["failing_shot:1"] + assertion.compare = failing_compare.new("failing_shot") + SnapDiff.session.add_assertion(assertion) + + raised = assert_raises(SnapDiff::ExpectationNotMet) { SnapDiff.session.verify } + SnapDiff.session.reset + raised + end + + passing_thread = Thread.new do + SnapDiff.session.add_assertion(build_assertion("passing_shot")) + SnapDiff.session.verify + SnapDiff.session.reset + :passed + end + + assert_match(/failing_shot/, failing_thread.value.message) + assert_equal :passed, passing_thread.value + end +end diff --git a/test/unit/removed_surface_test.rb b/test/unit/removed_surface_test.rb new file mode 100644 index 00000000..3dd563fc --- /dev/null +++ b/test/unit/removed_surface_test.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +require "test_helper" +require "open3" + +# THE 2.1 DELETION, ASSERTED AS ABSENCE. +# +# Its predecessor (legacy_deletion_test.rb) SIMULATED the deletion -- copy +# lib/, `rm` the trees, probe entry points -- because the trees were still +# there. They are not, so the simulation harness went with them: what every +# canonical entry point still LOADS is already checked against the real lib/ +# on every run by support_load_probe_test.rb. +# +# What nothing else checks is the negative, which is exactly the claim a bad +# rebase or an over-eager revert breaks quietly: +# +# 1. No removed path is back under lib/. Everything there is packaged +# (see the gemspec's `spec.files` glob), so a restored file SHIPS. +# 2. A fresh process that loads the gem defines none of the removed names. +# +# THE GATE LINE. A probe that cannot tell the deleted tree from an intact one +# proves nothing -- an earlier lane's "green" run turned out to have measured +# the INTACT tree, because BUNDLE_GEMFILE pointed at the gemspec, which +# unshifts the real lib/ onto $LOAD_PATH ahead of any -I. Here the subject IS +# the real lib/, so the trap is the opposite one: the probe must prove it +# loaded THIS repo's lib/ and not some installed copy of the gem. It asserts +# that before it asserts any absence -- otherwise "the constant is gone" and +# "nothing was ever loaded" look identical. +class RemovedSurfaceTest < ActiveSupport::TestCase + LIB = Pathname.new(File.expand_path("../../lib", __dir__)) + + # The 2.1 `git rm`, verbatim. Anything on this list reappearing under lib/ + # is a shipped regression, not a local mess. + REMOVED_PATHS = %w[ + capybara + capybara_screenshot_diff + capybara-screenshot-diff.rb + capybara_screenshot_diff.rb + snap_diff/legacy_shims.rb + snap_diff/deprecation.rb + snap_diff/removal.rb + snap_diff/driver.rb + snap_diff/drivers.rb + snap_diff/drivers/chunky_png_driver.rb + snap_diff/utils.rb + ].freeze + + # Removed CONSTANTS, by fully qualified name. + REMOVED_CONSTANTS = %w[ + Capybara::Screenshot + CapybaraScreenshotDiff + SnapDiff::Deprecation + SnapDiff::Removal + SnapDiff::Driver + SnapDiff::Utils + SnapDiff::Drivers::ChunkyPNGDriver + SnapDiff::Drivers::AVAILABLE_DRIVERS + ].freeze + + # Removed METHODS on SnapDiff itself. `start` yielded the two v1 config + # holders (SnapDiff.configure replaces it); `silence_deprecations` silenced + # a channel that no longer exists. + REMOVED_METHODS = %w[start silence_deprecations silence_deprecations=].freeze + + # Removed driver-registry methods. Named separately because SnapDiff::Drivers + # SURVIVES as the namespace SnapDiff::Drivers::VipsDriver is published under + # -- so "the module is gone" would be the wrong assertion. + REMOVED_DRIVERS_METHODS = %w[loaded available for registry detect_available].freeze + + # Removed CONFIG settings. Absence here means NoMethodError at boot, which is + # the point: `driver:` cannot select anything with one backend, and an + # accept-and-ignore knob would let a config claim a backend choice that does + # not exist. + REMOVED_SETTINGS = %w[driver driver= shift_distance_limit shift_distance_limit=].freeze + + # Runs FIRST, before any absence assertion. Proves the process really loaded + # THIS repo's lib/ -- otherwise every "constant is gone" below is vacuous. + GATE_SCRIPT = <<~'RUBY' + lib = ENV.fetch("LIB_UNDER_TEST") + gate = [] + + loaded = $LOADED_FEATURES.grep(%r{/lib/snap_diff(/|\.rb\z)}) + gate << "no snap_diff files loaded at all" if loaded.empty? + + # The BUNDLE_GEMFILE trap in reverse: files resolving from an INSTALLED + # copy of the gem while the repo's lib/ sits unused on the load path. + # + # Anchored on the library path, NOT a bare /snap_diff/ substring: CI checks + # this repo out at .../snap_diff-capybara/, so a bare match flags every gem + # under vendor/bundle and the gate fails everywhere except a dev machine + # whose directory happens to be named otherwise. + strays = loaded.reject { |f| f.start_with?(lib) } + gate << "loaded from outside the tree under test: #{strays.join(", ")}" unless strays.empty? + + # A positive control: something the gem still HAS must be present, or the + # process is too broken for an absence to mean anything. + gate << "SnapDiff.configure is missing -- the gem did not load" unless SnapDiff.respond_to?(:configure) + gate << "VipsDriver is missing -- the only backend did not load" unless + defined?(SnapDiff::Drivers::VipsDriver) + + unless gate.empty? + abort("GATE: this process is NOT measuring the repo's lib/, so nothing below is evidence:\n- " + gate.join("\n- ")) + end + RUBY + + test "no removed path is back under lib/" do + back = REMOVED_PATHS.select { |path| LIB.join(path).exist? } + + assert_empty back, <<~MSG + Path(s) 2.1 removed exist under lib/ again. Everything under lib/ is + packaged, so this SHIPS: + + #{back.join("\n")} + MSG + end + + test "a fresh process loading the gem defines none of the removed names" do + failure = probe(<<~RUBY) + require "snap_diff" + #{GATE_SCRIPT} + + back = [] + #{REMOVED_CONSTANTS.inspect}.each { |c| back << c if Object.const_defined?(c) } + #{REMOVED_METHODS.inspect}.each { |m| back << "SnapDiff.\#{m}" if SnapDiff.respond_to?(m) } + #{REMOVED_DRIVERS_METHODS.inspect}.each do |m| + back << "SnapDiff::Drivers.\#{m}" if SnapDiff::Drivers.respond_to?(m) + end + #{REMOVED_SETTINGS.inspect}.each do |s| + back << "SnapDiff.config.\#{s}" if SnapDiff.config.respond_to?(s) + end + + abort("still defined: \#{back.join(", ")}") unless back.empty? + RUBY + + assert_nil failure, <<~MSG + A name 2.1 removed is defined again in a fresh process: + + #{failure} + MSG + end + + # The gate line has to be able to FAIL, or it is a comment with an `if` + # around it. Same script, run with the repo's lib/ NOT on the load path and + # LIB_UNDER_TEST still pointing at it: every absence assertion would pass + # there (nothing is loaded, so nothing is defined), so only the gate can + # reject it. + test "the gate line rejects a process that never loaded the tree under test" do + script = <<~RUBY + module SnapDiff + def self.configure = nil + end + #{GATE_SCRIPT} + RUBY + env = {"LIB_UNDER_TEST" => LIB.to_s, "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil, "RUBYLIB" => nil} + out, status = Open3.capture2e(env, RbConfig.ruby, "-e", script, chdir: Dir.tmpdir) + + assert_not status.success?, "the gate line passed on a process that loaded nothing" + assert_includes out, "no snap_diff files loaded at all" + end + + private + + # A fresh process with ONLY the repo's lib/ on the load path. + # + # `chdir: Dir.tmpdir` is the load-bearing half, and NOT a detail. Scrubbing + # RUBYOPT/BUNDLE_GEMFILE is not sufficient on its own: with the cwd still + # inside the project, RubyGems auto-discovers gems.rb, puts `-rbundler/setup` + # BACK into RUBYOPT, and the gemspec unshifts lib/ ahead of the -I dir. + # Running from a tmpdir means there is no gems.rb to find. Both defenses are + # here because the gate line inside the script is the only thing that says so + # out loud when they stop working. + def probe(script) + env = {"LIB_UNDER_TEST" => LIB.to_s, "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil, "RUBYLIB" => nil} + out, status = Open3.capture2e( + env, RbConfig.ruby, "-I#{LIB}", "-e", script, chdir: Dir.tmpdir + ) + + out unless status.success? + end +end diff --git a/test/unit/reporter_interplay_test.rb b/test/unit/reporter_interplay_test.rb new file mode 100644 index 00000000..d4667992 --- /dev/null +++ b/test/unit/reporter_interplay_test.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff" + +# Guard #4 from the v2 core-redesign acceptance contract. +# +# Reporter tests used to exercise the wrong seam (reporter.record called +# with hand-built arrays). This pins the real wiring: a delayed +# assert_matches_screenshot lands in the registry, SnapDiff.reset +# notifies registered reporters BEFORE clearing the registry, and +# finalize_reporters! drives finalize/summary. +class ReporterInterplayTest < ActiveSupport::TestCase + include SnapDiff::DSL + include DSLStub + + class SpyReporter + attr_reader :recorded, :finalized + + def initialize + @recorded = [] + @finalized = false + end + + def record(assertions) + @recorded << assertions.dup + end + + def finalize + @finalized = true + end + + def summary + "spy reporter summary" + end + end + + setup do + @spy = SpyReporter.new + SnapDiff::Reporting.mutex.synchronize do + @original_reporters = SnapDiff::Reporting.reporters.dup + SnapDiff::Reporting.reporters.clear + SnapDiff::Reporting.reporters << @spy + end + end + + teardown do + SnapDiff::Reporting.mutex.synchronize do + SnapDiff::Reporting.reporters.clear + SnapDiff::Reporting.reporters.concat(@original_reporters) + end + end + + test "a delayed assertion reaches registered reporters through the real reset path" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + snap = create_snapshot_for(:a, :a) + + assert_matches_screenshot(snap.full_name) # delayed by default + assert_predicate SnapDiff.session, :assertions_present? + + SnapDiff.reset + + assert_equal 1, @spy.recorded.size, "reset must notify reporters with the pending assertions" + recorded_assertions = @spy.recorded.first + assert_equal [snap.full_name], recorded_assertions.map(&:name) + + assert_not_predicate SnapDiff.session, :assertions_present?, + "reset must clear the registry only after reporters were notified" + end + end + + test "reset without assertions does not notify reporters" do + SnapDiff.reset + + assert_empty @spy.recorded + end + + test "finalize_reporters! finalizes each reporter and prints its summary" do + assert_output(/spy reporter summary/) do + SnapDiff::Reporting.finalize! + end + + assert @spy.finalized + end + + class RaisingReporter + def record(_assertions) = raise "record boom" + + def finalize = raise "finalize boom" + + def summary = nil + end + + test "a raising reporter during reset warns and does not stop other reporters" do + SnapDiff::Reporting.mutex.synchronize do + SnapDiff::Reporting.reporters.unshift(RaisingReporter.new) + end + + SnapDiff::Vcs.stub(:checkout_vcs, true) do + snap = create_snapshot_for(:a, :a) + assert_matches_screenshot(snap.full_name) + + _out, err = capture_io { SnapDiff.reset } + + assert_match(/\[snap_diff\] Reporter \S*RaisingReporter failed \(RuntimeError: record boom\)/, err) + assert_equal 1, @spy.recorded.size, "reporters after the raising one must still be notified" + end + end + + test "a raising reporter during finalize_reporters! warns and does not stop other reporters" do + SnapDiff::Reporting.mutex.synchronize do + SnapDiff::Reporting.reporters.unshift(RaisingReporter.new) + end + + _out, err = capture_io { SnapDiff::Reporting.finalize! } + + assert_match(/RaisingReporter failed \(RuntimeError: finalize boom\)/, err) + assert @spy.finalized, "reporters after the raising one must still be finalized" + end +end diff --git a/test/unit/reporters/default_test.rb b/test/unit/reporters/default_test.rb new file mode 100644 index 00000000..825242e3 --- /dev/null +++ b/test/unit/reporters/default_test.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff/reporters/default" + +require "snap_diff/drivers/vips_driver" if defined?(Vips) + +class DefaultReporterTest < ActiveSupport::TestCase + setup do + skip "VIPS not present. Skipping VIPS driver tests." unless defined?(Vips) + @_tmpdir = Pathname.new(Dir.mktmpdir) + end + + teardown do + FileUtils.remove_entry @_tmpdir if @_tmpdir + end + + test "for vips driver generates heatmap diff file" do + driver = SnapDiff::Drivers::VipsDriver.new + comparison = build_comparison_for(driver, "a.png", "b.png") + reporter = SnapDiff::Reporters::Default.new(driver.find_difference_region(comparison)) + + reporter.generate + + assert_same_images "a-and-b.heatmap.diff.png", reporter.heatmap_diff_path + end + + test "#clean_tmp_files removes heatmap diff along with other diff artifacts" do + driver = SnapDiff::Drivers::VipsDriver.new + comparison = build_comparison_for(driver, "a.png", "b.png") + reporter = SnapDiff::Reporters::Default.new(driver.find_difference_region(comparison)) + reporter.generate + + assert_predicate reporter.heatmap_diff_path, :exist? + + reporter.clean_tmp_files + + assert_not reporter.annotated_image_path.exist?, "diff should be cleaned" + assert_not reporter.annotated_base_image_path.exist?, "base diff should be cleaned" + assert_not reporter.heatmap_diff_path.exist?, "heatmap diff should be cleaned" + end + + # The test above calls clean_tmp_files directly, so #generate's OWN call to + # it on the equal path had no coverage: deleting that line left the suite + # green while every passing comparison leaked the previous run's diff + # artifacts, which the next run then reports as stale output. + test "#generate removes the previous run's diff artifacts when the images are equal" do + driver = SnapDiff::Drivers::VipsDriver.new + reporter = SnapDiff::Reporters::Default.new(driver.find_difference_region(build_comparison_for(driver, "a.png", "b.png"))) + reporter.generate # a.png vs b.png differ: writes the artifacts + + assert_predicate reporter.annotated_image_path, :exist? + assert_predicate reporter.heatmap_diff_path, :exist? + + # Equal images, but the SAME artifact paths as the comparison above, so + # what gets cleaned is exactly what got written. + equal_image = driver.from_file(TEST_IMAGES_DIR.join("a.png")) + equal_difference = driver.find_difference_region( + SnapDiff::Comparison::Images.new(equal_image, equal_image, {}, driver, @_tmpdir / "a.png", @_tmpdir / "b.png") + ) + assert_predicate equal_difference, :equal? + + assert_nil SnapDiff::Reporters::Default.new(equal_difference).generate + + assert_not reporter.annotated_image_path.exist?, "diff should be cleaned on the equal path" + assert_not reporter.annotated_base_image_path.exist?, "base diff should be cleaned on the equal path" + assert_not reporter.heatmap_diff_path.exist?, "heatmap diff should be cleaned on the equal path" + end + + test "failure message reports metrics without leaking image objects" do + driver = SnapDiff::Drivers::VipsDriver.new + comparison = build_comparison_for(driver, "a.png", "b.png") + difference = driver.find_difference_region(comparison) + difference.meta[:difference_level] = 0.42 + + message = SnapDiff::Reporters::Default.new(difference).generate + metrics = message.lines.first + + assert_includes metrics, "area_size" + assert_includes metrics, "region" + assert_includes metrics, "difference_level" + assert_not_includes metrics, "Vips::Image" + assert_not_includes metrics, "0x" + end + + private + + def build_comparison_for(driver, *images) + new_image = driver.from_file(TEST_IMAGES_DIR.join(images.first)) + base_image = driver.from_file(TEST_IMAGES_DIR.join(images.last)) + + SnapDiff::Comparison::Images.new(new_image, base_image, {}, driver, @_tmpdir / images.first, @_tmpdir / images.last) + end +end diff --git a/test/unit/reporters/html_reporter_test.rb b/test/unit/reporters/html_reporter_test.rb new file mode 100644 index 00000000..f1ec2258 --- /dev/null +++ b/test/unit/reporters/html_reporter_test.rb @@ -0,0 +1,251 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff/reporters/html" + +class HTMLReporterTest < ActiveSupport::TestCase + include DSLStub + + setup do + @output_dir = Pathname.new(Dir.mktmpdir) + @output_path = @output_dir / "report.html" + end + + teardown do + FileUtils.remove_entry(@output_dir) + end + + # Requiring this file is the only thing that puts a reporter in the + # default list -- nothing else in the suite registers one. Guards the + # auto-registration block at the bottom of reporters/html.rb, which + # goes through SnapDiff::Reporting.register since ADR-008 step 6. + test "requiring the reporter auto-registers exactly one HTML reporter" do + registered = SnapDiff::Reporting.reporters.count { |reporter| reporter.is_a?(SnapDiff::Reporters::HTML) } + + assert_equal 1, registered + end + + test "#record with no assertions writes nothing" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + reporter.record([]) + + assert_not @output_path.exist? + end + + test "#record with passing assertions writes nothing" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + + reporter.record([build_passing_assertion("index")]) + reporter.finalize + + assert_not @output_path.exist? + end + + test "#record and #finalize with failing assertion generates HTML file" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + + reporter.record([build_failing_assertion("index")]) + reporter.finalize + + assert @output_path.exist? + html = @output_path.read + assert_includes html, "" + assert_includes html, "index" + end + + test "#record and #finalize includes summary stats" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + + reporter.record([ + build_failing_assertion("page_a"), + build_passing_assertion("page_b"), + build_failing_assertion("page_c") + ]) + reporter.finalize + + html = @output_path.read + assert_includes html, "2 failed" + assert_includes html, "1 passed" + assert_includes html, "3 total" + end + + test "#record tolerates broken assertions without crashing" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + + broken = SnapDiff::ScreenshotAssertion.new("broken") + broken.compare = Object.new # will raise on .difference + + valid = build_failing_assertion("valid") + + assert_nothing_raised do + reporter.record([broken, valid]) + end + + reporter.finalize + assert @output_path.exist? + assert_includes @output_path.read, "valid" + end + + test "HTML reporter defaults to screenshot root path" do + reporter = SnapDiff::Reporters::HTML.new + expected = SnapDiff.config.root / SnapDiff.config.save_path / "snap_diff_report.html" + assert_equal expected, reporter.output_path + end + + test "#record uses relative paths by default" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + + reporter.record([build_failing_assertion("rel")]) + reporter.finalize + + html = @output_path.read + assert_not_includes html, "data:image" + end + + test "#record embeds base64 images when embed_images: true" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path, embed_images: true) + + reporter.record([build_failing_assertion("embed")]) + reporter.finalize + + html = @output_path.read + assert_includes html, "data:image/png;base64," + end + + test "#record from multiple threads produces correct totals" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + threads = 10 + assertions_per_thread = 5 + + workers = threads.times.map do + Thread.new do + batch = assertions_per_thread.times.map { |i| build_passing_assertion("t#{Thread.current.object_id}_#{i}") } + reporter.record(batch) + end + end + workers.each(&:join) + + assert_equal threads * assertions_per_thread, reporter.total + end + + test "#record and #finalize synchronize internal state" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + + fake_mutex = Class.new do + attr_reader :synchronize_calls + + def initialize + @synchronize_calls = 0 + end + + def synchronize + @synchronize_calls += 1 + yield + end + end.new + + # Using private state and a fake mutex here is intentional. + # This test guards a tricky synchronization detail; avoid refactors unless behavior changes. + reporter.instance_variable_set(:@mutex, fake_mutex) + + reporter.record([build_passing_assertion("sync")]) + reporter.finalize + + assert_operator fake_mutex.synchronize_calls, :>=, 1 + end + + test "#finalize returns output_path when there are failures" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + reporter.record([build_failing_assertion("fail")]) + result = reporter.finalize + + assert_instance_of Pathname, result + end + + test "#finalize returns nil when no failures" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + reporter.record([build_passing_assertion("pass")]) + result = reporter.finalize + + assert_nil result + end + + test "#summary returns screenshot count and status" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + reporter.record([build_passing_assertion("ok"), build_failing_assertion("fail")]) + reporter.finalize + + summary = reporter.summary + assert_includes summary, "1 failure" + assert_includes summary, "2 screenshots" + assert_includes summary, @output_path.to_s + end + + test "#summary pluralizes failures label for multiple failures" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + reporter.record([ + build_failing_assertion("first failure"), + build_failing_assertion("second failure") + ]) + reporter.finalize + + summary = reporter.summary + assert_includes summary, "2 failures" + assert_includes summary, "2 screenshots" + assert_includes summary, @output_path.to_s + end + + test "#summary when all pass shows no failures" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + reporter.record([build_passing_assertion("ok")]) + reporter.finalize + + summary = reporter.summary + assert_includes summary, "1 screenshot" + assert_includes summary, "no failures" + refute_includes summary, @output_path.to_s + end + + test "#summary when no screenshots recorded" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + assert_nil reporter.summary + end + + test "#finalize can retry after write_report failure" do + reporter = SnapDiff::Reporters::HTML.new(output_path: @output_path) + reporter.record([build_failing_assertion("retry")]) + + # Make write_report fail on first attempt + FileUtils.rm_rf(@output_dir) + File.write(@output_dir.to_s, "not a directory") + + begin + reporter.finalize + rescue # rubocop:disable Lint/SuppressedException + end + + # Restore writable directory and retry + File.delete(@output_dir.to_s) + FileUtils.mkdir_p(@output_dir) + + result = reporter.finalize + assert_instance_of Pathname, result + assert @output_path.exist? + end + + private + + def build_passing_assertion(name) + compare = make_comparison(:a, :a, destination: "pass_#{name}") + compare.processed + + SnapDiff::ScreenshotAssertion.new(name).tap { |a| a.compare = compare } + end + + def build_failing_assertion(name) + compare = make_comparison(:a, :b, destination: "fail_#{name}") + compare.processed + + SnapDiff::ScreenshotAssertion.new(name).tap { |a| a.compare = compare } + end +end diff --git a/test/unit/reporters_mutex_test.rb b/test/unit/reporters_mutex_test.rb new file mode 100644 index 00000000..07010acd --- /dev/null +++ b/test/unit/reporters_mutex_test.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require "test_helper" + +class ReportersMutexTest < ActiveSupport::TestCase + setup do + @original_reporters = SnapDiff::Reporting.reporters.dup + SnapDiff::Reporting.reporters.clear + end + + teardown do + SnapDiff::Reporting.reporters.clear + SnapDiff::Reporting.reporters.concat(@original_reporters) + end + + test "reporters_mutex is eagerly initialized" do + assert_instance_of Mutex, SnapDiff::Reporting.mutex + end + + test "reporters_mutex returns the same instance" do + assert_same SnapDiff::Reporting.mutex, SnapDiff::Reporting.mutex + end + + # ADR-008 step 6: SnapDiff::Reporting.register is the canonical way in, + # and it returns the reporter it registered. + # (That the v1 CapybaraScreenshotDiff.reporters view is the SAME array is + # pinned in test/legacy/legacy_forwarders_test.rb.) + test "register appends to the array SnapDiff::Reporting.reporters exposes" do + reporter = Object.new + + assert_same reporter, SnapDiff::Reporting.register(reporter) + assert_includes SnapDiff::Reporting.reporters, reporter + end + + # Best-effort probe: MRI's GVL can make an unsynchronized Array#<< look + # safe, so a green run here is not proof. The mutex in .register is the + # actual defense (issue #217 item 2); this pins that no registration is + # dropped under contention. + test "concurrent register calls retain every reporter" do + reporters = 32.times.map { Object.new } + + reporters.map { |reporter| Thread.new { SnapDiff::Reporting.register(reporter) } }.each(&:join) + + assert_equal reporters.size, SnapDiff::Reporting.reporters.size + assert_empty reporters - SnapDiff::Reporting.reporters + end + + test "reporters notification iterates over snapshot" do + received = [] + + mutating_reporter = Class.new do + define_method :record do |assertions| + received << [:original, assertions] + SnapDiff::Reporting.reporters.clear + SnapDiff::Reporting.reporters << Class.new { + define_method(:record) { |a| received << [:added, a] } + }.new + end + end.new + + SnapDiff::Reporting.reporters << mutating_reporter + + assertions = [:some, :assertions] + + assert_nothing_raised do + SnapDiff::Reporting.notify(assertions) + end + + assert_equal [[:original, assertions]], received + end +end diff --git a/test/unit/screenshot_assertion_test.rb b/test/unit/screenshot_assertion_test.rb new file mode 100644 index 00000000..b4802289 --- /dev/null +++ b/test/unit/screenshot_assertion_test.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff" + +module SnapDiff + # Pins the baseline-archiving side effect of the verify flow: when a + # comparison passes, the base image is moved over the actual image + # (the baseline is "committed" and the temp base copy disappears). + # These guards protect the file-level behavior while the mutation is + # extracted from the read path into an explicit archive step. + class ScreenshotAssertionTest < ActiveSupport::TestCase + include SnapDiff::DSL + include DSLStub + + test "#validate! archives the baseline when the comparison passes" do + comparison = make_comparison(:a, :a) + assertion = build_assertion(comparison) + + assertion.validate! + + assert_not comparison.base_image_path.exist?, "base image must be archived (moved over the actual image) on pass" + assert_predicate comparison.image_path, :exist? + end + + test "#validate! keeps the baseline and raises when the comparison fails" do + comparison = make_comparison(:a, :b) + assertion = build_assertion(comparison) + + assert_raises(SnapDiff::ExpectationNotMet) { assertion.validate! } + + assert comparison.base_image_path.exist?, "base image must be kept for the reporter on failure" + end + + test "verify archives baselines of passing delayed assertions end-to-end" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + snap = create_snapshot_for(:a, :a) + + assert_matches_screenshot(snap.full_name) # delayed by default + assert comparison_for(snap.full_name).base_image_path.exist?, "verify has not run yet: base image must still be present" + + SnapDiff.session.verify + + assert_not snap.base_path.exist?, "verify must archive the baseline of a passing assertion" + assert_predicate snap.path, :exist? + end + end + + test "verify keeps baselines of failing delayed assertions end-to-end" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + snap = create_snapshot_for(:a, :b) + + assert_matches_screenshot(snap.full_name) # delayed by default + + assert_raises(SnapDiff::ExpectationNotMet) { SnapDiff.session.verify } + + assert snap.base_path.exist?, "base image must be kept for the reporter on failure" + end + end + + test "#archive_baseline! moves the base image over the actual image and is idempotent" do + comparison = make_comparison(:a, :a) + assertion = build_assertion(comparison) + + assertion.archive_baseline! + + assert_not comparison.base_image_path.exist? + assert_predicate comparison.image_path, :exist? + + assertion.archive_baseline! # second call is a no-op + + assert_predicate comparison.image_path, :exist? + end + + test "#archive_baseline! keeps the baseline when the comparison differs" do + comparison = make_comparison(:a, :b) + assertion = build_assertion(comparison) + + assertion.archive_baseline! + + assert comparison.base_image_path.exist? + end + + test "#inspect is a one-line summary that does not run the comparison" do + comparison = make_comparison(:a, :b) + assertion = build_assertion(comparison, name: "widget") + + line = assertion.inspect + + assert_includes line, '"widget"' + assert_includes line, "pending" + assert_includes line, comparison.image_path.to_s + assert_includes line, comparison.base_image_path.to_s + assert_not_includes line, "\n" + assert_not comparison.processed?, "#inspect must not trigger the comparison" + end + + test "#inspect shows the verified state after the comparison ran" do + comparison = make_comparison(:a, :b) + assertion = build_assertion(comparison) + assertion.validate + + assert_includes assertion.inspect, "different" + + assert_includes SnapDiff::ScreenshotAssertion.new("fresh").inspect, "no comparison" + end + + private + + def build_assertion(comparison, name: "name") + SnapDiff::ScreenshotAssertion.new(name).tap do |assertion| + assertion.compare = comparison + assertion.caller = ["my_test.rb:42"] + end + end + + def comparison_for(name) + SnapDiff.session.assertions.find { |assertion| assertion.name == name }.compare + end + end +end diff --git a/test/unit/screenshot_matcher_test.rb b/test/unit/screenshot_matcher_test.rb new file mode 100644 index 00000000..90ccebb9 --- /dev/null +++ b/test/unit/screenshot_matcher_test.rb @@ -0,0 +1,217 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff" + +# Guard #1 from the v2 core-redesign acceptance contract: pins +# ScreenshotMatcher's current external behavior (D1-D4) in isolation +# before the orchestration middle is redesigned. Until now the class had +# no dedicated unit test at all — it was exercised only indirectly +# through dsl_test stubs and full-browser integration tests. +class ScreenshotMatcherTest < ActiveSupport::TestCase + include DSLStub + + # A screenshoter probe that records the (capture_options, + # comparison_options) split it was built with (D1) and writes a real + # file so the rest of the flow proceeds. + def recording_screenshoter(calls) + Class.new do + define_method(:initialize) do |capture_options, comparison_options| + calls << [capture_options, comparison_options] + end + + def take_comparison_screenshot(snapshot) + snapshot.path.dirname.mkpath + FileUtils.cp(File.expand_path("a.png", TEST_IMAGES_DIR), snapshot.path) + end + end + end + + # D4: dual return shape — nil for the new-screenshot path, after + # side-effecting record_new_screenshot into the registry. + test "#build_screenshot_assertion returns nil and records a new screenshot when no baseline exists" do + # ScreenshoterStub resolves "c_" to the c.png fixture. + name = "c_#{Time.now.nsec}" + + SnapDiff::Vcs.stub(:checkout_vcs, false) do + assertion = SnapDiff::ScreenshotMatcher.new(name).build_screenshot_assertion + + assert_nil assertion + assert_includes SnapDiff.session.new_screenshots, name + assert_predicate SnapDiff::SnapManager.path_for(name).path, :exist?, + "the screenshot is still captured on the new-screenshot path" + end + end + + # D4: the other return shape — a fully wired ScreenshotAssertion. + test "#build_screenshot_assertion returns an assertion with compare and caller when a baseline exists" do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + snap = create_snapshot_for(:a, :c) + + assertion = SnapDiff::ScreenshotMatcher.new(snap.full_name).build_screenshot_assertion + + assert_instance_of SnapDiff::ScreenshotAssertion, assertion + assert_equal snap.full_name, assertion.name + assert_kind_of Array, assertion.caller + assert_match(/screenshot_matcher_test\.rb/, assertion.caller.first) + assert_equal snap.path, assertion.compare.image_path + assert_equal snap.base_path, assertion.compare.base_image_path + end + end + + # D1: the one-hash-carved-into-two option split. Stability/wait/crop + # are deleted into capture options; whatever is left over becomes the + # comparison options. + test "#build_screenshot_assertion splits capture options from comparison options" do + calls = [] + + SnapDiff::Vcs.stub(:checkout_vcs, true) do + SnapDiff.config.stub(:screenshoter, recording_screenshoter(calls)) do + snap = create_snapshot_for(:a, :c) + + SnapDiff::ScreenshotMatcher.new(snap.full_name, tolerance: 0.03, wait: 5).build_screenshot_assertion + end + end + + assert_equal 1, calls.size + capture_options, comparison_options = calls.first + + assert_equal 5, capture_options[:wait] + assert_nil capture_options[:stability_time_limit] + assert_includes capture_options, :crop + assert_includes capture_options, :screenshot_format + + assert_equal 0.03, comparison_options[:tolerance] + assert_not_includes comparison_options, :wait, "wait must be carved out of comparison options" + assert_not_includes comparison_options, :stability_time_limit + assert_not_includes comparison_options, :crop + end + + # D1 (fixed): the split is a pure partition — the input hash survives + # untouched. The frozen input would raise FrozenError under the old + # delete-based carve. + test "#extract_capture_and_comparison_options does not mutate the input options" do + matcher = SnapDiff::ScreenshotMatcher.new("a") + options = {tolerance: 0.03, wait: 5, crop: [0, 0, 2, 2], stability_time_limit: 1}.freeze + + capture_options, comparison_options = matcher.send(:extract_capture_and_comparison_options, options) + + assert_equal 5, capture_options[:wait] + assert_not_includes comparison_options, :wait + assert_equal({tolerance: 0.03, wait: 5, crop: [0, 0, 2, 2], stability_time_limit: 1}, options) + end + + # D2: screenshoter selection is by hash-key presence — no + # :stability_time_limit means the configured plain screenshoter. + test "#build_screenshot_assertion uses the configured screenshoter without stability_time_limit" do + calls = [] + + SnapDiff::Vcs.stub(:checkout_vcs, true) do + SnapDiff.config.stub(:screenshoter, recording_screenshoter(calls)) do + snap = create_snapshot_for(:a, :c) + SnapDiff::ScreenshotMatcher.new(snap.full_name).build_screenshot_assertion + end + end + + assert_equal 1, calls.size, "the configured plain screenshoter must take the shot" + end + + # D2: presence of :stability_time_limit switches to StableScreenshoter. + test "#build_screenshot_assertion uses StableScreenshoter when stability_time_limit is present" do + stable_calls = [] + fake_stable = Object.new + def fake_stable.take_comparison_screenshot(snapshot) + snapshot.path.dirname.mkpath + FileUtils.cp(File.expand_path("a.png", TEST_IMAGES_DIR), snapshot.path) + end + + SnapDiff::Vcs.stub(:checkout_vcs, true) do + SnapDiff::StableScreenshoter.stub(:new, lambda { |capture_options, comparison_options| + stable_calls << [capture_options, comparison_options] + fake_stable + }) do + snap = create_snapshot_for(:a, :c) + SnapDiff::ScreenshotMatcher.new(snap.full_name, stability_time_limit: 0.1, wait: 1).build_screenshot_assertion + end + end + + assert_equal 1, stable_calls.size + assert_equal 0.1, stable_calls.first.first[:stability_time_limit] + end + + # The raise-only window-size guard (relocated in the redesign into + # Capture::Viewport#prepare!): wrong window size fails fast, before + # any capture. + test "#build_screenshot_assertion raises WindowSizeMismatchError when the window size is wrong" do + SnapDiff::BrowserHelpers.stub(:window_size_is_wrong?, true) do + SnapDiff::BrowserHelpers.stub(:selenium?, false) do + assert_raises(SnapDiff::WindowSizeMismatchError) do + SnapDiff::ScreenshotMatcher.new("matcher_window_size").build_screenshot_assertion + end + end + end + + refute SnapDiff::SnapManager.path_for("matcher_window_size").path.exist?, + "no screenshot may be written when the window size is wrong" + end + + # Same guard on the compare-free #capture path. + test "#capture raises WindowSizeMismatchError when the window size is wrong" do + SnapDiff::BrowserHelpers.stub(:window_size_is_wrong?, true) do + SnapDiff::BrowserHelpers.stub(:selenium?, false) do + assert_raises(SnapDiff::WindowSizeMismatchError) do + SnapDiff::ScreenshotMatcher.new("matcher_window_size").capture + end + end + end + + refute SnapDiff::SnapManager.path_for("matcher_window_size").path.exist?, + "no screenshot may be written when the window size is wrong" + end + + # Pins ScreenshotMatcher's viewport-preparation cadence: exactly one + # window-size check per capture, before the screenshoter runs. The + # stable screenshoter is stubbed here, so this guard does not police + # the real retry loop — that it stays check-free is verified by + # reading (no window-size calls in stable_screenshoter.rb). + test "window size is checked exactly once per capture even when stability retries happen" do + checks = 0 + fake_stable = Object.new + def fake_stable.take_comparison_screenshot(snapshot) + # Simulates a stability loop that needed several attempts; nothing + # here may trigger another window-size check. + 2.times { snapshot.next_attempt_path! } + snapshot.path.dirname.mkpath + FileUtils.cp(File.expand_path("a.png", TEST_IMAGES_DIR), snapshot.path) + end + + SnapDiff::BrowserHelpers.stub(:window_size_is_wrong?, proc { |_expected| + checks += 1 + false + }) do + SnapDiff::Vcs.stub(:checkout_vcs, true) do + SnapDiff::StableScreenshoter.stub(:new, ->(*, **) { fake_stable }) do + snap = create_snapshot_for(:a, :c) + SnapDiff::ScreenshotMatcher.new(snap.full_name, stability_time_limit: 0.1, wait: 1).build_screenshot_assertion + end + end + end + + assert_equal 1, checks + end + + # #capture is the compare-free path: file written, no assertion built, + # nothing recorded in the registry. + test "#capture writes the screenshot without touching the registry" do + # ScreenshoterStub resolves "b_" to the b.png fixture. + name = "b_#{Time.now.nsec}" + + SnapDiff::Vcs.stub(:checkout_vcs, true) do + SnapDiff::ScreenshotMatcher.new(name).capture + + assert_predicate SnapDiff::SnapManager.path_for(name).path, :exist? + assert_not_predicate SnapDiff.session, :assertions_present? + assert_empty SnapDiff.session.new_screenshots + end + end +end diff --git a/test/unit/screenshot_namer_test.rb b/test/unit/screenshot_namer_test.rb new file mode 100644 index 00000000..741e4d01 --- /dev/null +++ b/test/unit/screenshot_namer_test.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require "test_helper" + +class ScreenshotNamerTest < ActiveSupport::TestCase + setup do + @screenshot_namer = SnapDiff::ScreenshotNamer.new + end + + test "#group= resets counter when group changes" do + @screenshot_namer.group = "group1" + assert_equal "group1/00_image", @screenshot_namer.full_name("image") + assert_equal "group1/01_image", @screenshot_namer.full_name("image") + + @screenshot_namer.group = "group2" + assert_equal "group2/00_image", @screenshot_namer.full_name("image") + end + + test "#group= handles nil group" do + @screenshot_namer.group = nil + assert_equal "image", @screenshot_namer.full_name("image") + assert_equal [], @screenshot_namer.directory_parts + end + + test "#group= handles empty string group" do + @screenshot_namer.group = "" + assert_equal "image", @screenshot_namer.full_name("image") + assert_equal [], @screenshot_namer.directory_parts + end + + test "#section= handles nil section" do + @screenshot_namer.section = nil + assert_equal [], @screenshot_namer.directory_parts + end + + test "#section= handles empty string section" do + @screenshot_namer.section = "" + assert_equal [], @screenshot_namer.directory_parts + end + + test "#full_name generates basic name when no group is set" do + assert_equal "image_a", @screenshot_namer.full_name("image_a") + assert_equal "image_b", @screenshot_namer.full_name("image_b") + end + + test "#full_name generates prefixed and incremented names when group is set" do + @screenshot_namer.group = "user_flow" + assert_equal "user_flow/00_step1", @screenshot_namer.full_name("step1") + assert_equal "user_flow/01_step2", @screenshot_namer.full_name("step2") + end + + test "#full_name handles symbol base_name and group" do + @screenshot_namer.group = "symbols" + assert_equal "symbols/00_my_symbol", @screenshot_namer.full_name(:my_symbol) + @screenshot_namer.group = nil + assert_equal "plain_symbol", @screenshot_namer.full_name(:plain_symbol) + end + + test "#full_name includes section and group" do + @screenshot_namer.section = "user_profile" + @screenshot_namer.group = "avatar_upload" + assert_equal File.join("user_profile", "avatar_upload", "00_new_image"), @screenshot_namer.full_name("new_image") + end + + test "#full_name adds counter for duplicated names with active group" do + @screenshot_namer.group = "user_flow" + assert_equal "user_flow/00_step1", @screenshot_namer.full_name("step1") + assert_equal "user_flow/01_step1", @screenshot_namer.full_name("step1") + assert_equal "user_flow/02_step1", @screenshot_namer.full_name("step1") + end + + test "#full_name ignores duplicate names without active group" do + @screenshot_namer.group = nil + assert_equal "step1", @screenshot_namer.full_name("step1") + assert_equal "step1", @screenshot_namer.full_name("step1") + end + + test "#directory_parts is empty initially" do + assert_equal [], @screenshot_namer.directory_parts + end + + test "#directory_parts contains section when set" do + @screenshot_namer.section = "s1" + assert_equal ["s1"], @screenshot_namer.directory_parts + end + + test "#directory_parts contains group when set" do + @screenshot_namer.group = "g1" + assert_equal ["g1"], @screenshot_namer.directory_parts + end + + test "#directory_parts contains section and group when both set" do + @screenshot_namer.section = "s1" + @screenshot_namer.group = "g1" + assert_equal ["s1", "g1"], @screenshot_namer.directory_parts + end +end diff --git a/test/unit/screenshot_test.rb b/test/unit/screenshot_test.rb new file mode 100644 index 00000000..04eb4cbb --- /dev/null +++ b/test/unit/screenshot_test.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require "test_helper" +require "minitest/mock" + +class ScreenshotTest < ActiveSupport::TestCase + test "SnapManager.root returns an absolute path" do + assert SnapDiff::SnapManager.root.absolute? + end + + test "Screenshot.root returns a Pathname when Rails.root is a Pathname" do + # NOTE: We test that Rails.root is Pathname, which is true. + assert_kind_of Pathname, SnapDiff.config.root + assert SnapDiff.config.root.absolute? + end + + test "Screenshot.root can be set to a relative path and is converted to absolute" do + @orig_root = SnapDiff.config.root + + SnapDiff.config.root = "./tmp" + assert_kind_of Pathname, SnapDiff.config.root + assert SnapDiff.config.root.absolute? + ensure + SnapDiff.config.root = @orig_root if @orig_root + end +end diff --git a/test/unit/screenshoter_test.rb b/test/unit/screenshoter_test.rb new file mode 100644 index 00000000..7f3a59d2 --- /dev/null +++ b/test/unit/screenshoter_test.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require "test_helper" +require "minitest/mock" + +class ScreenshoterTest < ActiveSupport::TestCase + include SnapDiff::DSL + include DSLStub + + test "#take_screenshot without wait skips image loading" do + screenshoter = SnapDiff::Screenshoter.new({wait: nil}) + + mock = ::Minitest::Mock.new + mock.expect(:save_screenshot, true) { |path| path.include?("01_a.png") } + + SnapDiff::BrowserHelpers.stub(:session, mock) do + screenshoter.stub(:process_screenshot, true) do + screenshoter.take_screenshot(Pathname.new("tmp/01_a.png")) + end + end + + assert mock.verify + end + + test "#take_screenshot with custom screenshot options" do + screenshoter = SnapDiff::Screenshoter.new({wait: nil, capybara_screenshot_options: {full: true}}) + + mock = ::Minitest::Mock.new + mock.expect(:save_screenshot, true) { |path, options| path.include?("01_a.png") && options[:full] } + + SnapDiff::BrowserHelpers.stub(:session, mock) do + screenshoter.stub(:process_screenshot, true) do + screenshoter.take_screenshot(Pathname.new("tmp/01_a.png")) + end + end + + assert mock.verify + end + + test "#prepare_page_for_screenshot without wait does not raise any error" do + screenshoter = SnapDiff::Screenshoter.new({wait: nil}) + + assert_nil screenshoter.prepare_page_for_screenshot(timeout: nil) # does not raise an error + end + + test "#resize_if_needed halves a non-square retina screenshot to the expected window size" do + screenshoter = SnapDiff::Screenshoter.new({}) + retina_image = Vips::Image.black(2560, 1600) # 2x window size, non-square + + resized = SnapDiff.config.stub(:window_size, [1280, 1024]) do + screenshoter.send(:resize_if_needed, retina_image) + end + + assert_equal [1280, 800], screenshoter.driver.dimension(resized) + end +end diff --git a/test/unit/snap_diff_config_test.rb b/test/unit/snap_diff_config_test.rb new file mode 100644 index 00000000..f9d2e553 --- /dev/null +++ b/test/unit/snap_diff_config_test.rb @@ -0,0 +1,175 @@ +# frozen_string_literal: true + +require "test_helper" + +# The Config-only half. Everything about the v1 mattr_accessor VIEW of this +# same storage -- LegacyShims::CONFIG_MAPPING's completeness and the +# write-through-either-surface round trips -- lives in +# test/legacy/legacy_config_accessors_test.rb and is deleted with the v1 +# trees in 2.1. What is here has to keep standing on its own after that. +class SnapDiffConfigTest < ActiveSupport::TestCase + def config + SnapDiff.config + end + + test "SnapDiff.config returns a SnapDiff::Config" do + assert_kind_of SnapDiff::Config, config + end + + test "SnapDiff.config memoizes the same instance across calls" do + assert_same config, SnapDiff.config + end + + # Catches both an unmapped ivar sneaking into Config and a declared + # setting whose ivar is missing (which would also silently escape + # test_helper's per-test ivar snapshot/restore). + test "SnapDiff.config stores exactly one ivar per declared setting" do + assert_equal SnapDiff::Config::SETTINGS.map { |k| :"@#{k}" }.sort, + SnapDiff.config.instance_variables.sort + end + + # screenshot_enabled and enabled are two independent settings that + # happened to share a bare name in their own v1 modules (see #active?, + # which reads both). Config is flat, so it cannot expose two attributes + # both called `enabled` -- the Screenshot-side one is renamed + # `screenshot_enabled`. This proves the rename didn't accidentally + # collapse them into one shared value. + test "screenshot_enabled and enabled stay independent settings under Config" do + original_screenshot = config.screenshot_enabled + original_diff = config.enabled + + begin + config.screenshot_enabled = true + config.enabled = false + + assert_equal true, config.screenshot_enabled + assert_equal false, config.enabled + + config.screenshot_enabled = false + config.enabled = true + + assert_equal false, config.screenshot_enabled + assert_equal true, config.enabled + ensure + config.screenshot_enabled = original_screenshot + config.enabled = original_diff + end + end + + # ADR-008 step 7b moved this precedence rule into Config#active?, and + # found it had no test at all: replacing the whole expression with a bare + # `enabled` kept all 529 unit tests green. + # + # The rule: the screenshot-side flag wins whenever it was set to anything + # at all; only a nil there falls through to the diff-side flag. + ACTIVE_TRUTH_TABLE = [ + [true, true, true], + [true, false, true], + [false, true, false], + [false, false, false], + [nil, true, true], + [nil, false, false] + ].freeze + + test "active? gives screenshot_enabled precedence and only falls through on nil" do + original_screenshot = config.screenshot_enabled + original_diff = config.enabled + + ACTIVE_TRUTH_TABLE.each do |screenshot_enabled, enabled, expected| + config.screenshot_enabled = screenshot_enabled + config.enabled = enabled + context = "screenshot_enabled=#{screenshot_enabled.inspect}, enabled=#{enabled.inspect}" + + assert_equal expected, !!config.active?, "Config#active? with #{context}" + end + ensure + config.screenshot_enabled = original_screenshot + config.enabled = original_diff + end + + # The two path-segment flags were only ever exercised TOGETHER (both true + # in test/system_test_case.rb and the rspec fixtures, both nil everywhere + # else), so either `if` could be deleted or swapped for the other flag and + # the whole suite stayed green. All four combinations, with the segment + # sources stubbed so the assertion names the expected path literally + # rather than recomputing it. + SCREENSHOT_AREA_COMBINATIONS = [ + [nil, nil, "doc/screenshots"], + [true, nil, "doc/screenshots/fake_os"], + [nil, true, "doc/screenshots/fake_driver"], + [true, true, "doc/screenshots/fake_os/fake_driver"] + ].freeze + + test "screenshot_area appends the os and driver segments independently" do + original_os, original_driver, original_save_path = + config.add_os_path, config.add_driver_path, config.save_path + + SnapDiff::Os.stub(:name, "fake_os") do + Capybara.stub(:current_driver, :fake_driver) do + config.save_path = "doc/screenshots" + + SCREENSHOT_AREA_COMBINATIONS.each do |add_os_path, add_driver_path, expected| + config.add_os_path = add_os_path + config.add_driver_path = add_driver_path + + assert_equal expected, config.screenshot_area, + "screenshot_area with add_os_path=#{add_os_path.inspect}, " \ + "add_driver_path=#{add_driver_path.inspect}" + end + end + end + ensure + config.add_os_path = original_os + config.add_driver_path = original_driver + config.save_path = original_save_path + end + + # The tolerance floor in Config#default_options is the one literal in there + # that is not a stored setting. It used to be conditional on + # `driver == :vips`; 2.1 made libvips the only backend, so the condition was + # always true and is gone. The "and only for vips" half of this test went + # with the `driver` setting. + test "default_options floors tolerance at 0.001, and an explicit tolerance still wins" do + original_tolerance = config.tolerance + + begin + config.tolerance = nil + assert_equal 0.001, config.default_options[:tolerance] + + config.tolerance = 0.5 + assert_equal 0.5, config.default_options[:tolerance] + ensure + config.tolerance = original_tolerance + end + end + + test "writing root through config round-trips through the same Pathname coercion" do + original = config.root + + begin + config.root = "/tmp" + + assert_equal Pathname("/tmp"), config.root + ensure + config.root = original + end + end + + test "SnapDiff.configure yields the SnapDiff.config object" do + yielded = nil + SnapDiff.configure { |c| yielded = c } + + assert_same config, yielded + end + + test "SnapDiff.configure lets callers set values through the yielded config" do + original = config.tolerance + + begin + SnapDiff.configure { |c| c.tolerance = 0.0321 } + assert_equal 0.0321, config.tolerance + ensure + config.tolerance = original + end + end +end diff --git a/test/unit/snap_diff_test.rb b/test/unit/snap_diff_test.rb new file mode 100644 index 00000000..c7f6bb98 --- /dev/null +++ b/test/unit/snap_diff_test.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require "test_helper" +require "open3" + +class SnapDiffTest < ActiveSupport::TestCase + # The ImageCompare alias claim and the v1-shaped SnapDiff.start / .configure + # pair live in test/legacy/legacy_forwarders_test.rb -- both are v1 surface + # and go with it in 2.1. + test ".compare returns the same kind of result as Diff.compare, forwarding options" do + result = SnapDiff.compare( + TEST_IMAGES_DIR / "a.png", + TEST_IMAGES_DIR / "b.png", + tolerance: 0.02 + ) + + assert_kind_of SnapDiff::Comparison, result + assert_equal 0.02, result.driver_options[:tolerance] + end + + # The test above only pins that an EXPLICIT option round-trips, so + # dropping `config.default_options.merge` from .compare entirely left the + # full suite green -- callers who configure once and then call .compare + # with no options silently lost every configured default. + test ".compare with no options still carries the configured defaults" do + original = SnapDiff.config.tolerance + + begin + SnapDiff.config.tolerance = 0.0123 + + result = SnapDiff.compare(TEST_IMAGES_DIR / "a.png", TEST_IMAGES_DIR / "b.png") + + assert_equal 0.0123, result.driver_options[:tolerance] + ensure + SnapDiff.config.tolerance = original + end + end + + # Regression test for a load-order bug: `require "snap_diff"` standalone + # (nothing else preloaded) used to raise + # `NameError: uninitialized constant ...ImageCompare::Drivers` because + # image_compare.rb called Drivers.for without requiring drivers.rb. + # test_helper.rb preloads the whole gem, so only a subprocess with a + # fresh load path can catch this class of bug. + test "require \"snap_diff\" is standalone-loadable in a fresh process" do + script = <<~RUBY + require "snap_diff" + result = SnapDiff.compare(#{(TEST_IMAGES_DIR / "a.png").to_s.inspect}, #{(TEST_IMAGES_DIR / "b.png").to_s.inspect}) + exit(result.is_a?(SnapDiff::Comparison) ? 0 : 1) + RUBY + + out, status = Open3.capture2e(RbConfig.ruby, "-Ilib", "-e", script) + + assert status.success?, "expected standalone `require \"snap_diff\"` to succeed, got:\n#{out}" + end + + # Regression test (#218 adversarial review): the probe above only builds a + # comparison; annotation runs when a difference is actually reported, and + # under bare `require "snap_diff"` that used to raise + # `NameError: uninitialized constant SnapDiff::RED_RGBA` -- + # the annotation colors were defined only in the umbrella + # capybara_screenshot_diff.rb, which this entry never loads. + test "bare require \"snap_diff\" can annotate a difference between differing images" do + script = <<~RUBY + require "snap_diff" + require "fileutils" + require "tmpdir" + + Dir.mktmpdir do |dir| + base = File.join(dir, "base.png") + new_image = File.join(dir, "new.png") + FileUtils.cp(#{(TEST_IMAGES_DIR / "a.png").to_s.inspect}, base) + FileUtils.cp(#{(TEST_IMAGES_DIR / "b.png").to_s.inspect}, new_image) + + comparison = SnapDiff.compare(base, new_image) + abort("expected a.png and b.png to differ") unless comparison.different? + comparison.error_message + end + RUBY + + out, status = Open3.capture2e(RbConfig.ruby, "-Ilib", "-e", script) + + assert status.success?, "expected bare `require \"snap_diff\"` to annotate a difference, got:\n#{out}" + end + + # The acyclicity contract ("bare require never loads the umbrella") is in + # test/legacy/legacy_forwarders_test.rb: its subject is the v1 umbrella + # file, and once 3.0 deletes lib/capybara_screenshot_diff.rb the + # $LOADED_FEATURES grep is empty by construction, so the guard could never + # fail again. + + # Dual-install guard: both gem names ship identical files, so with BOTH + # activated every require silently resolves from whichever gem activated + # first -- version skew between them is undetectable. The entry point + # refuses that setup outright. + test "raises when both capybara-screenshot-diff and snap_diff-capybara are activated" do + specs = {"capybara-screenshot-diff" => :spec, "snap_diff-capybara" => :spec} + + error = assert_raises(SnapDiff::DualInstallError) { SnapDiff.assert_single_gem!(specs) } + + assert_match(/capybara-screenshot-diff/, error.message) + assert_match(/snap_diff-capybara/, error.message) + assert_match(/remove one/i, error.message) + end + + test "dual-install guard passes single-gem installs and local dev from source" do + SnapDiff.assert_single_gem!({"capybara-screenshot-diff" => :spec}) + SnapDiff.assert_single_gem!({"snap_diff-capybara" => :spec}) + SnapDiff.assert_single_gem!({}) # local dev from source: neither spec loaded + end +end diff --git a/test/unit/snap_manager_cleanup_test.rb b/test/unit/snap_manager_cleanup_test.rb new file mode 100644 index 00000000..fc191223 --- /dev/null +++ b/test/unit/snap_manager_cleanup_test.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require "test_helper" + +# Guard #3 from the v2 core-redesign acceptance contract (D7). +# +# Documents the previously-dead class-method cleanup path: +# `SnapDiff::SnapManager.instance` used to build a brand-new manager on every call, so +# `SnapDiff::SnapManager.snapshot` tracked snapshots on one throwaway instance and +# `SnapDiff::SnapManager.cleanup!` iterated the empty set of another — test_helper's +# teardown cleanup had never deleted anything through the tracked path. +# These tests were RED before SnapDiff::SnapManager.instance was memoized per thread. +class SnapManagerCleanupTest < ActiveSupport::TestCase + test ".instance returns the same manager within a thread" do + assert_same SnapDiff::SnapManager.instance, SnapDiff::SnapManager.instance + end + + test ".instance is rebuilt when the screenshot root changes" do + original = SnapDiff::SnapManager.instance + + Dir.mktmpdir do |dir| + SnapDiff.config.root = dir + + rebuilt = SnapDiff::SnapManager.instance + + assert_not_same original, rebuilt + assert_equal Pathname.new(SnapDiff.config.screenshot_area_abs), rebuilt.root + end + end + + test ".cleanup! deletes files of snapshots tracked via the class-method path" do + snap = SnapDiff::SnapManager.snapshot("cleanup_guard") + provision(snap) + + assert_predicate snap.path, :exist? + assert_predicate snap.base_path, :exist? + + SnapDiff::SnapManager.cleanup! + + assert_not snap.path.exist?, "cleanup! must delete the actual screenshot tracked by SnapDiff::SnapManager.snapshot" + assert_not snap.base_path.exist?, "cleanup! must delete the base screenshot tracked by SnapDiff::SnapManager.snapshot" + end + + test ".cleanup! in one thread does not delete snapshots tracked by another thread" do + barrier = Queue.new + + thread_b_snap = nil + thread_b = Thread.new do + thread_b_snap = SnapDiff::SnapManager.snapshot("cleanup_guard_thread_b") + provision(thread_b_snap) + barrier.pop # wait until thread A has cleaned up + end + + thread_a = Thread.new do + snap = SnapDiff::SnapManager.snapshot("cleanup_guard_thread_a") + provision(snap) + Thread.pass until thread_b_snap&.path&.exist? + + SnapDiff::SnapManager.cleanup! + snap + end + + thread_a_snap = thread_a.value + barrier << :done + thread_b.join + + assert_not thread_a_snap.path.exist?, "thread A's own snapshot should be cleaned up" + assert_predicate thread_b_snap.path, :exist?, "thread B's snapshot must survive thread A's cleanup!" + ensure + thread_b_snap&.delete! + end + + private + + def provision(snap) + snap.path.dirname.mkpath + FileUtils.cp(fixture_image_path_from("a"), snap.path) + FileUtils.cp(fixture_image_path_from("a"), snap.base_path) + end +end diff --git a/test/unit/snap_manager_test.rb b/test/unit/snap_manager_test.rb new file mode 100644 index 00000000..35ce3c8e --- /dev/null +++ b/test/unit/snap_manager_test.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require "test_helper" + +class SnapManagerTest < ActiveSupport::TestCase + setup do + @manager = SnapDiff::SnapManager.new(Dir.mktmpdir("snap_diff-storage")) + end + + teardown do + @manager.cleanup! + end + + test "#provision_snap_with copies the file to the snap path" do + snap = @manager.snapshot("test_image") + path = fixture_image_path_from("a") + + @manager.provision_snap_with(snap, path) + + assert_predicate snap.path, :exist? + assert_not_predicate snap.base_path, :exist? + end + + test "#provision_snap_with populate the base version of the snapshot" do + snap = @manager.snapshot("test_image") + path = fixture_image_path_from("a") + + @manager.provision_snap_with(snap, path, version: :base) + + assert_not_predicate snap.path, :exist? + assert_predicate snap.base_path, :exist? + end + + test "#screenshots_dir returns all created snapshots" do + assert_equal [], @manager.snapshots.to_a + + snap = @manager.snapshot("test_image") + path = fixture_image_path_from("a") + + @manager.provision_snap_with(snap, path) + assert_equal [snap], @manager.snapshots.to_a + end + + test "#screenshots_dir ignores attempts" do + assert_equal [], @manager.snapshots.to_a + + snap = @manager.snapshot("test_image") + path = fixture_image_path_from("a") + + @manager.provision_snap_with(snap, path, version: :attempt) + + assert_equal [snap], @manager.snapshots.to_a + end + + test "#snapshot overrides the file extension" do + snap = @manager.snapshot("test_image") + assert_equal "test_image", snap.full_name + assert_includes snap.path.to_s, "test_image.png" + assert_includes snap.base_path.to_s, "test_image.base.png" + assert_includes snap.next_attempt_path!.to_s, "test_image.attempt_00.png" + end + + # With a per-thread long-lived manager, cleanup! must also empty the + # tracked set — otherwise it grows for the whole run and a later cleanup! + # can delete a path an earlier test tracked but this test re-provisioned. + test "#cleanup! empties the tracked set" do + snap = @manager.snapshot("tracked_then_cleaned") + @manager.provision_snap_with(snap, fixture_image_path_from("a")) + + @manager.cleanup! + + assert_empty @manager.snapshots + end + + # Store-API split guard (v2 amendment item 1): #path_for is the pure + # lookup half of what #snapshot used to be used for. Callers that only + # need paths (e.g. dsl_test's post-capture assertions) must be able to + # look up without registering the name for cleanup!. + test "#path_for resolves the same paths as #snapshot" do + registered = @manager.snapshot("split_guard") + looked_up = @manager.path_for("split_guard") + + assert_equal registered.path, looked_up.path + assert_equal registered.base_path, looked_up.base_path + end + + test "#path_for never registers the snapshot for cleanup" do + @manager.path_for("split_guard") + + assert_empty @manager.snapshots.to_a + end + + test ".path_for is a pure lookup on the shared instance" do + tracked_before = SnapDiff::SnapManager.instance.snapshots.dup + + looked_up = SnapDiff::SnapManager.path_for("split_guard_class") + + assert_equal tracked_before, SnapDiff::SnapManager.instance.snapshots + + registered = SnapDiff::SnapManager.snapshot("split_guard_class") + assert_includes SnapDiff::SnapManager.instance.snapshots, registered + assert_equal registered.path, looked_up.path + end + + test "#cleanup! removes diff artifacts created by reporters" do + snap = @manager.snapshot("test_image") + source = fixture_image_path_from("a") + @manager.provision_snap_with(snap, source) + @manager.provision_snap_with(snap, source, version: :base) + + diff_path = snap.path.sub_ext(".diff.png") + base_diff_path = snap.path.sub_ext(".base.diff.png") + heatmap_path = snap.path.sub_ext(".heatmap.diff.png") + [diff_path, base_diff_path, heatmap_path].each { |p| FileUtils.cp(source, p) } + + @manager.cleanup! + + assert_not diff_path.exist?, "diff artifact should be cleaned up" + assert_not base_diff_path.exist?, "base diff artifact should be cleaned up" + assert_not heatmap_path.exist?, "heatmap diff artifact should be cleaned up" + end +end diff --git a/test/unit/stable_screenshoter_test.rb b/test/unit/stable_screenshoter_test.rb new file mode 100644 index 00000000..3a73d3b3 --- /dev/null +++ b/test/unit/stable_screenshoter_test.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +require "test_helper" + +class StableScreenshoterTest < ActiveSupport::TestCase + include DSLStub + + def setup + super + @manager = SnapDiff::SnapManager.new(SnapDiff.config.root / "stable_screenshoter_test") + @manager.create_output_directory_for + end + + def teardown + @manager.cleanup! + super + end + + test "#take_stable_screenshot retries until images are stable across iterations" do + image_compare_stub = build_image_compare_stub + + mock = ::Minitest::Mock.new(image_compare_stub) + + mock.expect(:quick_equal?, false) + mock.expect(:quick_equal?, false) + mock.expect(:quick_equal?, true) + + SnapDiff::Comparison.stub :new, mock do + snap = @manager.snapshot("02_a") + take_stable_screenshot_with(snap) + end + + assert mock.verify + end + + test "#take_stable_screenshot raises ArgumentError when wait parameter is nil" do + assert_raises ArgumentError, "wait should be provided" do + take_stable_screenshot_with(@manager.snapshot("02_a"), wait: nil) + end + end + + test "#take_stable_screenshot raises ArgumentError when stability_time_limit is nil" do + assert_raises ArgumentError, "stability_time_limit should be provided" do + take_stable_screenshot_with(@manager.snapshot("02_a"), stability_time_limit: nil) + end + end + + test "#take_comparison_screenshot cleans up temporary files after successful comparison" do + image_compare_stub = build_image_compare_stub + + mock = ::Minitest::Mock.new(image_compare_stub) + mock.expect(:quick_equal?, false) + mock.expect(:quick_equal?, true) + + snap = @manager.snapshot("02_a") + assert_not_predicate snap.path, :exist? + + SnapDiff::Comparison.stub :new, mock do + SnapDiff::StableScreenshoter + .new({stability_time_limit: 0.5, wait: 1}, image_compare_stub.driver_options) + .take_comparison_screenshot(snap) + end + + mock.verify + assert_empty snap.find_attempts_paths + assert_predicate snap.path, :exist? + assert_not_predicate snap.path.size, :zero? + end + + test "#take_comparison_screenshot raises UnstableImage when stability timeout is reached" do + snap = @manager.snapshot("01_a") + + screenshot_path = snap.path + + # Stub annotated files for generated comparison annotations + # We need to have different from screenshot_path name because of other stubs + pseudo_snap_for_annotations = @manager.snapshot("02_a") + annotated_screenshot_path = pseudo_snap_for_annotations.path + annotated_attempts_paths = [ + [annotated_screenshot_path.sub_ext(".attempt_01.latest.png"), annotated_screenshot_path.sub_ext(".attempt_01.committed.png")], + [annotated_screenshot_path.sub_ext(".attempt_02.latest.png"), annotated_screenshot_path.sub_ext(".attempt_02.committed.png")] + ] + + FileUtils.touch(annotated_attempts_paths) + + mock = ::Minitest::Mock.new(build_image_compare_stub(equal: false)) + annotated_attempts_paths.reverse_each do |(actual_path, base_path)| + mock.reporter.expect(:annotated_image_path, actual_path.to_s) + mock.reporter.expect(:annotated_base_image_path, base_path.to_s) + end + + assert_raises SnapDiff::UnstableImage, "Could not get stable screenshot within 1s" do + SnapDiff::Comparison.stub :new, mock do + # Wait time is less then stability time, which will generate problem + SnapDiff::StableScreenshoter + .new({stability_time_limit: 0.5, wait: 1}, build_image_compare_stub(equal: false).driver_options) + .take_comparison_screenshot(snap) + end + end + + mock.verify + mock.reporter.verify + + # There are no runtime files to find difference on stabilization + assert_empty Dir["tmp/*_a*.latest.png"] + assert_empty Dir["tmp/*_a*.committed.png"] + + # All stabilization files should be annotated + last_annotation = screenshot_path.sub_ext(".attempt_02.png") + assert_equal 0, last_annotation.size, "#{last_annotation.to_path} should be override with annotated version" + last_annotation = screenshot_path.sub_ext(".attempt_01.png") + assert_equal 0, last_annotation.size, "#{last_annotation.to_path} should be override with annotated version" + ensure + snap&.delete! + pseudo_snap_for_annotations&.delete! + end +end diff --git a/test/unit/static_test.rb b/test/unit/static_test.rb new file mode 100644 index 00000000..7dbab817 --- /dev/null +++ b/test/unit/static_test.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require "test_helper" +require "snap_diff/static" + +class StaticTest < ActiveSupport::TestCase + setup do + @original_root = SnapDiff.config.root + end + + teardown do + Capybara.app = Rails.application + SnapDiff.config.root = @original_root + end + + test ".serve sets Capybara.app to serve the directory" do + SnapDiff.serve("test/fixtures") + + assert_kind_of Rack::Files, Capybara.app + end + + test ".serve sets Screenshot.root to pwd" do + SnapDiff.serve("test/fixtures") + + assert_equal Pathname(Dir.pwd), SnapDiff.config.root + end + + # The legacy CapybaraScreenshotDiff.serve forwarder over this is pinned in + # test/legacy/legacy_forwarders_test.rb. + test ".serve accepts custom root" do + SnapDiff.serve("test/fixtures", root: "/tmp") + + assert_equal Pathname("/tmp"), SnapDiff.config.root + end +end diff --git a/test/unit/support_load_probe_test.rb b/test/unit/support_load_probe_test.rb new file mode 100644 index 00000000..dda92bc8 --- /dev/null +++ b/test/unit/support_load_probe_test.rb @@ -0,0 +1,231 @@ +# frozen_string_literal: true + +require "test_helper" +require "open3" + +# Kaizen guard for support-file transitive-require gaps (same subprocess +# pattern as snap_diff_test.rb's standalone-load regression test). +# +# test_helper.rb preloads the whole gem, so a support file missing one of its +# own requires still loads fine in most suite runs and only breaks in the CI +# matrix cell with a different load order — exactly how +# setup_capybara_drivers.rb broke on selenium_chrome_headless + vips (it used +# SnapDiff::Os without requiring it, fixed in b7ada5e). This test requires +# every test/support file in a bare subprocess with only capybara core +# preloaded. Scope: it catches missing requires for constants referenced +# AT LOAD TIME under this process's env; references hidden behind env guards +# (e.g. CAPYBARA_DRIVER branches not taken here) or inside method bodies +# still escape it. +# +# Currently every support file is expected to be bare-loadable: each one +# requires what it uses (rack/action_controller in setup_rails_app, +# active_support/concern in dsl_stub and driver_contract_tests, the gem's own +# files elsewhere). If a future support file legitimately needs more context +# than "capybara is loaded", list it in SKIP with the reason. +# +# The v1 entry points -- their advertised constants, the +# CapybaraScreenshotDiff session surface, and the eager user-facing +# constants under the old names -- are probed the same way from +# test/legacy/legacy_entry_point_probe_test.rb, which is deleted with the v1 +# trees in 2.1. +class SupportLoadProbeTest < ActiveSupport::TestCase + SKIP = { + # "support/example" => "why it cannot load bare" + }.freeze + + test "every test/support file is requirable with only capybara core loaded" do + project_root = File.expand_path("../..", __dir__) + support_files = Dir.chdir(File.join(project_root, "test")) { Dir["support/**/*.rb"] }.sort + assert_operator support_files.size, :>=, 10, "probe should see the support files" + + failures = support_files.filter_map do |file| + require_path = file.sub(/\.rb\z/, "") + next if SKIP.key?(require_path) + + script = "require \"capybara\"; require #{require_path.inspect}" + out, status = Open3.capture2e(RbConfig.ruby, "-Ilib", "-Itest", "-e", script, chdir: project_root) + + "#{file}:\n#{out}" unless status.success? + end + + assert_empty failures, <<~MSG + Support file(s) rely on requires test_helper happens to load first. + Add the missing require to the support file itself (or document a SKIP here): + + #{failures.join("\n\n")} + MSG + end + + # --- beta3 blockers: entry points must load a COMPLETE surface --- + # + # The canonical `snap_diff/*` requires never loaded snap_diff.rb itself, + # so the docs' own quick start (`require "snap_diff/integrations/minitest"`) + # left SnapDiff.configure/.start/.compare undefined, SnapDiff::VERSION + # unresolvable, and the dual-install guard silent. + + # SnapDiff.serve is deliberately absent here: docs/snapdiff.md's object + # map gates it behind its own `require "snap_diff/static"` (which pulls + # rack + minitest), so the core must not carry it. + # + # SnapDiff.start is absent for a different reason: it is defined in + # snap_diff/legacy_shims.rb and yields the two v1 config holders, so it + # cannot outlive them (#235). A CANONICAL gate demanding a method 2.1 + # deletes is a gate that fails the day the deletion lands -- which is + # exactly what it did. `.start` keeps its coverage on the legacy side: + # test/legacy/legacy_forwarders_test.rb asserts what it yields and that it + # applies a setting, and LEGACY_SESSION_SURFACE pins it per v1 entry point. + CANONICAL_SURFACE = %w[ + config configure compare session reset pending_screenshots_message + ].freeze + + # snap_diff-capybara is the canonical gem's Bundler entry point, so it is + # probed here rather than with the v1 ones: 2.1 keeps it (repointed at + # snap_diff/integrations/minitest), it just stops carrying the + # CapybaraScreenshotDiff surface. + CANONICAL_ENTRY_POINTS = { + "snap_diff" => CANONICAL_SURFACE, + "snap_diff/dsl" => CANONICAL_SURFACE, + "snap_diff/integrations/minitest" => CANONICAL_SURFACE, + "snap_diff/integrations/rspec" => CANONICAL_SURFACE, + "snap_diff/integrations/cucumber" => CANONICAL_SURFACE, + "snap_diff/static" => CANONICAL_SURFACE + %w[serve], + "snap_diff-capybara" => CANONICAL_SURFACE + }.freeze + + test "every canonical snap_diff entry point loads the full SnapDiff surface" do + failures = CANONICAL_ENTRY_POINTS.filter_map do |entry, methods| + self.class.probe(entry, <<~RUBY) + require #{entry.inspect} + missing = #{methods.inspect}.reject { |m| SnapDiff.respond_to?(m) } + missing << "VERSION" unless defined?(SnapDiff::VERSION) + abort("missing: \#{missing.join(", ")}") unless missing.empty? + RUBY + end + + assert_empty failures, <<~MSG + Canonical entry point(s) load only a fragment of SnapDiff: + + #{failures.join("\n")} + MSG + end + + # Ported from the legacy entry-point probe (test/legacy/), which was the + # only place asserting that an entry point defines its advertised + # CONSTANTS when it is the ONLY require -- it just did so for the v1 names + # (CapybaraScreenshotDiff::DSL, Capybara::Screenshot::Os, ...). That is the + # f89cea2 bug class: the acyclic redesign once narrowed an entry point so + # consumers lost the DSL, and only one CI matrix leg noticed. Same claim, + # canonical names, canonical entries -- so 3.0 keeps the guard. + # + # Entry-specific on purpose: bare `snap_diff` deliberately carries neither + # the DSL nor the reporters (see CANONICAL_SURFACE above), so a flat list + # over all entries would be wrong rather than strict. + CANONICAL_ADVERTISED_CONSTANTS = { + "snap_diff/dsl" => %w[SnapDiff::DSL SnapDiff::Os SnapDiff::Comparison], + "snap_diff/integrations/minitest" => %w[ + SnapDiff::DSL SnapDiff::Minitest::Assertions SnapDiff::Os SnapDiff::Comparison + ], + "snap_diff/integrations/rspec" => %w[SnapDiff::DSL SnapDiff::Os SnapDiff::Comparison], + "snap_diff-capybara" => %w[ + SnapDiff::DSL SnapDiff::Minitest::Assertions SnapDiff::Os SnapDiff::Comparison + ] + }.freeze + + test "every canonical entry point defines its advertised constants standalone" do + failures = CANONICAL_ADVERTISED_CONSTANTS.filter_map do |entry, constants| + self.class.probe(entry, <<~RUBY) + require #{entry.inspect} + missing = #{constants.inspect}.reject { |c| Object.const_defined?(c) } + abort("missing: \#{missing.join(", ")}") unless missing.empty? + RUBY + end + + assert_empty failures, <<~MSG + Canonical entry point(s) no longer provide their advertised constants standalone: + + #{failures.join("\n")} + MSG + end + + test "every canonical snap_diff entry point runs the dual-install guard" do + failures = CANONICAL_ENTRY_POINTS.keys.filter_map do |entry| + self.class.probe(entry, <<~RUBY) + require "rubygems" + %w[capybara-screenshot-diff snap_diff-capybara].each do |name| + Gem.loaded_specs[name] ||= Gem::Specification.new(name, "0.0.0") + end + begin + require #{entry.inspect} + rescue SnapDiff::DualInstallError + exit 0 + end + abort("both gems activated, but the dual-install guard stayed silent") + RUBY + end + + assert_empty failures, <<~MSG + Entry point(s) bypass SnapDiff.assert_single_gem!: + + #{failures.join("\n")} + MSG + end + + # A circular `require` between two core files is invisible until $VERBOSE + # is on -- which Rake::TestTask turns on by default, i.e. the standard + # Rails/Minitest setup and how real adopters run. drivers.rb <-> utils.rb + # shouted this at every such user on every process start, with an 18-frame + # backtrace. Ruby only warns once per cycle per process, so this has to be + # a fresh process per entry point. + # + # Canonical entries only; the v1 ones are covered the same way from + # test/legacy/legacy_entry_point_probe_test.rb, which reuses verbose_load. + test "no canonical entry point emits a circular require warning under -w" do + failures = CANONICAL_ENTRY_POINTS.keys.filter_map do |entry| + noise = self.class.verbose_load(entry).lines.grep(/circular require/) + "require \"#{entry}\" ->\n#{noise.join}" unless noise.empty? + end + + assert_empty failures, <<~MSG + Entry point(s) load through a `require` cycle. Ruby shouts about it on + stderr for every user whose suite runs with warnings on. Break the + cycle (move the shared unit, or defer the require into the method that + needs it) rather than muting it: + + #{failures.join("\n")} + MSG + end + + # cucumber's World/Before/After/AfterAll only exist inside its runtime, so + # a bare probe of a cucumber entry has to supply them or the require dies + # before it reaches what is under test. + CUCUMBER_RUNTIME_STUB = <<~RUBY + def World(*) = nil + def Before(*) = nil + def After(*) = nil + def AfterAll(*) = nil + RUBY + + # Runs +script+ in a fresh process with only lib/ on the load path. + # Returns nil on success, a failure description otherwise. + # + # Public, and deliberately so: test/legacy/legacy_entry_point_probe_test.rb + # reuses it rather than keeping a second copy that would drift. + def self.probe(entry, script) + project_root = File.expand_path("../..", __dir__) + preamble = entry.include?("cucumber") ? CUCUMBER_RUNTIME_STUB : "" + out, status = Open3.capture2e(RbConfig.ruby, "-Ilib", "-e", preamble + script, chdir: project_root) + + "require \"#{entry}\" -> #{out}" unless status.success? + end + + # Loads +entry+ in a fresh process with warnings ON, returning everything + # it printed. Public for the same reason as .probe above. + def self.verbose_load(entry) + project_root = File.expand_path("../..", __dir__) + preamble = entry.include?("cucumber") ? CUCUMBER_RUNTIME_STUB : "" + out, _status = Open3.capture2e( + RbConfig.ruby, "-w", "-Ilib", "-e", preamble + "require #{entry.inspect}", chdir: project_root + ) + out + end +end diff --git a/test/unit/vcs_test.rb b/test/unit/vcs_test.rb new file mode 100644 index 00000000..4efb3efc --- /dev/null +++ b/test/unit/vcs_test.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require "test_helper" + +class VcsTest < ActiveSupport::TestCase + include SnapDiff::Vcs + + PROJECT_ROOT = Pathname.new(File.expand_path("../..", __dir__)) + + setup do + @tmp_dir = PROJECT_ROOT / "tmp" / "vcs_test_#{Process.pid}" + FileUtils.mkdir_p(@tmp_dir) + @base_screenshot = Tempfile.new(%w[vcs_base. .png], @tmp_dir.to_s) + end + + teardown do + @base_screenshot&.close + @base_screenshot&.unlink + FileUtils.rm_rf(@tmp_dir) + end + + test "#checkout_vcs checks out and verifies the original screenshot" do + screenshot_path = file_fixture("images/a.png") + base_screenshot_path = Pathname.new(@base_screenshot.path) + + assert SnapDiff::Vcs.checkout_vcs(@tmp_dir, screenshot_path, base_screenshot_path), + "checkout_vcs failed: root=#{@tmp_dir}" + + assert base_screenshot_path.exist? + assert_equal screenshot_path.size, base_screenshot_path.size + end +end