cli功能合并

.bazelignore

@@ -0,0 +1,25 @@
+# Bazel output directories
+bazel-out
+bazel-bin
+bazel-testlogs
+bazel-genfiles
+
+# Bazel workspace symlinks
+bazel-*
+
+# Bazel cache
+.bazel_cache
+
+# User-specific bazelrc
+.bazelrc.user
+
+# Generated files from crate_universe
+crates_io_*.json
+
+# Go module files (managed by Bazel)
+go.mod
+go.sum
+
+# Node modules (managed by Bazel)
+node_modules/
+package-lock.json

.bazelrc

@@ -0,0 +1,93 @@
+# Bazel configuration for Spider project
+
+# Build options
+build --incompatible_enable_cc_toolchain_resolution
+
+# Fix for aws-lc-sys jitterentropy: allow crates to control their own optimization
+build --repo_env=CRATE_CC_NO_DEFAULTS=1
+
+# Common build settings
+build --verbose_failures
+build --worker_verbose
+
+# Output formatting
+build --experimental_repository_cache_hardlinks
+
+# Feature flag configurations
+# These can be enabled with --config=<feature_name>
+
+# Chrome browser automation
+build:chrome --//:chrome=true
+
+# OpenAI integration
+build:openai --//:openai=true
+
+# io_uring support (Linux only)
+build:io_uring --//:io_uring=true
+
+# Gemini integration
+build:gemini --//:gemini=true
+
+# WebDriver support
+build:webdriver --//:webdriver=true
+
+# Serde serialization
+build:serde --//:serde=true
+
+# Cache support
+build:cache --//:cache=true
+
+# Sitemap support
+build:sitemap --//:sitemap=true
+
+# Smart crawling
+build:smart --//:smart=true
+
+# Cron scheduling
+build:cron --//:cron=true
+
+# Tracing
+build:tracing --//:tracing=true
+
+# Firewall
+build:firewall --//:firewall=true
+
+# Spider Cloud integration
+build:spider_cloud --//:spider_cloud=true
+
+# Agent features
+build:agent --//:agent=true
+build:agent_openai --//:agent_openai=true
+build:agent_chrome --//:agent_chrome=true
+build:agent_webdriver --//:agent_webdriver=true
+build:agent_skills --//:agent_skills=true
+build:agent_full --//:agent_full=true
+
+# Release build optimization
+build:release --compilation_mode=opt
+build:release --opt_level=3
+build:release --lto=full
+
+# Debug build
+build:debug --compilation_mode=dbg
+
+# Test configuration
+test --test_output=errors
+test --test_tag_filters=-manual
+
+# Platform-specific settings
+build:linux --platforms=@platforms//os:linux
+build:macos --platforms=@platforms//os:macos
+build:windows --platforms=@platforms//os:windows
+
+# Remote cache (optional - uncomment if using remote cache)
+# build --remote_cache=grpc://your-cache-server:9092
+
+# Try to use all available cores
+build --jobs=auto
+
+# Disk cache (separate from output root)
+build --disk_cache=~/.cache/bazel_cache
+
+# Documentation
+build:docs --//:docs=true

.cargo/audit.toml

@@ -0,0 +1,22 @@
+# https://rustsec.org/advisories/
+[advisories]
+ignore = [
+    # lru 0.13.0 unsound IterMut — transitive via wreq
+    "RUSTSEC-2026-0002",
+    # rustls-pemfile unmaintained — transitive via warp (spider_worker)
+    "RUSTSEC-2025-0134",
+    # rsa timing sidechannel — transitive via sqlx-mysql (no fix available)
+    "RUSTSEC-2023-0071",
+    # backoff unmaintained — transitive via async-openai, memvid-rs
+    "RUSTSEC-2025-0012",
+    # bincode unmaintained — transitive via memvid-rs, http-cache
+    "RUSTSEC-2025-0141",
+    # fxhash unmaintained — transitive via selectors (spider_scraper)
+    "RUSTSEC-2025-0057",
+    # instant unmaintained — transitive via backoff
+    "RUSTSEC-2024-0384",
+    # number_prefix unmaintained — transitive via indicatif (memvid-rs)
+    "RUSTSEC-2025-0119",
+    # paste unmaintained — transitive dep
+    "RUSTSEC-2024-0436",
+]

.editorconfig

@@ -0,0 +1,21 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.{yml,yaml}]
+indent_size = 2
+
+[*.{md,markdown}]
+trim_trailing_whitespace = false
+
+[*.toml]
+indent_size = 4
+
+[Makefile]
+indent_style = tab

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,60 @@
+name: Bug Report
+description: Report a bug or unexpected behavior
+labels: ["bug"]
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: What happened?
+      description: A clear description of the bug.
+      placeholder: Describe the issue...
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: Steps to reproduce
+      description: Minimal code or steps to reproduce the issue.
+      placeholder: |
+        1. Create a Website with...
+        2. Call crawl()...
+        3. Observe...
+      render: rust
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: What did you expect to happen?
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Spider version
+      placeholder: "2.45.25"
+    validations:
+      required: true
+  - type: dropdown
+    id: features
+    attributes:
+      label: Rendering mode
+      options:
+        - HTTP (default)
+        - Chrome (CDP)
+        - WebDriver
+        - Smart mode
+        - Not applicable
+    validations:
+      required: true
+  - type: input
+    id: rust-version
+    attributes:
+      label: Rust version
+      placeholder: "Output of `rustc --version`"
+  - type: input
+    id: os
+    attributes:
+      label: Operating system
+      placeholder: "e.g. macOS 15, Ubuntu 24.04, Windows 11"

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Discord
+    url: https://discord.spider.cloud
+    about: Ask questions and get help from the community
+  - name: Documentation
+    url: https://docs.rs/spider/latest/spider
+    about: API reference and crate documentation

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,24 @@
+name: Feature Request
+description: Suggest a new feature or improvement
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: What problem does this solve?
+      description: Describe the use case or pain point.
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed solution
+      description: How you'd like it to work. Include API examples if you have them.
+      render: rust
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+      description: Other approaches you've thought about or workarounds you're using.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,14 @@
+## What does this PR do?
+
+<!-- Brief description of the change. -->
+
+## How was this tested?
+
+<!-- How did you verify this works? Link to relevant tests if applicable. -->
+
+## Checklist
+
+- [ ] `cargo test` passes
+- [ ] `cargo fmt` applied
+- [ ] New public APIs have doc comments
+- [ ] Feature-gated behind a flag (if adding optional functionality)

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    labels:
+      - "dependencies"

.github/workflows/audit.yml

@@ -0,0 +1,19 @@
+name: Security Audit
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "0 6 * * 1" # Every Monday at 6am UTC
+
+jobs:
+  audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install cargo-audit
+        run: cargo install cargo-audit --locked || true
+      - name: Run audit
+        run: cargo audit

.github/workflows/bench.yml

@@ -0,0 +1,52 @@
+name: Benches
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_LOG: "off"
+  SPIDER_BENCH_URL: http://localhost:3000
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      - name: Setup Local Web server
+        id: server
+        run: |
+          set +e
+          git clone https://github.com/tailwindlabs/tailwindcss.com || exit 0
+          cd tailwindcss.com || exit 0
+          npm i --force || exit 0
+          npm run build
+          if [ $? -eq 0 ]; then
+            echo "status=ok" >> $GITHUB_OUTPUT
+            npm run start &
+          else
+            echo "Server build failed"
+            echo "status=failed" >> $GITHUB_OUTPUT
+          fi
+      - name: Run Benchmark Compare OSS
+        if: steps.server.outputs.status == 'ok'
+        run: cargo bench --bench crawl
+      - name: Run Benchmark Compare OSS - Features[decentralized]
+        if: steps.server.outputs.status == 'ok'
+        run: cargo bench --bench crawl --features decentralized

.github/workflows/lint.yml

@@ -0,0 +1,36 @@
+name: Lint
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-lint-${{ hashFiles('**/Cargo.lock') }}
+      - name: Clippy (default features)
+        run: cargo clippy --workspace -- -D warnings
+      - name: Clippy (chrome)
+        run: cargo clippy -p spider --features chrome -- -D warnings
+
+  fmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check formatting
+        run: cargo fmt --all -- --check