doc: Add an auto-generated dependency graph for internal crates

.github/workflows/dependencies.yml

@@ -66,4 +66,4 @@ jobs:
       - name: Install cargo-machete
         run: cargo install cargo-machete --version ^0.9 --locked
       - name: Detect unused dependencies
-        run: cargo machete --with-metadata
+        run: cargo machete --with-metadata

.github/workflows/docs.yaml

@@ -51,6 +51,12 @@ jobs:
           python3 -m venv venv
           source venv/bin/activate
           pip install -r docs/requirements.txt
+      - name: Install dependency graph tooling
+        run: |
+          set -x
+          sudo apt-get update
+          sudo apt-get install -y graphviz
+          cargo install cargo-depgraph --version ^1.6 --locked
 
       - name: Build docs
         run: |

.github/workflows/docs_pr.yaml

@@ -54,10 +54,15 @@ jobs:
           python3 -m venv venv
           source venv/bin/activate
           pip install -r docs/requirements.txt
+      - name: Install dependency graph tooling
+        run: |
+          set -x
+          sudo apt-get update
+          sudo apt-get install -y graphviz
+          cargo install cargo-depgraph --version ^1.6 --locked
       - name: Build docs html and check for warnings
         run: |
           set -x
           source venv/bin/activate
           cd docs
           ./build.sh # fails on errors
-

docs/.gitignore

@@ -20,3 +20,7 @@ build/
 venv/
 .python-version
 __pycache__/
+
+# Generated dependency graph artifacts (produced during docs CI)
+source/_static/data/deps.dot
+source/_static/data/deps.svg

docs/README.md

@@ -40,6 +40,11 @@ needing to create a virtual environment:
 uv run --with-requirements requirements.txt bash build.sh
 ```
 
+The docs build regenerates the workspace dependency graph via
+`docs/scripts/generate_dependency_graph.sh`, so ensure `cargo`, `cargo-depgraph`
+(`cargo install cargo-depgraph --version ^1.6 --locked`), and Graphviz `dot`
+(`brew install graphviz` or `sudo apt-get install -y graphviz`) are available.
+
 ## Build & Preview
 
 Run the provided script to build the HTML pages.

docs/build.sh

@@ -18,7 +18,14 @@
 # under the License.
 #
 
-set -e
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "${SCRIPT_DIR}"
+
 rm -rf build 2> /dev/null
 
+# Keep the workspace dependency graph in sync with the codebase.
+scripts/generate_dependency_graph.sh
+
 make html

docs/scripts/generate_dependency_graph.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# See `usage()` for details about this script.
+#
+# The key commands to generate the dependency graph SVG in this script are:
+#   cargo depgraph ... | dot -Tsvg > deps.svg
+# See below for the exact command used.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+OUTPUT_DIR="${REPO_DIR}/docs/source/_static/data"
+SVG_OUTPUT="${OUTPUT_DIR}/deps.svg"
+
+usage() {
+  cat <<EOF
+Generate the workspace dependency graph SVG for the docs.
+
+'deps.svg' is embedded in the DataFusion docs (Contributor Guide → Architecture → Workspace Dependency Graph).
+
+Output:
+  SVG: ${SVG_OUTPUT}
+
+Usage: $(basename "$0")
+
+Options:
+  -h, --help  Show this help message.
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1" >&2
+      usage
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+if ! command -v cargo >/dev/null 2>&1; then
+  echo "cargo is required to build the dependency graph." >&2
+  exit 1
+fi
+
+if ! command -v cargo-depgraph > /dev/null 2>&1; then
+  echo "cargo-depgraph is required (install with: cargo install cargo-depgraph)." >&2
+  exit 1
+fi
+
+if ! command -v dot >/dev/null 2>&1; then
+  echo "Graphviz 'dot' is required to render the SVG." >&2
+  exit 1
+fi
+
+mkdir -p "${OUTPUT_DIR}"
+
+(
+  cd "${REPO_DIR}"
+  cargo depgraph \
+    --workspace-only \
+    --all-deps \
+    --dedup-transitive-deps \
+    | dot \
+      -Grankdir=TB \
+      -Gconcentrate=true \
+      -Goverlap=false \
+      -Tsvg \
+      > "${SVG_OUTPUT}"
+)
+
+echo "Wrote dependency graph SVG to ${SVG_OUTPUT}"

docs/source/contributor-guide/architecture/dependency-graph.md

@@ -0,0 +1,180 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Workspace Dependency Graph
+
+This page shows the dependency relationships between DataFusion's workspace
+crates. This only includes internal dependencies, external crates like `Arrow` are not included
+
+The dependency graph is auto-generated by `docs/scripts/generate_dependency_graph.sh` to ensure it stays up-to-date, and the script now runs automatically as part of `docs/build.sh`.
+
+## Dependency Graph for Workspace Crates
+
+<!--
+  Below is an embedded .svg file, with interactive functionalities like drag/zoom-in/etc.
+  -->
+
+```{raw} html
+<div id="workspace-deps-wrapper" style="border:1px solid #d4d4d8;border-radius:10px;overflow:hidden;background:#fff;">
+  <div id="workspace-deps-inline" style="min-height:760px;width:100%;background:#f8fafc;overflow:hidden;padding:0;margin:0;">
+```
+
+```{eval-rst}
+.. raw:: html
+   :file: ../../_static/data/deps.svg
+```
+
+```{raw} html
+  </div>
+  <div style="padding:10px 12px;background:#f1f5f9;border-top:1px solid #e5e7eb;display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:8px;">
+    <span style="color:#334155;font-size:0.95rem;">Interactive SVG (pan, zoom, search)</span>
+    <div style="display:flex;align-items:center;gap:6px;">
+      <button id="workspace-deps-zoom-out" type="button" style="padding:6px 10px;border:1px solid #cbd5e1;border-radius:6px;background:#fff;color:#334155;cursor:pointer;">−</button>
+      <button id="workspace-deps-zoom-in" type="button" style="padding:6px 10px;border:1px solid #cbd5e1;border-radius:6px;background:#fff;color:#334155;cursor:pointer;">+</button>
+    </div>
+    <a href="../../_static/data/deps.svg" target="_blank" rel="noopener"
+       style="font-weight:600;color:#2563eb;text-decoration:none;">Open SVG ↗</a>
+  </div>
+</div>
+<script>
+  (function () {
+    const host = document.getElementById("workspace-deps-inline");
+    if (!host) {
+      return;
+    }
+
+    const svg = host.querySelector("svg");
+      if (!svg) {
+        host.textContent = "Unable to load dependency graph.";
+        host.style.display = "flex";
+        host.style.alignItems = "center";
+        host.style.justifyContent = "center";
+      host.style.background = "#f8fafc";
+      return;
+    }
+
+    svg.removeAttribute("width");
+    svg.removeAttribute("height");
+    svg.style.width = "100%";
+    svg.style.height = "100%";
+    svg.style.cursor = "grab";
+    svg.style.touchAction = "none";
+
+    const rawViewBox = (svg.getAttribute("viewBox") || "").split(/\s+/).map(Number);
+    if (rawViewBox.length !== 4 || rawViewBox.some((v) => Number.isNaN(v))) {
+      return;
+    }
+
+    const initial = {
+      x: rawViewBox[0],
+      y: rawViewBox[1],
+      width: rawViewBox[2],
+      height: rawViewBox[3],
+    };
+
+    const state = { ...initial };
+    const applyViewBox = () => {
+      svg.setAttribute("viewBox", `${state.x} ${state.y} ${state.width} ${state.height}`);
+    };
+
+    let isPanning = false;
+    let last = { x: 0, y: 0 };
+
+    svg.addEventListener("pointerdown", (event) => {
+      isPanning = true;
+      last = { x: event.clientX, y: event.clientY };
+      svg.setPointerCapture(event.pointerId);
+      svg.style.cursor = "grabbing";
+    });
+
+    const endPan = (event) => {
+      if (event && svg.hasPointerCapture(event.pointerId)) {
+        svg.releasePointerCapture(event.pointerId);
+      }
+      isPanning = false;
+      svg.style.cursor = "grab";
+    };
+
+    svg.addEventListener("pointerup", endPan);
+    svg.addEventListener("pointerleave", endPan);
+    svg.addEventListener("pointercancel", endPan);
+
+    const zoomBy = (factor) => {
+      const targetWidth = state.width * factor;
+      const targetHeight = state.height * factor;
+      const minSize = Math.max(initial.width * 0.05, 10);
+      const maxSize = initial.width * 20;
+      const clampedWidth = Math.min(Math.max(targetWidth, minSize), maxSize);
+      const clampedHeight = Math.min(Math.max(targetHeight, minSize), maxSize);
+
+      state.x += (state.width - clampedWidth) / 2;
+      state.y += (state.height - clampedHeight) / 2;
+      state.width = clampedWidth;
+      state.height = clampedHeight;
+      applyViewBox();
+    };
+
+    const normalizeDelta = (deltaY, deltaMode) => {
+      // Make trackpad/wheel zoom feel smooth across devices.
+      const multiplier = deltaMode === 1 ? 16 : deltaMode === 2 ? window.innerHeight : 1;
+      return deltaY * multiplier;
+    };
+
+    svg.addEventListener("pointermove", (event) => {
+      if (!isPanning) {
+        return;
+      }
+      const scaleX = state.width / svg.clientWidth;
+      const scaleY = state.height / svg.clientHeight;
+      state.x -= (event.clientX - last.x) * scaleX;
+      state.y -= (event.clientY - last.y) * scaleY;
+      last = { x: event.clientX, y: event.clientY };
+      applyViewBox();
+    });
+
+    svg.addEventListener("wheel", (event) => {
+      event.preventDefault();
+
+      const delta = normalizeDelta(event.deltaY, event.deltaMode);
+      const factor = Math.exp(delta * 0.0015); // smaller magnitude for smoother scrolling
+      zoomBy(factor);
+    }, { passive: false });
+
+    const zoomIn = document.getElementById("workspace-deps-zoom-in");
+    const zoomOut = document.getElementById("workspace-deps-zoom-out");
+    if (zoomIn) {
+      zoomIn.addEventListener("click", () => zoomBy(0.9));
+    }
+    if (zoomOut) {
+      zoomOut.addEventListener("click", () => zoomBy(1.1));
+    }
+  })();
+</script>
+```
+
+### Explanations
+
+- black lines: normal dependency
+- blue lines: dev-dependency
+- green lines: build-dependency
+- dotted lines: optional dependency (could be removed by disabling a cargo feature)
+
+The dependency graph is generated through `cargo depgraph`, see [cargo-depgraph] for details.
+
+[cargo-depgraph]: https://github.com/jplatte/cargo-depgraph?tab=readme-ov-file#output-explanation

docs/source/index.rst

@@ -157,6 +157,7 @@ To get started, see
    contributor-guide/communication
    contributor-guide/development_environment
    contributor-guide/architecture
+   contributor-guide/architecture/dependency-graph
    contributor-guide/testing
    contributor-guide/api-health
    contributor-guide/howtos