From 7f2c2c59107647f68a5305a3cecd45fd4c409eb0 Mon Sep 17 00:00:00 2001 From: Kevin Klopfenstein Date: Mon, 4 May 2026 16:10:14 +0200 Subject: [PATCH] feat(dependencies): add databricks-runtime extra This new optional dependency group includes all the soda-core-spark, databricks-sql-connector, and databricks-sdk dependencies from the databricks extra, but excludes pyspark (provided by the Databricks Runtime environment). Use `pip install "datacontract-cli[databricks-runtime]"` when running inside Databricks to avoid dependency conflicts. Updated CHANGELOG.md and README.md with the new extra. --- CHANGELOG.md | 3 +++ README.md | 1 + pyproject.toml | 9 ++++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 485b9700..16657b7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added +- New `databricks-runtime` extra that excludes `pyspark` (provided by the Databricks Runtime environment). Install with `pip install "datacontract-cli[databricks-runtime]"` when running the CLI inside Databricks notebooks or jobs. The `databricks` extra remains unchanged for use outside Databricks. + ### Fixed - `changelog` command help text now advertises `(url or path)` for V1/V2 arguments, clarifying that HTTP/HTTPS URLs are accepted (#1162) - **breaking:** `test` command now exits non-zero when a server is specified, but soda-core fails to connect or authenticate (#1181) diff --git a/README.md b/README.md index 611662ee..26d80e1f 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,7 @@ A list of available extras: | Avro Support | `pip install datacontract-cli[avro]` | | Google BigQuery | `pip install datacontract-cli[bigquery]` | | Databricks Integration | `pip install datacontract-cli[databricks]` | +| Databricks Runtime | `pip install datacontract-cli[databricks-runtime]` (use inside Databricks where pyspark is pre-installed) | | DuckDB (local/S3/GCS/Azure file testing) | `pip install datacontract-cli[duckdb]` | | Iceberg | `pip install datacontract-cli[iceberg]` | | Kafka Integration | `pip install datacontract-cli[kafka]` | diff --git a/pyproject.toml b/pyproject.toml index ab6ebe83..74051531 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,13 @@ databricks = [ "pyspark>=3.5.0,<5.0.0", ] +databricks-runtime = [ + "soda-core-spark-df>=3.3.20,<3.6.0", + "soda-core-spark[databricks]>=3.3.20,<3.6.0", + "databricks-sql-connector>=3.7.0,<4.3.0", + "databricks-sdk<0.106.0", +] + iceberg = [ "pyiceberg==0.11.1" ] @@ -192,4 +199,4 @@ extend-select = [ ] [tool.ruff.lint.per-file-ignores] -"__init__.py" = ["F401", "F403"] +"__init__.py" = ["F401", "F403"] \ No newline at end of file