diff --git a/CHANGELOG.md b/CHANGELOG.md index 78619f2..8f65c5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Change Log +## 2.2.0 + +### Create Source UX Improvements + +- Retain context so create source panel doesn't re init on tab switch. +- Make select dropdowns virtualized to improve page perf. +- Bug fixes with parsing trino output. + +### DBT Run UX improvements + +- **Defer works with any run scope.** In `DJ: Run Model`, enabling Defer no longer locks you to modified models — combine it with single model, multi-model, or full project. +- **Favor State with Defer.** When Defer is on, toggle Favor State to add `--favor-state` so deferred (state) versions win over local ones for overlapping nodes. + ## 2.1.1 ### Airflow diff --git a/docs/AGENT_SKILLS.md b/docs/AGENT_SKILLS.md index ee24346..2dd7c55 100644 --- a/docs/AGENT_SKILLS.md +++ b/docs/AGENT_SKILLS.md @@ -44,7 +44,7 @@ Point your AI coding tool at the workspace and the skills become available. Most DJ provides 17 skills, grouped below by what they help you do. -### Setup & onboarding +### Setup & configuration #### `dj-initialize` @@ -53,7 +53,7 @@ Interactive wizard that sets up and configures the DJ Framework in an existing d - **Use when:** you want to set up DJ in an existing dbt project, configure required settings, or diagnose why DJ is not working correctly. - **Example prompt:** _"Set up the DJ framework in this dbt project."_ -### Authoring SQL models +### Creating models & sources #### `dj-create-new-model` @@ -77,7 +77,7 @@ Registers a raw Trino table as a DJ `.source.json` by introspecting its exact co - **Use when:** a model needs a raw `catalog.schema.table` that isn't defined as a source yet, or you want to add a table or columns to an existing source. - **Example prompt:** _"Register the raw orders table as a DJ source."_ -### Python ETL models +### Python models (ETL) #### `dj-create-python-model` @@ -95,7 +95,7 @@ Scaffolds a `.python.json` for a pre-dbt Python ETL pipeline that extracts data - **Example prompt:** _"Review this Python model for production readiness."_ - **Bundled reference:** `review-checklist.md` — pass/fail examples and edge cases for every check. -### Lightdash BI & AI hints +### Lightdash & AI hints #### `dj-create-lightdash-yaml` @@ -120,7 +120,7 @@ Adds or updates Lightdash `ai_hint` values across a model's full dependency tree - **Use when:** you're working with AI hints in model or source JSON files. - **Example prompt:** _"Update the AI hints for this model's dependency tree from ai_hints.xlsx."_ -### Refactoring & maintenance +### Reviewing, refactoring & governance #### `dj-review-and-refactor-model` @@ -138,16 +138,6 @@ Detects legacy ephemeral models and inlines them as Common Table Expressions (CT - **Example prompt:** _"Inline the ephemeral models under intermediate/ as CTEs."_ - **Bundled reference:** `transformation-matrix.md` — per-type inline recipes and CTE-naming rules. -#### `dj-resolve-merge-conflicts` - -Resolves git merge, rebase, or cherry-pick conflicts the DJ way — hand-merging only the `.model.json` / `.source.json` sources of truth and regenerating the `.sql` / `.yml` siblings (never hand-merging generated files). It also helps when an incoming branch is old or diverged and you must choose between a full merge and porting specific models. - -- **Use when:** you hit conflicts in DJ files while merging/rebasing/cherry-picking, or say "resolve the merge conflicts" or "help me rebase". -- **Example prompt:** _"Resolve the merge conflicts in these .model.json files."_ -- **Bundled reference:** `staleness-and-porting.md` — the staleness assessment and guided-port recipe. - -### Governance - #### `dj-govern-model` **Read-only** audit of governance posture across a model, folder, dependency tree, or the whole workspace — ownership coverage, PII / classification / compliance tagging, registered-group conformance, and prod-write posture. It reports gaps and points you at the skill to fix each one; it never edits files and never forces a project to adopt metadata it hasn't chosen. @@ -155,17 +145,7 @@ Resolves git merge, rebase, or cherry-pick conflicts the DJ way — hand-merging - **Use when:** you want to review data ownership, check PII / sensitivity / compliance tagging, find models with no owner, or assess governance coverage for a group or project. - **Example prompt:** _"Which models in the finance group have no owner or PII tags?"_ -### Performance diagnostics - -#### `dj-trino-analyzer` - -**Read-only** diagnosis of Trino query performance from the `QueryInfo` JSON that DJ's Query Control Center writes to `.dj/diagnostics/`. It explains slowness — broadcast-join blow-ups, data skew, blocked time, object-store scan latency, and more — and suggests `.model.json` knobs; it never edits generated SQL. Run **`DJ: Analyze Trino Query with AI`** first to produce the diagnostics. - -- **Use when:** a query is slow, you want to understand a query plan, compare two queries (for example before vs. after a config change), or investigate a specific Trino query ID. -- **Example prompt:** _"Explain why Trino query 20260101_120000_00001_abcde is slow."_ -- **Bundled references:** a six-file Trino field reference — `query-info.md`, `query-stats.md`, `stage-and-task-stats.md`, `operator-stats.md`, `types-and-enums.md`, and `recipes.md`. - -### Running dbt, Trino & git +### Running commands & diagnostics #### `dj-run-dbt` @@ -181,6 +161,16 @@ Runs a **read-only** Trino query from the terminal to inspect warehouse data or - **Use when:** you want to query Trino, preview rows, `DESCRIBE` / `SHOW` a table, or sanity-check a value. For diagnosing captured query performance, use `dj-trino-analyzer` instead. - **Example prompt:** _"Preview 20 rows from the orders table."_ +#### `dj-trino-analyzer` + +**Read-only** diagnosis of Trino query performance from the `QueryInfo` JSON that DJ's Query Control Center writes to `.dj/diagnostics/`. It explains slowness — broadcast-join blow-ups, data skew, blocked time, object-store scan latency, and more — and suggests `.model.json` knobs; it never edits generated SQL. Run **`DJ: Analyze Trino Query with AI`** first to produce the diagnostics. + +- **Use when:** a query is slow, you want to understand a query plan, compare two queries (for example before vs. after a config change), or investigate a specific Trino query ID. +- **Example prompt:** _"Explain why Trino query 20260101_120000_00001_abcde is slow."_ +- **Bundled references:** a six-file Trino field reference — `query-info.md`, `query-stats.md`, `stage-and-task-stats.md`, `operator-stats.md`, `types-and-enums.md`, and `recipes.md`. + +### Git & collaboration + #### `dj-git-workflow` Commits DJ work the right way — staging each `.model.json` / `.source.json` together with its generated `.sql` / `.yml` after a sync, ignoring DJ's local `.dj/` state, and following the downstream project's own commit conventions. It stops at the commit and guards against staging secrets; pushing needs your go-ahead. For merge conflicts, use `dj-resolve-merge-conflicts` instead. @@ -188,6 +178,14 @@ Commits DJ work the right way — staging each `.model.json` / `.source.json` to - **Use when:** you want to commit, stage, branch, or check in your DJ models, or ask what should be committed. - **Example prompt:** _"Commit these model changes."_ +#### `dj-resolve-merge-conflicts` + +Resolves git merge, rebase, or cherry-pick conflicts the DJ way — hand-merging only the `.model.json` / `.source.json` sources of truth and regenerating the `.sql` / `.yml` siblings (never hand-merging generated files). It also helps when an incoming branch is old or diverged and you must choose between a full merge and porting specific models. + +- **Use when:** you hit conflicts in DJ files while merging/rebasing/cherry-picking, or say "resolve the merge conflicts" or "help me rebase". +- **Example prompt:** _"Resolve the merge conflicts in these .model.json files."_ +- **Bundled reference:** `staleness-and-porting.md` — the staleness assessment and guided-port recipe. + ## Feedback & more - The full framework reference the skills build on is generated to `.agents/dj/AGENTS.md` in your workspace once `dj.codingAgent` is enabled. diff --git a/package-lock.json b/package-lock.json index b32e056..05dffda 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dj", - "version": "2.1.1", + "version": "2.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dj", - "version": "2.1.1", + "version": "2.2.0", "license": "Apache-2.0", "workspaces": [ "web" @@ -1358,9 +1358,9 @@ "license": "MIT" }, "node_modules/@eslint/config-array/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -1456,9 +1456,9 @@ "license": "MIT" }, "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -1789,9 +1789,9 @@ } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { - "version": "3.15.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.0.tgz", - "integrity": "sha512-ttBQIIQPDeLjpPOohtUdXuXUVoA2uIB6fEH9HyJ7234s5mBJ5wTx20njxplLZQgLaOfpmPQA7X2t5AX6tIPbog==", + "version": "3.15.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.1.tgz", + "integrity": "sha512-S99WuO3HlhO3XN41EtYUNl9zzXjoJx7QvmipxsJVxtCBT0YHEFy+iOJhjSvrmV12nYhWpZaM8lPHkJm0yUMbag==", "dev": true, "license": "MIT", "dependencies": { @@ -2092,9 +2092,9 @@ "license": "MIT" }, "node_modules/@jest/reporters/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -4788,9 +4788,9 @@ "license": "BSD-2-Clause" }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -5977,9 +5977,9 @@ } }, "node_modules/dompurify": { - "version": "3.4.12", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.12.tgz", - "integrity": "sha512-zQvGet8Z2sWbQhCmfFz/T5QWH2oBmjnqK3qvOjaqaNLrLEF912WamU+ohnTp0TCep/MFVHpdJuCZEdFOdTnEFg==", + "version": "3.4.13", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz", + "integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -6488,9 +6488,9 @@ "license": "MIT" }, "node_modules/eslint-plugin-jsx-a11y/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -6628,9 +6628,9 @@ "license": "MIT" }, "node_modules/eslint/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -6903,9 +6903,9 @@ "integrity": "sha512-HPtaa38cPgWvaCFmRNhlc6NG7pv6NUHqjPgVAkWGoB9mQMwYB27/K0CvOM5Czy+qpT3e8XJ6Q4aPAnzpNpzNaw==" }, "node_modules/fast-uri": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", - "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", "funding": [ { "type": "github", @@ -8720,9 +8720,9 @@ "license": "MIT" }, "node_modules/jest-config/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -9069,9 +9069,9 @@ "license": "MIT" }, "node_modules/jest-runtime/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -9271,9 +9271,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", - "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", + "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", "dev": true, "funding": [ { @@ -9892,12 +9892,12 @@ } }, "node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.5" + "brace-expansion": "^5.0.8" }, "engines": { "node": "18 || 20 || >=22" @@ -9986,9 +9986,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -10221,9 +10221,9 @@ "license": "MIT" }, "node_modules/npm-run-all2/node_modules/brace-expansion": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", - "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", "dev": true, "license": "MIT", "dependencies": { @@ -10896,9 +10896,9 @@ } }, "node_modules/postcss": { - "version": "8.5.19", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.19.tgz", - "integrity": "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==", + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", "dev": true, "funding": [ { @@ -10916,7 +10916,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -11432,9 +11432,9 @@ } }, "node_modules/react-router": { - "version": "7.18.1", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.1.tgz", - "integrity": "sha512-GDLgg3i3uM0aeJO3Fm+TCS+sDQ7gu12T6x0qdTEzcwqEfleci7JwugVNIF3U//0FWKnJT7ptG+20B2jfDqnZAg==", + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.2.tgz", + "integrity": "sha512-aUVMjFm3GAPTTZL7oYr5E7ETiqfQCHRLH+B+5afnICvf0r7kkK4eR6SMuwbSTJw/7t+12khT/Kahij49fqOCIg==", "license": "MIT", "dependencies": { "cookie": "^1.0.1", @@ -13084,9 +13084,9 @@ "license": "MIT" }, "node_modules/test-exclude/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -14136,9 +14136,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", "dev": true, "license": "MIT", "engines": { @@ -15364,4 +15364,4 @@ } } } -} \ No newline at end of file +} diff --git a/package.json b/package.json index a8f028d..aa81f9c 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "type": "git", "url": "https://github.com/Workday/dj.git" }, - "version": "2.1.1", + "version": "2.2.0", "workspaces": [ "web" ], @@ -1049,7 +1049,7 @@ "typescript-eslint": "^8.50.1" }, "overrides": { - "fast-uri": "3.1.4", - "brace-expansion@5": "5.0.8" + "fast-uri": "3.1.5", + "brace-expansion@5": "5.0.9" } -} \ No newline at end of file +} diff --git a/src/services/dbt.ts b/src/services/dbt.ts index 35d7285..996b3c0 100644 --- a/src/services/dbt.ts +++ b/src/services/dbt.ts @@ -3132,7 +3132,7 @@ ${macro.macro_sql}`; VIEW_ID.SOURCE_CREATE, DBT_MSG.CREATE_SOURCE, vscode.ViewColumn.One, - { enableScripts: true }, + { enableScripts: true, retainContextWhenHidden: true }, ); panel.onDidDispose(() => { this.webviewPanelSourceCreate = undefined; diff --git a/src/services/framework/index.ts b/src/services/framework/index.ts index e6237ff..54e8faa 100644 --- a/src/services/framework/index.ts +++ b/src/services/framework/index.ts @@ -66,7 +66,7 @@ import { findDangerousSqlStatements, frameworkGetModelId, frameworkGetSourceIds, - frameworkMakeSourcePrefix, + frameworkMakeSourceId, generateAutoTests, generatePythonModelConfigPy, generatePythonModelScaffoldPy, @@ -730,7 +730,13 @@ export class Framework implements ApiEnabledService<'framework'> { context.subscriptions.push( vscode.commands.registerCommand( COMMAND_ID.SOURCE_REFRESH, - async ({ sourceId }: { sourceId: string }) => { + async ({ + sourceId, + tableName: providedTableName, + }: { + sourceId: string; + tableName?: string; + }) => { await vscode.window.withProgress( { title: 'DJ Loading', @@ -754,7 +760,14 @@ export class Framework implements ApiEnabledService<'framework'> { return; } - const tableName = sourceId.split('.')[3]; + const tableName = + providedTableName ?? sourceId.split('.').pop(); + if (!tableName) { + vscode.window.showErrorMessage( + 'Could not determine the source table to refresh.', + ); + return; + } const trinoColumnsResponse = await this.getApi().handleApi({ type: 'trino-fetch-columns', @@ -997,21 +1010,19 @@ export class Framework implements ApiEnabledService<'framework'> { new RegExp(regex), ); if (range) { - const sourceId = - frameworkMakeSourcePrefix({ - database: sourceDatabase, - schema: sourceSchema, - project, - }) + - '.' + - sourceTable.name; + const sourceId = frameworkMakeSourceId({ + database: sourceDatabase, + schema: sourceSchema, + table: sourceTable.name, + project, + }); // Add refresh source as first code lens codeLenses.push( new vscode.CodeLens(range, { title: 'Refresh Source $(refresh)', command: COMMAND_ID.SOURCE_REFRESH, - arguments: [{ sourceId }], + arguments: [{ sourceId, tableName: sourceTable.name }], }), ); @@ -1025,19 +1036,14 @@ export class Framework implements ApiEnabledService<'framework'> { }), ); } else { - // Use dbtSourcePropertiesString helper - const currentProperties = `${sourceDatabase}.${sourceSchema}.${sourceTable.name}`; - const registeredProperties = etlSource.properties; - const propertiesEqual = - currentProperties === registeredProperties; - if (propertiesEqual) { - codeLenses.push( - new vscode.CodeLens(range, { - title: 'Source Registered $(pass-filled)', - command: '', - }), - ); - } + // Present in the dbt_sources registry but not actively + // ETL'd -- surface it as registered. + codeLenses.push( + new vscode.CodeLens(range, { + title: 'Source Registered $(pass-filled)', + command: '', + }), + ); } } } diff --git a/src/shared/dbt/types.ts b/src/shared/dbt/types.ts index 1f11abb..a841e7d 100644 --- a/src/shared/dbt/types.ts +++ b/src/shared/dbt/types.ts @@ -643,6 +643,7 @@ export type DbtRunConfig = { // Advanced Flags build: boolean; defer: boolean; + favorState: boolean; fullRefresh: boolean; // Run Scope scope: DbtRunScope; diff --git a/src/shared/dbt/utils.ts b/src/shared/dbt/utils.ts index fe13151..a3f27e4 100644 --- a/src/shared/dbt/utils.ts +++ b/src/shared/dbt/utils.ts @@ -246,6 +246,9 @@ export function buildDbtRunCommand(config: DbtRunConfig): string { } if (config.defer && config.statePath) { finalCommand += ` --defer --state ${config.statePath}`; + if (config.favorState) { + finalCommand += ' --favor-state'; + } } // Add date variables if provided diff --git a/src/shared/sql/__tests__/identifier.test.ts b/src/shared/sql/__tests__/identifier.test.ts index 5676d5b..a66ee17 100644 --- a/src/shared/sql/__tests__/identifier.test.ts +++ b/src/shared/sql/__tests__/identifier.test.ts @@ -99,4 +99,13 @@ describe('quoteTrinoIdentifier', () => { '"x"" union select secret --"', ); }); + + it('throws a clear error when the identifier is missing', () => { + // A missing catalog/schema/table (e.g. an undefined table name reaching + // the query builder) must fail with a descriptive message rather than the + // opaque "Cannot read properties of undefined (reading 'replace')". + expect(() => + quoteTrinoIdentifier(undefined as unknown as string), + ).toThrow(/expected an identifier/); + }); }); diff --git a/src/shared/sql/identifier.ts b/src/shared/sql/identifier.ts index ad5b893..0059c81 100644 --- a/src/shared/sql/identifier.ts +++ b/src/shared/sql/identifier.ts @@ -52,5 +52,16 @@ export function assertSqlIdentifier( * catalog / schema / table name into a query. */ export function quoteTrinoIdentifier(identifier: string): string { + if (typeof identifier !== 'string') { + // Guard against a missing catalog/schema/table name reaching the query + // builder. Without this, an `undefined` identifier throws the opaque + // "Cannot read properties of undefined (reading 'replace')" deep in the + // Trino CLI path instead of a message that names the real problem. + throw new Error( + `Cannot build Trino query: expected an identifier but received ${JSON.stringify( + identifier, + )}.`, + ); + } return `"${identifier.replace(/"/g, '""')}"`; } diff --git a/web/src/forms/FieldSelectSingle.tsx b/web/src/forms/FieldSelectSingle.tsx index aa10af1..bd4325b 100644 --- a/web/src/forms/FieldSelectSingle.tsx +++ b/web/src/forms/FieldSelectSingle.tsx @@ -11,6 +11,14 @@ export type FieldSelectSingleProps = ControllerRenderProps & { inputClassName?: string; labelClass?: string; helpIcon?: React.ReactNode; // Help icon for Assist Me + /** + * Virtualize the options list. Enable for large option sets (hundreds+), + * e.g. Trino tables, so the dropdown doesn't render every row into the DOM + * and hang the webview. + */ + virtualized?: boolean; + /** Debounce (ms) applied to the typed query before filtering large lists. */ + filterDebounceMs?: number; }; export const FieldSelectSingle = forwardRef< diff --git a/web/src/hooks/useModelRunValidation.ts b/web/src/hooks/useModelRunValidation.ts index d35a55b..fcdd1f4 100644 --- a/web/src/hooks/useModelRunValidation.ts +++ b/web/src/hooks/useModelRunValidation.ts @@ -84,15 +84,12 @@ export function useModelRunValidation({ errors.statePath = 'State path is required when defer is enabled'; } - // Validate that modified models exist when defer is enabled with 'modified' scope + // Validate that modified models exist when 'modified' scope is selected // Only show error after we've completed at least one fetch attempt - const shouldValidateDeferModels = - defer && - scope === 'modified' && - !fetchingModifiedModels && - hasFetchedModels; + const shouldValidateModifiedModels = + scope === 'modified' && !fetchingModifiedModels && hasFetchedModels; - if (shouldValidateDeferModels && modifiedModels.length === 0) { + if (shouldValidateModifiedModels && modifiedModels.length === 0) { errors.defer = 'No model changes detected from master branch. All models are up to date.'; } diff --git a/web/src/pages/ModelRun/ModelRun.tsx b/web/src/pages/ModelRun/ModelRun.tsx index 62c0613..325e5c0 100644 --- a/web/src/pages/ModelRun/ModelRun.tsx +++ b/web/src/pages/ModelRun/ModelRun.tsx @@ -81,6 +81,7 @@ export function ModelRun() { seed: false, build: false, defer: false, + favorState: false, fullRefresh: false, scope: 'single', lineage: 'model-only', @@ -132,16 +133,6 @@ export function ModelRun() { })); }, [modelInfo]); - // Auto-switch scope to 'modified' when defer is enabled - useEffect(() => { - setConfig((prev) => { - if (prev.defer && prev.scope !== 'modified') { - return { ...prev, scope: 'modified' }; - } - return prev; - }); - }, [config.defer]); - // Fetch available models when project is set in config useEffect(() => { if (!config.projectName) return; @@ -162,24 +153,23 @@ export function ModelRun() { void fetchAvailableModels(); }, [config.projectName, api]); - // Debounce the scope and defer changes to avoid rapid API calls + // Debounce scope changes to avoid rapid API calls const debouncedScope = useDebounce(config.scope, 300); - const debouncedDefer = useDebounce(config.defer, 300); - // Fetch modified models when defer is enabled OR scope is 'modified' + // Fetch modified models when scope is 'modified' // Uses state to track if we've already fetched to avoid redundant API calls useEffect(() => { const shouldFetchModifiedModels = - (debouncedDefer || debouncedScope === 'modified') && config.projectName; + debouncedScope === 'modified' && config.projectName; - const shouldClearModels = !debouncedDefer && debouncedScope !== 'modified'; + const shouldClearModels = debouncedScope !== 'modified'; if (shouldClearModels) { // Cancel any in-flight request if (abortControllerRef.current) { abortControllerRef.current.abort(); } - // Reset modified models and fetch flag when neither defer nor modified scope is active + // Reset modified models and fetch flag when modified scope is not active setModifiedModels([]); setSelectedModifiedModels([]); setHasFetchedModels(false); @@ -228,13 +218,7 @@ export function ModelRun() { abortControllerRef.current.abort(); } }; - }, [ - debouncedDefer, - debouncedScope, - config.projectName, - hasFetchedModels, - api, - ]); + }, [debouncedScope, config.projectName, hasFetchedModels, api]); // Listen for theme changes in the document useEffect(() => { @@ -432,6 +416,11 @@ export function ModelRun() { */ const handleBooleanConfigChange = useCallback( (key: string, value: boolean) => { + if (key === 'defer' && !value) { + // Favor State only applies with Defer — clear it when Defer is turned off + setConfig((prev) => ({ ...prev, defer: false, favorState: false })); + return; + } handleConfigChange(key as keyof DbtRunConfig, value); }, [handleConfigChange], @@ -606,8 +595,17 @@ export function ModelRun() { tooltipText={TOOLTIPS.fullRefresh} onChange={handleBooleanConfigChange} /> + {config.defer && ( + + )} - {/* State Path Input - Show when defer is enabled */} + {/* State Path only applies with Defer */} {config.defer && ( void; // For single model scope - the model from active editor activeModelName?: string | null; @@ -36,7 +35,6 @@ interface ScopeSelectorProps { export const ScopeSelector = React.memo( ({ currentScope, - isDeferEnabled, onScopeChange, activeModelName, currentLineage = 'model-only', @@ -60,22 +58,16 @@ export const ScopeSelector = React.memo(
{availableScopeOptions.map((option) => { - // When defer is enabled, disable all scopes except 'modified' - const isDisabled = isDeferEnabled && option.value !== 'modified'; - return (