Feat/tpcc dual

.gitignore

@@ -24,4 +24,5 @@ sqlite_bench
 kite_sql_tpcc
 copy.csv
 
-tests/data/row_20000.csv
+tests/data/row_20000.csv
+tests/data/distinct_rows.csv

AGENT.md

@@ -88,6 +88,9 @@ PRs that modify logic but leave obvious test gaps untouched may be rejected.
 
 If something can be written clearly using fewer concepts, do that.
 
+- Prefer incremental fixes: solve the problem at hand with the smallest reasonable change
+  before introducing new abstractions or generalizations.
+
 ---
 
 ### 3.2 Low Comments, High Signal

Cargo.toml

@@ -71,7 +71,7 @@ tokio                 = { version = "1.36", features = ["full"], optional = true
 
 
 [target.'cfg(unix)'.dev-dependencies]
-pprof = { version = "0.13", features = ["flamegraph", "criterion"] }
+pprof = { version = "0.15", features = ["flamegraph", "criterion"] }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 criterion             = { version = "0.5", features = ["html_reports"] }

Makefile

@@ -3,7 +3,7 @@ CARGO ?= cargo
 WASM_PACK ?= wasm-pack
 SQLLOGIC_PATH ?= tests/slt/**/*.slt
 
-.PHONY: test test-wasm test-slt test-all wasm-build check tpcc cargo-check build wasm-examples native-examples fmt clippy
+.PHONY: test test-wasm test-slt test-all wasm-build check tpcc tpcc-dual cargo-check build wasm-examples native-examples fmt clippy
 
 ## Run default Rust tests in the current environment (non-WASM).
 test:
@@ -47,6 +47,10 @@ check: fmt clippy
 tpcc:
 	$(CARGO) run -p tpcc --release
 
+## Execute TPCC while mirroring every statement to an in-memory SQLite instance for validation.
+tpcc-dual:
+	$(CARGO) run -p tpcc --release -- --backend dual --measure-time 60
+
 ## Run JavaScript-based Wasm example scripts.
 wasm-examples:
 	node examples/wasm_hello_world.test.mjs

README.md

@@ -82,7 +82,8 @@ for tuple in iter {
 - [transaction](examples/transaction.rs)
 
 ## TPC-C
-run `cargo run -p tpcc --release` to run tpcc
+Run `make tpcc` (or `cargo run -p tpcc --release`) to execute the benchmark against the default KiteSQL storage.  
+Run `make tpcc-dual` to mirror every TPCC statement to an in-memory SQLite database alongside KiteSQL and assert the two engines return identical results; this target runs for 60 seconds (`--measure-time 60`). Use `cargo run -p tpcc --release -- --backend dual --measure-time <secs>` for a custom duration.
 
 - i9-13900HX
 - 32.0 GB
@@ -92,13 +93,13 @@ run `cargo run -p tpcc --release` to run tpcc
 All cases have been fully optimized.
 ```shell
 <90th Percentile RT (MaxRT)>
-   New-Order : 0.002  (0.012)
-     Payment : 0.001  (0.002)
-Order-Status : 0.002  (0.019)
-    Delivery : 0.001  (0.001)
- Stock-Level : 0.002  (0.018)
+   New-Order : 0.002  (0.006)
+     Payment : 0.001  (0.019)
+Order-Status : 0.001  (0.003)
+    Delivery : 0.022  (0.038)
+ Stock-Level : 0.002  (0.005)
 <TpmC>
-37166 Tpmc
+18432 Tpmc
 ```
 #### 👉[check more](tpcc/README.md)
 

src/db.rs

@@ -203,9 +203,12 @@ fn default_optimizer_pipeline() -> HepOptimizerPipeline {
             vec![NormalizationRuleImpl::TopK],
         )
         .after_batch(
-            "Eliminate Redundant Sort".to_string(),
+            "Eliminate Aggregate".to_string(),
             HepBatchStrategy::once_topdown(),
-            vec![NormalizationRuleImpl::EliminateRedundantSort],
+            vec![
+                NormalizationRuleImpl::EliminateRedundantSort,
+                NormalizationRuleImpl::UseStreamDistinct,
+            ],
         )
         .after_batch(
             "Expression Remapper".to_string(),

src/execution/dql/aggregate/mod.rs

@@ -17,6 +17,7 @@ mod count;
 pub mod hash_agg;
 mod min_max;
 pub mod simple_agg;
+pub mod stream_distinct;
 mod sum;
 
 use crate::errors::DatabaseError;