diff --git a/Cargo.lock b/Cargo.lock index 01d82c9ab2..34eab17397 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1468,17 +1468,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "diesel_migrations" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745fd255645f0f1135f9ec55c7b00e0882192af9683ab4731e4bba3da82b8f9c" -dependencies = [ - "diesel", - "migrations_internals", - "migrations_macros", -] - [[package]] name = "diesel_table_macro_syntax" version = "0.3.0" @@ -3407,7 +3396,6 @@ dependencies = [ "deadpool", "deadpool-diesel", "diesel", - "diesel_migrations", "fs-err", "hex", "indexmap", @@ -3433,7 +3421,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-stream", - "toml 1.1.2+spec-1.1.0", + "toml", "tonic", "tonic-reflection", "tower-http", @@ -3581,7 +3569,7 @@ dependencies = [ "serde", "serde-untagged", "thiserror 2.0.18", - "toml 1.1.2+spec-1.1.0", + "toml", ] [[package]] @@ -3609,7 +3597,7 @@ dependencies = [ "semver 1.0.28", "serde", "thiserror 2.0.18", - "toml 1.1.2+spec-1.1.0", + "toml", "walkdir", ] @@ -3884,27 +3872,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "migrations_internals" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c791ecdf977c99f45f23280405d7723727470f6689a5e6dbf513ac547ae10d" -dependencies = [ - "serde", - "toml 0.9.12+spec-1.1.0", -] - -[[package]] -name = "migrations_macros" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fc5ac76be324cfd2d3f2cf0fdf5d5d3c4f14ed8aaebadb09e304ba42282703" -dependencies = [ - "migrations_internals", - "proc-macro2", - "quote", -] - [[package]] name = "mime" version = "0.3.17" @@ -6366,19 +6333,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.9.12+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" -dependencies = [ - "serde_core", - "serde_spanned", - "toml_datetime 0.7.5+spec-1.1.0", - "toml_parser", - "winnow 0.7.15", -] - [[package]] name = "toml" version = "1.1.2+spec-1.1.0" @@ -6388,19 +6342,10 @@ dependencies = [ "indexmap", "serde_core", "serde_spanned", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime", "toml_parser", "toml_writer", - "winnow 1.0.1", -] - -[[package]] -name = "toml_datetime" -version = "0.7.5+spec-1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" -dependencies = [ - "serde_core", + "winnow", ] [[package]] @@ -6419,9 +6364,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ "indexmap", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime", "toml_parser", - "winnow 1.0.1", + "winnow", ] [[package]] @@ -6430,7 +6375,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.1", + "winnow", ] [[package]] @@ -6810,7 +6755,7 @@ dependencies = [ "serde_json", "target-triple", "termcolor", - "toml 1.1.2+spec-1.1.0", + "toml", ] [[package]] @@ -7490,12 +7435,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" -[[package]] -name = "winnow" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" - [[package]] name = "winnow" version = "1.0.1" @@ -7622,7 +7561,7 @@ dependencies = [ "anyhow", "clap", "fs-err", - "toml 1.1.2+spec-1.1.0", + "toml", "tree-sitter", "tree-sitter-rust", ] diff --git a/Cargo.toml b/Cargo.toml index 91c51bc386..4e90f07a16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,6 @@ deadpool = { default-features = false, version = "0.12" } deadpool-diesel = { version = "0.6" } deadpool-sync = { default-features = false, version = "0.1" } diesel = { version = "2.3" } -diesel_migrations = { version = "2.3" } fs-err = { version = "3" } futures = { version = "0.3" } hex = { version = "0.4" } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 2c72ffc7ff..44edd5357f 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -23,7 +23,6 @@ async-trait = { workspace = true } deadpool = { features = ["managed", "rt_tokio_1"], workspace = true } deadpool-diesel = { features = ["sqlite"], workspace = true } diesel = { features = ["numeric", "sqlite"], workspace = true } -diesel_migrations = { features = ["sqlite"], workspace = true } fs-err = { workspace = true } hex = { workspace = true } indexmap = { workspace = true } @@ -55,8 +54,9 @@ tracing = { workspace = true } url = { workspace = true } [build-dependencies] -build-rs = { workspace = true } -fs-err = { workspace = true } +build-rs = { workspace = true } +fs-err = { workspace = true } +miden-node-db = { workspace = true } # TODO: consider removing the `testing` from relevant functions in miden-agglayer miden-agglayer = { features = ["testing"], workspace = true } miden-protocol = { features = ["std"], workspace = true } diff --git a/crates/store/build.rs b/crates/store/build.rs index 09bf745d32..5df52daa5b 100644 --- a/crates/store/build.rs +++ b/crates/store/build.rs @@ -1,6 +1,3 @@ -// This build.rs is required to trigger the `diesel_migrations::embed_migrations!` proc-macro in -// `store/src/db/migrations.rs` to include the latest version of the migrations into the binary, see . - use std::path::PathBuf; use std::sync::Arc; @@ -18,8 +15,9 @@ use miden_protocol::{Felt, Word}; use miden_standards::AuthMethod; use miden_standards::account::wallets::create_basic_wallet; -fn main() { - build_rs::output::rerun_if_changed("src/db/migrations"); +fn main() -> Result<(), Box> { + miden_node_db::migration::Migrator::generate("src/db/migrations")?; + // If we do one re-write, the default rules are disabled, // hence we need to trigger explicitly on `Cargo.toml`. // @@ -27,6 +25,8 @@ fn main() { // Generate sample agglayer account files for genesis config samples. generate_agglayer_sample_accounts(); + + Ok(()) } /// Generates sample agglayer account files for the `02-with-account-files` genesis config sample. diff --git a/crates/store/src/db/migrations.rs b/crates/store/src/db/migrations.rs index 10ce01409e..c9ee04ebff 100644 --- a/crates/store/src/db/migrations.rs +++ b/crates/store/src/db/migrations.rs @@ -1,31 +1,93 @@ -use diesel::SqliteConnection; -use diesel_migrations::{EmbeddedMigrations, MigrationHarness, embed_migrations}; +use std::path::Path; + +use miden_node_db::DatabaseError; use tracing::instrument; use crate::COMPONENT; -use crate::db::schema_hash::verify_schema; -// The rebuild is automatically triggered by `build.rs` as described in -// . -pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("src/db/migrations"); +include!(concat!(env!("OUT_DIR"), "/db_migrator.rs")); -// TODO we have not tested this in practice! #[instrument(level = "debug", target = COMPONENT, skip_all, err)] -pub fn apply_migrations( - conn: &mut SqliteConnection, -) -> std::result::Result<(), miden_node_db::DatabaseError> { - let migrations = conn.pending_migrations(MIGRATIONS).expect("In memory migrations never fail"); - tracing::info!(target = COMPONENT, migrations = migrations.len(), "Applying migrations"); - - let Err(e) = conn.run_pending_migrations(MIGRATIONS) else { - // Migrations applied successfully, verify schema hash - verify_schema(conn)?; - return Ok(()); - }; - tracing::warn!(target = COMPONENT, error = ?e, "Failed to apply migration"); - // something went wrong, MIGRATIONS contains - conn.revert_last_migration(MIGRATIONS) - .expect("Duality is maintained by the developer"); +pub fn apply_migrations(database_filepath: &Path) -> std::result::Result<(), DatabaseError> { + let migrator = migrator().map_err(DatabaseError::migration)?; + tracing::info!( + target: COMPONENT, + migration_count = migrator.schema_hashes().len(), + "Applying database migrations" + ); + + migrator.migrate(database_filepath).map_err(DatabaseError::migration)?; Ok(()) } + +#[cfg(test)] +pub(crate) fn test_connection() -> diesel::SqliteConnection { + use diesel::{Connection, SqliteConnection}; + + let database_filepath = tempfile::NamedTempFile::new() + .expect("failed to create temp database file") + .into_temp_path(); + apply_migrations(&database_filepath).expect("migrations should apply on empty database"); + + let conn = SqliteConnection::establish( + database_filepath.to_str().expect("temp database path should be valid UTF-8"), + ) + .expect("temp file sqlite should always work"); + database_filepath + .keep() + .expect("failed to keep temp database file for test connection"); + conn +} + +#[cfg(test)] +mod tests { + use std::process::Command; + + use anyhow::{Context, Result, ensure}; + use miden_node_db::migration::SchemaHash; + + use super::*; + + const EXPECTED_SCHEMA_HASHES: [SchemaHash; 1] = [SchemaHash::from_hex( + "b3bdd2e530fbb66c9146cda9f3bf79df49c6a6bf99f7432aae0a8927a15406ac", + )]; + + #[test] + fn migration_schema_hashes_are_stable() -> Result<()> { + let migrator = migrator()?; + + assert_eq!(migrator.schema_hashes(), &EXPECTED_SCHEMA_HASHES); + Ok(()) + } + + #[test] + #[ignore = "requires diesel CLI; CI runs this in the diesel-schema job"] + fn diesel_schema_is_in_sync_with_migrations() -> Result<()> { + let temp_dir = tempfile::tempdir()?; + let database_filepath = temp_dir.path().join("store.sqlite3"); + apply_migrations(&database_filepath)?; + + let output = Command::new("diesel") + .arg("print-schema") + .arg("--database-url") + .arg(&database_filepath) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .output() + .context( + "failed to run diesel CLI; install it with \ + `cargo install diesel_cli --no-default-features --features sqlite`", + )?; + + ensure!( + output.status.success(), + "diesel print-schema failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let generated = + String::from_utf8(output.stdout).context("diesel CLI output is not UTF-8")?; + assert_eq!(generated, include_str!("schema.rs")); + Ok(()) + } +} diff --git a/crates/store/src/db/migrations/2025062000000_setup/up.sql b/crates/store/src/db/migrations/001_initial.sql similarity index 76% rename from crates/store/src/db/migrations/2025062000000_setup/up.sql rename to crates/store/src/db/migrations/001_initial.sql index 2f8538d988..f0b4a8fb89 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/up.sql +++ b/crates/store/src/db/migrations/001_initial.sql @@ -1,18 +1,13 @@ CREATE TABLE block_headers ( - block_num INTEGER NOT NULL, - block_header BLOB NOT NULL, - signature BLOB NOT NULL, - commitment BLOB NOT NULL, - proving_inputs BLOB, -- Serialized BlockProofRequest needed for deferred proving. NULL if it has been proven or never proven (genesis block). - proven_in_sequence BOOLEAN NOT NULL DEFAULT FALSE, -- TRUE when this block and all its ancestors have been proven. + block_num BIGINT NOT NULL, + block_header BLOB NOT NULL, + signature BLOB NOT NULL, + commitment BLOB NOT NULL, PRIMARY KEY (block_num), CONSTRAINT block_header_block_num_is_u32 CHECK (block_num BETWEEN 0 AND 0xFFFFFFFF) ); -CREATE INDEX block_headers_proven_desc ON block_headers(block_num DESC) WHERE proving_inputs IS NULL; -CREATE INDEX block_headers_proven_in_sequence ON block_headers(block_num DESC) WHERE proven_in_sequence = TRUE; - CREATE TABLE account_codes ( code_commitment BLOB NOT NULL, code BLOB NOT NULL, @@ -22,14 +17,14 @@ CREATE TABLE account_codes ( CREATE TABLE accounts ( account_id BLOB NOT NULL, network_account_type INTEGER NOT NULL, -- 0-not a network account, 1-network account - block_num INTEGER NOT NULL, + block_num BIGINT NOT NULL, account_commitment BLOB NOT NULL, code_commitment BLOB, - nonce INTEGER, + nonce BIGINT, storage_header BLOB, -- Serialized AccountStorageHeader from miden-objects vault_root BLOB, -- Vault root commitment is_latest BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id - created_at_block INTEGER NOT NULL, + created_at_block BIGINT NOT NULL, PRIMARY KEY (account_id, block_num), CONSTRAINT all_null_or_none_null CHECK @@ -49,11 +44,15 @@ CREATE INDEX idx_accounts_created_at_block ON accounts(created_at_block); CREATE INDEX idx_accounts_block_num ON accounts(block_num); -- Index for joining with account_codes CREATE INDEX idx_accounts_code_commitment ON accounts(code_commitment) WHERE code_commitment IS NOT NULL; --- Covering index for the prune_account_codes subquery: filters rows by block_num/is_latest and projects code_commitment -CREATE INDEX idx_accounts_prune_code ON accounts(block_num, is_latest, code_commitment) WHERE code_commitment IS NOT NULL; +-- Covering index for the prune_account_codes recent-history branch. +CREATE INDEX idx_accounts_prune_code ON accounts(block_num, code_commitment) WHERE code_commitment IS NOT NULL; +-- Covering index for the prune_account_codes latest-state branch. +CREATE INDEX idx_accounts_latest_code_commitment + ON accounts(code_commitment) + WHERE is_latest = 1 AND code_commitment IS NOT NULL; CREATE TABLE notes ( - committed_at INTEGER NOT NULL, -- Block number when the note was committed + committed_at BIGINT NOT NULL, -- Block number when the note was committed batch_index INTEGER NOT NULL, -- Index of batch in block, starting from 0 note_index INTEGER NOT NULL, -- Index of note in batch, starting from 0 note_id BLOB NOT NULL, @@ -65,7 +64,7 @@ CREATE TABLE notes ( target_account_id BLOB, -- Full target account ID for single-target network notes attachment BLOB NOT NULL, -- Serialized note attachment data inclusion_path BLOB NOT NULL, -- Serialized sparse Merkle path of the note in the block's note tree - consumed_at INTEGER, -- Block number when the note was consumed + consumed_at BIGINT, -- Block number when the note was consumed nullifier BLOB, -- Only known for public notes, null for private notes assets BLOB, storage BLOB, @@ -102,7 +101,7 @@ CREATE TABLE note_scripts ( CREATE TABLE account_storage_map_values ( account_id BLOB NOT NULL, - block_num INTEGER NOT NULL, + block_num BIGINT NOT NULL, slot_name TEXT NOT NULL, key BLOB NOT NULL, value BLOB NOT NULL, @@ -119,7 +118,7 @@ CREATE INDEX idx_account_storage_latest ON account_storage_map_values(account_id CREATE TABLE account_vault_assets ( account_id BLOB NOT NULL, - block_num INTEGER NOT NULL, + block_num BIGINT NOT NULL, vault_key BLOB NOT NULL, asset BLOB, is_latest BOOLEAN NOT NULL, @@ -136,7 +135,7 @@ CREATE INDEX idx_vault_assets_latest ON account_vault_assets(account_id, is_late CREATE TABLE nullifiers ( nullifier BLOB NOT NULL, nullifier_prefix INTEGER NOT NULL, - block_num INTEGER NOT NULL, + block_num BIGINT NOT NULL, PRIMARY KEY (nullifier), CONSTRAINT nullifiers_nullifier_is_digest CHECK (length(nullifier) = 32), @@ -148,15 +147,15 @@ CREATE INDEX idx_nullifiers_prefix ON nullifiers(nullifier_prefix); CREATE INDEX idx_nullifiers_block_num ON nullifiers(block_num); CREATE TABLE transactions ( - transaction_id BLOB NOT NULL, - account_id BLOB NOT NULL, - block_num INTEGER NOT NULL, -- Block number in which the transaction was included. - initial_state_commitment BLOB NOT NULL, -- State of the account before applying the transaction. - final_state_commitment BLOB NOT NULL, -- State of the account after applying the transaction. - input_notes BLOB NOT NULL, -- Serialized Vec (nullifier + optional NoteHeader). - output_notes BLOB NOT NULL, -- Serialized Vec (NoteId + NoteMetadata). - size_in_bytes INTEGER NOT NULL, -- Estimated size of the row in bytes, considering the size of the input and output notes. - fee BLOB NOT NULL, -- Serialized FungibleAsset representing the fee paid by the transaction. + transaction_id BLOB NOT NULL, + account_id BLOB NOT NULL, + block_num BIGINT NOT NULL, -- Block number in which the transaction was included. + initial_state_commitment BLOB NOT NULL, -- State of the account before applying the transaction. + final_state_commitment BLOB NOT NULL, -- State of the account after applying the transaction. + input_notes BLOB NOT NULL, -- Serialized Vec (nullifier + optional NoteHeader). + output_notes BLOB NOT NULL, -- Serialized Vec (NoteId + NoteMetadata). + size_in_bytes BIGINT NOT NULL, -- Estimated size of the row in bytes, considering the size of the input and output notes. + fee BLOB NOT NULL, -- Serialized FungibleAsset representing the fee paid by the transaction. PRIMARY KEY (transaction_id) ) WITHOUT ROWID; @@ -165,6 +164,9 @@ CREATE TABLE transactions ( CREATE INDEX idx_transactions_account_id ON transactions(account_id); -- Index for joining with block_headers CREATE INDEX idx_transactions_block_num ON transactions(block_num); +-- Composite index to speed up select_transactions_records. +CREATE INDEX idx_transactions_account_block_txid + ON transactions(account_id, block_num, transaction_id); CREATE INDEX idx_vault_cleanup ON account_vault_assets(block_num) WHERE is_latest = 0; CREATE INDEX idx_storage_cleanup ON account_storage_map_values(block_num) WHERE is_latest = 0; diff --git a/crates/store/src/db/migrations/2025062000000_setup/down.sql b/crates/store/src/db/migrations/2025062000000_setup/down.sql deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/store/src/db/migrations/2026042800000_add_idx_transactions_sync/down.sql b/crates/store/src/db/migrations/2026042800000_add_idx_transactions_sync/down.sql deleted file mode 100644 index b9ea916663..0000000000 --- a/crates/store/src/db/migrations/2026042800000_add_idx_transactions_sync/down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP INDEX IF EXISTS idx_transactions_account_block_txid; diff --git a/crates/store/src/db/migrations/2026042800000_add_idx_transactions_sync/up.sql b/crates/store/src/db/migrations/2026042800000_add_idx_transactions_sync/up.sql deleted file mode 100644 index 4a84a1a830..0000000000 --- a/crates/store/src/db/migrations/2026042800000_add_idx_transactions_sync/up.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Composite index to speed up select_transactions_records. --- --- The query filters by account_id (IN), block_num range, and paginates via a --- (block_num, transaction_id) cursor, then orders by (block_num, transaction_id). --- The existing idx_transactions_account_id index only covers account_id, forcing --- a full scan over matching rows to apply the block_num filter and sort. --- --- With (account_id, block_num, transaction_id) SQLite can: --- 1. Seek to each account_id bucket directly, --- 2. Range-scan block_num within that bucket, and --- 3. Use transaction_id for cursor comparison and ORDER BY — all index-only. -CREATE INDEX idx_transactions_account_block_txid - ON transactions(account_id, block_num, transaction_id); diff --git a/crates/store/src/db/migrations/2026051200000_remove_proving_inputs/down.sql b/crates/store/src/db/migrations/2026051200000_remove_proving_inputs/down.sql deleted file mode 100644 index 62614bdd90..0000000000 --- a/crates/store/src/db/migrations/2026051200000_remove_proving_inputs/down.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Restore proving_inputs and proven_in_sequence columns (data is not recovered). -ALTER TABLE block_headers ADD COLUMN proving_inputs BLOB; -ALTER TABLE block_headers ADD COLUMN proven_in_sequence BOOLEAN NOT NULL DEFAULT FALSE; -CREATE INDEX block_headers_proven_desc ON block_headers(block_num DESC) WHERE proving_inputs IS NULL; -CREATE INDEX block_headers_proven_in_sequence ON block_headers(block_num DESC) WHERE proven_in_sequence = TRUE; diff --git a/crates/store/src/db/migrations/2026051200000_remove_proving_inputs/up.sql b/crates/store/src/db/migrations/2026051200000_remove_proving_inputs/up.sql deleted file mode 100644 index 1591dd0f48..0000000000 --- a/crates/store/src/db/migrations/2026051200000_remove_proving_inputs/up.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Move proving inputs out of the database. --- --- Proving inputs are large BLOBs that only serve the proof scheduler; they now live as --- `inputs_.dat` files in the block store alongside block data and proofs. --- The proven-in-sequence tip is tracked by a small `proven_tip` file in the data directory. --- --- Drop indexes that reference the columns being removed first (required by SQLite before DROP --- COLUMN can succeed for indexed columns). -DROP INDEX block_headers_proven_desc; -DROP INDEX block_headers_proven_in_sequence; -ALTER TABLE block_headers DROP COLUMN proving_inputs; -ALTER TABLE block_headers DROP COLUMN proven_in_sequence; diff --git a/crates/store/src/db/migrations/2026051800000_add_idx_accounts_latest_code_commitment/down.sql b/crates/store/src/db/migrations/2026051800000_add_idx_accounts_latest_code_commitment/down.sql deleted file mode 100644 index ce0a67d4ec..0000000000 --- a/crates/store/src/db/migrations/2026051800000_add_idx_accounts_latest_code_commitment/down.sql +++ /dev/null @@ -1,6 +0,0 @@ -DROP INDEX IF EXISTS idx_accounts_latest_code_commitment; -DROP INDEX IF EXISTS idx_accounts_prune_code; - -CREATE INDEX idx_accounts_prune_code - ON accounts(block_num, is_latest, code_commitment) - WHERE code_commitment IS NOT NULL; diff --git a/crates/store/src/db/migrations/2026051800000_add_idx_accounts_latest_code_commitment/up.sql b/crates/store/src/db/migrations/2026051800000_add_idx_accounts_latest_code_commitment/up.sql deleted file mode 100644 index d406a82b52..0000000000 --- a/crates/store/src/db/migrations/2026051800000_add_idx_accounts_latest_code_commitment/up.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Backfill/reshape the recent-history covering index for prune_account_codes. The old definition --- included is_latest, but the split recent-history branch only filters by block_num and projects --- code_commitment. -DROP INDEX IF EXISTS idx_accounts_prune_code; -CREATE INDEX idx_accounts_prune_code - ON accounts(block_num, code_commitment) - WHERE code_commitment IS NOT NULL; - --- Covering partial index for prune_account_codes. --- --- prune_account_codes keeps account code rows that are referenced either by recent account history --- or by the latest account state. The recent-history branch is covered by idx_accounts_prune_code. --- This index covers the latest-state branch without scanning historical public account rows. -CREATE INDEX idx_accounts_latest_code_commitment - ON accounts(code_commitment) - WHERE is_latest = 1 AND code_commitment IS NOT NULL; diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 965ef439d9..306f4a0e16 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -52,7 +52,6 @@ fn default_storage_map_entries_limit() -> usize { } mod migrations; -mod schema_hash; #[cfg(test)] mod tests; @@ -61,23 +60,8 @@ pub(crate) mod models; /// [diesel](https://diesel.rs) generated schema /// -/// ```sh -/// cargo binstall diesel_cli -/// sqlite3 -init ./src/db/migrations/001-init.sql ephemeral_setup.db "" -/// diesel setup --database-url=./ephemeral_setup.db -/// diesel print-schema > src/db/schema.rs -/// ``` -/// -/// which assumes an _existing_ database. -/// -/// Unfortunately, there is no systematic way of modifying the schema other -/// than patching (in the diff sense) which is brittle at best. -/// So the above must be followed by a manual editing step, for now it's -/// limited to: -/// -/// * `i64`/`u64` being represented as `BigInt` -/// -/// The list might be extended. +/// The ignored `diesel_schema_is_in_sync_with_migrations` test verifies that this file matches the +/// schema produced by the current migrations. pub(crate) mod schema; pub type Result = std::result::Result; @@ -261,11 +245,11 @@ impl Db { err, )] pub fn bootstrap(database_filepath: PathBuf, genesis: GenesisBlock) -> anyhow::Result<()> { - // Create database. - // - // This will create the file if it does not exist, but will also happily open it if already - // exists. In the latter case we will error out when attempting to insert the genesis - // block so this isn't such a problem. + apply_migrations(&database_filepath).context("failed to apply database migrations")?; + + // Open the database. This will create the file if it does not exist, but will also happily + // open it if already exists. In the latter case we will error out when attempting to insert + // the genesis block so this isn't such a problem. let mut conn: SqliteConnection = diesel::sqlite::SqliteConnection::establish( database_filepath.to_str().context("database filepath is invalid")?, ) @@ -273,9 +257,6 @@ impl Db { miden_node_db::configure_connection_on_creation(&mut conn)?; - // Run migrations. - apply_migrations(&mut conn).context("failed to apply database migrations")?; - // Insert genesis block data. let genesis_block = genesis.into_inner(); conn.transaction(move |conn| models::queries::apply_block(conn, &genesis_block, &[])) @@ -296,6 +277,8 @@ impl Db { database_filepath: PathBuf, connection_pool_size: NonZeroUsize, ) -> Result { + apply_migrations(&database_filepath)?; + let db = miden_node_db::Db::new_with_pool_size(&database_filepath, connection_pool_size)?; info!( target: COMPONENT, @@ -304,7 +287,6 @@ impl Db { "Connected to the database" ); - db.query("migrations", apply_migrations).await?; Ok(Self { db }) } diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 53dacbeab3..c7b7464a1e 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -10,6 +10,8 @@ use diesel::{ BoolExpressionMethods, ExpressionMethods, Insertable, + JoinOnDsl, + NullableExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, @@ -176,15 +178,17 @@ pub(crate) fn select_full_account( account_id: AccountId, ) -> Result { // Get account metadata (nonce, code_commitment) and code in a single join query - let (nonce, code_bytes): (Option, Vec) = SelectDsl::select( - schema::accounts::table.inner_join(schema::account_codes::table), - (schema::accounts::nonce, schema::account_codes::code), - ) - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::is_latest.eq(true)) - .get_result(conn) - .optional()? - .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + let joined = schema::accounts::table.inner_join(schema::account_codes::table.on( + schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), + )); + + let (nonce, code_bytes): (Option, Vec) = + SelectDsl::select(joined, (schema::accounts::nonce, schema::account_codes::code)) + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .get_result(conn) + .optional()? + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; let nonce = raw_sql_to_nonce(nonce.ok_or_else(|| { DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) diff --git a/crates/store/src/db/models/queries/accounts/delta/tests.rs b/crates/store/src/db/models/queries/accounts/delta/tests.rs index 3cacb46b0d..2f9b69ea94 100644 --- a/crates/store/src/db/models/queries/accounts/delta/tests.rs +++ b/crates/store/src/db/models/queries/accounts/delta/tests.rs @@ -4,8 +4,7 @@ use std::collections::BTreeMap; use assert_matches::assert_matches; -use diesel::{Connection, ExpressionMethods, QueryDsl, RunQueryDsl, SqliteConnection}; -use diesel_migrations::MigrationHarness; +use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl, SqliteConnection}; use miden_node_utils::fee::test_fee_params; use miden_protocol::account::auth::{AuthScheme, PublicKeyCommitment}; use miden_protocol::account::component::AccountComponentMetadata; @@ -40,7 +39,6 @@ use miden_protocol::{EMPTY_WORD, Felt, Word}; use miden_standards::account::auth::AuthSingleSig; use miden_standards::code_builder::CodeBuilder; -use crate::db::migrations::MIGRATIONS; use crate::db::models::queries::accounts::tests::select_account_vault_at_block; use crate::db::models::queries::accounts::{ select_account_header_with_storage_header_at_block, @@ -50,12 +48,7 @@ use crate::db::models::queries::accounts::{ use crate::db::schema::accounts; fn setup_test_db() -> SqliteConnection { - let mut conn = - SqliteConnection::establish(":memory:").expect("Failed to create in-memory database"); - - conn.run_pending_migrations(MIGRATIONS).expect("Failed to run migrations"); - - conn + crate::db::migrations::test_connection() } fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { diff --git a/crates/store/src/db/models/queries/accounts/tests.rs b/crates/store/src/db/models/queries/accounts/tests.rs index eee9f69795..4b0e8f2da4 100644 --- a/crates/store/src/db/models/queries/accounts/tests.rs +++ b/crates/store/src/db/models/queries/accounts/tests.rs @@ -3,15 +3,7 @@ use std::collections::BTreeMap; use diesel::query_dsl::methods::SelectDsl; -use diesel::{ - BoolExpressionMethods, - Connection, - ExpressionMethods, - OptionalExtension, - QueryDsl, - RunQueryDsl, -}; -use diesel_migrations::MigrationHarness; +use diesel::{BoolExpressionMethods, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; use miden_node_utils::fee::test_fee_params; use miden_protocol::account::auth::{AuthScheme, PublicKeyCommitment}; use miden_protocol::account::component::AccountComponentMetadata; @@ -46,18 +38,12 @@ use miden_standards::account::auth::AuthSingleSig; use miden_standards::code_builder::CodeBuilder; use super::*; -use crate::db::migrations::MIGRATIONS; use crate::db::models::conv::SqlTypeConvert; use crate::db::schema; use crate::errors::DatabaseError; fn setup_test_db() -> SqliteConnection { - let mut conn = - SqliteConnection::establish(":memory:").expect("Failed to create in-memory database"); - - conn.run_pending_migrations(MIGRATIONS).expect("Failed to run migrations"); - - conn + crate::db::migrations::test_connection() } /// Test helper: reconstructs account storage at a given block from DB. diff --git a/crates/store/src/db/schema.rs b/crates/store/src/db/schema.rs index 09e3621383..82faec3924 100644 --- a/crates/store/src/db/schema.rs +++ b/crates/store/src/db/schema.rs @@ -1,5 +1,12 @@ // @generated automatically by Diesel CLI. +diesel::table! { + account_codes (code_commitment) { + code_commitment -> Binary, + code -> Binary, + } +} + diesel::table! { account_storage_map_values (account_id, block_num, slot_name, key) { account_id -> Binary, @@ -25,24 +32,17 @@ diesel::table! { accounts (account_id, block_num) { account_id -> Binary, network_account_type -> Integer, + block_num -> BigInt, account_commitment -> Binary, code_commitment -> Nullable, nonce -> Nullable, storage_header -> Nullable, vault_root -> Nullable, - block_num -> BigInt, is_latest -> Bool, created_at_block -> BigInt, } } -diesel::table! { - account_codes (code_commitment) { - code_commitment -> Binary, - code -> Binary, - } -} - diesel::table! { block_headers (block_num) { block_num -> BigInt, @@ -104,21 +104,11 @@ diesel::table! { } } -diesel::joinable!(accounts -> account_codes (code_commitment)); -diesel::joinable!(accounts -> block_headers (block_num)); -// Note: Cannot use diesel::joinable! with accounts table due to composite primary key -// diesel::joinable!(notes -> accounts (sender)); diesel::joinable!(transactions -> accounts -// (account_id)); -diesel::joinable!(notes -> block_headers (committed_at)); -diesel::joinable!(notes -> note_scripts (script_root)); -diesel::joinable!(nullifiers -> block_headers (block_num)); -diesel::joinable!(transactions -> block_headers (block_num)); - diesel::allow_tables_to_appear_in_same_query!( account_codes, account_storage_map_values, - accounts, account_vault_assets, + accounts, block_headers, note_scripts, notes, diff --git a/crates/store/src/db/schema_hash.rs b/crates/store/src/db/schema_hash.rs deleted file mode 100644 index 9a5ad1328a..0000000000 --- a/crates/store/src/db/schema_hash.rs +++ /dev/null @@ -1,198 +0,0 @@ -//! Schema verification to detect database schema changes. -//! -//! Detects: -//! -//! - Direct modifications to the database schema outside of migrations -//! - Running a node against a database created with different set of migrations -//! - Forgetting to reset the database after schema changes i.e. for a specific migration -//! -//! The verification works by creating an in-memory reference database, applying all -//! migrations to it, and comparing its schema against the actual database schema. - -use diesel::{Connection, RunQueryDsl, SqliteConnection}; -use diesel_migrations::MigrationHarness; -use miden_node_db::SchemaVerificationError; -use tracing::instrument; - -use crate::COMPONENT; -use crate::db::migrations::MIGRATIONS; - -/// Represents a schema object for comparison. -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] -struct SchemaObject { - object_type: String, - name: String, - sql: String, -} - -/// Represents a row from the `sqlite_schema` table. -#[derive(diesel::QueryableByName, Debug)] -struct SqliteSchemaRow { - #[diesel(sql_type = diesel::sql_types::Text)] - schema_type: String, - #[diesel(sql_type = diesel::sql_types::Text)] - name: String, - #[diesel(sql_type = diesel::sql_types::Nullable)] - sql: Option, -} - -/// Extracts all schema objects from a database connection. -fn extract_schema( - conn: &mut SqliteConnection, -) -> Result, SchemaVerificationError> { - let rows: Vec = diesel::sql_query( - "SELECT type as schema_type, name, sql FROM sqlite_schema \ - WHERE type IN ('table', 'index') \ - AND name NOT LIKE 'sqlite_%' \ - AND name NOT LIKE '__diesel_%' \ - ORDER BY type, name", - ) - .load(conn) - .map_err(SchemaVerificationError::SchemaExtraction)?; - - let mut objects: Vec = rows - .into_iter() - .filter_map(|row| { - row.sql.map(|sql| SchemaObject { - object_type: row.schema_type, - name: row.name, - sql, - }) - }) - .collect(); - - objects.sort(); - Ok(objects) -} - -/// Computes the expected schema by applying migrations to an in-memory database. -fn compute_expected_schema() -> Result, SchemaVerificationError> { - let mut conn = SqliteConnection::establish(":memory:") - .map_err(SchemaVerificationError::InMemoryDbCreation)?; - - conn.run_pending_migrations(MIGRATIONS) - .map_err(SchemaVerificationError::MigrationApplication)?; - - extract_schema(&mut conn) -} - -/// Verifies that the database schema matches the expected schema. -/// -/// Creates an in-memory database, applies all migrations, and compares schemas. -/// -/// # Errors -/// -/// Returns `SchemaVerificationError::Mismatch` if schemas differ. -#[instrument(level = "info", target = COMPONENT, skip_all, err)] -pub fn verify_schema(conn: &mut SqliteConnection) -> Result<(), SchemaVerificationError> { - let expected = compute_expected_schema()?; - let actual = extract_schema(conn)?; - - if actual != expected { - let expected_names: Vec<_> = expected.iter().map(|o| &o.name).collect(); - let actual_names: Vec<_> = actual.iter().map(|o| &o.name).collect(); - - // Find differences for better error messages - let missing: Vec<_> = expected.iter().filter(|e| !actual.contains(e)).collect(); - let extra: Vec<_> = actual.iter().filter(|a| !expected.contains(a)).collect(); - - tracing::error!( - target: COMPONENT, - ?expected_names, - ?actual_names, - missing_count = missing.len(), - extra_count = extra.len(), - "Database schema mismatch detected" - ); - - // Log specific differences at debug level - for obj in &missing { - tracing::debug!( - target: COMPONENT, - name = %obj.name, - sql = %obj.sql, - "Missing or modified" - ); - } - for obj in &extra { - tracing::debug!( - target: COMPONENT, - name = %obj.name, - sql = %obj.sql, - "Extra or modified" - ); - } - - return Err(SchemaVerificationError::Mismatch { - expected_count: expected.len(), - actual_count: actual.len(), - missing_count: missing.len(), - extra_count: extra.len(), - }); - } - - tracing::info!(target: COMPONENT, objects = expected.len(), "Database schema verification passed"); - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::db::migrations::apply_migrations; - - #[test] - fn verify_schema_passes_for_correct_schema() { - let mut conn = SqliteConnection::establish(":memory:").unwrap(); - conn.run_pending_migrations(MIGRATIONS).unwrap(); - verify_schema(&mut conn).expect("Should pass for correct schema"); - } - - #[test] - fn verify_schema_fails_for_added_object() { - let mut conn = SqliteConnection::establish(":memory:").unwrap(); - conn.run_pending_migrations(MIGRATIONS).unwrap(); - - diesel::sql_query("CREATE TABLE rogue_table (id INTEGER PRIMARY KEY)") - .execute(&mut conn) - .unwrap(); - - assert!(matches!( - verify_schema(&mut conn), - Err(SchemaVerificationError::Mismatch { .. }) - )); - } - - #[test] - fn verify_schema_fails_for_removed_object() { - let mut conn = SqliteConnection::establish(":memory:").unwrap(); - conn.run_pending_migrations(MIGRATIONS).unwrap(); - - diesel::sql_query("DROP TABLE transactions").execute(&mut conn).unwrap(); - - assert!(matches!( - verify_schema(&mut conn), - Err(SchemaVerificationError::Mismatch { .. }) - )); - } - - #[test] - fn apply_migrations_succeeds_on_fresh_database() { - let mut conn = SqliteConnection::establish(":memory:").unwrap(); - apply_migrations(&mut conn).expect("Should succeed on fresh database"); - } - - #[test] - fn apply_migrations_fails_on_tampered_database() { - let mut conn = SqliteConnection::establish(":memory:").unwrap(); - conn.run_pending_migrations(MIGRATIONS).unwrap(); - - diesel::sql_query("CREATE TABLE tampered (id INTEGER)") - .execute(&mut conn) - .unwrap(); - - assert!(matches!( - apply_migrations(&mut conn), - Err(miden_node_db::DatabaseError::SchemaVerification(_)) - )); - } -} diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 5a8b4df197..93153f0a39 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -80,15 +80,12 @@ use tempfile::tempdir; use super::{AccountInfo, NoteRecord, NoteSyncRecord, NullifierInfo, TransactionRecord}; use crate::account_state_forest::HISTORICAL_BLOCK_RETENTION; -use crate::db::migrations::apply_migrations; use crate::db::models::queries::{StorageMapValue, insert_account_storage_map_value}; use crate::db::models::{Page, queries, utils}; use crate::errors::DatabaseError; fn create_db() -> SqliteConnection { - let mut conn = SqliteConnection::establish(":memory:").expect("In memory sqlite always works"); - apply_migrations(&mut conn).expect("Migrations always work on an empty database"); - conn + crate::db::migrations::test_connection() } fn create_block(conn: &mut SqliteConnection, block_num: BlockNumber) { @@ -1584,7 +1581,6 @@ async fn reconstruct_storage_map_from_db_pages_until_latest() { let slot_name_for_db = slot_name.clone(); db.query("insert paged values", move |db_conn| { db_conn.transaction(|db_conn| { - apply_migrations(db_conn)?; create_block(db_conn, block1); create_block(db_conn, block2); create_block(db_conn, block3); @@ -1652,7 +1648,6 @@ async fn reconstruct_storage_map_from_db_returns_limit_exceeded_for_single_block let slot_name_for_db = slot_name.clone(); db.query("insert entries in single block", move |db_conn| { db_conn.transaction(|db_conn| { - apply_migrations(db_conn)?; create_block(db_conn, block5); queries::upsert_accounts(db_conn, &[mock_block_account_update(account_id, 0)], block5)?; @@ -2010,7 +2005,9 @@ async fn genesis_with_account_assets() { GenesisState::new(vec![account], test_fee_params(), 1, 0, signer.public_key()); let genesis_block = genesis_state.into_block(&signer).unwrap(); - crate::db::Db::bootstrap(":memory:".into(), genesis_block).unwrap(); + let temp_dir = tempdir().unwrap(); + let db_path = temp_dir.path().join("store.sqlite"); + crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); } /// Verifies genesis block with account containing storage maps can be inserted. @@ -2066,7 +2063,9 @@ async fn genesis_with_account_storage_map() { GenesisState::new(vec![account], test_fee_params(), 1, 0, signer.public_key()); let genesis_block = genesis_state.into_block(&signer).unwrap(); - crate::db::Db::bootstrap(":memory:".into(), genesis_block).unwrap(); + let temp_dir = tempdir().unwrap(); + let db_path = temp_dir.path().join("store.sqlite"); + crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); } /// Verifies genesis block with account containing both vault assets and storage maps. @@ -2120,7 +2119,9 @@ async fn genesis_with_account_assets_and_storage() { GenesisState::new(vec![account], test_fee_params(), 1, 0, signer.public_key()); let genesis_block = genesis_state.into_block(&signer).unwrap(); - crate::db::Db::bootstrap(":memory:".into(), genesis_block).unwrap(); + let temp_dir = tempdir().unwrap(); + let db_path = temp_dir.path().join("store.sqlite"); + crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); } /// Verifies genesis block with multiple accounts of different types. Tests realistic genesis @@ -2217,7 +2218,9 @@ async fn genesis_with_multiple_accounts() { ); let genesis_block = genesis_state.into_block(&signer).unwrap(); - crate::db::Db::bootstrap(":memory:".into(), genesis_block).unwrap(); + let temp_dir = tempdir().unwrap(); + let db_path = temp_dir.path().join("store.sqlite"); + crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); } #[test]