feat(store): port SQLite migrations

Cargo.toml

@@ -83,7 +83,6 @@ deadpool          = { default-features = false, version = "0.12" }
 deadpool-diesel   = { version = "0.6" }
 deadpool-sync     = { default-features = false, version = "0.1" }
 diesel            = { version = "2.3" }
-diesel_migrations = { version = "2.3" }
 fs-err            = { version = "3" }
 futures           = { version = "0.3" }
 hex               = { version = "0.4" }

crates/store/Cargo.toml

@@ -23,7 +23,6 @@ async-trait                     = { workspace = true }
 deadpool                        = { features = ["managed", "rt_tokio_1"], workspace = true }
 deadpool-diesel                 = { features = ["sqlite"], workspace = true }
 diesel                          = { features = ["numeric", "sqlite"], workspace = true }
-diesel_migrations               = { features = ["sqlite"], workspace = true }
 fs-err                          = { workspace = true }
 hex                             = { workspace = true }
 indexmap                        = { workspace = true }
@@ -55,8 +54,9 @@ tracing           = { workspace = true }
 url               = { workspace = true }
 
 [build-dependencies]
-build-rs = { workspace = true }
-fs-err   = { workspace = true }
+build-rs      = { workspace = true }
+fs-err        = { workspace = true }
+miden-node-db = { workspace = true }
 # TODO: consider removing the `testing` from relevant functions in miden-agglayer
 miden-agglayer  = { features = ["testing"], workspace = true }
 miden-protocol  = { features = ["std"], workspace = true }

crates/store/build.rs

@@ -1,6 +1,3 @@
-// This build.rs is required to trigger the `diesel_migrations::embed_migrations!` proc-macro in
-// `store/src/db/migrations.rs` to include the latest version of the migrations into the binary, see <https://docs.rs/diesel_migrations/latest/diesel_migrations/macro.embed_migrations.html#automatic-rebuilds>.
-
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -18,15 +15,18 @@ use miden_protocol::{Felt, Word};
 use miden_standards::AuthMethod;
 use miden_standards::account::wallets::create_basic_wallet;
 
-fn main() {
-    build_rs::output::rerun_if_changed("src/db/migrations");
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    miden_node_db::migration::Migrator::generate("src/db/migrations")?;
+
     // If we do one re-write, the default rules are disabled,
     // hence we need to trigger explicitly on `Cargo.toml`.
     // <https://doc.rust-lang.org/cargo/reference/build-scripts.html#rerun-if-changed>
     build_rs::output::rerun_if_changed("Cargo.toml");
 
     // Generate sample agglayer account files for genesis config samples.
     generate_agglayer_sample_accounts();
+
+    Ok(())
 }
 
 /// Generates sample agglayer account files for the `02-with-account-files` genesis config sample.

crates/store/src/db/migrations.rs

@@ -1,31 +1,93 @@
-use diesel::SqliteConnection;
-use diesel_migrations::{EmbeddedMigrations, MigrationHarness, embed_migrations};
+use std::path::Path;
+
+use miden_node_db::DatabaseError;
 use tracing::instrument;
 
 use crate::COMPONENT;
-use crate::db::schema_hash::verify_schema;
 
-// The rebuild is automatically triggered by `build.rs` as described in
-// <https://docs.rs/diesel_migrations/latest/diesel_migrations/macro.embed_migrations.html#automatic-rebuilds>.
-pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("src/db/migrations");
+include!(concat!(env!("OUT_DIR"), "/db_migrator.rs"));
 
-// TODO we have not tested this in practice!
 #[instrument(level = "debug", target = COMPONENT, skip_all, err)]
-pub fn apply_migrations(
-    conn: &mut SqliteConnection,
-) -> std::result::Result<(), miden_node_db::DatabaseError> {
-    let migrations = conn.pending_migrations(MIGRATIONS).expect("In memory migrations never fail");
-    tracing::info!(target = COMPONENT, migrations = migrations.len(), "Applying migrations");
-
-    let Err(e) = conn.run_pending_migrations(MIGRATIONS) else {
-        // Migrations applied successfully, verify schema hash
-        verify_schema(conn)?;
-        return Ok(());
-    };
-    tracing::warn!(target = COMPONENT, error = ?e, "Failed to apply migration");
-    // something went wrong, MIGRATIONS contains
-    conn.revert_last_migration(MIGRATIONS)
-        .expect("Duality is maintained by the developer");
+pub fn apply_migrations(database_filepath: &Path) -> std::result::Result<(), DatabaseError> {
+    let migrator = migrator().map_err(DatabaseError::migration)?;
+    tracing::info!(
+        target: COMPONENT,
+        migration_count = migrator.schema_hashes().len(),
+        "Applying database migrations"
+    );
+
+    migrator.migrate(database_filepath).map_err(DatabaseError::migration)?;
 
     Ok(())
 }
+
+#[cfg(test)]
+pub(crate) fn test_connection() -> diesel::SqliteConnection {
+    use diesel::{Connection, SqliteConnection};
+
+    let database_filepath = tempfile::NamedTempFile::new()
+        .expect("failed to create temp database file")
+        .into_temp_path();
+    apply_migrations(&database_filepath).expect("migrations should apply on empty database");
+
+    let conn = SqliteConnection::establish(
+        database_filepath.to_str().expect("temp database path should be valid UTF-8"),
+    )
+    .expect("temp file sqlite should always work");
+    database_filepath
+        .keep()
+        .expect("failed to keep temp database file for test connection");
+    conn
+}
+
+#[cfg(test)]
+mod tests {
+    use std::process::Command;
+
+    use anyhow::{Context, Result, ensure};
+    use miden_node_db::migration::SchemaHash;
+
+    use super::*;
+
+    const EXPECTED_SCHEMA_HASHES: [SchemaHash; 1] = [SchemaHash::from_hex(
+        "b3bdd2e530fbb66c9146cda9f3bf79df49c6a6bf99f7432aae0a8927a15406ac",
+    )];
+
+    #[test]
+    fn migration_schema_hashes_are_stable() -> Result<()> {
+        let migrator = migrator()?;
+
+        assert_eq!(migrator.schema_hashes(), &EXPECTED_SCHEMA_HASHES);
+        Ok(())
+    }
+
+    #[test]
+    #[ignore = "requires diesel CLI; CI runs this in the diesel-schema job"]
+    fn diesel_schema_is_in_sync_with_migrations() -> Result<()> {
+        let temp_dir = tempfile::tempdir()?;
+        let database_filepath = temp_dir.path().join("store.sqlite3");
+        apply_migrations(&database_filepath)?;
+
+        let output = Command::new("diesel")
+            .arg("print-schema")
+            .arg("--database-url")
+            .arg(&database_filepath)
+            .current_dir(env!("CARGO_MANIFEST_DIR"))
+            .output()
+            .context(
+                "failed to run diesel CLI; install it with \
+                 `cargo install diesel_cli --no-default-features --features sqlite`",
+            )?;
+
+        ensure!(
+            output.status.success(),
+            "diesel print-schema failed: {}",
+            String::from_utf8_lossy(&output.stderr)
+        );
+
+        let generated =
+            String::from_utf8(output.stdout).context("diesel CLI output is not UTF-8")?;
+        assert_eq!(generated, include_str!("schema.rs"));
+        Ok(())
+    }
+}