Faster SQL data saving

.github/workflows/pypi.yml

@@ -17,6 +17,7 @@ jobs:
     outputs:
       should_build: ${{ steps.check.outputs.should_build }}
       version: ${{ steps.extract.outputs.version }}
+      publish_version: ${{ steps.plan.outputs.publish_version }}
 
     steps:
     - name: Checkout repository
@@ -32,19 +33,36 @@ jobs:
         echo "version=${VERSION}" >> $GITHUB_OUTPUT
         echo "Extracted version: ${VERSION}"
 
-    - name: Check if version exists on PyPI or TestPyPI
-      id: check
+    - name: Plan publish version
+      id: plan
       run: |
-        VERSION="${{ steps.extract.outputs.version }}"
-
-        # Determine which PyPI to check based on event type
+        BASE_VERSION="${{ steps.extract.outputs.version }}"
+
         if [ "${{ github.event_name }}" = "pull_request" ]; then
-          PYPI_URL="https://test.pypi.org/pypi/dsf-mobility/${VERSION}/json"
-          PYPI_NAME="TestPyPI"
+          # TestPyPI only: use a numeric dev suffix so the version is PEP 440 compliant.
+          # Concatenating run id and attempt keeps it unique across PRs and re-runs.
+          PUBLISH_VERSION="${BASE_VERSION}.dev${GITHUB_RUN_ID}${GITHUB_RUN_ATTEMPT}"
         else
-          PYPI_URL="https://pypi.org/pypi/dsf-mobility/${VERSION}/json"
-          PYPI_NAME="PyPI"
+          # PyPI: publish the exact release version with no run-id suffix.
+          PUBLISH_VERSION="${BASE_VERSION}"
+        fi
+
+        echo "publish_version=${PUBLISH_VERSION}" >> "$GITHUB_OUTPUT"
+        echo "Planned publish version: ${PUBLISH_VERSION}"
+
+    - name: Check publish policy for PyPI
+      id: check
+      run: |
+        VERSION="${{ steps.plan.outputs.publish_version }}"
+
+        if [ "${{ github.event_name }}" = "pull_request" ]; then
+          echo "PR build uses unique TestPyPI version ${VERSION}; skipping existence check."
+          echo "should_build=true" >> "$GITHUB_OUTPUT"
+          exit 0
         fi
+
+        PYPI_URL="https://pypi.org/pypi/dsf-mobility/${VERSION}/json"
+        PYPI_NAME="PyPI"
 
         echo "Checking if dsf-mobility version ${VERSION} exists on ${PYPI_NAME}..."
 
@@ -53,7 +71,8 @@ jobs:
 
         if [ "$HTTP_STATUS" = "200" ]; then
           echo "Version ${VERSION} already exists on ${PYPI_NAME}"
-          echo "should_build=false" >> $GITHUB_OUTPUT
+          echo "ERROR: Refusing to overwrite an existing version on PyPI."
+          exit 1
         else
           echo "Version ${VERSION} does not exist on ${PYPI_NAME} (HTTP ${HTTP_STATUS})"
           echo "should_build=true" >> $GITHUB_OUTPUT
@@ -90,6 +109,7 @@ jobs:
     - name: Build wheel
       env:
         CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF"
+        DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }}
       run: |
         rm -rf dist wheelhouse
         python -m build --wheel
@@ -145,6 +165,7 @@ jobs:
     - name: Build wheel
       env:
         CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF"
+        DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }}
       run: python -m build --wheel
 
     - name: Repair wheel (bundle libraries)
@@ -195,6 +216,7 @@ jobs:
     - name: Build wheel
       env:
         CMAKE_ARGS: "-DDSF_OPTIMIZE_ARCH=OFF"
+        DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }}
       run: python -m build --wheel
 
     - name: Upload wheels as artifacts
@@ -229,6 +251,8 @@ jobs:
         python -m pip install build
 
     - name: Build source distribution
+      env:
+        DSF_PACKAGE_VERSION: ${{ needs.check-version.outputs.publish_version }}
       run: python -m build --sdist
 
     - name: Upload sdist as artifact

setup.py

@@ -485,8 +485,8 @@ def run_stubgen(self):
     with open("README.md", "r", encoding="utf-8") as f:
         LONG_DESCRIPTION = f.read()
 
-# Get version from header file
-PROJECT_VERSION = get_version_from_header()
+# Get version from header file, unless explicitly overridden for CI pre-releases.
+PROJECT_VERSION = os.environ.get("DSF_PACKAGE_VERSION", get_version_from_header())
 
 setup(
     name="dsf-mobility",

src/dsf/base/Dynamics.hpp

@@ -103,13 +103,17 @@ namespace dsf {
     inline auto concurrency() const {
       return static_cast<std::size_t>(m_taskArena.max_concurrency());
     }
-
-    inline void connectDataBase(std::string const& dbPath) {
+    /// @brief Connect to a SQLite database, creating it if it doesn't exist, and executing optional initialization queries
+    /// @param dbPath The path to the SQLite database file
+    /// @param queries Optional SQL queries to execute upon connecting to the database (default is a set of pragmas for performance optimization : "PRAGMA busy_timeout = 5000;PRAGMA journal_mode = WAL;PRAGMA synchronous=NORMAL;PRAGMA temp_store=MEMORY;PRAGMA cache_size=-20000;")
+    inline void connectDataBase(
+        std::string const& dbPath,
+        std::string const& queries =
+            "PRAGMA busy_timeout = 5000;PRAGMA journal_mode = WAL;PRAGMA "
+            "synchronous=NORMAL;PRAGMA temp_store=MEMORY;PRAGMA cache_size=-20000;") {
       m_database = std::make_unique<SQLite::Database>(
           dbPath, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE);
-      // Enable WAL mode for better concurrency and set busy timeout
-      m_database->exec("PRAGMA journal_mode = WAL;");
-      m_database->exec("PRAGMA busy_timeout = 5000;");  // 5 seconds
+      m_database->exec(queries);
     }
 
     /// @brief Get the graph

src/dsf/bindings.cpp

@@ -514,6 +514,9 @@ PYBIND11_MODULE(dsf_cpp, m) {
       .def("connectDataBase",
            &dsf::mobility::FirstOrderDynamics::connectDataBase,
            pybind11::arg("dbPath"),
+           pybind11::arg("queries") =
+               "PRAGMA busy_timeout = 5000;PRAGMA journal_mode = WAL;PRAGMA "
+               "synchronous=NORMAL;PRAGMA temp_store=MEMORY;PRAGMA cache_size=-20000;",
            dsf::g_docstrings.at("dsf::Dynamics::connectDataBase").c_str())
       .def("setForcePriorities",
            &dsf::mobility::FirstOrderDynamics::setForcePriorities,

src/dsf/dsf.hpp

@@ -9,7 +9,7 @@
 
 static constexpr uint8_t DSF_VERSION_MAJOR = 5;
 static constexpr uint8_t DSF_VERSION_MINOR = 3;
-static constexpr uint8_t DSF_VERSION_PATCH = 0;
+static constexpr uint8_t DSF_VERSION_PATCH = 1;
 
 static auto const DSF_VERSION =
     std::format("{}.{}.{}", DSF_VERSION_MAJOR, DSF_VERSION_MINOR, DSF_VERSION_PATCH);

src/dsf/mobility/FirstOrderDynamics.cpp

@@ -1531,9 +1531,21 @@ namespace dsf::mobility {
     this->m_evolveAgents();
 
     if (bComputeStats) {
+      auto const datetime = this->strDateTime();
+      auto const step = static_cast<std::int64_t>(this->time_step());
+      auto const simulationId = static_cast<std::int64_t>(this->id());
+
+      bool const hasWritePayload = (m_bSaveStreetData && !streetDataRecords.empty()) ||
+                                   (m_bSaveTravelData && !m_travelDTs.empty()) ||
+                                   m_bSaveAverageStats;
+
+      std::optional<SQLite::Transaction> transaction;
+      if (hasWritePayload) {
+        transaction.emplace(*this->database());
+      }
+
       // Batch insert street data collected during parallel section
-      if (m_bSaveStreetData) {
-        SQLite::Transaction transaction(*this->database());
+      if (m_bSaveStreetData && !streetDataRecords.empty()) {
         SQLite::Statement insertStmt(
             *this->database(),
             "INSERT INTO road_data (datetime, time_step, simulation_id, street_id, "
@@ -1542,9 +1554,9 @@ namespace dsf::mobility {
             "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
 
         for (auto const& record : streetDataRecords) {
-          insertStmt.bind(1, this->strDateTime());
-          insertStmt.bind(2, static_cast<std::int64_t>(this->time_step()));
-          insertStmt.bind(3, static_cast<std::int64_t>(this->id()));
+          insertStmt.bind(1, datetime);
+          insertStmt.bind(2, step);
+          insertStmt.bind(3, simulationId);
           insertStmt.bind(4, static_cast<std::int64_t>(record.streetId));
           if (record.coilName.has_value()) {
             insertStmt.bind(5, record.coilName.value());
@@ -1569,71 +1581,71 @@ namespace dsf::mobility {
           insertStmt.exec();
           insertStmt.reset();
         }
-        transaction.commit();
       }
 
-      if (m_bSaveTravelData) {  // Begin transaction for better performance
-        SQLite::Transaction transaction(*this->database());
+      if (m_bSaveTravelData && !m_travelDTs.empty()) {
         SQLite::Statement insertStmt(*this->database(),
                                      "INSERT INTO travel_data (datetime, time_step, "
                                      "simulation_id, distance_m, travel_time_s) "
                                      "VALUES (?, ?, ?, ?, ?)");
 
         for (auto const& [distance, time] : m_travelDTs) {
-          insertStmt.bind(1, this->strDateTime());
-          insertStmt.bind(2, static_cast<int64_t>(this->time_step()));
-          insertStmt.bind(3, static_cast<int64_t>(this->id()));
+          insertStmt.bind(1, datetime);
+          insertStmt.bind(2, step);
+          insertStmt.bind(3, simulationId);
           insertStmt.bind(4, distance);
           insertStmt.bind(5, time);
           insertStmt.exec();
           insertStmt.reset();
         }
-        transaction.commit();
         m_travelDTs.clear();
       }
 
       if (m_bSaveAverageStats) {  // Average Stats Table
-        mean_speed.store(mean_speed.load() / nValidEdges.load());
-        mean_density.store(mean_density.load() / numEdges);
-        mean_traveltime.store(mean_traveltime.load() / nValidEdges.load());
-        mean_queue_length.store(mean_queue_length.load() / numEdges);
-        {
-          double std_speed_val = std_speed.load();
-          double mean_speed_val = mean_speed.load();
-          std_speed.store(std::sqrt(std_speed_val / nValidEdges.load() -
-                                    mean_speed_val * mean_speed_val));
-        }
-        {
-          double std_density_val = std_density.load();
-          double mean_density_val = mean_density.load();
-          std_density.store(std::sqrt(std_density_val / numEdges -
-                                      mean_density_val * mean_density_val));
-        }
+        auto const validEdges = nValidEdges.load();
+        auto const edgeCount = static_cast<double>(numEdges);
+        auto const meanDensity = mean_density.load() / edgeCount;
+        auto const meanQueueLength = mean_queue_length.load() / edgeCount;
+        auto const densityVariance =
+            std::max(0.0, std_density.load() / edgeCount - meanDensity * meanDensity);
+
         SQLite::Statement insertStmt(
             *this->database(),
             "INSERT INTO avg_stats ("
             "simulation_id, datetime, time_step, n_ghost_agents, n_agents, "
             "mean_speed_kph, std_speed_kph, mean_density_vpk, std_density_vpk, "
             "mean_travel_time_s, mean_queue_length) "
             "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
-        insertStmt.bind(1, static_cast<std::int64_t>(this->id()));
-        insertStmt.bind(2, this->strDateTime());
-        insertStmt.bind(3, static_cast<std::int64_t>(this->time_step()));
+        insertStmt.bind(1, simulationId);
+        insertStmt.bind(2, datetime);
+        insertStmt.bind(3, step);
         insertStmt.bind(4, static_cast<std::int64_t>(m_agents.size()));
         insertStmt.bind(5, static_cast<std::int64_t>(this->nAgents()));
-        if (nValidEdges.load() > 0) {
-          insertStmt.bind(6, mean_speed);
-          insertStmt.bind(7, std_speed);
+
+        if (validEdges > 0) {
+          auto const validEdgeCount = static_cast<double>(validEdges);
+          auto const meanSpeed = mean_speed.load() / validEdgeCount;
+          auto const meanTravelTime = mean_traveltime.load() / validEdgeCount;
+          auto const speedVariance =
+              std::max(0.0, std_speed.load() / validEdgeCount - meanSpeed * meanSpeed);
+          insertStmt.bind(6, meanSpeed);
+          insertStmt.bind(7, std::sqrt(speedVariance));
+          insertStmt.bind(10, meanTravelTime);
         } else {
           insertStmt.bind(6);
           insertStmt.bind(7);
+          insertStmt.bind(10);
         }
-        insertStmt.bind(8, mean_density);
-        insertStmt.bind(9, std_density);
-        insertStmt.bind(10, mean_traveltime);
-        insertStmt.bind(11, mean_queue_length);
+        insertStmt.bind(8, meanDensity);
+        insertStmt.bind(9, std::sqrt(densityVariance));
+        insertStmt.bind(11, meanQueueLength);
         insertStmt.exec();
       }
+
+      if (transaction.has_value()) {
+        transaction->commit();
+      }
+
       // Special case: if m_savingInterval == 0, it was a triggered saveData() call, so we need to reset all flags
       if (m_savingInterval.value() == 0) {
         m_savingInterval.reset();

webapp/script.js

@@ -1677,15 +1677,16 @@ document.addEventListener('DOMContentLoaded', () => {
     loadDbBtn.disabled = true;
 
     try {
+      // Snapshot and read the selected file first so later async work cannot invalidate access.
+      const fileSnapshot = file.slice(0, file.size);
+      const arrayBuffer = await fileSnapshot.arrayBuffer();
+      const uint8Array = new Uint8Array(arrayBuffer);
+
       // Initialize sql.js
       const SQL = await initSqlJs({
         locateFile: file => `https://cdnjs.cloudflare.com/ajax/libs/sql.js/1.10.3/${file}`
       });
 
-      // Read the file
-      const arrayBuffer = await file.arrayBuffer();
-      const uint8Array = new Uint8Array(arrayBuffer);
-
       // Open the database
       db = new SQL.Database(uint8Array);
 
@@ -1721,7 +1722,11 @@ document.addEventListener('DOMContentLoaded', () => {
     } catch (error) {
       console.error('Database loading error:', error);
       dbStatus.className = 'db-status error';
-      dbStatus.textContent = `Error: ${error.message}`;
+      if (error instanceof DOMException && error.name === 'NotReadableError') {
+        dbStatus.textContent = 'Error: Could not read the selected file. Re-select it, or move it to a local folder you own and try again.';
+      } else {
+        dbStatus.textContent = `Error: ${error.message}`;
+      }
       loadDbBtn.disabled = false;
     }
   });