From fd86ec0f0c0dfe2d80b329dc7a0b810f77ed7e0a Mon Sep 17 00:00:00 2001 From: Liza2030-good <“ec220983@meiji.ac.jp”> Date: Mon, 27 Apr 2026 14:27:08 +0900 Subject: [PATCH] fix: resolve #75 - Bank Sync Connector Architecture Closes #75 --- packages/backend/app/db/schema.sql | 14 ++ packages/backend/app/models.py | 13 ++ packages/backend/app/routes/__init__.py | 2 + packages/backend/app/routes/bank_sync.py | 210 ++++++++++++++++++ .../app/services/bank_sync/__init__.py | 41 ++++ .../backend/app/services/bank_sync/base.py | 79 +++++++ .../backend/app/services/bank_sync/mock.py | 98 ++++++++ .../app/services/bank_sync/registry.py | 41 ++++ packages/backend/tests/test_bank_sync.py | 198 +++++++++++++++++ 9 files changed, 696 insertions(+) create mode 100644 packages/backend/app/routes/bank_sync.py create mode 100644 packages/backend/app/services/bank_sync/__init__.py create mode 100644 packages/backend/app/services/bank_sync/base.py create mode 100644 packages/backend/app/services/bank_sync/mock.py create mode 100644 packages/backend/app/services/bank_sync/registry.py create mode 100644 packages/backend/tests/test_bank_sync.py diff --git a/packages/backend/app/db/schema.sql b/packages/backend/app/db/schema.sql index 410189def..fb05a86aa 100644 --- a/packages/backend/app/db/schema.sql +++ b/packages/backend/app/db/schema.sql @@ -123,3 +123,17 @@ CREATE TABLE IF NOT EXISTS audit_logs ( action VARCHAR(100) NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT NOW() ); + +CREATE TABLE IF NOT EXISTS bank_accounts ( + id SERIAL PRIMARY KEY, + user_id INT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + provider VARCHAR(50) NOT NULL, + external_id VARCHAR(255) NOT NULL, + name VARCHAR(200) NOT NULL, + account_type VARCHAR(50), + currency VARCHAR(10) NOT NULL DEFAULT 'USD', + last_synced_at TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + UNIQUE(user_id, provider, external_id) +); +CREATE INDEX IF NOT EXISTS idx_bank_accounts_user ON bank_accounts(user_id); diff --git a/packages/backend/app/models.py b/packages/backend/app/models.py index 64d448104..c2002ccb0 100644 --- a/packages/backend/app/models.py +++ b/packages/backend/app/models.py @@ -133,3 +133,16 @@ class AuditLog(db.Model): user_id = db.Column(db.Integer, db.ForeignKey("users.id"), nullable=True) action = db.Column(db.String(100), nullable=False) created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + + +class BankAccount(db.Model): + __tablename__ = "bank_accounts" + id = db.Column(db.Integer, primary_key=True) + user_id = db.Column(db.Integer, db.ForeignKey("users.id"), nullable=False) + provider = db.Column(db.String(50), nullable=False) + external_id = db.Column(db.String(255), nullable=False) + name = db.Column(db.String(200), nullable=False) + account_type = db.Column(db.String(50), nullable=True) + currency = db.Column(db.String(10), default="USD", nullable=False) + last_synced_at = db.Column(db.DateTime, nullable=True) + created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) diff --git a/packages/backend/app/routes/__init__.py b/packages/backend/app/routes/__init__.py index f13b0f897..7c2aef26a 100644 --- a/packages/backend/app/routes/__init__.py +++ b/packages/backend/app/routes/__init__.py @@ -7,6 +7,7 @@ from .categories import bp as categories_bp from .docs import bp as docs_bp from .dashboard import bp as dashboard_bp +from .bank_sync import bp as bank_sync_bp def register_routes(app: Flask): @@ -18,3 +19,4 @@ def register_routes(app: Flask): app.register_blueprint(categories_bp, url_prefix="/categories") app.register_blueprint(docs_bp, url_prefix="/docs") app.register_blueprint(dashboard_bp, url_prefix="/dashboard") + app.register_blueprint(bank_sync_bp, url_prefix="/bank-sync") diff --git a/packages/backend/app/routes/bank_sync.py b/packages/backend/app/routes/bank_sync.py new file mode 100644 index 000000000..4ea41de9e --- /dev/null +++ b/packages/backend/app/routes/bank_sync.py @@ -0,0 +1,210 @@ +import logging +from datetime import date, datetime +from decimal import Decimal + +from flask import Blueprint, jsonify, request +from flask_jwt_extended import get_jwt_identity, jwt_required + +from ..extensions import db +from ..models import BankAccount, Expense, User +from ..services import bank_sync +from ..services.cache import cache_delete_patterns, monthly_summary_key + +bp = Blueprint("bank_sync", __name__) +logger = logging.getLogger("finmind.bank_sync") + + +@bp.get("/connectors") +@jwt_required() +def list_connectors(): + return jsonify(bank_sync.list_connectors()) + + +@bp.get("/accounts") +@jwt_required() +def list_accounts(): + uid = int(get_jwt_identity()) + items = ( + db.session.query(BankAccount) + .filter_by(user_id=uid) + .order_by(BankAccount.created_at.desc()) + .all() + ) + return jsonify([_account_to_dict(a) for a in items]) + + +@bp.post("/accounts") +@jwt_required() +def link_account(): + uid = int(get_jwt_identity()) + data = request.get_json() or {} + provider = (data.get("provider") or "").strip().lower() + credentials = data.get("credentials") or {} + if not provider: + return jsonify(error="provider required"), 400 + try: + connector = bank_sync.get_connector(provider) + except KeyError: + return jsonify(error=f"unknown provider: {provider}"), 404 + try: + accounts = connector.connect(credentials) + except bank_sync.ConnectorError as exc: + return jsonify(error=str(exc)), 400 + + requested_id = data.get("external_id") + if requested_id: + accounts = [a for a in accounts if a.external_id == requested_id] + if not accounts: + return jsonify(error="account not available from connector"), 404 + + created: list[BankAccount] = [] + for info in accounts: + existing = ( + db.session.query(BankAccount) + .filter_by(user_id=uid, provider=provider, external_id=info.external_id) + .first() + ) + if existing: + created.append(existing) + continue + account = BankAccount( + user_id=uid, + provider=provider, + external_id=info.external_id, + name=info.name, + account_type=info.account_type, + currency=info.currency, + ) + db.session.add(account) + created.append(account) + db.session.commit() + logger.info( + "Linked bank accounts user=%s provider=%s count=%s", + uid, + provider, + len(created), + ) + return jsonify([_account_to_dict(a) for a in created]), 201 + + +@bp.delete("/accounts/") +@jwt_required() +def unlink_account(account_id: int): + uid = int(get_jwt_identity()) + account = db.session.get(BankAccount, account_id) + if not account or account.user_id != uid: + return jsonify(error="not found"), 404 + db.session.delete(account) + db.session.commit() + return jsonify(message="deleted") + + +@bp.post("/accounts//import") +@jwt_required() +def import_account(account_id: int): + return _sync_account(account_id, mode="import") + + +@bp.post("/accounts//refresh") +@jwt_required() +def refresh_account(account_id: int): + return _sync_account(account_id, mode="refresh") + + +def _sync_account(account_id: int, *, mode: str): + uid = int(get_jwt_identity()) + account = db.session.get(BankAccount, account_id) + if not account or account.user_id != uid: + return jsonify(error="not found"), 404 + try: + connector = bank_sync.get_connector(account.provider) + except KeyError: + return jsonify(error=f"unknown provider: {account.provider}"), 400 + + since: date | None = None + if mode == "refresh" and account.last_synced_at: + since = account.last_synced_at.date() + + try: + transactions = connector.fetch_transactions( + account.external_id, since=since + ) + except bank_sync.ConnectorError as exc: + return jsonify(error=str(exc)), 400 + + user = db.session.get(User, uid) + fallback_currency = ( + account.currency or (user.preferred_currency if user else "USD") + ) + + inserted = 0 + duplicates = 0 + touched_months: set[str] = set() + for tx in transactions: + if _is_duplicate_tx(uid, tx): + duplicates += 1 + continue + expense = Expense( + user_id=uid, + amount=Decimal(str(tx.amount)).quantize(Decimal("0.01")), + currency=tx.currency or fallback_currency, + expense_type=(tx.expense_type or "EXPENSE").upper(), + notes=tx.description[:500], + spent_at=tx.date, + ) + db.session.add(expense) + inserted += 1 + touched_months.add(tx.date.strftime("%Y-%m")) + + account.last_synced_at = datetime.utcnow() + db.session.commit() + for ym in touched_months: + cache_delete_patterns( + [ + monthly_summary_key(uid, ym), + f"insights:{uid}:*", + f"user:{uid}:dashboard_summary:*", + ] + ) + logger.info( + "Bank sync %s user=%s account=%s inserted=%s duplicates=%s", + mode, + uid, + account_id, + inserted, + duplicates, + ) + return jsonify( + mode=mode, + inserted=inserted, + duplicates=duplicates, + last_synced_at=account.last_synced_at.isoformat(), + ) + + +def _is_duplicate_tx(uid: int, tx) -> bool: + amount = Decimal(str(tx.amount)).quantize(Decimal("0.01")) + return ( + db.session.query(Expense) + .filter_by( + user_id=uid, + spent_at=tx.date, + amount=amount, + notes=tx.description[:500], + ) + .first() + is not None + ) + + +def _account_to_dict(a: BankAccount) -> dict: + return { + "id": a.id, + "provider": a.provider, + "external_id": a.external_id, + "name": a.name, + "account_type": a.account_type, + "currency": a.currency, + "last_synced_at": a.last_synced_at.isoformat() if a.last_synced_at else None, + "created_at": a.created_at.isoformat() if a.created_at else None, + } diff --git a/packages/backend/app/services/bank_sync/__init__.py b/packages/backend/app/services/bank_sync/__init__.py new file mode 100644 index 000000000..44d2eadfb --- /dev/null +++ b/packages/backend/app/services/bank_sync/__init__.py @@ -0,0 +1,41 @@ +"""Pluggable bank sync connector architecture. + +A connector is a class that implements :class:`BankConnector` and is +registered via :func:`register_connector`. Connectors expose two operations +needed for bank integration: + +- ``connect``: validate credentials and return the linkable bank accounts +- ``fetch_transactions``: pull transactions for a linked account, optionally + filtered by ``since`` for incremental refresh + +Concrete connectors live in sibling modules (e.g. ``mock.py``). Importing +this package eagerly imports the bundled mock connector so it is always +available. +""" + +from .base import ( + BankAccountInfo, + BankConnector, + BankTransaction, + ConnectorError, +) +from .registry import ( + available_connectors, + get_connector, + list_connectors, + register_connector, +) + +# Eagerly import bundled connectors so they self-register. +from . import mock # noqa: F401 + +__all__ = [ + "BankAccountInfo", + "BankConnector", + "BankTransaction", + "ConnectorError", + "available_connectors", + "get_connector", + "list_connectors", + "register_connector", +] diff --git a/packages/backend/app/services/bank_sync/base.py b/packages/backend/app/services/bank_sync/base.py new file mode 100644 index 000000000..62664d2e2 --- /dev/null +++ b/packages/backend/app/services/bank_sync/base.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import date +from typing import Any + + +class ConnectorError(Exception): + """Raised when a connector cannot complete an operation.""" + + +@dataclass +class BankAccountInfo: + """Account metadata returned by a connector during ``connect``.""" + + external_id: str + name: str + account_type: str | None = None + currency: str = "USD" + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class BankTransaction: + """A single transaction returned by ``fetch_transactions``. + + ``external_id`` should be stable across refreshes so the caller can + deduplicate. ``amount`` is always positive; ``expense_type`` is one of + ``"EXPENSE"`` or ``"INCOME"``. + """ + + external_id: str + date: date + amount: float + description: str + currency: str = "USD" + expense_type: str = "EXPENSE" + + def to_import_row(self) -> dict[str, Any]: + return { + "external_id": self.external_id, + "date": self.date.isoformat(), + "amount": float(self.amount), + "description": self.description, + "currency": self.currency, + "expense_type": self.expense_type, + } + + +class BankConnector(ABC): + """Interface every bank connector must implement. + + Subclasses must define ``provider_name`` and ``display_name`` and + implement ``connect`` and ``fetch_transactions``. + """ + + provider_name: str = "" + display_name: str = "" + required_credentials: tuple[str, ...] = () + + @abstractmethod + def connect(self, credentials: dict[str, Any]) -> list[BankAccountInfo]: + """Validate credentials and return the accounts available to link.""" + + @abstractmethod + def fetch_transactions( + self, + external_account_id: str, + *, + since: date | None = None, + ) -> list[BankTransaction]: + """Return transactions for an account. + + ``since`` is an inclusive lower bound on the transaction date. When + ``None`` the connector should return its full available history, + which is what ``import`` uses for the initial pull. ``refresh`` will + pass the account's ``last_synced_at`` date. + """ diff --git a/packages/backend/app/services/bank_sync/mock.py b/packages/backend/app/services/bank_sync/mock.py new file mode 100644 index 000000000..8174e7a72 --- /dev/null +++ b/packages/backend/app/services/bank_sync/mock.py @@ -0,0 +1,98 @@ +"""In-memory mock bank connector used for development and tests. + +The mock connector accepts any non-empty ``api_key`` and exposes two +deterministic accounts. ``fetch_transactions`` returns a fixed list of +transactions which can be filtered by ``since``. Tests can override the +canned data by mutating :data:`MOCK_ACCOUNTS` or :data:`MOCK_TRANSACTIONS` +before calling the connector. +""" + +from __future__ import annotations + +from datetime import date +from typing import Any + +from .base import BankAccountInfo, BankConnector, BankTransaction, ConnectorError +from .registry import register_connector + + +MOCK_ACCOUNTS: list[BankAccountInfo] = [ + BankAccountInfo( + external_id="mock-checking-001", + name="Mock Checking", + account_type="checking", + currency="USD", + ), + BankAccountInfo( + external_id="mock-savings-002", + name="Mock Savings", + account_type="savings", + currency="USD", + ), +] + + +MOCK_TRANSACTIONS: dict[str, list[BankTransaction]] = { + "mock-checking-001": [ + BankTransaction( + external_id="mock-tx-1", + date=date(2026, 2, 1), + amount=12.50, + description="Coffee Shop", + currency="USD", + expense_type="EXPENSE", + ), + BankTransaction( + external_id="mock-tx-2", + date=date(2026, 2, 5), + amount=2500.00, + description="Payroll Deposit", + currency="USD", + expense_type="INCOME", + ), + BankTransaction( + external_id="mock-tx-3", + date=date(2026, 2, 12), + amount=42.00, + description="Grocery Run", + currency="USD", + expense_type="EXPENSE", + ), + ], + "mock-savings-002": [ + BankTransaction( + external_id="mock-tx-s1", + date=date(2026, 2, 3), + amount=500.00, + description="Transfer In", + currency="USD", + expense_type="INCOME", + ), + ], +} + + +@register_connector +class MockConnector(BankConnector): + provider_name = "mock" + display_name = "Mock Bank" + required_credentials = ("api_key",) + + def connect(self, credentials: dict[str, Any]) -> list[BankAccountInfo]: + api_key = (credentials or {}).get("api_key") + if not api_key: + raise ConnectorError("api_key is required") + return list(MOCK_ACCOUNTS) + + def fetch_transactions( + self, + external_account_id: str, + *, + since: date | None = None, + ) -> list[BankTransaction]: + if external_account_id not in MOCK_TRANSACTIONS: + raise ConnectorError(f"unknown account: {external_account_id}") + txs = MOCK_TRANSACTIONS[external_account_id] + if since is not None: + txs = [t for t in txs if t.date >= since] + return list(txs) diff --git a/packages/backend/app/services/bank_sync/registry.py b/packages/backend/app/services/bank_sync/registry.py new file mode 100644 index 000000000..be2bb9b5d --- /dev/null +++ b/packages/backend/app/services/bank_sync/registry.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import Type + +from .base import BankConnector + +_REGISTRY: dict[str, Type[BankConnector]] = {} + + +def register_connector(cls: Type[BankConnector]) -> Type[BankConnector]: + """Class decorator that registers a connector by its ``provider_name``.""" + name = (cls.provider_name or "").strip().lower() + if not name: + raise ValueError(f"{cls.__name__} must define a non-empty provider_name") + _REGISTRY[name] = cls + return cls + + +def get_connector(provider_name: str) -> BankConnector: + """Return a fresh instance of the connector for ``provider_name``.""" + key = (provider_name or "").strip().lower() + cls = _REGISTRY.get(key) + if cls is None: + raise KeyError(f"Unknown bank connector: {provider_name!r}") + return cls() + + +def available_connectors() -> list[Type[BankConnector]]: + return list(_REGISTRY.values()) + + +def list_connectors() -> list[dict]: + """Public-facing summary of registered connectors.""" + return [ + { + "provider": cls.provider_name, + "display_name": cls.display_name or cls.provider_name, + "required_credentials": list(cls.required_credentials), + } + for cls in _REGISTRY.values() + ] diff --git a/packages/backend/tests/test_bank_sync.py b/packages/backend/tests/test_bank_sync.py new file mode 100644 index 000000000..91aa2117a --- /dev/null +++ b/packages/backend/tests/test_bank_sync.py @@ -0,0 +1,198 @@ +from datetime import date + +import pytest + +from app.services import bank_sync +from app.services.bank_sync.base import ( + BankAccountInfo, + BankConnector, + BankTransaction, + ConnectorError, +) + + +def test_mock_connector_is_registered(): + names = [c["provider"] for c in bank_sync.list_connectors()] + assert "mock" in names + + +def test_mock_connector_connect_and_fetch(): + connector = bank_sync.get_connector("mock") + accounts = connector.connect({"api_key": "test"}) + assert len(accounts) >= 1 + assert all(isinstance(a, BankAccountInfo) for a in accounts) + + txs = connector.fetch_transactions("mock-checking-001") + assert len(txs) >= 1 + assert all(isinstance(t, BankTransaction) for t in txs) + + +def test_mock_connector_requires_credentials(): + connector = bank_sync.get_connector("mock") + with pytest.raises(ConnectorError): + connector.connect({}) + + +def test_mock_connector_filters_by_since(): + connector = bank_sync.get_connector("mock") + txs = connector.fetch_transactions( + "mock-checking-001", since=date(2026, 2, 6) + ) + assert all(t.date >= date(2026, 2, 6) for t in txs) + + +def test_get_connector_unknown(): + with pytest.raises(KeyError): + bank_sync.get_connector("does-not-exist") + + +def test_register_custom_connector(): + @bank_sync.register_connector + class CustomConnector(BankConnector): + provider_name = "custom-test" + display_name = "Custom" + required_credentials = ("token",) + + def connect(self, credentials): + return [BankAccountInfo(external_id="x", name="X")] + + def fetch_transactions(self, external_account_id, *, since=None): + return [] + + connector = bank_sync.get_connector("custom-test") + assert isinstance(connector, CustomConnector) + assert connector.connect({"token": "t"})[0].external_id == "x" + + +def test_list_connectors_endpoint(client, auth_header): + r = client.get("/bank-sync/connectors", headers=auth_header) + assert r.status_code == 200 + providers = [c["provider"] for c in r.get_json()] + assert "mock" in providers + + +def test_link_account_requires_provider(client, auth_header): + r = client.post("/bank-sync/accounts", json={}, headers=auth_header) + assert r.status_code == 400 + + +def test_link_account_unknown_provider(client, auth_header): + r = client.post( + "/bank-sync/accounts", + json={"provider": "nope", "credentials": {}}, + headers=auth_header, + ) + assert r.status_code == 404 + + +def test_link_account_invalid_credentials(client, auth_header): + r = client.post( + "/bank-sync/accounts", + json={"provider": "mock", "credentials": {}}, + headers=auth_header, + ) + assert r.status_code == 400 + + +def test_link_specific_account_then_import_then_refresh(client, auth_header): + # Link only the checking account + r = client.post( + "/bank-sync/accounts", + json={ + "provider": "mock", + "credentials": {"api_key": "test"}, + "external_id": "mock-checking-001", + }, + headers=auth_header, + ) + assert r.status_code == 201 + accounts = r.get_json() + assert len(accounts) == 1 + account_id = accounts[0]["id"] + assert accounts[0]["external_id"] == "mock-checking-001" + assert accounts[0]["last_synced_at"] is None + + # Initial import pulls everything + r = client.post( + f"/bank-sync/accounts/{account_id}/import", headers=auth_header + ) + assert r.status_code == 200 + body = r.get_json() + assert body["mode"] == "import" + assert body["inserted"] == 3 + assert body["duplicates"] == 0 + assert body["last_synced_at"] + + # Verify expenses landed in the user's expense list + r = client.get("/expenses", headers=auth_header) + assert r.status_code == 200 + items = r.get_json() + assert len(items) == 3 + descriptions = {it["description"] for it in items} + assert "Coffee Shop" in descriptions + assert "Payroll Deposit" in descriptions + + # Refresh re-runs the connector but dedupes against existing rows + r = client.post( + f"/bank-sync/accounts/{account_id}/refresh", headers=auth_header + ) + assert r.status_code == 200 + refresh_body = r.get_json() + assert refresh_body["mode"] == "refresh" + assert refresh_body["inserted"] == 0 + assert refresh_body["duplicates"] >= 0 + + # Account list reflects last_synced_at + r = client.get("/bank-sync/accounts", headers=auth_header) + assert r.status_code == 200 + listed = r.get_json() + assert len(listed) == 1 + assert listed[0]["last_synced_at"] is not None + + +def test_link_all_accounts_default(client, auth_header): + r = client.post( + "/bank-sync/accounts", + json={"provider": "mock", "credentials": {"api_key": "test"}}, + headers=auth_header, + ) + assert r.status_code == 201 + accounts = r.get_json() + assert len(accounts) == 2 + + # Re-linking is idempotent: returns the same accounts, no duplicates + r = client.post( + "/bank-sync/accounts", + json={"provider": "mock", "credentials": {"api_key": "test"}}, + headers=auth_header, + ) + assert r.status_code == 201 + again = r.get_json() + assert {a["id"] for a in again} == {a["id"] for a in accounts} + + +def test_unlink_account(client, auth_header): + r = client.post( + "/bank-sync/accounts", + json={ + "provider": "mock", + "credentials": {"api_key": "test"}, + "external_id": "mock-savings-002", + }, + headers=auth_header, + ) + assert r.status_code == 201 + account_id = r.get_json()[0]["id"] + + r = client.delete( + f"/bank-sync/accounts/{account_id}", headers=auth_header + ) + assert r.status_code == 200 + + r = client.get("/bank-sync/accounts", headers=auth_header) + assert r.get_json() == [] + + +def test_import_unknown_account_returns_404(client, auth_header): + r = client.post("/bank-sync/accounts/9999/import", headers=auth_header) + assert r.status_code == 404