From b785e30b4557f075648dd7b259c89ce3d6d9a84e Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Sat, 23 May 2026 20:48:31 +0530 Subject: [PATCH 1/8] feat(analytics): create the analytics related apis --- backend/app/api/docs/analytics/monthly.md | 167 ++++++ .../app/api/docs/analytics/monthly_chart.md | 147 +++++ backend/app/api/main.py | 2 + backend/app/api/routes/analytics.py | 513 ++++++++++++++++++ backend/app/models/__init__.py | 8 + backend/app/models/analytics.py | 86 +++ 6 files changed, 923 insertions(+) create mode 100644 backend/app/api/docs/analytics/monthly.md create mode 100644 backend/app/api/docs/analytics/monthly_chart.md create mode 100644 backend/app/api/routes/analytics.py create mode 100644 backend/app/models/analytics.py diff --git a/backend/app/api/docs/analytics/monthly.md b/backend/app/api/docs/analytics/monthly.md new file mode 100644 index 000000000..cabb4da23 --- /dev/null +++ b/backend/app/api/docs/analytics/monthly.md @@ -0,0 +1,167 @@ +Read live monthly analytics for the current organization. + +The response is shaped as a list of data points, one per +`(month, modality, provider)` combination — aggregated across every project +in the caller's organization. Each point contains a single numeric `value` — +what that value represents depends on the `metric` query parameter. This +lets the frontend pivot the response directly into chart series without +further post-processing. + +Data is computed on-demand from `llm_call`, `llm_chain`, and +`evaluation_run`, so every request reflects the current database state with +no caching layer in between. A row inserted seconds ago will already be +visible in the response. + +--- + +## Authentication & default scope + +Any authenticated user with an organization context can call this endpoint. +Scope is decided per-request from the caller's auth context: + +| Caller's context | Default scope | +| ---------------- | ------------- | +| Currently selected project | Analytics for **just that project**. | +| Org-level (no project selected) | Analytics across **all projects in the caller's org**. | + +The implicit org-id filter is always applied first, so data from other +organizations is never returned. To override the default and look at a +specific project (e.g. an org admin comparing two projects), pass the +`project_id` query parameter — it must reference a project inside the +caller's organization. A `project_id` from a different org returns an +empty result, not a leak. + +--- + +## Query parameters + +| Parameter | Type | Required | Default | Description | +| ----------- | -------- | -------- | ------- | ----------- | +| `metric` | enum | **yes** | — | Which metric the `value` field carries on each point. One of: `requests`, `cost`, `eval_runs`, `eval_cost`. | +| `from_month`| date | no | 24 months before `to_month` (or before today if `to_month` is also omitted) | Inclusive lower bound. Must be a first-of-month date, e.g. `2026-01-01`. Pass an explicit value to query further back. The default exists to cap worst-case scan size as `llm_call` grows. | +| `to_month` | date | no | — (no upper bound) | Inclusive upper bound. Must be a first-of-month date, e.g. `2026-05-01`. | +| `modality` | enum | no | — (all) | Filter to a single modality bucket. One of: `T-FS-T`, `S-FS-S`, `STT`, `TTS`, `OTHER`. | +| `provider` | string | no | — (all) | Filter to a single provider, e.g. `openai`, `google`, `sarvamai`, `elevenlabs`. | +| `project_id`| integer | no | Caller's current project, if any; else all projects in the org. | Override the default scope. Must reference a project inside the caller's organization. Cross-organization access is rejected (the org filter is always applied first). | + +### `metric` values + +| Value | What `value` contains on each point | +| ------------ | ----------------------------------- | +| `requests` | `total_llm_call_requests + total_llm_chain_requests` — the total number of inference requests in the bucket (LLM calls plus chain orchestrations). | +| `cost` | Sum of LLM call cost in USD for the bucket. Chains are NOT added on top — a chain's cost equals the sum of its child calls, which are already counted. | +| `eval_runs` | Count of evaluation runs in the bucket. | +| `eval_cost` | Sum of evaluation run cost in USD for the bucket. | + +### `modality` values and how they're derived + +| Modality | LLM call (`input_type` → `output_type`) | Evaluation run `type` | +| -------- | --------------------------------------- | --------------------- | +| `T-FS-T` | `text` → `text` | `text` | +| `S-FS-S` | `audio` → `audio` | — | +| `STT` | `audio` → `text` | `stt` | +| `TTS` | `text` → `audio` | `tts` | +| `OTHER` | anything else (image, pdf, multimodal) | `assessment`, any other type | + +LLM chains are attributed to the modality of their **first child call**. + +--- + +## Response shape + +```json +{ + "success": true, + "data": [ + { + "month": "2026-03-01", + "modality": "T-FS-T", + "provider": "openai", + "value": "12450", + "input_tokens": 1250000, + "output_tokens": 820000, + "total_tokens": 2070000 + }, + { + "month": "2026-04-01", + "modality": "T-FS-T", + "provider": "openai", + "value": "18230", + "input_tokens": 1840000, + "output_tokens": 1210000, + "total_tokens": 3050000 + }, + { + "month": "2026-04-01", + "modality": "STT", + "provider": "sarvamai", + "value": "1402", + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0 + } + ], + "error": null, + "metadata": null +} +``` + +Rows are sorted by `month`, then `modality`, then `provider`. Cost values +are decimal strings with up to 6 decimal places (e.g. `"12.450000"`). + +Token fields (`input_tokens`, `output_tokens`, `total_tokens`) are sourced +from `llm_call.usage` and are independent of the chosen `metric` — they +are populated on every point regardless of whether you asked for +`requests`, `cost`, or eval metrics. This lets the frontend render token +usage in a tooltip or secondary axis without a second API call. + +Tokens contributed only by `llm_call` rows. Chains and evaluation runs +add nothing to token totals — chain tokens are the sum of their child +calls (would double-count), and eval tokens live in a separate domain. + +If no data matches the filters, `data` is an empty array — this is not an +error. + +--- + +## Example requests + +### 1. Total monthly cost across all modalities and providers + +``` +GET /api/analytics/monthly?metric=cost&from_month=2026-01-01&to_month=2026-05-01 +``` + +### 2. Just the OpenAI text-to-text request volume + +``` +GET /api/analytics/monthly?metric=requests&modality=T-FS-T&provider=openai +``` + +### 3. STT evaluation run costs this year + +``` +GET /api/analytics/monthly?metric=eval_cost&modality=STT&from_month=2026-01-01 +``` + +--- + +## Notes on accuracy + +- **Live reads**: every request runs a fresh `GROUP BY` against the source + tables, so the response always reflects the current database. There is + no daily aggregation cron and no staleness window. +- **Default time window** is the last 24 months. When `from_month` is + omitted, the query is bounded to that range so an unfiltered call can't + trigger a full-table scan as the source tables grow. Pass an explicit + `from_month` to query further back. +- **Missing pricing** for a provider/model yields a cost of `0` for those + rows rather than failing the whole query. Make sure your + `ModelConfig.pricing` is populated for every provider/model you use if + you want accurate cost numbers. +- **Cost is not double-counted across chains**: a chain row contributes + only to the `requests` metric (via the chain count), never to `cost` — + its dollars come from the underlying `llm_call` rows. +- **Cost computed on summed tokens per (provider, model) group**, which is + equivalent to per-row pricing because `estimate_model_cost` is linear in + token counts. diff --git a/backend/app/api/docs/analytics/monthly_chart.md b/backend/app/api/docs/analytics/monthly_chart.md new file mode 100644 index 000000000..36211d726 --- /dev/null +++ b/backend/app/api/docs/analytics/monthly_chart.md @@ -0,0 +1,147 @@ +Chart-shaped live monthly analytics for the current organization. + +Use this endpoint when you want to render the data directly as a line, bar, +or stacked-area chart. Numbers are computed on-demand from `llm_call`, +`llm_chain`, and `evaluation_run` — no caching layer, so the chart always +reflects the current database state. The response shape is compatible with +most chart libraries (Recharts, Chart.js, ApexCharts, Highcharts, ECharts): + +- `labels[]` — the x-axis values (one entry per month). +- `series[]` — one entry per chart line/bar, each with a human-readable + `name` and a `data[]` array. `series[i].data[j]` corresponds to + `labels[j]`. Missing months are filled with `0` so every series has the + same length as `labels`. + +For a flat row-per-bucket shape (suitable when you want to do your own +pivoting), use `GET /api/analytics/monthly` instead. + +--- + +## Authentication & default scope + +Any authenticated user with an organization context can call this endpoint. +By default it returns data scoped to the caller's **currently selected +project**; if the caller has no project selected, it falls back to all +projects in the caller's organization. Pass `project_id` to override the +default — it must reference a project inside the caller's organization, so +cross-organization access is never possible. + +--- + +## Query parameters + +| Parameter | Type | Required | Default | Description | +| ----------- | ------- | -------- | ---------------------- | ----------- | +| `metric` | enum | **yes** | — | Which metric to plot. One of: `requests`, `cost`, `eval_runs`, `eval_cost`. | +| `group_by` | enum | no | `modality_provider` | How to split the data into series. See the table below. | +| `from_month`| date | no | 24 months before `to_month` (or before today if `to_month` is also omitted) | Inclusive lower bound (first-of-month), e.g. `2026-01-01`. Pass an explicit value to query further back. The default exists to cap worst-case scan size as the source tables grow. | +| `to_month` | date | no | — (no upper bound) | Inclusive upper bound (first-of-month), e.g. `2026-05-01`. | +| `modality` | enum | no | — (all) | Pre-filter to a single modality bucket. | +| `provider` | string | no | — (all) | Pre-filter to a single provider. | +| `project_id`| integer | no | Caller's current project, if any; else all projects in the org. | Override the default scope. Must reference a project inside the caller's organization. | + +### `group_by` values + +| Value | Series produced | +| --------------------- | ---------------------------------------------------------------------------- | +| `modality_provider` | One series per `(modality, provider)` combination. Series name: `"T-FS-T · openai"`. | +| `modality` | One series per modality, summed across providers. Series name: `"T-FS-T"`. | +| `provider` | One series per provider, summed across modalities. Series name: `"openai"`. | +| `total` | A single series containing the per-month grand total. Series name: `"total"`. | + +--- + +## Response shape + +```json +{ + "success": true, + "data": { + "metric": "cost", + "group_by": "modality_provider", + "labels": ["2026-01-01", "2026-02-01", "2026-03-01", "2026-04-01"], + "series": [ + { + "name": "T-FS-T · openai", + "data": ["10.500000", "15.400000", "18.700000", "22.100000"], + "total_input_tokens": 4250000, + "total_output_tokens": 2810000, + "total_tokens": 7060000 + }, + { + "name": "T-FS-T · google", + "data": ["5.100000", "6.300000", "8.200000", "12.400000"], + "total_input_tokens": 1820000, + "total_output_tokens": 1240000, + "total_tokens": 3060000 + }, + { + "name": "STT · sarvamai", + "data": ["0", "0.800000", "1.200000", "1.900000"], + "total_input_tokens": 0, + "total_output_tokens": 0, + "total_tokens": 0 + } + ] + }, + "error": null, + "metadata": null +} +``` + +- `labels` are sorted chronologically (oldest → newest). +- `series` are sorted alphabetically by `name`. +- All `series[].data` arrays have the same length as `labels`. Months with + no data for a given series are filled with `0`, so the chart library + doesn't have to align points itself. +- Cost values are decimal strings with up to 6 decimal places. +- `total_input_tokens`, `total_output_tokens`, and `total_tokens` on each + series are series-wide sums across every label, sourced from + `llm_call.usage`. They are independent of the chosen `metric` — populated + whether you're charting requests, cost, or eval numbers. Chains and + evaluation runs contribute zero to token totals. +- An empty result returns `labels: []` and `series: []`. + +--- + +## Example requests + +### 1. Monthly cost grouped by provider (one line per provider) + +``` +GET /api/analytics/monthly/chart?metric=cost&group_by=provider +``` + +### 2. Total request volume across all dimensions (single line) + +``` +GET /api/analytics/monthly/chart?metric=requests&group_by=total +``` + +### 3. STT-only eval cost trend for the year + +``` +GET /api/analytics/monthly/chart?metric=eval_cost&modality=STT&from_month=2026-01-01 +``` + +### 4. Cost split by modality for a specific project + +``` +GET /api/analytics/monthly/chart?metric=cost&group_by=modality&project_id=42 +``` + +--- + +## Frontend integration tips + +**Recharts**: pass `labels` as the X-axis source and render one ``, +``, or `` per item in `series`, using `series[i].name` as the +key and the values from `series[i].data`. + +**Chart.js / ApexCharts**: the shape is almost their native config — the +`labels` array maps to their `labels`/`categories`, and each series object +maps to `datasets[]` / `series[]`. + +For a **stacked area chart** of cost by provider over time, use +`metric=cost` and `group_by=provider` — the response is already +chart-ready. diff --git a/backend/app/api/main.py b/backend/app/api/main.py index c35005704..8ccf9b8d6 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -1,6 +1,7 @@ from fastapi import APIRouter from app.api.routes import ( + analytics, api_keys, assessment as assessment_routes, assistants, @@ -35,6 +36,7 @@ from app.core.config import settings api_router = APIRouter() +api_router.include_router(analytics.router) api_router.include_router(api_keys.router) api_router.include_router(assessment_routes.router) api_router.include_router(assistants.router) diff --git a/backend/app/api/routes/analytics.py b/backend/app/api/routes/analytics.py new file mode 100644 index 000000000..d6995ac1e --- /dev/null +++ b/backend/app/api/routes/analytics.py @@ -0,0 +1,513 @@ +import logging +from collections import defaultdict +from datetime import date +from decimal import Decimal + +import sqlalchemy as sa +from fastapi import APIRouter, Depends, Query +from sqlmodel import Session, select + +from app.api.deps import AuthContextDep, SessionDep +from app.api.permissions import Permission, require_permission +from app.crud.model_config import estimate_model_cost +from app.models import ( + AnalyticsChartGroupBy, + AnalyticsChartResponse, + AnalyticsChartSeries, + AnalyticsMetric, + AnalyticsMonthlyMetricPoint, + Modality, +) +from app.models.evaluation import EvaluationRun +from app.models.llm.request import LlmCall, LlmChain +from app.utils import APIResponse, load_description + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/analytics", tags=["Analytics"]) + + +# (input_type, output_type) -> modality bucket for llm_call rows. +_LLM_MODALITY: dict[tuple[str | None, str | None], Modality] = { + ("text", "text"): Modality.T_FS_T, + ("audio", "audio"): Modality.S_FS_S, + ("audio", "text"): Modality.STT, + ("text", "audio"): Modality.TTS, +} + +# evaluation_run.type (lowercased) -> modality bucket. +_EVAL_TYPE_TO_MODALITY: dict[str, Modality] = { + "text": Modality.T_FS_T, + "stt": Modality.STT, + "tts": Modality.TTS, +} + + +def _derive_llm_modality(input_type: str | None, output_type: str | None) -> Modality: + return _LLM_MODALITY.get((input_type, output_type), Modality.OTHER) + + +def _first_of_next_month(d: date) -> date: + if d.month == 12: + return date(d.year + 1, 1, 1) + return date(d.year, d.month + 1, 1) + + +# Default lookback when the caller omits `from_month`. Caps the worst-case +# scan size so an unfiltered request can't trigger a full-table scan on +# llm_call / llm_chain / evaluation_run as the source tables grow. +DEFAULT_LOOKBACK_MONTHS = 24 + + +def _default_from_month(anchor: date) -> date: + """First-of-month DEFAULT_LOOKBACK_MONTHS calendar months before anchor.""" + year = anchor.year + month = anchor.month - DEFAULT_LOOKBACK_MONTHS + while month <= 0: + month += 12 + year -= 1 + return date(year, month, 1) + + +def _llm_modality_case() -> sa.sql.ColumnElement[str]: + """SQL CASE mapping llm_call.input_type/output_type to a modality string.""" + return sa.case( + ( + sa.and_(LlmCall.input_type == "text", LlmCall.output_type == "text"), + Modality.T_FS_T.value, + ), + ( + sa.and_(LlmCall.input_type == "audio", LlmCall.output_type == "audio"), + Modality.S_FS_S.value, + ), + ( + sa.and_(LlmCall.input_type == "audio", LlmCall.output_type == "text"), + Modality.STT.value, + ), + ( + sa.and_(LlmCall.input_type == "text", LlmCall.output_type == "audio"), + Modality.TTS.value, + ), + else_=Modality.OTHER.value, + ) + + +def _empty_bucket() -> dict[str, int | Decimal]: + return { + "llm_call_requests": 0, + "llm_chain_requests": 0, + "cost_usd": Decimal("0"), + "input_tokens": 0, + "output_tokens": 0, + "eval_runs": 0, + "eval_cost_usd": Decimal("0"), + } + + +def _aggregate_live( + session: Session, + *, + organization_id: int, + from_month: date | None, + to_month: date | None, + modality_filter: Modality | None, + provider_filter: str | None, + project_id: int | None, +) -> dict[tuple[date, Modality, str], dict[str, int | Decimal]]: + """Live aggregation against llm_call, llm_chain, evaluation_run. + + Each source is GROUP BY'd in Postgres; per-group cost for llm_call is + computed in Python via estimate_model_cost using the summed tokens. + estimate_model_cost is linear in tokens, so summing first and pricing + once per (provider, model) is equivalent to per-row pricing. + + Returns: {(month, modality, provider) -> totals dict}. + """ + end_date = _first_of_next_month(to_month) if to_month else None + buckets: dict[tuple[date, Modality, str], dict[str, int | Decimal]] = defaultdict( + _empty_bucket + ) + + # ---- llm_call -------------------------------------------------------- + month_col = ( + sa.func.date_trunc("month", LlmCall.inserted_at).cast(sa.Date).label("month") + ) + modality_col = _llm_modality_case().label("modality") + provider_col = sa.func.coalesce(LlmCall.provider, "unknown").label("provider") + input_tokens_col = sa.func.coalesce( + sa.func.sum(sa.cast(LlmCall.usage["input_tokens"].astext, sa.Integer)), + 0, + ).label("input_tokens") + output_tokens_col = sa.func.coalesce( + sa.func.sum(sa.cast(LlmCall.usage["output_tokens"].astext, sa.Integer)), + 0, + ).label("output_tokens") + count_col = sa.func.count().label("request_count") + + llm_stmt = ( + select( + month_col, + modality_col, + provider_col, + LlmCall.model, + count_col, + input_tokens_col, + output_tokens_col, + ) + .where( + LlmCall.deleted_at.is_(None), + LlmCall.organization_id == organization_id, + ) + .group_by(month_col, modality_col, provider_col, LlmCall.model) + ) + if from_month is not None: + llm_stmt = llm_stmt.where(LlmCall.inserted_at >= from_month) + if end_date is not None: + llm_stmt = llm_stmt.where(LlmCall.inserted_at < end_date) + if project_id is not None: + llm_stmt = llm_stmt.where(LlmCall.project_id == project_id) + if provider_filter is not None: + llm_stmt = llm_stmt.where(LlmCall.provider == provider_filter) + + for row in session.exec(llm_stmt).all(): + modality_enum = Modality(row.modality) + if modality_filter is not None and modality_enum is not modality_filter: + continue + key = (row.month, modality_enum, row.provider) + bucket = buckets[key] + bucket["llm_call_requests"] += row.request_count + + input_tokens = int(row.input_tokens or 0) + output_tokens = int(row.output_tokens or 0) + bucket["input_tokens"] += input_tokens + bucket["output_tokens"] += output_tokens + if input_tokens or output_tokens: + estimate = estimate_model_cost( + session=session, + provider=row.provider, # type: ignore[arg-type] + model_name=row.model, + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + if estimate is not None: + bucket["cost_usd"] += Decimal(str(estimate.get("total_cost", 0))) + + # ---- llm_chain ------------------------------------------------------- + # A chain is attributed to the modality+provider of its first child call. + # Fetch chains with the first-block UUID, then do one batched lookup + # against llm_call to resolve modality+provider. + chain_first_block = LlmChain.block_sequences[0].astext.label("first_call_id") + chain_month_col = ( + sa.func.date_trunc("month", LlmChain.inserted_at).cast(sa.Date).label("month") + ) + + chain_stmt = select(chain_month_col, chain_first_block).where( + LlmChain.organization_id == organization_id, + ) + if from_month is not None: + chain_stmt = chain_stmt.where(LlmChain.inserted_at >= from_month) + if end_date is not None: + chain_stmt = chain_stmt.where(LlmChain.inserted_at < end_date) + if project_id is not None: + chain_stmt = chain_stmt.where(LlmChain.project_id == project_id) + + chain_rows = session.exec(chain_stmt).all() + first_call_ids = {row.first_call_id for row in chain_rows if row.first_call_id} + + first_call_map: dict[str, sa.Row] = {} + if first_call_ids: + lookup_stmt = select( + LlmCall.id, LlmCall.input_type, LlmCall.output_type, LlmCall.provider + ).where(LlmCall.id.in_(first_call_ids)) + for call_row in session.exec(lookup_stmt).all(): + first_call_map[str(call_row.id)] = call_row + + for row in chain_rows: + first = first_call_map.get(row.first_call_id) if row.first_call_id else None + if first is not None: + chain_modality = _derive_llm_modality(first.input_type, first.output_type) + chain_provider = first.provider or "unknown" + else: + chain_modality = Modality.OTHER + chain_provider = "unknown" + + if modality_filter is not None and chain_modality is not modality_filter: + continue + if provider_filter is not None and chain_provider != provider_filter: + continue + buckets[(row.month, chain_modality, chain_provider)]["llm_chain_requests"] += 1 + + # ---- evaluation_run -------------------------------------------------- + eval_month_col = ( + sa.func.date_trunc("month", EvaluationRun.inserted_at) + .cast(sa.Date) + .label("month") + ) + eval_type_lower = sa.func.lower(sa.func.coalesce(EvaluationRun.type, "")).label( + "type_lower" + ) + eval_provider_col = sa.func.coalesce( + EvaluationRun.providers[0].astext, "unknown" + ).label("provider") + eval_count_col = sa.func.count().label("eval_count") + eval_cost_col = sa.func.coalesce( + sa.func.sum(sa.cast(EvaluationRun.cost["total_cost_usd"].astext, sa.Numeric)), + 0, + ).label("eval_cost_usd") + + eval_stmt = ( + select( + eval_month_col, + eval_type_lower, + eval_provider_col, + eval_count_col, + eval_cost_col, + ) + .where(EvaluationRun.organization_id == organization_id) + .group_by(eval_month_col, eval_type_lower, eval_provider_col) + ) + if from_month is not None: + eval_stmt = eval_stmt.where(EvaluationRun.inserted_at >= from_month) + if end_date is not None: + eval_stmt = eval_stmt.where(EvaluationRun.inserted_at < end_date) + if project_id is not None: + eval_stmt = eval_stmt.where(EvaluationRun.project_id == project_id) + if provider_filter is not None: + eval_stmt = eval_stmt.where( + sa.func.coalesce(EvaluationRun.providers[0].astext, "unknown") + == provider_filter + ) + + for row in session.exec(eval_stmt).all(): + eval_modality = _EVAL_TYPE_TO_MODALITY.get(row.type_lower, Modality.OTHER) + if modality_filter is not None and eval_modality is not modality_filter: + continue + key = (row.month, eval_modality, row.provider) + bucket = buckets[key] + bucket["eval_runs"] += row.eval_count + bucket["eval_cost_usd"] += Decimal(str(row.eval_cost_usd or 0)) + + return buckets + + +def _bucket_value(bucket: dict[str, int | Decimal], metric: AnalyticsMetric) -> Decimal: + if metric is AnalyticsMetric.REQUESTS: + return Decimal( + int(bucket["llm_call_requests"]) + int(bucket["llm_chain_requests"]) + ) + if metric is AnalyticsMetric.COST: + return Decimal(bucket["cost_usd"]) + if metric is AnalyticsMetric.EVAL_RUNS: + return Decimal(int(bucket["eval_runs"])) + return Decimal(bucket["eval_cost_usd"]) # EVAL_COST + + +def _series_name( + modality: Modality, provider: str, group_by: AnalyticsChartGroupBy +) -> str: + if group_by is AnalyticsChartGroupBy.MODALITY_PROVIDER: + return f"{modality.value} · {provider}" + if group_by is AnalyticsChartGroupBy.MODALITY: + return modality.value + if group_by is AnalyticsChartGroupBy.PROVIDER: + return provider + return "total" # AnalyticsChartGroupBy.TOTAL + + +@router.get( + "/monthly", + description=load_description("analytics/monthly.md"), + response_model=APIResponse[list[AnalyticsMonthlyMetricPoint]], + dependencies=[Depends(require_permission(Permission.REQUIRE_ORGANIZATION))], +) +def get_monthly_analytics( + session: SessionDep, + current_user: AuthContextDep, + metric: AnalyticsMetric = Query( + ..., + description="Which metric to return (requests | cost | eval_runs | eval_cost)", + ), + from_month: date + | None = Query( + None, + description=( + "Inclusive lower bound (first-of-month). When omitted, defaults to " + f"{DEFAULT_LOOKBACK_MONTHS} months before `to_month` (or before today " + "if `to_month` is also omitted). Pass an explicit value to query " + "further back." + ), + ), + to_month: date + | None = Query( + None, + description="Inclusive upper bound (first-of-month). Defaults to no upper bound.", + ), + modality: Modality + | None = Query(None, description="Filter to a single modality bucket."), + provider: str + | None = Query( + None, description="Filter to a single provider (e.g. 'openai', 'google')." + ), + project_id: int + | None = Query( + None, + description=( + "Optional: scope to a single project within the organization. " + "Defaults to the caller's current project if one is selected; " + "otherwise aggregates across every project in the caller's org." + ), + ), +): + """Live monthly analytics for the caller's current project (or whole org + if no project is selected), shaped per-point. + + Each point is `{month, modality, provider, value, ...tokens}`. Data is + computed on-demand from llm_call/llm_chain/evaluation_run — no + aggregation table or background job, so reads always reflect the + current database state. + """ + effective_project_id = ( + project_id + if project_id is not None + else (current_user.project.id if current_user.project else None) + ) + effective_from_month = from_month or _default_from_month(to_month or date.today()) + buckets = _aggregate_live( + session=session, + organization_id=current_user.organization_.id, + from_month=effective_from_month, + to_month=to_month, + modality_filter=modality, + provider_filter=provider, + project_id=effective_project_id, + ) + + points: list[AnalyticsMonthlyMetricPoint] = [] + for key in sorted(buckets.keys(), key=lambda k: (k[0], k[1].value, k[2])): + month, mod, prov = key + bucket = buckets[key] + input_tokens = int(bucket["input_tokens"]) + output_tokens = int(bucket["output_tokens"]) + points.append( + AnalyticsMonthlyMetricPoint( + month=month, + modality=mod, + provider=prov, + value=_bucket_value(bucket, metric), + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, + ) + ) + + return APIResponse.success_response(points) + + +@router.get( + "/monthly/chart", + description=load_description("analytics/monthly_chart.md"), + response_model=APIResponse[AnalyticsChartResponse], + dependencies=[Depends(require_permission(Permission.REQUIRE_ORGANIZATION))], +) +def get_monthly_analytics_chart( + session: SessionDep, + current_user: AuthContextDep, + metric: AnalyticsMetric = Query( + ..., + description="Which metric to plot (requests | cost | eval_runs | eval_cost).", + ), + from_month: date + | None = Query( + None, + description=( + "Inclusive lower bound (first-of-month). When omitted, defaults to " + f"{DEFAULT_LOOKBACK_MONTHS} months before `to_month` (or before today " + "if `to_month` is also omitted). Pass an explicit value to query " + "further back." + ), + ), + to_month: date + | None = Query( + None, + description="Inclusive upper bound (first-of-month). Defaults to no upper bound.", + ), + modality: Modality + | None = Query(None, description="Filter to a single modality bucket."), + provider: str | None = Query(None, description="Filter to a single provider."), + project_id: int + | None = Query( + None, + description=( + "Optional: scope to a single project within the organization. " + "Defaults to the caller's current project if one is selected; " + "otherwise aggregates across every project in the caller's org." + ), + ), + group_by: AnalyticsChartGroupBy = Query( + AnalyticsChartGroupBy.MODALITY_PROVIDER, + description=( + "How to split data into chart series. " + "`modality_provider` = one series per (modality, provider) combo. " + "`modality` = one per modality, summing across providers. " + "`provider` = one per provider, summing across modalities. " + "`total` = a single series with the grand total per month." + ), + ), +): + """Live analytics shaped for direct rendering as a chart. + + Scoped to the caller's current project by default, or to the whole + organization if the caller has no project selected. + """ + effective_project_id = ( + project_id + if project_id is not None + else (current_user.project.id if current_user.project else None) + ) + effective_from_month = from_month or _default_from_month(to_month or date.today()) + buckets = _aggregate_live( + session=session, + organization_id=current_user.organization_.id, + from_month=effective_from_month, + to_month=to_month, + modality_filter=modality, + provider_filter=provider, + project_id=effective_project_id, + ) + + labels: list[date] = sorted({month for (month, _, _) in buckets.keys()}) + label_index = {m: i for i, m in enumerate(labels)} + + series_acc: dict[str, list[Decimal]] = {} + series_tokens: dict[str, dict[str, int]] = {} + for (month, mod, prov), bucket in buckets.items(): + name = _series_name(mod, prov, group_by) + if name not in series_acc: + series_acc[name] = [Decimal("0")] * len(labels) + series_tokens[name] = {"input_tokens": 0, "output_tokens": 0} + series_acc[name][label_index[month]] += _bucket_value(bucket, metric) + series_tokens[name]["input_tokens"] += int(bucket["input_tokens"]) + series_tokens[name]["output_tokens"] += int(bucket["output_tokens"]) + + series = [ + AnalyticsChartSeries( + name=name, + data=values, + total_input_tokens=series_tokens[name]["input_tokens"], + total_output_tokens=series_tokens[name]["output_tokens"], + total_tokens=( + series_tokens[name]["input_tokens"] + + series_tokens[name]["output_tokens"] + ), + ) + for name, values in sorted(series_acc.items()) + ] + + return APIResponse.success_response( + AnalyticsChartResponse( + metric=metric, + group_by=group_by, + labels=labels, + series=series, + ) + ) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 3d9a2c4c6..ae5bf3cf0 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -1,5 +1,13 @@ from sqlmodel import SQLModel +from app.models.analytics import ( # noqa: F401 + AnalyticsChartGroupBy, + AnalyticsChartResponse, + AnalyticsChartSeries, + AnalyticsMetric, + AnalyticsMonthlyMetricPoint, + Modality, +) from app.models.assessment import Assessment, AssessmentRun # noqa: F401 from .api_key import ( diff --git a/backend/app/models/analytics.py b/backend/app/models/analytics.py new file mode 100644 index 000000000..c730e26ce --- /dev/null +++ b/backend/app/models/analytics.py @@ -0,0 +1,86 @@ +from datetime import date +from decimal import Decimal +from enum import Enum + +from sqlmodel import SQLModel + + +class Modality(str, Enum): + """High-level modality bucket for analytics grouping. + + Derived from llm_call.input_type + output_type, or from evaluation_run.type. + """ + + T_FS_T = "T-FS-T" # text -> text + S_FS_S = "S-FS-S" # audio -> audio + STT = "STT" # audio -> text + TTS = "TTS" # text -> audio + OTHER = "OTHER" # anything else (image, pdf, multimodal, assessment, ...) + + +class AnalyticsMetric(str, Enum): + """Metric selector for the analytics endpoints.""" + + REQUESTS = "requests" + COST = "cost" + EVAL_RUNS = "eval_runs" + EVAL_COST = "eval_cost" + + +class AnalyticsMonthlyMetricPoint(SQLModel): + """One data point in a metric-shaped response. + + `value` carries the metric the caller asked for. Token fields are + sourced from `llm_call.usage` and are independent of the chosen metric, + so the frontend can render token usage alongside any metric without an + extra API call. Chains and evaluation runs do not contribute tokens — + chain tokens are the sum of their child calls (would double-count), and + eval tokens live in a separate domain. + """ + + month: date + modality: Modality + provider: str + value: Decimal + input_tokens: int = 0 + output_tokens: int = 0 + total_tokens: int = 0 + + +class AnalyticsChartGroupBy(str, Enum): + """Dimension to split the chart series by.""" + + MODALITY_PROVIDER = "modality_provider" + MODALITY = "modality" + PROVIDER = "provider" + TOTAL = "total" + + +class AnalyticsChartSeries(SQLModel): + """A single line / bar on the chart. + + `data[i]` aligns with `labels[i]` from the parent response. The + `total_*_tokens` fields are series-wide sums (across every month in + `labels`) sourced from `llm_call.usage`, so the chart can render a + secondary axis or tooltip totals without an extra API call. + """ + + name: str + data: list[Decimal] + total_input_tokens: int = 0 + total_output_tokens: int = 0 + total_tokens: int = 0 + + +class AnalyticsChartResponse(SQLModel): + """Chart-shaped analytics response. + + Compatible with most chart libraries (Recharts, Chart.js, ApexCharts, + Highcharts, ECharts): each series has the same length as `labels`, and + `data[i]` corresponds to `labels[i]`. + """ + + metric: AnalyticsMetric + group_by: AnalyticsChartGroupBy + labels: list[date] + series: list[AnalyticsChartSeries] From 5ca387bdb22d38f9f4c84dea890962a78c0a408f Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Sat, 23 May 2026 22:55:18 +0530 Subject: [PATCH 2/8] fix(analytics): few fixes --- backend/app/api/routes/analytics.py | 44 +++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/backend/app/api/routes/analytics.py b/backend/app/api/routes/analytics.py index d6995ac1e..adbe31f35 100644 --- a/backend/app/api/routes/analytics.py +++ b/backend/app/api/routes/analytics.py @@ -18,8 +18,10 @@ AnalyticsMonthlyMetricPoint, Modality, ) +from app.models.config.version import ConfigVersion from app.models.evaluation import EvaluationRun from app.models.llm.request import LlmCall, LlmChain +from app.models.model_config import ModelConfig from app.utils import APIResponse, load_description logger = logging.getLogger(__name__) @@ -237,7 +239,26 @@ def _aggregate_live( continue buckets[(row.month, chain_modality, chain_provider)]["llm_chain_requests"] += 1 - # ---- evaluation_run -------------------------------------------------- + # ---- evaluation_run ---- + mc_lookup = ( + select(ModelConfig.model_name, ModelConfig.provider) + .distinct(ModelConfig.model_name) + .order_by(ModelConfig.model_name, ModelConfig.provider) + .subquery() + ) + + cv_provider_normalized = sa.func.split_part( + sa.cast(ConfigVersion.config_blob["completion"]["provider"].astext, sa.String), + "-native", + 1, + ) + + eval_provider_expr = sa.func.coalesce( + mc_lookup.c.provider, + sa.func.nullif(cv_provider_normalized, ""), + "unknown", + ) + eval_month_col = ( sa.func.date_trunc("month", EvaluationRun.inserted_at) .cast(sa.Date) @@ -246,9 +267,7 @@ def _aggregate_live( eval_type_lower = sa.func.lower(sa.func.coalesce(EvaluationRun.type, "")).label( "type_lower" ) - eval_provider_col = sa.func.coalesce( - EvaluationRun.providers[0].astext, "unknown" - ).label("provider") + eval_provider_col = eval_provider_expr.label("provider") eval_count_col = sa.func.count().label("eval_count") eval_cost_col = sa.func.coalesce( sa.func.sum(sa.cast(EvaluationRun.cost["total_cost_usd"].astext, sa.Numeric)), @@ -263,6 +282,18 @@ def _aggregate_live( eval_count_col, eval_cost_col, ) + .select_from(EvaluationRun) + .outerjoin( + mc_lookup, + mc_lookup.c.model_name == EvaluationRun.providers[0].astext, + ) + .outerjoin( + ConfigVersion, + sa.and_( + ConfigVersion.config_id == EvaluationRun.config_id, + ConfigVersion.version == EvaluationRun.config_version, + ), + ) .where(EvaluationRun.organization_id == organization_id) .group_by(eval_month_col, eval_type_lower, eval_provider_col) ) @@ -273,10 +304,7 @@ def _aggregate_live( if project_id is not None: eval_stmt = eval_stmt.where(EvaluationRun.project_id == project_id) if provider_filter is not None: - eval_stmt = eval_stmt.where( - sa.func.coalesce(EvaluationRun.providers[0].astext, "unknown") - == provider_filter - ) + eval_stmt = eval_stmt.where(eval_provider_expr == provider_filter) for row in session.exec(eval_stmt).all(): eval_modality = _EVAL_TYPE_TO_MODALITY.get(row.type_lower, Modality.OTHER) From ef396c935dc62770a988c6ae2a79516cd09a0d48 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Tue, 26 May 2026 08:13:52 +0530 Subject: [PATCH 3/8] fix(analytics): remove the unwanted js comments --- backend/app/api/routes/analytics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/api/routes/analytics.py b/backend/app/api/routes/analytics.py index adbe31f35..80c0bc944 100644 --- a/backend/app/api/routes/analytics.py +++ b/backend/app/api/routes/analytics.py @@ -130,7 +130,7 @@ def _aggregate_live( _empty_bucket ) - # ---- llm_call -------------------------------------------------------- + # ---- For the llm_call ---- month_col = ( sa.func.date_trunc("month", LlmCall.inserted_at).cast(sa.Date).label("month") ) @@ -194,7 +194,7 @@ def _aggregate_live( if estimate is not None: bucket["cost_usd"] += Decimal(str(estimate.get("total_cost", 0))) - # ---- llm_chain ------------------------------------------------------- + # ---- llm_chain ---- # A chain is attributed to the modality+provider of its first child call. # Fetch chains with the first-block UUID, then do one batched lookup # against llm_call to resolve modality+provider. From 351d632422a28d13debe64387297af9859e4f7ea Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Tue, 26 May 2026 08:38:06 +0530 Subject: [PATCH 4/8] fix(analytics): added the test cases for this --- .../app/tests/api/routes/test_analytics.py | 668 ++++++++++++++++++ 1 file changed, 668 insertions(+) create mode 100644 backend/app/tests/api/routes/test_analytics.py diff --git a/backend/app/tests/api/routes/test_analytics.py b/backend/app/tests/api/routes/test_analytics.py new file mode 100644 index 000000000..cd7c5f825 --- /dev/null +++ b/backend/app/tests/api/routes/test_analytics.py @@ -0,0 +1,668 @@ +from datetime import date, datetime +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from sqlmodel import Session + +from app.core.config import settings +from app.models import Job +from app.models.evaluation import EvaluationDataset, EvaluationRun +from app.models.llm.request import LlmCall, LlmChain +from app.models.model_config import ModelConfig +from app.tests.utils.auth import TestAuthContext +from app.tests.utils.llm import create_llm_job +from app.tests.utils.test_data import create_test_evaluation_dataset + +MONTHLY_URL = f"{settings.API_V1_STR}/analytics/monthly" +CHART_URL = f"{settings.API_V1_STR}/analytics/monthly/chart" + + +@pytest.fixture +def llm_job(db: Session) -> Job: + return create_llm_job(db) + + +@pytest.fixture +def eval_dataset(db: Session, user_api_key: TestAuthContext) -> EvaluationDataset: + return create_test_evaluation_dataset( + db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + + +@pytest.fixture +def model_pricing(db: Session) -> ModelConfig: + """Seed pricing so estimate_model_cost returns a value during tests.""" + model = ModelConfig( + provider="openai", + model_name=f"gpt-4o-analytics-test-{uuid4().hex[:8]}", + pricing={ + "response": {"input_token_cost": 1.0, "output_token_cost": 2.0}, + }, + is_active=True, + ) + db.add(model) + db.commit() + db.refresh(model) + return model + + +# ----- Helpers ---------------------------------------------------------------- + + +def _make_llm_call( + db: Session, + *, + job_id, + project_id: int, + organization_id: int, + provider: str = "openai", + model: str = "gpt-4o", + input_type: str = "text", + output_type: str | None = "text", + input_tokens: int = 100, + output_tokens: int = 50, + inserted_at: datetime | None = None, +) -> LlmCall: + call = LlmCall( + job_id=job_id, + project_id=project_id, + organization_id=organization_id, + input="hi", + input_type=input_type, + output_type=output_type, + provider=provider, + model=model, + usage={"input_tokens": input_tokens, "output_tokens": output_tokens}, + ) + if inserted_at is not None: + call.inserted_at = inserted_at + db.add(call) + db.commit() + db.refresh(call) + return call + + +def _make_llm_chain( + db: Session, + *, + job_id, + project_id: int, + organization_id: int, + first_call_id: str | None = None, +) -> LlmChain: + chain = LlmChain( + job_id=job_id, + project_id=project_id, + organization_id=organization_id, + total_blocks=1, + input="chain", + block_sequences=[first_call_id] if first_call_id else [], + ) + db.add(chain) + db.commit() + db.refresh(chain) + return chain + + +def _make_eval_run( + db: Session, + *, + dataset_id: int, + project_id: int, + organization_id: int, + type_: str = "text", + cost_usd: float = 1.50, +) -> EvaluationRun: + run = EvaluationRun( + run_name=f"run_{uuid4().hex[:8]}", + dataset_name="ds", + dataset_id=dataset_id, + type=type_, + status="completed", + organization_id=organization_id, + project_id=project_id, + cost={"total_cost_usd": cost_usd}, + ) + db.add(run) + db.commit() + db.refresh(run) + return run + + +def _ok(response): + assert response.status_code == 200, response.text + body = response.json() + assert body["success"] is True + return body["data"] + + +# ----- /analytics/monthly ---- +class TestMonthlyAnalytics: + def test_requires_authentication(self, client: TestClient): + response = client.get(MONTHLY_URL, params={"metric": "requests"}) + assert response.status_code in (401, 403) + + def test_metric_requests_counts_llm_calls( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + ): + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="anlz-provider-A", + input_tokens=100, + output_tokens=50, + ) + + data = _ok( + client.get( + MONTHLY_URL, + params={"metric": "requests", "provider": "anlz-provider-A"}, + headers=user_api_key_header, + ) + ) + assert len(data) == 1 + point = data[0] + assert point["modality"] == "T-FS-T" + assert point["provider"] == "anlz-provider-A" + assert int(point["value"]) == 1 + assert point["input_tokens"] == 100 + assert point["output_tokens"] == 50 + assert point["total_tokens"] == 150 + + def test_metric_cost_uses_estimate_model_cost( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + model_pricing: ModelConfig, + ): + # 1M input @ $1 + 500k output @ $2 = $2.00 + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="openai", + model=model_pricing.model_name, + input_tokens=1_000_000, + output_tokens=500_000, + ) + + data = _ok( + client.get( + MONTHLY_URL, + params={"metric": "cost", "provider": "openai"}, + headers=user_api_key_header, + ) + ) + total = sum(float(p["value"]) for p in data) + assert total == pytest.approx(2.0) + + @pytest.mark.parametrize( + "input_type,output_type,expected_modality", + [ + ("text", "text", "T-FS-T"), + ("audio", "audio", "S-FS-S"), + ("audio", "text", "STT"), + ("text", "audio", "TTS"), + ("image", "text", "OTHER"), + ], + ) + def test_modality_derivation_from_io_types( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + input_type: str, + output_type: str, + expected_modality: str, + ): + provider = f"mod-{expected_modality.lower()}" + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider=provider, + input_type=input_type, + output_type=output_type, + ) + + data = _ok( + client.get( + MONTHLY_URL, + params={"metric": "requests", "provider": provider}, + headers=user_api_key_header, + ) + ) + assert len(data) == 1 + assert data[0]["modality"] == expected_modality + + def test_filter_modality( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + ): + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="filter-mod", + input_type="text", + output_type="text", + ) + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="filter-mod", + input_type="audio", + output_type="text", + ) + + data = _ok( + client.get( + MONTHLY_URL, + params={ + "metric": "requests", + "modality": "STT", + "provider": "filter-mod", + }, + headers=user_api_key_header, + ) + ) + assert len(data) == 1 + assert data[0]["modality"] == "STT" + + def test_filter_provider( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + ): + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="prov-only-this", + ) + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="prov-other", + ) + + data = _ok( + client.get( + MONTHLY_URL, + params={"metric": "requests", "provider": "prov-only-this"}, + headers=user_api_key_header, + ) + ) + assert len(data) == 1 + assert data[0]["provider"] == "prov-only-this" + + def test_filter_excludes_data_outside_window( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + ): + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="window-test", + ) + + # to_month far in the past — current row should be out of range. + data = _ok( + client.get( + MONTHLY_URL, + params={ + "metric": "requests", + "provider": "window-test", + "from_month": "2020-01-01", + "to_month": "2020-01-01", + }, + headers=user_api_key_header, + ) + ) + assert data == [] + + def test_metric_eval_runs( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + eval_dataset: EvaluationDataset, + ): + _make_eval_run( + db, + dataset_id=eval_dataset.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + type_="text", + cost_usd=3.0, + ) + + data = _ok( + client.get( + MONTHLY_URL, + params={"metric": "eval_runs"}, + headers=user_api_key_header, + ) + ) + # Find at least one bucket with eval_runs >= 1 in text modality. + assert any(p["modality"] == "T-FS-T" and int(p["value"]) >= 1 for p in data) + + def test_metric_eval_cost_stt( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + eval_dataset: EvaluationDataset, + ): + _make_eval_run( + db, + dataset_id=eval_dataset.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + type_="stt", + cost_usd=3.5, + ) + + data = _ok( + client.get( + MONTHLY_URL, + params={"metric": "eval_cost"}, + headers=user_api_key_header, + ) + ) + assert any(p["modality"] == "STT" and float(p["value"]) >= 3.5 for p in data) + + def test_llm_chain_attributed_to_first_block( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + ): + call = _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="chain-provider", + input_type="audio", + output_type="text", + ) + _make_llm_chain( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + first_call_id=str(call.id), + ) + + data = _ok( + client.get( + MONTHLY_URL, + params={"metric": "requests", "provider": "chain-provider"}, + headers=user_api_key_header, + ) + ) + # Both the call and the chain should be attributed to STT/chain-provider. + assert len(data) == 1 + assert data[0]["modality"] == "STT" + assert int(data[0]["value"]) == 2 + + def test_llm_chain_without_first_block_falls_back_to_other( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + ): + _make_llm_chain( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + first_call_id=None, + ) + + data = _ok( + client.get( + MONTHLY_URL, + params={"metric": "requests", "provider": "unknown"}, + headers=user_api_key_header, + ) + ) + assert any( + p["modality"] == "OTHER" and p["provider"] == "unknown" for p in data + ) + + def test_explicit_nonexistent_project_id_returns_empty( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + ): + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="proj-scoping", + ) + + data = _ok( + client.get( + MONTHLY_URL, + params={ + "metric": "requests", + "project_id": 999_999_999, + "provider": "proj-scoping", + }, + headers=user_api_key_header, + ) + ) + assert data == [] + + def test_soft_deleted_llm_calls_excluded( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + ): + call = _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="soft-deleted", + ) + call.deleted_at = datetime.now() + db.add(call) + db.commit() + + data = _ok( + client.get( + MONTHLY_URL, + params={"metric": "requests", "provider": "soft-deleted"}, + headers=user_api_key_header, + ) + ) + assert data == [] + + +# ----- /analytics/monthly/chart ---- +class TestMonthlyChartAnalytics: + def test_requires_authentication(self, client: TestClient): + response = client.get(CHART_URL, params={"metric": "requests"}) + assert response.status_code in (401, 403) + + def test_chart_default_group_by_modality_provider( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + ): + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="chart-default", + ) + + data = _ok( + client.get( + CHART_URL, + params={"metric": "requests", "provider": "chart-default"}, + headers=user_api_key_header, + ) + ) + assert data["metric"] == "requests" + assert data["group_by"] == "modality_provider" + assert isinstance(data["labels"], list) + assert len(data["labels"]) >= 1 + series_names = {s["name"] for s in data["series"]} + assert "T-FS-T · chart-default" in series_names + # Each series.data is aligned with labels. + for s in data["series"]: + assert len(s["data"]) == len(data["labels"]) + + @pytest.mark.parametrize( + "group_by,expected_name", + [ + ("modality", "T-FS-T"), + ("provider", "chart-gb"), + ("total", "total"), + ], + ) + def test_chart_group_by_variants( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + group_by: str, + expected_name: str, + ): + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="chart-gb", + ) + + data = _ok( + client.get( + CHART_URL, + params={ + "metric": "requests", + "group_by": group_by, + "provider": "chart-gb", + }, + headers=user_api_key_header, + ) + ) + assert data["group_by"] == group_by + names = {s["name"] for s in data["series"]} + assert expected_name in names + + def test_chart_token_totals_aggregate_across_months( + self, + client: TestClient, + db: Session, + user_api_key: TestAuthContext, + user_api_key_header: dict[str, str], + llm_job: Job, + ): + _make_llm_call( + db, + job_id=llm_job.id, + project_id=user_api_key.project_id, + organization_id=user_api_key.organization_id, + provider="chart-tokens", + input_tokens=200, + output_tokens=100, + ) + + data = _ok( + client.get( + CHART_URL, + params={ + "metric": "requests", + "group_by": "total", + "provider": "chart-tokens", + }, + headers=user_api_key_header, + ) + ) + total = next(s for s in data["series"] if s["name"] == "total") + assert total["total_input_tokens"] == 200 + assert total["total_output_tokens"] == 100 + assert total["total_tokens"] == 300 + + def test_chart_with_no_data_returns_empty_labels_and_series( + self, + client: TestClient, + user_api_key_header: dict[str, str], + ): + # Far-future window guarantees no data. + future = date(date.today().year + 5, 1, 1) + data = _ok( + client.get( + CHART_URL, + params={ + "metric": "requests", + "from_month": future.isoformat(), + "to_month": future.isoformat(), + }, + headers=user_api_key_header, + ) + ) + assert data["labels"] == [] + assert data["series"] == [] From 84d11230a190bf26fa198e1ecf2f5f75e875440b Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Tue, 26 May 2026 08:56:56 +0530 Subject: [PATCH 5/8] fix(analytics): added the test cases for this --- backend/app/api/routes/analytics.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/app/api/routes/analytics.py b/backend/app/api/routes/analytics.py index 80c0bc944..dc3d97b11 100644 --- a/backend/app/api/routes/analytics.py +++ b/backend/app/api/routes/analytics.py @@ -2,6 +2,7 @@ from collections import defaultdict from datetime import date from decimal import Decimal +from typing import get_args import sqlalchemy as sa from fastapi import APIRouter, Depends, Query @@ -9,7 +10,7 @@ from app.api.deps import AuthContextDep, SessionDep from app.api.permissions import Permission, require_permission -from app.crud.model_config import estimate_model_cost +from app.crud.model_config import Provider, estimate_model_cost from app.models import ( AnalyticsChartGroupBy, AnalyticsChartResponse, @@ -44,6 +45,9 @@ "tts": Modality.TTS, } +# Values accepted by the `global.provider_enum` column on model_config. +_KNOWN_PROVIDERS: frozenset[str] = frozenset(get_args(Provider)) + def _derive_llm_modality(input_type: str | None, output_type: str | None) -> Modality: return _LLM_MODALITY.get((input_type, output_type), Modality.OTHER) @@ -183,7 +187,7 @@ def _aggregate_live( output_tokens = int(row.output_tokens or 0) bucket["input_tokens"] += input_tokens bucket["output_tokens"] += output_tokens - if input_tokens or output_tokens: + if (input_tokens or output_tokens) and row.provider in _KNOWN_PROVIDERS: estimate = estimate_model_cost( session=session, provider=row.provider, # type: ignore[arg-type] @@ -254,7 +258,7 @@ def _aggregate_live( ) eval_provider_expr = sa.func.coalesce( - mc_lookup.c.provider, + sa.cast(mc_lookup.c.provider, sa.String), sa.func.nullif(cv_provider_normalized, ""), "unknown", ) From b8768379e68fe254ac7448f4507005ecf8ee80d6 Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Tue, 26 May 2026 09:11:01 +0530 Subject: [PATCH 6/8] fix(analytics): added the test cases for this --- backend/app/tests/api/routes/test_analytics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/app/tests/api/routes/test_analytics.py b/backend/app/tests/api/routes/test_analytics.py index cd7c5f825..11e2103d2 100644 --- a/backend/app/tests/api/routes/test_analytics.py +++ b/backend/app/tests/api/routes/test_analytics.py @@ -38,6 +38,7 @@ def model_pricing(db: Session) -> ModelConfig: model = ModelConfig( provider="openai", model_name=f"gpt-4o-analytics-test-{uuid4().hex[:8]}", + completion_type="text", pricing={ "response": {"input_token_cost": 1.0, "output_token_cost": 2.0}, }, From 07059e63e2cef41a9dfbc7012eaa53d4ded28e3a Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Tue, 26 May 2026 11:23:33 +0530 Subject: [PATCH 7/8] fix(analytics): refactor business --- backend/app/api/routes/analytics.py | 331 +++------------- backend/app/crud/model_config.py | 17 +- backend/app/services/analytics/__init__.py | 7 + backend/app/services/analytics/aggregation.py | 354 ++++++++++++++++++ 4 files changed, 425 insertions(+), 284 deletions(-) create mode 100644 backend/app/services/analytics/__init__.py create mode 100644 backend/app/services/analytics/aggregation.py diff --git a/backend/app/api/routes/analytics.py b/backend/app/api/routes/analytics.py index dc3d97b11..d9b76922c 100644 --- a/backend/app/api/routes/analytics.py +++ b/backend/app/api/routes/analytics.py @@ -1,16 +1,12 @@ import logging -from collections import defaultdict from datetime import date from decimal import Decimal -from typing import get_args -import sqlalchemy as sa -from fastapi import APIRouter, Depends, Query -from sqlmodel import Session, select +from fastapi import APIRouter, Depends, HTTPException, Query from app.api.deps import AuthContextDep, SessionDep from app.api.permissions import Permission, require_permission -from app.crud.model_config import Provider, estimate_model_cost +from app.crud.model_config import KNOWN_PROVIDERS from app.models import ( AnalyticsChartGroupBy, AnalyticsChartResponse, @@ -19,10 +15,7 @@ AnalyticsMonthlyMetricPoint, Modality, ) -from app.models.config.version import ConfigVersion -from app.models.evaluation import EvaluationRun -from app.models.llm.request import LlmCall, LlmChain -from app.models.model_config import ModelConfig +from app.services.analytics import Bucket, aggregate_monthly_metrics from app.utils import APIResponse, load_description logger = logging.getLogger(__name__) @@ -30,35 +23,6 @@ router = APIRouter(prefix="/analytics", tags=["Analytics"]) -# (input_type, output_type) -> modality bucket for llm_call rows. -_LLM_MODALITY: dict[tuple[str | None, str | None], Modality] = { - ("text", "text"): Modality.T_FS_T, - ("audio", "audio"): Modality.S_FS_S, - ("audio", "text"): Modality.STT, - ("text", "audio"): Modality.TTS, -} - -# evaluation_run.type (lowercased) -> modality bucket. -_EVAL_TYPE_TO_MODALITY: dict[str, Modality] = { - "text": Modality.T_FS_T, - "stt": Modality.STT, - "tts": Modality.TTS, -} - -# Values accepted by the `global.provider_enum` column on model_config. -_KNOWN_PROVIDERS: frozenset[str] = frozenset(get_args(Provider)) - - -def _derive_llm_modality(input_type: str | None, output_type: str | None) -> Modality: - return _LLM_MODALITY.get((input_type, output_type), Modality.OTHER) - - -def _first_of_next_month(d: date) -> date: - if d.month == 12: - return date(d.year + 1, 1, 1) - return date(d.year, d.month + 1, 1) - - # Default lookback when the caller omits `from_month`. Caps the worst-case # scan size so an unfiltered request can't trigger a full-table scan on # llm_call / llm_chain / evaluation_run as the source tables grow. @@ -75,254 +39,38 @@ def _default_from_month(anchor: date) -> date: return date(year, month, 1) -def _llm_modality_case() -> sa.sql.ColumnElement[str]: - """SQL CASE mapping llm_call.input_type/output_type to a modality string.""" - return sa.case( - ( - sa.and_(LlmCall.input_type == "text", LlmCall.output_type == "text"), - Modality.T_FS_T.value, - ), - ( - sa.and_(LlmCall.input_type == "audio", LlmCall.output_type == "audio"), - Modality.S_FS_S.value, - ), - ( - sa.and_(LlmCall.input_type == "audio", LlmCall.output_type == "text"), - Modality.STT.value, - ), - ( - sa.and_(LlmCall.input_type == "text", LlmCall.output_type == "audio"), - Modality.TTS.value, - ), - else_=Modality.OTHER.value, - ) - - -def _empty_bucket() -> dict[str, int | Decimal]: - return { - "llm_call_requests": 0, - "llm_chain_requests": 0, - "cost_usd": Decimal("0"), - "input_tokens": 0, - "output_tokens": 0, - "eval_runs": 0, - "eval_cost_usd": Decimal("0"), - } - +def _snap_to_first_of_month(d: date | None) -> date | None: + """Coerce a date to the first of its month. -def _aggregate_live( - session: Session, - *, - organization_id: int, - from_month: date | None, - to_month: date | None, - modality_filter: Modality | None, - provider_filter: str | None, - project_id: int | None, -) -> dict[tuple[date, Modality, str], dict[str, int | Decimal]]: - """Live aggregation against llm_call, llm_chain, evaluation_run. - - Each source is GROUP BY'd in Postgres; per-group cost for llm_call is - computed in Python via estimate_model_cost using the summed tokens. - estimate_model_cost is linear in tokens, so summing first and pricing - once per (provider, model) is equivalent to per-row pricing. - - Returns: {(month, modality, provider) -> totals dict}. + The analytics window is bucketed monthly, so a caller passing + `2026-03-15` would otherwise filter `inserted_at >= 2026-03-15` and + return a partial March bucket that looks indistinguishable from a + real month. Snap to `2026-03-01` so the response always represents + whole months. """ - end_date = _first_of_next_month(to_month) if to_month else None - buckets: dict[tuple[date, Modality, str], dict[str, int | Decimal]] = defaultdict( - _empty_bucket - ) - - # ---- For the llm_call ---- - month_col = ( - sa.func.date_trunc("month", LlmCall.inserted_at).cast(sa.Date).label("month") - ) - modality_col = _llm_modality_case().label("modality") - provider_col = sa.func.coalesce(LlmCall.provider, "unknown").label("provider") - input_tokens_col = sa.func.coalesce( - sa.func.sum(sa.cast(LlmCall.usage["input_tokens"].astext, sa.Integer)), - 0, - ).label("input_tokens") - output_tokens_col = sa.func.coalesce( - sa.func.sum(sa.cast(LlmCall.usage["output_tokens"].astext, sa.Integer)), - 0, - ).label("output_tokens") - count_col = sa.func.count().label("request_count") - - llm_stmt = ( - select( - month_col, - modality_col, - provider_col, - LlmCall.model, - count_col, - input_tokens_col, - output_tokens_col, - ) - .where( - LlmCall.deleted_at.is_(None), - LlmCall.organization_id == organization_id, - ) - .group_by(month_col, modality_col, provider_col, LlmCall.model) - ) - if from_month is not None: - llm_stmt = llm_stmt.where(LlmCall.inserted_at >= from_month) - if end_date is not None: - llm_stmt = llm_stmt.where(LlmCall.inserted_at < end_date) - if project_id is not None: - llm_stmt = llm_stmt.where(LlmCall.project_id == project_id) - if provider_filter is not None: - llm_stmt = llm_stmt.where(LlmCall.provider == provider_filter) - - for row in session.exec(llm_stmt).all(): - modality_enum = Modality(row.modality) - if modality_filter is not None and modality_enum is not modality_filter: - continue - key = (row.month, modality_enum, row.provider) - bucket = buckets[key] - bucket["llm_call_requests"] += row.request_count - - input_tokens = int(row.input_tokens or 0) - output_tokens = int(row.output_tokens or 0) - bucket["input_tokens"] += input_tokens - bucket["output_tokens"] += output_tokens - if (input_tokens or output_tokens) and row.provider in _KNOWN_PROVIDERS: - estimate = estimate_model_cost( - session=session, - provider=row.provider, # type: ignore[arg-type] - model_name=row.model, - input_tokens=input_tokens, - output_tokens=output_tokens, - ) - if estimate is not None: - bucket["cost_usd"] += Decimal(str(estimate.get("total_cost", 0))) + if d is None: + return None + return date(d.year, d.month, 1) - # ---- llm_chain ---- - # A chain is attributed to the modality+provider of its first child call. - # Fetch chains with the first-block UUID, then do one batched lookup - # against llm_call to resolve modality+provider. - chain_first_block = LlmChain.block_sequences[0].astext.label("first_call_id") - chain_month_col = ( - sa.func.date_trunc("month", LlmChain.inserted_at).cast(sa.Date).label("month") - ) - - chain_stmt = select(chain_month_col, chain_first_block).where( - LlmChain.organization_id == organization_id, - ) - if from_month is not None: - chain_stmt = chain_stmt.where(LlmChain.inserted_at >= from_month) - if end_date is not None: - chain_stmt = chain_stmt.where(LlmChain.inserted_at < end_date) - if project_id is not None: - chain_stmt = chain_stmt.where(LlmChain.project_id == project_id) - - chain_rows = session.exec(chain_stmt).all() - first_call_ids = {row.first_call_id for row in chain_rows if row.first_call_id} - - first_call_map: dict[str, sa.Row] = {} - if first_call_ids: - lookup_stmt = select( - LlmCall.id, LlmCall.input_type, LlmCall.output_type, LlmCall.provider - ).where(LlmCall.id.in_(first_call_ids)) - for call_row in session.exec(lookup_stmt).all(): - first_call_map[str(call_row.id)] = call_row - - for row in chain_rows: - first = first_call_map.get(row.first_call_id) if row.first_call_id else None - if first is not None: - chain_modality = _derive_llm_modality(first.input_type, first.output_type) - chain_provider = first.provider or "unknown" - else: - chain_modality = Modality.OTHER - chain_provider = "unknown" - - if modality_filter is not None and chain_modality is not modality_filter: - continue - if provider_filter is not None and chain_provider != provider_filter: - continue - buckets[(row.month, chain_modality, chain_provider)]["llm_chain_requests"] += 1 - - # ---- evaluation_run ---- - mc_lookup = ( - select(ModelConfig.model_name, ModelConfig.provider) - .distinct(ModelConfig.model_name) - .order_by(ModelConfig.model_name, ModelConfig.provider) - .subquery() - ) - - cv_provider_normalized = sa.func.split_part( - sa.cast(ConfigVersion.config_blob["completion"]["provider"].astext, sa.String), - "-native", - 1, - ) - - eval_provider_expr = sa.func.coalesce( - sa.cast(mc_lookup.c.provider, sa.String), - sa.func.nullif(cv_provider_normalized, ""), - "unknown", - ) - eval_month_col = ( - sa.func.date_trunc("month", EvaluationRun.inserted_at) - .cast(sa.Date) - .label("month") - ) - eval_type_lower = sa.func.lower(sa.func.coalesce(EvaluationRun.type, "")).label( - "type_lower" - ) - eval_provider_col = eval_provider_expr.label("provider") - eval_count_col = sa.func.count().label("eval_count") - eval_cost_col = sa.func.coalesce( - sa.func.sum(sa.cast(EvaluationRun.cost["total_cost_usd"].astext, sa.Numeric)), - 0, - ).label("eval_cost_usd") +def _validate_provider_filter(provider: str | None) -> None: + """Reject provider filters that aren't one of the canonical enum values. - eval_stmt = ( - select( - eval_month_col, - eval_type_lower, - eval_provider_col, - eval_count_col, - eval_cost_col, - ) - .select_from(EvaluationRun) - .outerjoin( - mc_lookup, - mc_lookup.c.model_name == EvaluationRun.providers[0].astext, - ) - .outerjoin( - ConfigVersion, - sa.and_( - ConfigVersion.config_id == EvaluationRun.config_id, - ConfigVersion.version == EvaluationRun.config_version, + A typo like `opena1` would otherwise silently return an empty result + set, which is indistinguishable from "no activity for openai". Surface + it as a 400 instead. + """ + if provider is not None and provider not in KNOWN_PROVIDERS: + raise HTTPException( + status_code=400, + detail=( + f"Unknown provider '{provider}'. " + f"Expected one of: {sorted(KNOWN_PROVIDERS)}." ), ) - .where(EvaluationRun.organization_id == organization_id) - .group_by(eval_month_col, eval_type_lower, eval_provider_col) - ) - if from_month is not None: - eval_stmt = eval_stmt.where(EvaluationRun.inserted_at >= from_month) - if end_date is not None: - eval_stmt = eval_stmt.where(EvaluationRun.inserted_at < end_date) - if project_id is not None: - eval_stmt = eval_stmt.where(EvaluationRun.project_id == project_id) - if provider_filter is not None: - eval_stmt = eval_stmt.where(eval_provider_expr == provider_filter) - - for row in session.exec(eval_stmt).all(): - eval_modality = _EVAL_TYPE_TO_MODALITY.get(row.type_lower, Modality.OTHER) - if modality_filter is not None and eval_modality is not modality_filter: - continue - key = (row.month, eval_modality, row.provider) - bucket = buckets[key] - bucket["eval_runs"] += row.eval_count - bucket["eval_cost_usd"] += Decimal(str(row.eval_cost_usd or 0)) - - return buckets -def _bucket_value(bucket: dict[str, int | Decimal], metric: AnalyticsMetric) -> Decimal: +def _bucket_value(bucket: Bucket, metric: AnalyticsMetric) -> Decimal: if metric is AnalyticsMetric.REQUESTS: return Decimal( int(bucket["llm_call_requests"]) + int(bucket["llm_chain_requests"]) @@ -378,7 +126,12 @@ def get_monthly_analytics( | None = Query(None, description="Filter to a single modality bucket."), provider: str | None = Query( - None, description="Filter to a single provider (e.g. 'openai', 'google')." + None, + description=( + "Filter to a single provider. Must be one of the canonical " + "model_config values: 'openai', 'google', 'sarvamai', 'elevenlabs'. " + "Anything else returns 400." + ), ), project_id: int | None = Query( @@ -403,8 +156,11 @@ def get_monthly_analytics( if project_id is not None else (current_user.project.id if current_user.project else None) ) + _validate_provider_filter(provider) + from_month = _snap_to_first_of_month(from_month) + to_month = _snap_to_first_of_month(to_month) effective_from_month = from_month or _default_from_month(to_month or date.today()) - buckets = _aggregate_live( + buckets = aggregate_monthly_metrics( session=session, organization_id=current_user.organization_.id, from_month=effective_from_month, @@ -465,7 +221,15 @@ def get_monthly_analytics_chart( ), modality: Modality | None = Query(None, description="Filter to a single modality bucket."), - provider: str | None = Query(None, description="Filter to a single provider."), + provider: str + | None = Query( + None, + description=( + "Filter to a single provider. Must be one of the canonical " + "model_config values: 'openai', 'google', 'sarvamai', 'elevenlabs'. " + "Anything else returns 400." + ), + ), project_id: int | None = Query( None, @@ -496,8 +260,11 @@ def get_monthly_analytics_chart( if project_id is not None else (current_user.project.id if current_user.project else None) ) + _validate_provider_filter(provider) + from_month = _snap_to_first_of_month(from_month) + to_month = _snap_to_first_of_month(to_month) effective_from_month = from_month or _default_from_month(to_month or date.today()) - buckets = _aggregate_live( + buckets = aggregate_monthly_metrics( session=session, organization_id=current_user.organization_.id, from_month=effective_from_month, diff --git a/backend/app/crud/model_config.py b/backend/app/crud/model_config.py index 9c627f7f4..3c72d34e6 100644 --- a/backend/app/crud/model_config.py +++ b/backend/app/crud/model_config.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Any, Literal, get_args from fastapi import HTTPException from sqlmodel import Session, select @@ -9,10 +9,23 @@ Provider = Literal["openai", "google", "sarvamai", "elevenlabs"] +# Runtime view of the Provider Literal. Use this anywhere the `global.provider_enum` +# values are needed (filter validation, cost-lookup guards) so the set stays in sync +# with the Literal definition. +KNOWN_PROVIDERS: frozenset[str] = frozenset(get_args(Provider)) + +# Suffix that distinguishes a NativeCompletionConfig provider (e.g. "openai-native") +# from the canonical provider name stored in model_config ("openai"). +NATIVE_PROVIDER_SUFFIX = "-native" + def _normalize_provider(raw: str) -> str: """Map NativeCompletionConfig providers (e.g. 'openai-native') to model_config provider names.""" - return raw[: -len("-native")] if raw.endswith("-native") else raw + return ( + raw[: -len(NATIVE_PROVIDER_SUFFIX)] + if raw.endswith(NATIVE_PROVIDER_SUFFIX) + else raw + ) def list_active_model_configs( diff --git a/backend/app/services/analytics/__init__.py b/backend/app/services/analytics/__init__.py new file mode 100644 index 000000000..87f209b7e --- /dev/null +++ b/backend/app/services/analytics/__init__.py @@ -0,0 +1,7 @@ +from app.services.analytics.aggregation import ( + Bucket, + BucketKey, + aggregate_monthly_metrics, +) + +__all__ = ["Bucket", "BucketKey", "aggregate_monthly_metrics"] diff --git a/backend/app/services/analytics/aggregation.py b/backend/app/services/analytics/aggregation.py new file mode 100644 index 000000000..89b9e2357 --- /dev/null +++ b/backend/app/services/analytics/aggregation.py @@ -0,0 +1,354 @@ +"""Live monthly aggregation across llm_call, llm_chain, and evaluation_run. + +The route handlers in `app/api/routes/analytics.py` call +`aggregate_monthly_metrics`, which fans out to three private helpers — one +per source table. Each helper does its own GROUP BY in Postgres and merges +results into the shared `buckets` dict in place. Splitting the sources lets +future endpoints (e.g. daily rollups, scheduled jobs) reuse individual +aggregations without dragging in the others. +""" + +import logging +from collections import defaultdict +from datetime import date +from decimal import Decimal + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID as PG_UUID +from sqlalchemy.orm import aliased +from sqlmodel import Session, select + +from app.crud.model_config import ( + KNOWN_PROVIDERS, + NATIVE_PROVIDER_SUFFIX, + estimate_model_cost, +) +from app.models.analytics import Modality +from app.models.config.version import ConfigVersion +from app.models.evaluation import EvaluationRun +from app.models.llm.request import LlmCall, LlmChain +from app.models.model_config import ModelConfig + +logger = logging.getLogger(__name__) + + +BucketKey = tuple[date, Modality, str] +Bucket = dict[str, int | Decimal] + + +# evaluation_run.type (lowercased) -> modality bucket. +_EVAL_TYPE_TO_MODALITY: dict[str, Modality] = { + "text": Modality.T_FS_T, + "stt": Modality.STT, + "tts": Modality.TTS, +} + + +def _first_of_next_month(d: date) -> date: + if d.month == 12: + return date(d.year + 1, 1, 1) + return date(d.year, d.month + 1, 1) + + +def _empty_bucket() -> Bucket: + return { + "llm_call_requests": 0, + "llm_chain_requests": 0, + "cost_usd": Decimal("0"), + "input_tokens": 0, + "output_tokens": 0, + "eval_runs": 0, + "eval_cost_usd": Decimal("0"), + } + + +def _llm_modality_case(call=LlmCall) -> sa.sql.ColumnElement[str]: + """SQL CASE mapping llm_call.input_type/output_type to a modality string. + + Accepts the `LlmCall` class or an `aliased(LlmCall)` so callers that + join llm_call into another query (e.g. chain → first-block call) can + reuse the same classification logic. + """ + return sa.case( + ( + sa.and_(call.input_type == "text", call.output_type == "text"), + Modality.T_FS_T.value, + ), + ( + sa.and_(call.input_type == "audio", call.output_type == "audio"), + Modality.S_FS_S.value, + ), + ( + sa.and_(call.input_type == "audio", call.output_type == "text"), + Modality.STT.value, + ), + ( + sa.and_(call.input_type == "text", call.output_type == "audio"), + Modality.TTS.value, + ), + else_=Modality.OTHER.value, + ) + + +def _aggregate_llm_calls( + session: Session, + buckets: dict[BucketKey, Bucket], + *, + organization_id: int, + from_month: date | None, + end_date: date | None, + modality_filter: Modality | None, + provider_filter: str | None, + project_id: int | None, +) -> None: + """GROUP BY llm_call → merge counts, tokens, and per-group cost into buckets. + + Cost is computed per (provider, model) group via `estimate_model_cost` + using the summed tokens. The pricing function is linear in tokens, so + summing first and pricing once is equivalent to per-row pricing. Skipped + for providers outside `KNOWN_PROVIDERS` (the `model_config.provider` + enum) since the lookup would raise InvalidTextRepresentation. + """ + month_col = ( + sa.func.date_trunc("month", LlmCall.inserted_at).cast(sa.Date).label("month") + ) + modality_col = _llm_modality_case().label("modality") + provider_col = sa.func.coalesce(LlmCall.provider, "unknown").label("provider") + input_tokens_col = sa.func.coalesce( + sa.func.sum(sa.cast(LlmCall.usage["input_tokens"].astext, sa.Integer)), + 0, + ).label("input_tokens") + output_tokens_col = sa.func.coalesce( + sa.func.sum(sa.cast(LlmCall.usage["output_tokens"].astext, sa.Integer)), + 0, + ).label("output_tokens") + count_col = sa.func.count().label("request_count") + + stmt = ( + select( + month_col, + modality_col, + provider_col, + LlmCall.model, + count_col, + input_tokens_col, + output_tokens_col, + ) + .where( + LlmCall.deleted_at.is_(None), + LlmCall.organization_id == organization_id, + ) + .group_by(month_col, modality_col, provider_col, LlmCall.model) + ) + if from_month is not None: + stmt = stmt.where(LlmCall.inserted_at >= from_month) + if end_date is not None: + stmt = stmt.where(LlmCall.inserted_at < end_date) + if project_id is not None: + stmt = stmt.where(LlmCall.project_id == project_id) + if provider_filter is not None: + stmt = stmt.where(LlmCall.provider == provider_filter) + + for row in session.exec(stmt).all(): + modality_enum = Modality(row.modality) + if modality_filter is not None and modality_enum is not modality_filter: + continue + bucket = buckets[(row.month, modality_enum, row.provider)] + bucket["llm_call_requests"] += row.request_count + + input_tokens = int(row.input_tokens or 0) + output_tokens = int(row.output_tokens or 0) + bucket["input_tokens"] += input_tokens + bucket["output_tokens"] += output_tokens + if (input_tokens or output_tokens) and row.provider in KNOWN_PROVIDERS: + estimate = estimate_model_cost( + session=session, + provider=row.provider, # type: ignore[arg-type] + model_name=row.model, + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + if estimate is not None: + bucket["cost_usd"] += Decimal(str(estimate.get("total_cost", 0))) + + +def _aggregate_chains( + session: Session, + buckets: dict[BucketKey, Bucket], + *, + organization_id: int, + from_month: date | None, + end_date: date | None, + modality_filter: Modality | None, + provider_filter: str | None, + project_id: int | None, +) -> None: + """GROUP BY llm_chain → merge counts into buckets. + + A chain is attributed to the modality+provider of its first child call. + Single LEFT JOIN (llm_chain → llm_call on the first block UUID) + + GROUP BY in Postgres — no per-row Python materialization. Chains + with no resolvable first call land in (OTHER, "unknown") because the + joined columns are NULL. + """ + first_call = aliased(LlmCall) + first_block_uuid = sa.cast( + LlmChain.block_sequences[0].astext, PG_UUID(as_uuid=True) + ) + month_col = ( + sa.func.date_trunc("month", LlmChain.inserted_at).cast(sa.Date).label("month") + ) + modality_col = _llm_modality_case(first_call).label("modality") + provider_col = sa.func.coalesce(first_call.provider, "unknown").label("provider") + count_col = sa.func.count().label("chain_count") + + stmt = ( + select(month_col, modality_col, provider_col, count_col) + .select_from(LlmChain) + .outerjoin(first_call, first_call.id == first_block_uuid) + .where(LlmChain.organization_id == organization_id) + .group_by(month_col, modality_col, provider_col) + ) + if from_month is not None: + stmt = stmt.where(LlmChain.inserted_at >= from_month) + if end_date is not None: + stmt = stmt.where(LlmChain.inserted_at < end_date) + if project_id is not None: + stmt = stmt.where(LlmChain.project_id == project_id) + if provider_filter is not None: + stmt = stmt.where( + sa.func.coalesce(first_call.provider, "unknown") == provider_filter + ) + + for row in session.exec(stmt).all(): + chain_modality = Modality(row.modality) + if modality_filter is not None and chain_modality is not modality_filter: + continue + buckets[(row.month, chain_modality, row.provider)][ + "llm_chain_requests" + ] += row.chain_count + + +def _aggregate_evals( + session: Session, + buckets: dict[BucketKey, Bucket], + *, + organization_id: int, + from_month: date | None, + end_date: date | None, + modality_filter: Modality | None, + provider_filter: str | None, + project_id: int | None, +) -> None: + """GROUP BY evaluation_run → merge counts and cost into buckets. + + `EvaluationRun.providers` is misnamed: per its column comment it actually + stores *model names* (e.g. ['gemini-2.5-pro']), not providers. To recover + the real provider we look up the model in `model_config`. + + CAVEAT: `DISTINCT ON (model_name) ORDER BY (model_name, provider)` picks + the alphabetically-first provider when the same model_name exists under + multiple providers (the unique key is (provider, model_name), so this is + legal). Attribution is therefore best-effort and may be wrong for + models shared across providers. The `ConfigVersion.config_blob` branch + below is the authoritative source when present. + """ + mc_lookup = ( + select(ModelConfig.model_name, ModelConfig.provider) + .distinct(ModelConfig.model_name) + .order_by(ModelConfig.model_name, ModelConfig.provider) + .subquery() + ) + + cv_provider_normalized = sa.func.split_part( + sa.cast(ConfigVersion.config_blob["completion"]["provider"].astext, sa.String), + NATIVE_PROVIDER_SUFFIX, + 1, + ) + + provider_expr = sa.func.coalesce( + sa.cast(mc_lookup.c.provider, sa.String), + sa.func.nullif(cv_provider_normalized, ""), + "unknown", + ) + + month_col = ( + sa.func.date_trunc("month", EvaluationRun.inserted_at) + .cast(sa.Date) + .label("month") + ) + type_lower = sa.func.lower(sa.func.coalesce(EvaluationRun.type, "")).label( + "type_lower" + ) + provider_col = provider_expr.label("provider") + count_col = sa.func.count().label("eval_count") + cost_col = sa.func.coalesce( + sa.func.sum(sa.cast(EvaluationRun.cost["total_cost_usd"].astext, sa.Numeric)), + 0, + ).label("eval_cost_usd") + + stmt = ( + select(month_col, type_lower, provider_col, count_col, cost_col) + .select_from(EvaluationRun) + .outerjoin( + mc_lookup, + mc_lookup.c.model_name == EvaluationRun.providers[0].astext, + ) + .outerjoin( + ConfigVersion, + sa.and_( + ConfigVersion.config_id == EvaluationRun.config_id, + ConfigVersion.version == EvaluationRun.config_version, + ), + ) + .where(EvaluationRun.organization_id == organization_id) + .group_by(month_col, type_lower, provider_col) + ) + if from_month is not None: + stmt = stmt.where(EvaluationRun.inserted_at >= from_month) + if end_date is not None: + stmt = stmt.where(EvaluationRun.inserted_at < end_date) + if project_id is not None: + stmt = stmt.where(EvaluationRun.project_id == project_id) + if provider_filter is not None: + stmt = stmt.where(provider_expr == provider_filter) + + for row in session.exec(stmt).all(): + eval_modality = _EVAL_TYPE_TO_MODALITY.get(row.type_lower, Modality.OTHER) + if modality_filter is not None and eval_modality is not modality_filter: + continue + bucket = buckets[(row.month, eval_modality, row.provider)] + bucket["eval_runs"] += row.eval_count + bucket["eval_cost_usd"] += Decimal(str(row.eval_cost_usd or 0)) + + +def aggregate_monthly_metrics( + session: Session, + *, + organization_id: int, + from_month: date | None, + to_month: date | None, + modality_filter: Modality | None, + provider_filter: str | None, + project_id: int | None, +) -> dict[BucketKey, Bucket]: + """Live aggregation across llm_call, llm_chain, and evaluation_run. + + Returns a dict keyed by (month, modality, provider) with per-bucket + totals (requests, tokens, cost, eval runs, eval cost). The caller + decides which fields to surface for a given metric. + """ + end_date = _first_of_next_month(to_month) if to_month else None + buckets: dict[BucketKey, Bucket] = defaultdict(_empty_bucket) + common = dict( + organization_id=organization_id, + from_month=from_month, + end_date=end_date, + modality_filter=modality_filter, + provider_filter=provider_filter, + project_id=project_id, + ) + _aggregate_llm_calls(session, buckets, **common) + _aggregate_chains(session, buckets, **common) + _aggregate_evals(session, buckets, **common) + return buckets From 917608d7732e39570a5ad67eea49f1cbfe7ccb6d Mon Sep 17 00:00:00 2001 From: Ayush8923 <80516839+Ayush8923@users.noreply.github.com> Date: Tue, 26 May 2026 11:38:47 +0530 Subject: [PATCH 8/8] fix(analytics): test cases --- backend/app/api/routes/analytics.py | 34 ++++++----------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/backend/app/api/routes/analytics.py b/backend/app/api/routes/analytics.py index d9b76922c..4d85c3933 100644 --- a/backend/app/api/routes/analytics.py +++ b/backend/app/api/routes/analytics.py @@ -2,11 +2,10 @@ from datetime import date from decimal import Decimal -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Depends, Query from app.api.deps import AuthContextDep, SessionDep from app.api.permissions import Permission, require_permission -from app.crud.model_config import KNOWN_PROVIDERS from app.models import ( AnalyticsChartGroupBy, AnalyticsChartResponse, @@ -53,23 +52,6 @@ def _snap_to_first_of_month(d: date | None) -> date | None: return date(d.year, d.month, 1) -def _validate_provider_filter(provider: str | None) -> None: - """Reject provider filters that aren't one of the canonical enum values. - - A typo like `opena1` would otherwise silently return an empty result - set, which is indistinguishable from "no activity for openai". Surface - it as a 400 instead. - """ - if provider is not None and provider not in KNOWN_PROVIDERS: - raise HTTPException( - status_code=400, - detail=( - f"Unknown provider '{provider}'. " - f"Expected one of: {sorted(KNOWN_PROVIDERS)}." - ), - ) - - def _bucket_value(bucket: Bucket, metric: AnalyticsMetric) -> Decimal: if metric is AnalyticsMetric.REQUESTS: return Decimal( @@ -128,9 +110,9 @@ def get_monthly_analytics( | None = Query( None, description=( - "Filter to a single provider. Must be one of the canonical " - "model_config values: 'openai', 'google', 'sarvamai', 'elevenlabs'. " - "Anything else returns 400." + "Filter to a single provider (exact match against " + "`llm_call.provider`, e.g. 'openai', 'google', 'openai-native'). " + "Free-form: passes through to the SQL filter as-is." ), ), project_id: int @@ -156,7 +138,6 @@ def get_monthly_analytics( if project_id is not None else (current_user.project.id if current_user.project else None) ) - _validate_provider_filter(provider) from_month = _snap_to_first_of_month(from_month) to_month = _snap_to_first_of_month(to_month) effective_from_month = from_month or _default_from_month(to_month or date.today()) @@ -225,9 +206,9 @@ def get_monthly_analytics_chart( | None = Query( None, description=( - "Filter to a single provider. Must be one of the canonical " - "model_config values: 'openai', 'google', 'sarvamai', 'elevenlabs'. " - "Anything else returns 400." + "Filter to a single provider (exact match against " + "`llm_call.provider`, e.g. 'openai', 'google', 'openai-native'). " + "Free-form: passes through to the SQL filter as-is." ), ), project_id: int @@ -260,7 +241,6 @@ def get_monthly_analytics_chart( if project_id is not None else (current_user.project.id if current_user.project else None) ) - _validate_provider_filter(provider) from_month = _snap_to_first_of_month(from_month) to_month = _snap_to_first_of_month(to_month) effective_from_month = from_month or _default_from_month(to_month or date.today())