From f531dbba2391a823ce897bb569226f6fc2a3158d Mon Sep 17 00:00:00 2001 From: salmonumbrella <182032677+salmonumbrella@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:07:43 -0400 Subject: [PATCH 1/2] docs(meetings): design generic ingestion endpoint Generic single-meeting ingestion needs one coherent change so the daemon API, persistence layer, TUI presentation, generated clients, and published contract evolve together. Keeping the browser contract generated from the same OpenAPI source prevents typed web consumers from silently missing the endpoint. Follow-up fixes consolidated: - keep imported sources out of account scheduling - preserve scheduled Teams and Discord sources - keep unchanged retries idempotent while repairing statistics - scope statistics recomputation to the affected conversation - refresh and preserve identity-derived attribution - support slow bounded uploads without weakening unauthenticated limits - make the import lifecycle and participant persistence cancellation-safe - preserve reversible source and identity attribution provenance - migrate legacy meeting attribution safely across SQLite and PostgreSQL - preserve configured calendar-account attribution - copy subset messages safely across upgraded schema column order - avoid no-op source-wide attribution rewrites - derive attribution atomically during initial ingestion and re-sync - preserve legacy Calendar organizer attribution provenance - refresh attribution after participant and identifier repoints - reconcile confirmed identities during upgrades and invalidate stale caches Generated with Codex Co-authored-by: Wes McKinney Co-authored-by: Codex --- api/openapi.yaml | 194 +++- cmd/msgvault/cmd/build_cache.go | 19 +- cmd/msgvault/cmd/build_cache_identity_test.go | 8 + cmd/msgvault/cmd/meeting_import_e2e_test.go | 123 +++ cmd/msgvault/cmd/serve.go | 26 +- cmd/msgvault/cmd/store_adapter_test.go | 29 + docs/api-server.md | 49 +- docs/usage/meetings.md | 46 +- internal/api/explore.go | 2 +- internal/api/handlers.go | 76 +- internal/api/handlers_test.go | 114 +++ internal/api/json_body.go | 6 +- internal/api/meeting_import.go | 187 ++++ internal/api/meeting_import_test.go | 489 ++++++++++ internal/api/middleware.go | 2 +- internal/api/openapi.go | 64 +- internal/api/openapi_test.go | 128 +++ internal/api/operation_gate.go | 13 +- internal/api/routes.go | 11 +- internal/api/scheduler_jobs.go | 35 + internal/api/server.go | 26 +- internal/api/server_test.go | 37 +- internal/api/settings_test.go | 2 +- internal/calsync/calsync.go | 39 +- internal/calsync/calsync_test.go | 42 + internal/calsync/incremental.go | 4 + internal/calsync/persist.go | 26 +- internal/circleback/importer.go | 19 +- internal/granola/importer.go | 19 +- internal/meetingimport/decode.go | 51 + internal/meetingimport/decode_test.go | 144 +++ internal/meetingimport/format.go | 204 ++++ internal/meetingimport/format_test.go | 152 +++ internal/meetingimport/importer.go | 295 ++++++ internal/meetingimport/importer_test.go | 885 ++++++++++++++++++ internal/meetingimport/models.go | 302 ++++++ internal/meetingimport/models_test.go | 181 ++++ internal/store/account_identities.go | 74 +- internal/store/account_identities_test.go | 162 ++++ .../attribution_provenance_migration_test.go | 325 +++++++ internal/store/dialect_pg.go | 2 + internal/store/dialect_sqlite.go | 2 + internal/store/messages.go | 492 ++++++++-- .../store/migrate_init_schema_ledger_test.go | 11 +- internal/store/migrate_legacy_identity.go | 7 + .../store/migrate_legacy_identity_test.go | 28 + internal/store/migrate_phone_unique.go | 18 +- internal/store/migrate_phone_unique_test.go | 14 + internal/store/migrations.go | 177 ++++ .../store/participant_identifiers_test.go | 46 + internal/store/participant_links_test.go | 29 + internal/store/schema.sql | 2 + internal/store/schema_pg.sql | 2 + internal/store/store.go | 60 +- internal/store/store_test.go | 69 ++ internal/store/subset.go | 16 +- internal/store/subset_test.go | 66 ++ internal/store/sync.go | 30 +- internal/store/sync_context_test.go | 115 +++ internal/tui/meeting_mode_test.go | 18 +- internal/tui/meeting_state.go | 3 +- internal/tui/meeting_view.go | 12 +- internal/tui/meeting_view_test.go | 28 +- pkg/client/client.go | 24 + pkg/client/client_test.go | 35 + pkg/client/generated/client.go | 68 ++ pkg/client/generated/client_options.go | 44 + pkg/client/generated/client_with_response.go | 66 ++ pkg/client/generated/enums.go | 17 + pkg/client/generated/payloads.go | 2 + pkg/client/generated/responses.go | 14 + pkg/client/generated/types.go | 129 +++ pkg/client/openapi.yaml | 157 +++- web/src/lib/api/generated/schema.d.ts | 116 +++ .../sources/SourcesWorkspace.svelte | 8 + .../sources/SourcesWorkspace.test.ts | 18 + 76 files changed, 6309 insertions(+), 246 deletions(-) create mode 100644 cmd/msgvault/cmd/meeting_import_e2e_test.go create mode 100644 internal/api/meeting_import.go create mode 100644 internal/api/meeting_import_test.go create mode 100644 internal/meetingimport/decode.go create mode 100644 internal/meetingimport/decode_test.go create mode 100644 internal/meetingimport/format.go create mode 100644 internal/meetingimport/format_test.go create mode 100644 internal/meetingimport/importer.go create mode 100644 internal/meetingimport/importer_test.go create mode 100644 internal/meetingimport/models.go create mode 100644 internal/meetingimport/models_test.go create mode 100644 internal/store/attribution_provenance_migration_test.go create mode 100644 internal/store/sync_context_test.go diff --git a/api/openapi.yaml b/api/openapi.yaml index 18554c1b6..3bf94ea0c 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -2626,6 +2626,134 @@ components: strongest_excerpt: type: string type: object + Meeting: + additionalProperties: false + allOf: + - anyOf: + - properties: + summary_markdown: + minLength: 1 + type: string + required: + - summary_markdown + type: object + - properties: + summary_text: + minLength: 1 + type: string + required: + - summary_text + type: object + - properties: + transcript: + minLength: 1 + type: string + required: + - transcript + type: object + - properties: + transcript_segments: + minItems: 1 + type: array + required: + - transcript_segments + type: object + - not: + properties: + transcript: + minLength: 1 + type: string + transcript_segments: + minItems: 1 + type: array + required: + - transcript + - transcript_segments + type: object + properties: + attendees: + items: + $ref: "#/components/schemas/MeetingPerson" + type: + - array + - "null" + ended_at: + format: date-time + type: string + external_id: + maxLength: 256 + type: string + metadata: + additionalProperties: {} + type: object + organizer: + $ref: "#/components/schemas/MeetingPerson" + started_at: + format: date-time + type: string + summary_markdown: + type: string + summary_text: + type: string + title: + maxLength: 4096 + type: string + transcript: + type: string + transcript_segments: + items: + $ref: "#/components/schemas/TranscriptSegment" + type: + - array + - "null" + required: + - external_id + - started_at + type: object + MeetingImportRequest: + additionalProperties: false + properties: + meeting: + $ref: "#/components/schemas/Meeting" + source: + $ref: "#/components/schemas/Source" + required: + - source + - meeting + type: object + MeetingImportResponse: + additionalProperties: true + properties: + message_id: + format: int64 + type: integer + source_id: + format: int64 + type: integer + source_message_id: + type: string + status: + enum: + - created + - updated + type: string + required: + - status + - source_id + - message_id + - source_message_id + type: object + MeetingPerson: + additionalProperties: false + properties: + email: + format: email + type: string + name: + type: string + required: + - email + type: object MessageDetail: additionalProperties: true properties: @@ -3798,6 +3926,22 @@ components: - generation - messages type: object + Source: + additionalProperties: false + properties: + account_email: + format: email + type: string + display_name: + maxLength: 256 + type: string + identifier: + maxLength: 128 + type: string + required: + - identifier + - account_email + type: object SourceCount: additionalProperties: true properties: @@ -4506,6 +4650,21 @@ components: - label_count - account_count type: object + TranscriptSegment: + additionalProperties: false + properties: + offset_seconds: + format: double + minimum: 0 + type: number + speaker: + type: string + text: + type: string + required: + - speaker + - text + type: object UpdateRequest: additionalProperties: false properties: @@ -4545,7 +4704,7 @@ components: type: apiKey info: title: msgvault API - version: 1.32.0 + version: 1.33.0 openapi: 3.1.0 paths: /api/ping: @@ -7263,6 +7422,39 @@ paths: summary: Remove a link edge between two participants tags: - API + /api/v1/import/meeting: + post: + operationId: importMeeting + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportResponse" + description: OK + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportResponse" + description: Created + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Import one meeting + tags: + - API /api/v1/integrations/tasks/search: get: operationId: searchIntegrationTasks diff --git a/cmd/msgvault/cmd/build_cache.go b/cmd/msgvault/cmd/build_cache.go index c3a7a09c8..be0261140 100644 --- a/cmd/msgvault/cmd/build_cache.go +++ b/cmd/msgvault/cmd/build_cache.go @@ -735,6 +735,17 @@ func buildCacheLocked( return nil, fmt.Errorf("inspect attachment MIME schema: %w", err) } sourceSnapshot.hasAttachmentMIME = attachmentMIMEColumnCount > 0 + var messageSourceAttributionColumnCount int + if err := sourceSnapshot.QueryRow(` + SELECT COUNT(*) FROM pragma_table_info('messages') + WHERE name = 'source_is_from_me' + `).Scan(&messageSourceAttributionColumnCount); err != nil { + return nil, fmt.Errorf("inspect message attribution schema: %w", err) + } + messageSourceAttribution := "COALESCE(m.is_from_me, FALSE)" + if messageSourceAttributionColumnCount > 0 { + messageSourceAttribution = "COALESCE(m.source_is_from_me, FALSE)" + } if err := sourceSnapshot.Prepare(); err != nil { return nil, err } @@ -1067,7 +1078,7 @@ func buildCacheLocked( m.deleted_from_source_at, m.sender_id, COALESCE(TRY_CAST(m.message_type AS VARCHAR), '') as message_type, - (COALESCE(m.is_from_me, FALSE) OR EXISTS ( + (%s OR EXISTS ( SELECT 1 FROM sqlite_db.account_identities ai JOIN sqlite_db.participants sp ON sp.id = m.sender_id WHERE ai.source_id = m.source_id @@ -1090,7 +1101,7 @@ func buildCacheLocked( OVERWRITE_OR_IGNORE, COMPRESSION 'zstd' ) - `, idFilter, escapedMessagesDir)); err != nil { + `, messageSourceAttribution, idFilter, escapedMessagesDir)); err != nil { return nil, fmt.Errorf("export messages: %w", err) } @@ -1126,7 +1137,7 @@ func buildCacheLocked( m.deleted_from_source_at, m.sender_id, COALESCE(TRY_CAST(m.message_type AS VARCHAR), '') as message_type, - (COALESCE(m.is_from_me, FALSE) OR EXISTS ( + (%s OR EXISTS ( SELECT 1 FROM sqlite_db.account_identities ai JOIN sqlite_db.participants sp ON sp.id = m.sender_id WHERE ai.source_id = m.source_id @@ -1143,7 +1154,7 @@ func buildCacheLocked( FROM sqlite_db.messages m WHERE 1 = 0 ) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd') - `, escapedEmptyShard)); err != nil { + `, messageSourceAttribution, escapedEmptyShard)); err != nil { return nil, fmt.Errorf("export empty messages shard: %w", err) } } diff --git a/cmd/msgvault/cmd/build_cache_identity_test.go b/cmd/msgvault/cmd/build_cache_identity_test.go index 6fae598b5..fd1a804b8 100644 --- a/cmd/msgvault/cmd/build_cache_identity_test.go +++ b/cmd/msgvault/cmd/build_cache_identity_test.go @@ -129,6 +129,14 @@ func TestBuildCache_DerivesIsFromMeAndIdentityDatasets(t *testing.T) { }) require.NoError(err) require.NoError(st.ReplaceMessageRecipients(controlMsgID, "from", []int64{otherParticipantID}, []string{""})) + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages + SET is_from_me = TRUE, + source_is_from_me = FALSE, + identity_is_from_me = TRUE + WHERE id = ? + `), controlMsgID) + require.NoError(err, "simulate stale persisted effective attribution") require.NoError(st.Close()) diff --git a/cmd/msgvault/cmd/meeting_import_e2e_test.go b/cmd/msgvault/cmd/meeting_import_e2e_test.go new file mode 100644 index 000000000..f71e38c43 --- /dev/null +++ b/cmd/msgvault/cmd/meeting_import_e2e_test.go @@ -0,0 +1,123 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/meetingimport" + "go.kenn.io/msgvault/internal/testutil" +) + +const meetingImportE2EBody = `{ + "source": { + "identifier": "local-meetings", + "display_name": "Local Meetings", + "account_email": "user@example.com" + }, + "meeting": { + "external_id": "42", + "title": "Weekly planning", + "started_at": "2026-07-23T18:00:00Z", + "summary_text": "Initial summary.", + "transcript": "Speaker 1: initial transcript", + "organizer": {"name": "Test Organizer", "email": "organizer@example.com"}, + "attendees": [{"name": "Test Attendee", "email": "attendee@example.com"}] + } +}` + +func postMeetingImport( + t *testing.T, + srv *api.Server, + body string, +) api.MeetingImportResponse { + t.Helper() + req := httptest.NewRequest( + http.MethodPost, + "/api/v1/import/meeting", + bytes.NewBufferString(body), + ) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, req) + require.Contains(t, []int{http.StatusOK, http.StatusCreated}, resp.Code, "body: %s", resp.Body.String()) + var decoded api.MeetingImportResponse + require.NoError(t, json.NewDecoder(resp.Body).Decode(&decoded)) + return decoded +} + +func TestMeetingImportAPIToStoreUpdatesCanonicalMessage(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + adapter := &storeAPIAdapter{ + store: st, + meetingImporter: meetingimport.NewImporter(st, meetingimport.Hooks{ + AfterSourceSetup: func() error { return nil }, + RefreshCache: func(_ context.Context, _ string) error { return nil }, + }), + } + srv := api.NewServer( + &config.Config{Server: config.ServerConfig{}}, + adapter, + nil, + slog.New(slog.DiscardHandler), + ) + + created := postMeetingImport(t, srv, meetingImportE2EBody) + assert.Equal(meetingimport.StatusCreated, created.Status) + + getReq := httptest.NewRequest( + http.MethodGet, + fmt.Sprintf("/api/v1/messages/%d", created.MessageID), + nil, + ) + getResp := httptest.NewRecorder() + srv.Router().ServeHTTP(getResp, getReq) + require.Equal(http.StatusOK, getResp.Code, "body: %s", getResp.Body.String()) + var initial api.MessageDetail + require.NoError(json.NewDecoder(getResp.Body).Decode(&initial)) + assert.Equal("Weekly planning", initial.Subject) + assert.Contains(initial.Body, "Initial summary.") + assert.Contains(initial.Body, "Speaker 1: initial transcript") + assert.Contains(initial.To, "Test Attendee ") + + replacement := strings.ReplaceAll(meetingImportE2EBody, "Weekly planning", "Replacement title") + replacement = strings.ReplaceAll(replacement, "Initial summary.", "Replacement summary.") + replacement = strings.ReplaceAll(replacement, "Speaker 1: initial transcript", "Speaker 2: replacement transcript") + replacement = strings.Replace( + replacement, + `"attendees": [{"name": "Test Attendee", "email": "attendee@example.com"}]`, + `"attendees": []`, + 1, + ) + updated := postMeetingImport(t, srv, replacement) + assert.Equal(meetingimport.StatusUpdated, updated.Status) + assert.Equal(created.MessageID, updated.MessageID) + + getReq = httptest.NewRequest( + http.MethodGet, + fmt.Sprintf("/api/v1/messages/%d", updated.MessageID), + nil, + ) + getResp = httptest.NewRecorder() + srv.Router().ServeHTTP(getResp, getReq) + require.Equal(http.StatusOK, getResp.Code, "body: %s", getResp.Body.String()) + var current api.MessageDetail + require.NoError(json.NewDecoder(getResp.Body).Decode(¤t)) + assert.Equal("Replacement title", current.Subject) + assert.Contains(current.Body, "Replacement summary.") + assert.Contains(current.Body, "Speaker 2: replacement transcript") + assert.Empty(current.To) +} diff --git a/cmd/msgvault/cmd/serve.go b/cmd/msgvault/cmd/serve.go index d4b0b7c86..ff2e712bc 100644 --- a/cmd/msgvault/cmd/serve.go +++ b/cmd/msgvault/cmd/serve.go @@ -24,6 +24,7 @@ import ( "go.kenn.io/msgvault/internal/discord" "go.kenn.io/msgvault/internal/gmail" "go.kenn.io/msgvault/internal/granola" + "go.kenn.io/msgvault/internal/meetingimport" "go.kenn.io/msgvault/internal/microsoft" "go.kenn.io/msgvault/internal/oauth" "go.kenn.io/msgvault/internal/query" @@ -432,7 +433,18 @@ func runServe(cmd *cobra.Command, args []string) error { sched.Start() // Create adapters for the API interfaces - storeAdapter := &storeAPIAdapter{store: s, attachmentMaintenance: attachmentMaint, analyticsDir: cfg.AnalyticsDir()} + meetingImporter := meetingimport.NewImporter(s, meetingimport.Hooks{ + AfterSourceSetup: func() error { + return runPostSourceCreateMigrations(s) + }, + RefreshCache: rebuildCacheAfterScheduledSync, + }) + storeAdapter := &storeAPIAdapter{ + store: s, + attachmentMaintenance: attachmentMaint, + meetingImporter: meetingImporter, + analyticsDir: cfg.AnalyticsDir(), + } schedAdapter := &schedulerAdapter{scheduler: sched} // Create and start API server @@ -835,6 +847,7 @@ func newDaemonIdleTracker(c *config.Config, stop context.CancelFunc) *api.IdleTr type storeAPIAdapter struct { store *store.Store attachmentMaintenance *attachmentMaintenance + meetingImporter *meetingimport.Importer // analyticsDir is the daemon's Parquet analytics cache directory, used // to read the revision committed by the derived-refresh child. analyticsDir string @@ -842,6 +855,7 @@ type storeAPIAdapter struct { var _ api.MessageStore = (*storeAPIAdapter)(nil) var _ api.CtxMessageStore = (*storeAPIAdapter)(nil) +var _ api.MeetingImporter = (*storeAPIAdapter)(nil) var _ api.SourceStatusStore = (*storeAPIAdapter)(nil) var _ api.CLIStore = (*storeAPIAdapter)(nil) var _ api.ContextCLIStore = (*storeAPIAdapter)(nil) @@ -883,6 +897,16 @@ func (a *storeAPIAdapter) GetStats() (*api.StoreStats, error) { return a.store.GetStats() } +func (a *storeAPIAdapter) ImportMeeting( + ctx context.Context, + req meetingimport.Request, +) (meetingimport.Result, error) { + if a == nil || a.meetingImporter == nil { + return meetingimport.Result{}, meetingimport.ErrUnavailable + } + return a.meetingImporter.Import(ctx, req) +} + func (a *storeAPIAdapter) GetStatsContext(ctx context.Context) (*api.StoreStats, error) { return a.store.GetStatsContext(ctx) } diff --git a/cmd/msgvault/cmd/store_adapter_test.go b/cmd/msgvault/cmd/store_adapter_test.go index 6b38d6f8b..f846e7dc2 100644 --- a/cmd/msgvault/cmd/store_adapter_test.go +++ b/cmd/msgvault/cmd/store_adapter_test.go @@ -14,6 +14,7 @@ import ( "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/apiprotocol" "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/meetingimport" "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" @@ -30,6 +31,7 @@ func TestStoreAPIAdapterImplementsCtxMessageStore(t *testing.T) { } var _ api.CtxMessageStore = (*storeAPIAdapter)(nil) +var _ api.MeetingImporter = (*storeAPIAdapter)(nil) type scopedStatsProductionAdapter struct { *storeAPIAdapter @@ -382,6 +384,33 @@ func TestStoreAPIAdapterContextReadsHonorCancellation(t *testing.T) { require.ErrorIs(err, context.Canceled, "GetMessagesSummariesByIDsContext must honor a cancelled context") } +func TestStoreAPIAdapterMeetingImport(t *testing.T) { + require := require.New(t) + + st := testutil.NewTestStore(t) + req, err := meetingimport.DecodeRequest(strings.NewReader(`{ + "source": { + "identifier": "local-meetings", + "account_email": "user@example.com" + }, + "meeting": { + "external_id": "42", + "started_at": "2026-07-23T18:00:00Z", + "transcript": "Speaker 1: synthetic transcript" + } + }`), meetingimport.MaxRequestBytes) + require.NoError(err) + adapter := &storeAPIAdapter{ + store: st, + meetingImporter: meetingimport.NewImporter(st, meetingimport.Hooks{}), + } + + result, err := adapter.ImportMeeting(context.Background(), req) + require.NoError(err) + require.Equal(meetingimport.StatusCreated, result.Status) + require.NotZero(result.MessageID) +} + var _ api.ConversationWindowStore = (*storeAPIAdapter)(nil) // TestStoreAPIAdapterImplementsConversationWindowStore is a compile-time diff --git a/docs/api-server.md b/docs/api-server.md index cc8dba9e5..cbe3251a0 100644 --- a/docs/api-server.md +++ b/docs/api-server.md @@ -579,6 +579,53 @@ with a run-level error. --- +### Import a meeting {#post-apiv1importmeeting} + +**Endpoint:** `POST /api/v1/import/meeting` + +Import one provider-neutral meeting without configuring a provider account. +The authenticated, on-demand endpoint accepts at most 16 MiB of JSON and is +idempotent on `source.identifier` plus `meeting.external_id`: the first import +returns `201` / `created`, and retries return `200` / `updated`. + +```bash +curl http://localhost:8080/api/v1/import/meeting \ + -H "Authorization: Bearer your-secret-key" \ + -H "Content-Type: application/json" \ + --data '{ + "source": { + "identifier": "local-meetings", + "display_name": "Local Meetings", + "account_email": "you@example.com" + }, + "meeting": { + "external_id": "planning-2026-07-29", + "title": "Planning", + "started_at": "2026-07-29T09:00:00-04:00", + "summary_text": "Reviewed the launch plan.", + "organizer": {"name": "You", "email": "you@example.com"}, + "attendees": [{"name": "Teammate", "email": "teammate@example.com"}] + } + }' +``` + +```json +{ + "status": "created", + "source_id": 12, + "message_id": 901, + "source_message_id": "meeting:planning-2026-07-29" +} +``` + +Timestamps must be RFC 3339 values with explicit offsets. A meeting must +contain at least one non-empty `summary_markdown`, `summary_text`, `transcript`, +or `transcript_segments` value; plain and segmented transcripts are mutually +exclusive. Segment offsets must be finite, non-negative, and non-decreasing. +Unknown fields are rejected except within `meeting.metadata`. + +--- + ### OAuth token exchange {#post-apiv1authtokenemail} **Endpoint:** `POST /api/v1/auth/token/{email}` @@ -749,7 +796,7 @@ explore contract is in the generated OpenAPI document (`/openapi.json`). { "action": "open_in_source", "reason": "trusted_source_link_unavailable" } ], "action_targets": [], - "operation_token": "3q2fF0kaVYlIuXQ8yYb-KzGH5mo2vNc1", + "operation_token": "", "expires_at": "2026-07-06T15:35:00Z" } ``` diff --git a/docs/usage/meetings.md b/docs/usage/meetings.md index d62e4e722..c55be3fb4 100644 --- a/docs/usage/meetings.md +++ b/docs/usage/meetings.md @@ -37,14 +37,52 @@ source. Add other confirmed aliases with the identity command: msgvault identity add work you+meetings@example.com ``` -When the primary email or aliases change, run a full provider sync to repair -`is_from_me` on meetings already in the archive: +Adding a new confirmed identity immediately repairs `is_from_me` on matching +messages already stored for that source. No provider resync is required. + +## Import from any meeting source + +The provider-neutral import API archives one meeting at a time and requires no +`[[granola]]`, `[[circleback]]`, or other provider configuration. Configure an +API key, start `msgvault serve`, then send authenticated JSON to +`POST /api/v1/import/meeting`: ```bash -msgvault sync-granola work --full -msgvault sync-circleback work --full +curl http://localhost:8080/api/v1/import/meeting \ + -H "Authorization: Bearer your-secret-key" \ + -H "Content-Type: application/json" \ + --data '{ + "source": { + "identifier": "local-meetings", + "display_name": "Local Meetings", + "account_email": "you@example.com" + }, + "meeting": { + "external_id": "weekly-planning-42", + "title": "Weekly planning", + "started_at": "2026-07-29T09:00:00-04:00", + "summary_markdown": "## Decisions\n\nShip the new importer.", + "transcript_segments": [ + {"speaker": "Alex", "text": "Let's ship it.", "offset_seconds": 4} + ] + } + }' ``` +Choose a stable `source.identifier` for the upstream dataset and preserve the +upstream meeting ID as `meeting.external_id`. That pair is the idempotency key: +the first import returns `201` with status `created`; unchanged retries and +replacements return `200` with status `updated` without creating duplicates. +`source.account_email` identifies you for sender attribution and becomes a +confirmed identity for the whole source. + +Each meeting needs at least one summary, a plain transcript, or segmented +transcript. Plain and segmented transcripts are mutually exclusive. Timestamps +use RFC 3339 with an explicit offset, request bodies are limited to 16 MiB, and +provider-specific fields belong under `meeting.metadata`. These sources are +on-demand: import through the API again to add or update meetings rather than +using **Sync now** or a scheduler. + ## Granola ### Prerequisites diff --git a/internal/api/explore.go b/internal/api/explore.go index 19a4010c5..a300106a1 100644 --- a/internal/api/explore.go +++ b/internal/api/explore.go @@ -237,7 +237,7 @@ func registerExploreRoute[Req any, Resp any](api huma.API, operationID, path, su op.Responses[httpStatusKey(http.StatusServiceUnavailable)] = &huma.Response{ Description: http.StatusText(http.StatusServiceUnavailable), Content: map[string]*huma.MediaType{ - "application/json": {Schema: &huma.Schema{AnyOf: []*huma.Schema{ + applicationJSONMediaType: {Schema: &huma.Schema{AnyOf: []*huma.Schema{ schemaFor[ExploreCacheUnavailableResponse](api), schemaFor[ErrorResponse](api), }}}, diff --git a/internal/api/handlers.go b/internal/api/handlers.go index fc1f9f89a..716a126ec 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -366,7 +366,7 @@ type scoreBreakdown struct { // writeJSON writes a JSON response. func writeJSON(w http.ResponseWriter, status int, data any) { - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", applicationJSONMediaType) w.WriteHeader(status) // Headers already sent; if Encode fails mid-stream (broken pipe, // non-serializable value) there's no meaningful recovery beyond @@ -1364,17 +1364,13 @@ func (s *Server) sourceStatus(statusStore SourceStatusStore, source *store.Sourc if err := s.hydrateSyncRunStatus(statusStore, status.ActiveSync); err != nil { return SourceStatus{}, err } + scheduling := classifySourceScheduling(source.SourceType, source.Identifier) schedulerRunning := false if s.scheduler != nil { - // Dispatch by source type first: generic-job source types - // (synctech-sms, gcal, granola, circleback, beeper) are never - // governed by the account scheduler, even if their identifier - // happens to collide with a scheduled account email. Only fall - // back to the account-scheduler path for source types that don't - // map to a generic job, so gmail/imap behavior stays byte-identical. - if _, ok := SchedulerJobNameForSource(source.SourceType, source.Identifier); ok { - schedulerRunning = s.applyGenericJobStatus(&status, source) - } else { + switch scheduling.kind { + case sourceScheduleGeneric: + schedulerRunning = s.applyGenericJobStatus(&status, scheduling.jobName) + case sourceScheduleAccount: status.Scheduled = s.scheduler.IsScheduled(source.Identifier) for _, scheduled := range s.scheduler.Status() { if scheduled.Email != source.Identifier { @@ -1388,9 +1384,12 @@ func (s *Server) sourceStatus(statusStore SourceStatusStore, source *store.Sourc } break } + case sourceScheduleNonSchedulable: } } switch { + case scheduling.kind == sourceScheduleNonSchedulable: + status.SyncUnavailableReason = "source_not_schedulable" case status.ActiveSync != nil || schedulerRunning: status.SyncUnavailableReason = "sync_already_running" case s.scheduler == nil: @@ -1427,11 +1426,7 @@ func (s *Server) sourceStatus(statusStore SourceStatusStore, source *store.Sourc // SchedulerJobNameForSource) when source's type is driven by one of those // jobs rather than the account scheduler. It reports whether that job is // currently running (false if no matching job exists). -func (s *Server) applyGenericJobStatus(status *SourceStatus, source *store.Source) bool { - jobName, ok := SchedulerJobNameForSource(source.SourceType, source.Identifier) - if !ok { - return false - } +func (s *Server) applyGenericJobStatus(status *SourceStatus, jobName string) bool { for _, job := range s.scheduler.JobStatus() { if job.Name != jobName { continue @@ -1527,45 +1522,44 @@ func (s *Server) handleTriggerSync(w http.ResponseWriter, r *http.Request) { } sourceType := r.URL.Query().Get("source_type") - // Generic (non-account) sources — synctech-sms, gcal, granola, circleback, - // beeper — are driven by named generic scheduler jobs. The caller passes - // the source type explicitly so this dispatches authoritatively, without - // scanning the store for a matching identifier (which could otherwise - // collide with a scheduled account email and trigger the wrong sync). - if jobName, ok := SchedulerJobNameForSource(sourceType, account); ok { - if !s.scheduler.IsJobScheduled(jobName) { + scheduling := classifySourceScheduling(sourceType, account) + switch scheduling.kind { + case sourceScheduleNonSchedulable: + writeError(w, http.StatusBadRequest, "source_not_schedulable", "Source type cannot be scheduled: "+sourceType) + return + case sourceScheduleGeneric: + if !s.scheduler.IsJobScheduled(scheduling.jobName) { writeError(w, http.StatusNotFound, "not_found", "Account is not scheduled: "+account) return } - if err := s.scheduler.StartJob(jobName); err != nil { + if err := s.scheduler.StartJob(scheduling.jobName); err != nil { s.logger.Error("failed to trigger generic sync job", - "job", jobName, "identifier", account, "error", err) + "job", scheduling.jobName, "identifier", account, "error", err) writeError(w, http.StatusConflict, "sync_error", err.Error()) return } - s.logger.Info("generic sync triggered via API", "job", jobName, "identifier", account) + s.logger.Info("generic sync triggered via API", "job", scheduling.jobName, "identifier", account) writeJSON(w, http.StatusAccepted, StatusMessageResponse{ Status: "accepted", Message: "Sync started for " + account, }) return + case sourceScheduleAccount: + if !s.scheduler.IsScheduled(account) { + writeError(w, http.StatusNotFound, "not_found", "Account is not scheduled: "+account) + return + } + if err := s.scheduler.TriggerSync(account); err != nil { + s.logger.Error("failed to trigger sync", "account", account, "error", err) + writeError(w, http.StatusConflict, "sync_error", err.Error()) + return + } + s.logger.Info("sync triggered via API", "account", account) + writeJSON(w, http.StatusAccepted, StatusMessageResponse{ + Status: "accepted", + Message: "Sync started for " + account, + }) } - - // Account-scheduler sources (gmail, imap, ...) are keyed by email. - if !s.scheduler.IsScheduled(account) { - writeError(w, http.StatusNotFound, "not_found", "Account is not scheduled: "+account) - return - } - if err := s.scheduler.TriggerSync(account); err != nil { - s.logger.Error("failed to trigger sync", "account", account, "error", err) - writeError(w, http.StatusConflict, "sync_error", err.Error()) - return - } - s.logger.Info("sync triggered via API", "account", account) - writeJSON(w, http.StatusAccepted, StatusMessageResponse{ - Status: "accepted", - Message: "Sync started for " + account, - }) } // handleSchedulerStatus returns the scheduler status. diff --git a/internal/api/handlers_test.go b/internal/api/handlers_test.go index fdcd67690..b3a3ed904 100644 --- a/internal/api/handlers_test.go +++ b/internal/api/handlers_test.go @@ -38,6 +38,7 @@ import ( "go.kenn.io/msgvault/internal/deletion" "go.kenn.io/msgvault/internal/gcal" "go.kenn.io/msgvault/internal/granola" + "go.kenn.io/msgvault/internal/meetingimport" "go.kenn.io/msgvault/internal/opserr" "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/query/querytest" @@ -3614,6 +3615,119 @@ func TestHandleSourceStatusGenericIdentifierCollidesWithScheduledAccount(t *test assert.True(got.CanSync, "generic job is scheduled and not running") } +func TestMeetingImportIdentifierCollisionDoesNotEnableAccountSync(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + const collidingIdentifier = "shared@example.com" + + st := testutil.NewTestStore(t) + sched := newMockScheduler() + sched.scheduled[collidingIdentifier] = true + sched.statuses = []AccountStatus{{ + Email: collidingIdentifier, + Schedule: "0 3 * * *", + }} + _, err := st.GetOrCreateSource(meetingimport.SourceType, collidingIdentifier) + require.NoError(err, "GetOrCreateSource meeting import") + srv := NewServer( + &config.Config{Server: config.ServerConfig{APIPort: 8080}}, + st, + sched, + testLogger(), + ) + + statusReq := httptest.NewRequest( + http.MethodGet, + "/api/v1/sources/status?source_type="+meetingimport.SourceType, + nil, + ) + statusResp := httptest.NewRecorder() + srv.Router().ServeHTTP(statusResp, statusReq) + + require.Equal(http.StatusOK, statusResp.Code, statusResp.Body.String()) + var status SourceStatusResponse + require.NoError(json.NewDecoder(statusResp.Body).Decode(&status), "decode response") + require.Len(status.Sources, 1, "sources") + assert.False(status.Sources[0].Scheduled, "meeting imports are not scheduler jobs") + assert.False(status.Sources[0].CanSync, "meeting imports cannot be synced") + assert.Equal("source_not_schedulable", status.Sources[0].SyncUnavailableReason) + + triggerResp := servePOSTTestRequest( + srv, + "/api/v1/sync/"+collidingIdentifier+"?source_type="+meetingimport.SourceType, + ) + + assert.Equal(http.StatusBadRequest, triggerResp.Code, triggerResp.Body.String()) + assert.Empty(sched.startedJobs, "must not start a generic job") + assert.Empty(sched.triggeredJobs, "must not trigger the colliding account") +} + +func TestAccountScheduledSourceTypesSupportStatusAndTrigger(t *testing.T) { + cases := []struct { + name string + sourceType string + identifier string + }{ + {"teams", "teams", "alice@example.com"}, + {"discord", "discord", "113456789012345678"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + sched := newMockScheduler() + sched.scheduled[tc.identifier] = true + sched.statuses = []AccountStatus{{ + Email: tc.identifier, + Schedule: "0 */6 * * *", + }} + var triggeredIdentifier string + sched.triggerFn = func(identifier string) error { + triggeredIdentifier = identifier + return nil + } + + source, err := st.GetOrCreateSource(tc.sourceType, tc.identifier) + require.NoError(err, "GetOrCreateSource") + srv := NewServer( + &config.Config{Server: config.ServerConfig{APIPort: 8080}}, + st, + sched, + testLogger(), + ) + + statusReq := httptest.NewRequest( + http.MethodGet, + "/api/v1/sources/status?source_type="+tc.sourceType, + nil, + ) + statusResp := httptest.NewRecorder() + srv.Router().ServeHTTP(statusResp, statusReq) + + require.Equal(http.StatusOK, statusResp.Code, statusResp.Body.String()) + var status SourceStatusResponse + require.NoError(json.NewDecoder(statusResp.Body).Decode(&status), "decode response") + require.Len(status.Sources, 1, "sources") + assert.Equal(source.ID, status.Sources[0].ID, "source ID") + assert.True(status.Sources[0].Scheduled, "Scheduled") + assert.Equal("0 */6 * * *", status.Sources[0].Schedule, "Schedule") + assert.True(status.Sources[0].CanSync, "CanSync") + assert.Empty(status.Sources[0].SyncUnavailableReason, "SyncUnavailableReason") + + triggerResp := servePOSTTestRequest( + srv, + "/api/v1/sync/"+tc.identifier+"?source_type="+tc.sourceType, + ) + + require.Equal(http.StatusAccepted, triggerResp.Code, triggerResp.Body.String()) + assert.Equal(tc.identifier, triggeredIdentifier, "triggered account identifier") + assert.Empty(sched.startedJobs, "must not start a generic scheduler job") + }) + } +} + // TestSchedulerJobNameForSource covers every generic-job source type plus an // account-scheduler type (gmail), which must report ok=false since it's // governed by the account scheduler, not a generic job. diff --git a/internal/api/json_body.go b/internal/api/json_body.go index 60028d046..cebaa5942 100644 --- a/internal/api/json_body.go +++ b/internal/api/json_body.go @@ -10,6 +10,8 @@ import ( "github.com/danielgtaylor/huma/v2" ) +const applicationJSONMediaType = "application/json" + // operationDeclaresJSONRequestBody reports whether a raw-route operation // documents an application/json request body, which is how every JSON // mutation route in this package registers (jsonRequestBodyFor). It keys the @@ -19,7 +21,7 @@ func operationDeclaresJSONRequestBody(op *huma.Operation) bool { if op.RequestBody == nil { return false } - _, ok := op.RequestBody.Content["application/json"] + _, ok := op.RequestBody.Content[applicationJSONMediaType] return ok } @@ -43,7 +45,7 @@ func enforceJSONRequestMediaType(next http.HandlerFunc) http.HandlerFunc { func hasJSONContentType(r *http.Request) bool { mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) - return err == nil && mediaType == "application/json" + return err == nil && mediaType == applicationJSONMediaType } // requireSingleJSONValue verifies no second JSON value follows the one dec diff --git a/internal/api/meeting_import.go b/internal/api/meeting_import.go new file mode 100644 index 000000000..5fd0ea85a --- /dev/null +++ b/internal/api/meeting_import.go @@ -0,0 +1,187 @@ +package api + +import ( + "context" + "errors" + "mime" + "net/http" + + "github.com/danielgtaylor/huma/v2" + "go.kenn.io/msgvault/internal/meetingimport" +) + +const meetingImportEndpointPath = "/api/v1/import/meeting" + +type MeetingImporter interface { + ImportMeeting(ctx context.Context, req meetingimport.Request) (meetingimport.Result, error) +} + +type MeetingImportResponse struct { + Status meetingimport.Status `json:"status" enum:"created,updated"` + SourceID int64 `json:"source_id"` + MessageID int64 `json:"message_id"` + SourceMessageID string `json:"source_message_id"` +} + +func (s *Server) registerMeetingImportRoute(api huma.API) { + op := rawAPIV1Operation( + "importMeeting", + http.MethodPost, + "/import/meeting", + "Import one meeting", + ) + op.RequestBody = jsonRequestBodyFor[meetingimport.Request](api) + hardenMeetingImportSchemas(api.OpenAPI()) + op.Responses = jsonResponsesFor[MeetingImportResponse]( + api, + http.StatusOK, + http.StatusCreated, + ) + op.Errors = []int{ + http.StatusBadRequest, + http.StatusUnauthorized, + http.StatusRequestEntityTooLarge, + http.StatusUnsupportedMediaType, + http.StatusUnprocessableEntity, + http.StatusInternalServerError, + http.StatusServiceUnavailable, + } + registerRawHumaRoute(api, op, s.handleMeetingImport) +} + +func hardenMeetingImportSchemas(doc *huma.OpenAPI) { + if doc == nil || doc.Components == nil || doc.Components.Schemas == nil { + return + } + meeting := doc.Components.Schemas.Map()["Meeting"] + if meeting == nil { + return + } + one := 1 + contentRequired := []*huma.Schema{ + { + Type: huma.TypeObject, + Required: []string{"summary_markdown"}, + Properties: map[string]*huma.Schema{ + "summary_markdown": {Type: huma.TypeString, MinLength: &one}, + }, + }, + { + Type: huma.TypeObject, + Required: []string{"summary_text"}, + Properties: map[string]*huma.Schema{ + "summary_text": {Type: huma.TypeString, MinLength: &one}, + }, + }, + { + Type: huma.TypeObject, + Required: []string{"transcript"}, + Properties: map[string]*huma.Schema{ + "transcript": {Type: huma.TypeString, MinLength: &one}, + }, + }, + { + Type: huma.TypeObject, + Required: []string{"transcript_segments"}, + Properties: map[string]*huma.Schema{ + "transcript_segments": {Type: huma.TypeArray, MinItems: &one}, + }, + }, + } + transcriptsExclusive := &huma.Schema{ + Type: huma.TypeObject, + Required: []string{"transcript", "transcript_segments"}, + Properties: map[string]*huma.Schema{ + "transcript": {Type: huma.TypeString, MinLength: &one}, + "transcript_segments": {Type: huma.TypeArray, MinItems: &one}, + }, + } + meeting.AllOf = []*huma.Schema{ + {AnyOf: contentRequired}, + {Not: transcriptsExclusive}, + } +} + +func (s *Server) handleMeetingImport(w http.ResponseWriter, r *http.Request) { + mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != applicationJSONMediaType { + writeError(w, http.StatusUnsupportedMediaType, "unsupported_media_type", + "Content-Type must be application/json") + return + } + + req, err := meetingimport.DecodeRequest(r.Body, meetingimport.MaxRequestBytes) + switch { + case errors.Is(err, meetingimport.ErrRequestTooLarge): + writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", + "Meeting import request exceeds 16 MiB") + return + case errors.Is(err, meetingimport.ErrMalformedRequest): + writeError(w, http.StatusBadRequest, "bad_request", "Invalid meeting import JSON") + return + case err != nil: + writeError(w, http.StatusBadRequest, "bad_request", "Invalid meeting import request") + return + } + if _, err := req.Normalize(); err != nil { + writeError(w, http.StatusUnprocessableEntity, "validation_failed", + "Meeting import request failed validation") + return + } + + importer, ok := s.store.(MeetingImporter) + if !ok || importer == nil { + writeError(w, http.StatusServiceUnavailable, "service_unavailable", + "Meeting import is unavailable") + return + } + + gateCtx, cancel := context.WithTimeout(r.Context(), operationGateWaitLimit) + defer cancel() + done, ok := s.beginLabeledOperationGateWork( + gateCtx, + operationGateLabelFromPath(meetingImportEndpointPath), + ) + if !ok { + writeOperationGateBusy(w, s.operationGate) + return + } + defer done() + + result, err := importer.ImportMeeting(r.Context(), req) + if err != nil { + if s.writeIfContextError(w, err) { + return + } + if errors.Is(err, meetingimport.ErrUnavailable) { + writeError(w, http.StatusServiceUnavailable, "service_unavailable", + "Meeting import is unavailable") + return + } + if errors.Is(err, meetingimport.ErrValidation) { + writeError(w, http.StatusUnprocessableEntity, "validation_failed", + "Meeting import request failed validation") + return + } + if s.logger != nil { + s.logger.Error("meeting import failed", + "source", req.Source.Identifier, + "external_id", req.Meeting.ExternalID, + "error_class", "internal") + } + writeError(w, http.StatusInternalServerError, "internal_error", + "Meeting import failed") + return + } + + status := http.StatusOK + if result.Status == meetingimport.StatusCreated { + status = http.StatusCreated + } + writeJSON(w, status, MeetingImportResponse{ + Status: result.Status, + SourceID: result.SourceID, + MessageID: result.MessageID, + SourceMessageID: result.SourceMessageID, + }) +} diff --git a/internal/api/meeting_import_test.go b/internal/api/meeting_import_test.go new file mode 100644 index 000000000..eab6f91b9 --- /dev/null +++ b/internal/api/meeting_import_test.go @@ -0,0 +1,489 @@ +package api + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/meetingimport" +) + +const ( + meetingImportTestAPIKey = "synthetic-api-key-for-tests" + validMeetingImportBody = `{ + "source": { + "identifier": "local-meetings", + "display_name": "Local Meetings", + "account_email": "user@example.com" + }, + "meeting": { + "external_id": "42", + "title": "Weekly planning", + "started_at": "2026-07-23T18:00:00Z", + "ended_at": "2026-07-23T18:30:00Z", + "summary_markdown": "## Summary\n\nReviewed the launch plan.", + "transcript_segments": [ + {"speaker": "Test Speaker", "text": "Review the launch plan.", "offset_seconds": 4} + ], + "organizer": {"name": "Test Organizer", "email": "organizer@example.com"}, + "attendees": [{"name": "Test Attendee", "email": "attendee@example.com"}], + "metadata": {"provider_key": "synthetic-value"} + } + }` +) + +type fakeMeetingImportStore struct { + *mockStore + + result meetingimport.Result + err error + calls int + req meetingimport.Request +} + +func (s *fakeMeetingImportStore) ImportMeeting( + _ context.Context, + req meetingimport.Request, +) (meetingimport.Result, error) { + s.calls++ + s.req = req + return s.result, s.err +} + +func newMeetingImportTestServer(t *testing.T, importer MeetingImporter) *Server { + t.Helper() + base := &mockStore{stats: &StoreStats{}} + var store MessageStore = base + if importer != nil { + typed, ok := importer.(MessageStore) + require.True(t, ok, "test importer must implement MessageStore") + store = typed + } + return NewServer( + &config.Config{Server: config.ServerConfig{APIKey: meetingImportTestAPIKey}}, + store, + nil, + testLogger(), + ) +} + +func meetingImportRequest(body string) *http.Request { + req := httptest.NewRequest(http.MethodPost, "/api/v1/import/meeting", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", meetingImportTestAPIKey) + return req +} + +func TestMeetingImportRequiresAuthentication(t *testing.T) { + store := &fakeMeetingImportStore{ + mockStore: &mockStore{stats: &StoreStats{}}, + result: meetingimport.Result{Status: meetingimport.StatusCreated}, + } + srv := newMeetingImportTestServer(t, store) + + for _, key := range []string{"", "wrong-key"} { + req := meetingImportRequest(validMeetingImportBody) + if key == "" { + req.Header.Del("X-Api-Key") + } else { + req.Header.Set("X-Api-Key", key) + } + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, req) + + assert.Equal(t, http.StatusUnauthorized, resp.Code, "key=%q body=%s", key, resp.Body.String()) + } + assert.Equal(t, 0, store.calls) +} + +func TestMeetingImportReturnsCreatedAndUpdated(t *testing.T) { + tests := []struct { + name string + status meetingimport.Status + wantStatus int + }{ + {name: "created", status: meetingimport.StatusCreated, wantStatus: http.StatusCreated}, + {name: "updated", status: meetingimport.StatusUpdated, wantStatus: http.StatusOK}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + store := &fakeMeetingImportStore{ + mockStore: &mockStore{stats: &StoreStats{}}, + result: meetingimport.Result{ + Status: tt.status, + SourceID: 3, + MessageID: 901, + SourceMessageID: "meeting:42", + }, + } + srv := newMeetingImportTestServer(t, store) + req := meetingImportRequest(validMeetingImportBody) + req.Header.Set("Content-Type", "application/json; charset=utf-8") + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, req) + + require.Equal(tt.wantStatus, resp.Code, "body: %s", resp.Body.String()) + assert.Equal(1, store.calls) + assert.Equal("42", store.req.Meeting.ExternalID) + var body MeetingImportResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal(tt.status, body.Status) + assert.Equal(int64(3), body.SourceID) + assert.Equal(int64(901), body.MessageID) + assert.Equal("meeting:42", body.SourceMessageID) + }) + } +} + +func TestMeetingImportRejectsInvalidRequests(t *testing.T) { + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := newMeetingImportTestServer(t, store) + + unknown := strings.Replace(validMeetingImportBody, `"external_id":`, `"unknown": true, "external_id":`, 1) + noContent := strings.Replace( + validMeetingImportBody, + `"summary_markdown": "## Summary\n\nReviewed the launch plan.",`, + `"summary_markdown": "",`, + 1, + ) + noContent = strings.Replace(noContent, + `"transcript_segments": [ + {"speaker": "Test Speaker", "text": "Review the launch plan.", "offset_seconds": 4} + ],`, + `"transcript_segments": [],`, + 1, + ) + + tests := []struct { + name string + body string + mediaType string + wantCode int + wantError string + }{ + {name: "malformed", body: `{"source":`, mediaType: "application/json", wantCode: http.StatusBadRequest, wantError: "bad_request"}, + {name: "trailing", body: validMeetingImportBody + `{}`, mediaType: "application/json", wantCode: http.StatusBadRequest, wantError: "bad_request"}, + {name: "unknown field", body: unknown, mediaType: "application/json", wantCode: http.StatusBadRequest, wantError: "bad_request"}, + {name: "semantic validation", body: noContent, mediaType: "application/json", wantCode: http.StatusUnprocessableEntity, wantError: "validation_failed"}, + {name: "wrong media type", body: validMeetingImportBody, mediaType: "text/plain", wantCode: http.StatusUnsupportedMediaType, wantError: "unsupported_media_type"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := meetingImportRequest(tt.body) + req.Header.Set("Content-Type", tt.mediaType) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, req) + + require.Equal(tt.wantCode, resp.Code, "body: %s", resp.Body.String()) + var body ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal(tt.wantError, body.Error) + assert.NotContains(body.Message, "Review the launch plan") + }) + } + assert.Equal(t, 0, store.calls) +} + +func TestMeetingImportRejectsOversizedBody(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := newMeetingImportTestServer(t, store) + req := meetingImportRequest(strings.Repeat("x", int(meetingimport.MaxRequestBytes)+1)) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, req) + + require.Equal(http.StatusRequestEntityTooLarge, resp.Code, "body: %s", resp.Body.String()) + var body ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal("request_too_large", body.Error) + assert.Equal(0, store.calls) +} + +func TestMeetingImportReturnsUnavailableWithoutCapability(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + srv := newMeetingImportTestServer(t, nil) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, meetingImportRequest(validMeetingImportBody)) + + require.Equal(http.StatusServiceUnavailable, resp.Code, "body: %s", resp.Body.String()) + var body ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal("service_unavailable", body.Error) +} + +func TestMeetingImportSanitizesInternalErrors(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + store := &fakeMeetingImportStore{ + mockStore: &mockStore{stats: &StoreStats{}}, + err: errors.New("database failed near secret transcript content"), + } + srv := newMeetingImportTestServer(t, store) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, meetingImportRequest(validMeetingImportBody)) + + require.Equal(http.StatusInternalServerError, resp.Code, "body: %s", resp.Body.String()) + var body ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal("internal_error", body.Error) + assert.NotContains(body.Message, "secret transcript content") +} + +func TestMeetingImportOpenAPIDocument(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := newMeetingImportTestServer(t, store) + req := httptest.NewRequest(http.MethodGet, "/openapi.json", nil) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, req) + + require.Equal(http.StatusOK, resp.Code) + var doc map[string]any + require.NoError(json.NewDecoder(resp.Body).Decode(&doc)) + paths, ok := doc["paths"].(map[string]any) + require.True(ok, "paths object") + path, ok := paths["/api/v1/import/meeting"].(map[string]any) + require.True(ok, "meeting import path object") + post, ok := path["post"].(map[string]any) + require.True(ok, "meeting import operation object") + assert.Equal("importMeeting", post["operationId"]) + assert.NotEmpty(post["security"]) + responses, ok := post["responses"].(map[string]any) + require.True(ok, "responses object") + assert.Contains(responses, "200") + assert.Contains(responses, "201") + + components, ok := doc["components"].(map[string]any) + require.True(ok, "components object") + schemas, ok := components["schemas"].(map[string]any) + require.True(ok, "schemas object") + meeting, ok := schemas["Meeting"].(map[string]any) + require.True(ok, "meeting component") + allOf, ok := meeting["allOf"].([]any) + require.True(ok, "meeting allOf constraints") + assert.Len(allOf, 2) + response, ok := schemas["MeetingImportResponse"].(map[string]any) + require.True(ok, "meeting response component") + properties, ok := response["properties"].(map[string]any) + require.True(ok, "meeting response properties") + status, ok := properties["status"].(map[string]any) + require.True(ok, "meeting response status") + assert.Equal([]any{"created", "updated"}, status["enum"]) +} + +func TestMeetingImportBodyLimitDoesNotReadPastBoundary(t *testing.T) { + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := newMeetingImportTestServer(t, store) + body := bytes.NewReader(bytes.Repeat([]byte("x"), int(meetingimport.MaxRequestBytes)+1)) + req := httptest.NewRequest(http.MethodPost, "/api/v1/import/meeting", body) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", meetingImportTestAPIKey) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, req) + + assert.Equal(t, http.StatusRequestEntityTooLarge, resp.Code) +} + +func TestMeetingImportReadsBodyBeforeWaitingOnOperationGate(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + const ordinaryReadTimeout = 100 * time.Millisecond + + gate := NewSerialOperationGate() + releaseGate, ok := gate.BeginWork() + require.True(ok, "hold operation gate") + releaseGateOnce := sync.OnceFunc(releaseGate) + defer releaseGateOnce() + + store := &fakeMeetingImportStore{ + mockStore: &mockStore{stats: &StoreStats{}}, + result: meetingimport.Result{ + Status: meetingimport.StatusCreated, + SourceID: 3, + MessageID: 901, + SourceMessageID: "meeting:42", + }, + } + srv := NewServerWithOptions(ServerOptions{ + Config: &config.Config{Server: config.ServerConfig{ + APIKey: meetingImportTestAPIKey, + }}, + Store: store, + Logger: testLogger(), + OperationGate: gate, + }) + srv.readTimeout = ordinaryReadTimeout + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(err, "listen") + serveErr := make(chan error, 1) + go func() { + serveErr <- srv.StartOnListener(listener) + }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(srv.Shutdown(ctx), "shutdown") + require.ErrorIs(<-serveErr, http.ErrServerClosed, "serve result") + }) + + conn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(err, "dial server") + defer func() { _ = conn.Close() }() + require.NoError(conn.SetDeadline(time.Now().Add(5*time.Second)), "bound test connection") + + body := []byte(validMeetingImportBody) + _, err = fmt.Fprintf(conn, + "POST /api/v1/import/meeting HTTP/1.1\r\n"+ + "Host: %s\r\n"+ + "Content-Type: application/json\r\n"+ + "X-Api-Key: %s\r\n"+ + "Content-Length: %d\r\n\r\n"+ + "%s", + listener.Addr().String(), + meetingImportTestAPIKey, + len(body), + body[:len(body)-1], + ) + require.NoError(err, "write headers and partial body") + + time.Sleep(2 * ordinaryReadTimeout) + assert.False(gate.HasRequestWaiters(), "partial upload must not hold or queue on the mutation gate") + _, err = conn.Write(body[len(body)-1:]) + require.NoError(err, "finish body after ordinary server read timeout") + require.Eventually(gate.HasRequestWaiters, time.Second, time.Millisecond, + "validated meeting import waits on operation gate") + releaseGateOnce() + + resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: http.MethodPost}) + require.NoError(err, "read meeting import response") + defer func() { _ = resp.Body.Close() }() + assert.Equal(http.StatusCreated, resp.StatusCode) + assert.Equal(1, store.calls) +} + +func TestMeetingImportValidatesBodyBeforeWaitingOnOperationGate(t *testing.T) { + require := require.New(t) + + oldLimit := operationGateWaitLimit + operationGateWaitLimit = 20 * time.Millisecond + t.Cleanup(func() { operationGateWaitLimit = oldLimit }) + + gate := NewSerialOperationGate() + releaseGate, ok := gate.BeginWork() + require.True(ok, "hold operation gate") + defer releaseGate() + + srv := NewServerWithOptions(ServerOptions{ + Config: &config.Config{Server: config.ServerConfig{ + APIKey: meetingImportTestAPIKey, + }}, + Store: &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}}, + Logger: testLogger(), + OperationGate: gate, + }) + + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, meetingImportRequest(`{}`)) + + require.Equal(http.StatusUnprocessableEntity, resp.Code, "body: %s", resp.Body.String()) + require.False(gate.HasRequestWaiters(), "invalid body must not queue on the mutation gate") +} + +func TestMeetingImportReturnsBusyWithinBoundAfterValidation(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + oldLimit := operationGateWaitLimit + operationGateWaitLimit = 20 * time.Millisecond + t.Cleanup(func() { operationGateWaitLimit = oldLimit }) + + gate := NewSerialOperationGate() + releaseGate, ok := gate.BeginLabeledWorkContext(context.Background(), "scheduled source sync") + require.True(ok, "hold operation gate") + defer releaseGate() + + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := NewServerWithOptions(ServerOptions{ + Config: &config.Config{Server: config.ServerConfig{ + APIKey: meetingImportTestAPIKey, + }}, + Store: store, + Logger: testLogger(), + OperationGate: gate, + }) + + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, meetingImportRequest(validMeetingImportBody)) + + require.Equal(http.StatusServiceUnavailable, resp.Code, "body: %s", resp.Body.String()) + var body ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal("operation_in_progress", body.Error) + assert.Contains(body.Message, "scheduled source sync") + assert.Equal(0, store.calls) +} + +func TestMeetingImportUnavailableDoesNotWaitOnOperationGate(t *testing.T) { + require := require.New(t) + + oldLimit := operationGateWaitLimit + operationGateWaitLimit = 20 * time.Millisecond + t.Cleanup(func() { operationGateWaitLimit = oldLimit }) + + gate := NewSerialOperationGate() + releaseGate, ok := gate.BeginWork() + require.True(ok, "hold operation gate") + defer releaseGate() + + srv := NewServerWithOptions(ServerOptions{ + Config: &config.Config{Server: config.ServerConfig{ + APIKey: meetingImportTestAPIKey, + }}, + Store: &mockStore{stats: &StoreStats{}}, + Logger: testLogger(), + OperationGate: gate, + }) + + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, meetingImportRequest(validMeetingImportBody)) + + require.Equal(http.StatusServiceUnavailable, resp.Code, "body: %s", resp.Body.String()) + require.Contains(resp.Body.String(), `"error":"service_unavailable"`) + require.False(gate.HasRequestWaiters(), "unavailable importer must not queue on the mutation gate") +} diff --git a/internal/api/middleware.go b/internal/api/middleware.go index d1579f960..8eba3d6e7 100644 --- a/internal/api/middleware.go +++ b/internal/api/middleware.go @@ -295,7 +295,7 @@ func RateLimitMiddleware( ip := clientIP(r) if !limiter.Allow(ip) { - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", applicationJSONMediaType) w.Header().Set("Retry-After", "1") w.WriteHeader(http.StatusTooManyRequests) _, _ = w.Write([]byte(`{"error":"rate_limit_exceeded","message":"Too many requests. Please slow down."}`)) diff --git a/internal/api/openapi.go b/internal/api/openapi.go index e5e831d39..dd3c9df14 100644 --- a/internal/api/openapi.go +++ b/internal/api/openapi.go @@ -152,7 +152,10 @@ import ( // profiles, update the display-name override and delete a profile with // revision-tag optimistic concurrency, and surface the covering profile on // the /people/{id} analytical detail. -const APISchemaVersion = "1.32.0" +// 1.33.0 adds provider-neutral single-meeting ingestion with strict request +// schemas and idempotent create/update responses. +// Additive (minor bump): the major-version compatibility gate stays at 1. +const APISchemaVersion = "1.33.0" // OpenAPIDocument builds the API schema from the same Huma route registration // used by the daemon. It binds no socket and needs no database. @@ -434,6 +437,36 @@ func applyClientCodegenExtensions(doc *huma.OpenAPI) { return } schemas := doc.Components.Schemas.Map() + const emailProperty = "email" + if meeting := schemas["Meeting"]; meeting != nil { + // The Go client generator treats composed object schemas as union + // wrappers. Runtime validation and the public schema retain these + // cross-field rules; the generated request keeps its useful struct shape. + meeting.AllOf = nil + for _, property := range []string{"started_at", "ended_at"} { + if timestamp := meeting.Properties[property]; timestamp != nil { + setCodegenGoType(timestamp, "string") + } + } + } + for schemaName, property := range map[string]string{ + "MeetingPerson": emailProperty, + "Source": "account_email", + } { + if schema := schemas[schemaName]; schema != nil { + if email := schema.Properties[property]; email != nil { + setCodegenGoType(email, "string") + } + } + } + queryResult := schemas["QueryResult"] + if queryResult != nil && queryResult.Properties != nil { + rows := queryResult.Properties["rows"] + if rows != nil && rows.Items != nil && rows.Items.Items != nil { + setCodegenGoType(rows.Items.Items, "any") + } + } + for _, schemaName := range []string{"FileSearchRow", "FileMetadataResponse"} { if schema := schemas[schemaName]; schema != nil { for _, property := range []string{"filename", "mime_type"} { @@ -478,6 +511,12 @@ func applyClientCodegenExtensions(doc *huma.OpenAPI) { "ExploreGroupDimensionSource", "ExploreGroupDimensionParticipant", "ExploreGroupDimensionDomain", "ExploreGroupDimensionMessageType", "ExploreGroupDimensionKind", "ExploreGroupDimensionYear", "ExploreGroupDimensionMonth", }) + if response := schemas["MeetingImportResponse"]; response != nil { + setEnumNames(response.Properties["status"], []any{ + "MeetingImportResponseStatusCreated", + "MeetingImportResponseStatusUpdated", + }) + } for schemaName, properties := range map[string]map[string][]any{ "ExploreCacheUnavailableResponse": { "readiness": {"ExploreCacheUnavailableResponseReadinessAbsent", "ExploreCacheUnavailableResponseReadinessInterrupted", "ExploreCacheUnavailableResponseReadinessStaleSchema", "ExploreCacheUnavailableResponseReadinessDrifted"}, @@ -517,19 +556,26 @@ func applyClientCodegenExtensions(doc *huma.OpenAPI) { setEnumNames(schema.Properties[propertyName], enumNames) } } - queryResult := schemas["QueryResult"] - if queryResult == nil || queryResult.Properties == nil { + meeting := schemas["Meeting"] + if meeting == nil || meeting.Properties == nil { return } - rows := queryResult.Properties["rows"] - if rows == nil || rows.Items == nil || rows.Items.Items == nil { + metadata := meeting.Properties["metadata"] + if metadata == nil { return } - cell := rows.Items.Items - if cell.Extensions == nil { - cell.Extensions = map[string]any{} + values, ok := metadata.AdditionalProperties.(*huma.Schema) + if !ok { + return + } + setCodegenGoType(values, "any") +} + +func setCodegenGoType(schema *huma.Schema, goType string) { + if schema.Extensions == nil { + schema.Extensions = map[string]any{} } - cell.Extensions["x-go-type"] = "any" + schema.Extensions["x-go-type"] = goType } func replaceStrictResponseAdditionalProperties(doc *huma.OpenAPI, replacement any) { diff --git a/internal/api/openapi_test.go b/internal/api/openapi_test.go index c5421af8d..6060ce64b 100644 --- a/internal/api/openapi_test.go +++ b/internal/api/openapi_test.go @@ -181,6 +181,109 @@ func TestOpenAPIFastSearchDocumentsSourceIDs(t *testing.T) { assert.Fail("source_ids query parameter is not documented for fastSearch") } +func TestOpenAPIMeetingImportContract(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + assert.Equal("1.33.0", APISchemaVersion, "meeting import is an additive schema release") + + doc := OpenAPIDocument() + path := doc.Paths["/api/v1/import/meeting"] + require.NotNil(path, "meeting import path") + op := path.Post + require.NotNil(op, "meeting import operation") + assert.Equal("importMeeting", op.OperationID) + require.Len(op.Security, 1, "API-key security requirement") + _, secured := op.Security[0]["apiKey"] + assert.True(secured, "apiKey security requirement") + + require.NotNil(op.RequestBody, "request body") + assert.True(op.RequestBody.Required, "request body is required") + requestMedia := op.RequestBody.Content["application/json"] + require.NotNil(requestMedia, "JSON request media type") + require.NotNil(requestMedia.Schema, "request schema") + assert.Equal("#/components/schemas/MeetingImportRequest", requestMedia.Schema.Ref) + + schemas := doc.Components.Schemas.Map() + requestSchema := schemas["MeetingImportRequest"] + require.NotNil(requestSchema, "request component") + requestAdditionalProperties, ok := requestSchema.AdditionalProperties.(bool) + require.True(ok, "request additionalProperties is boolean") + assert.False(requestAdditionalProperties, "request rejects unknown fields") + assert.ElementsMatch([]string{"source", "meeting"}, requestSchema.Required) + + for _, name := range []string{"Source", "Meeting", "MeetingPerson", "TranscriptSegment"} { + schema := schemas[name] + require.NotNil(schema, "%s component", name) + additionalProperties, ok := schema.AdditionalProperties.(bool) + require.True(ok, "%s additionalProperties is boolean", name) + assert.False(additionalProperties, "%s rejects unknown fields", name) + } + assert.ElementsMatch( + []string{"external_id", "started_at"}, + schemas["Meeting"].Required, + ) + source := schemas["Source"] + require.NotNil(source.Properties["identifier"].MaxLength) + assert.Equal(128, *source.Properties["identifier"].MaxLength) + require.NotNil(source.Properties["display_name"].MaxLength) + assert.Equal(256, *source.Properties["display_name"].MaxLength) + assert.Equal("email", source.Properties["account_email"].Format) + + meeting := schemas["Meeting"] + require.NotNil(meeting.Properties["external_id"].MaxLength) + assert.Equal(256, *meeting.Properties["external_id"].MaxLength) + require.NotNil(meeting.Properties["title"].MaxLength) + assert.Equal(4096, *meeting.Properties["title"].MaxLength) + assert.Equal("date-time", meeting.Properties["started_at"].Format) + assert.Equal("date-time", meeting.Properties["ended_at"].Format) + require.Len(meeting.AllOf, 2, "meeting cross-field constraints") + require.Len(meeting.AllOf[0].AnyOf, 4, "meeting requires a non-empty summary or transcript") + assert.ElementsMatch( + []string{"summary_markdown", "summary_text", "transcript", "transcript_segments"}, + []string{ + meeting.AllOf[0].AnyOf[0].Required[0], + meeting.AllOf[0].AnyOf[1].Required[0], + meeting.AllOf[0].AnyOf[2].Required[0], + meeting.AllOf[0].AnyOf[3].Required[0], + }, + ) + for idx, property := range []string{"summary_markdown", "summary_text", "transcript"} { + require.NotNil(meeting.AllOf[0].AnyOf[idx].Properties[property].MinLength) + assert.Equal(1, *meeting.AllOf[0].AnyOf[idx].Properties[property].MinLength) + } + require.NotNil(meeting.AllOf[0].AnyOf[3].Properties["transcript_segments"].MinItems) + assert.Equal(1, *meeting.AllOf[0].AnyOf[3].Properties["transcript_segments"].MinItems) + require.NotNil(meeting.AllOf[1].Not, "plain and segmented transcripts are mutually exclusive") + assert.ElementsMatch( + []string{"transcript", "transcript_segments"}, + meeting.AllOf[1].Not.Required, + ) + + assert.Equal("email", schemas["MeetingPerson"].Properties["email"].Format) + offset := schemas["TranscriptSegment"].Properties["offset_seconds"] + require.NotNil(offset.Minimum) + assert.Zero(*offset.Minimum) + + metadata := schemas["Meeting"].Properties["metadata"] + require.NotNil(metadata, "metadata schema") + _, extensible := metadata.AdditionalProperties.(*huma.Schema) + assert.True(extensible, "metadata accepts provider-specific values") + + responseSchema := schemas["MeetingImportResponse"] + require.NotNil(responseSchema, "meeting import response component") + assert.Equal([]any{"created", "updated"}, responseSchema.Properties["status"].Enum) + + for _, status := range []string{"200", "201"} { + response := op.Responses[status] + require.NotNil(response, "response %s", status) + media := response.Content["application/json"] + require.NotNil(media, "response %s JSON media type", status) + require.NotNil(media.Schema, "response %s schema", status) + assert.Equal("#/components/schemas/MeetingImportResponse", media.Schema.Ref) + } +} + func TestOpenAPIBinaryRoutesDocumentJSONErrors(t *testing.T) { doc := OpenAPIDocument() routes := map[string]struct { @@ -459,6 +562,31 @@ func TestOpenAPIClientArtifactUpToDate(t *testing.T) { } } +func TestOpenAPIGeneratedMeetingImportClient(t *testing.T) { + assertGeneratedFileContains(t, "client.go", + "ImportMeeting(ctx context.Context, options *ImportMeetingRequestOptions") + assertGeneratedFileContains(t, "client_options.go", + "type ImportMeetingRequestOptions struct") + assertGeneratedFileContains(t, "payloads.go", + "type ImportMeetingBody = MeetingImportRequest") + assertGeneratedFileContains(t, "responses.go", + "type ImportMeetingResp struct") + assertGeneratedFileContains(t, "types.go", + "type MeetingImportResponse struct") + assertGeneratedFileContains(t, "types.go", + "Metadata map[string]any") + assertGeneratedFileContains(t, "enums.go", + `MeetingImportResponseStatusUpdated MeetingImportResponseStatus = "updated"`) +} + +func assertGeneratedFileContains(t *testing.T, name, expected string) { + t.Helper() + content, err := os.ReadFile(filepath.Join(openAPIClientGeneratedDir, name)) + require.NoError(t, err, "read generated client file %s", name) + assert.Contains(t, string(content), expected, + "%s is missing the meeting import contract; run `make api-generate`", name) +} + func normalizeGeneratedArtifact(src []byte) string { return strings.ReplaceAll(string(src), "\r\n", "\n") } diff --git a/internal/api/operation_gate.go b/internal/api/operation_gate.go index fd11deebf..4a0e0bfa7 100644 --- a/internal/api/operation_gate.go +++ b/internal/api/operation_gate.go @@ -293,20 +293,21 @@ func writeOperationGateBusy(w http.ResponseWriter, gate OperationGate) { writeError(w, http.StatusServiceUnavailable, "operation_in_progress", message) } -// operationGateExemptPaths are non-GET endpoints that do not mutate the -// archive: they must not queue behind long archive operations. Most only read; +// operationGateExemptPaths bypass the generic mutation gate. Most only read; // the session endpoints mutate process-local authentication state. Verify is // NOT exempt: its subprocess opens the store read-write and runs schema // init/migrations. // -// The backup freeze endpoints are exempt for a different reason: begin -// acquires the operation gate itself (see beginLabeledOperationGateWork in -// handleBackupFreezeBegin), so routing them through the generic middleware -// gate as well would deadlock begin against its own acquisition. +// Backup freeze begin and meeting import coordinate the gate in their handlers. +// Meeting import first reads and validates its bounded request body so a slow +// authenticated upload cannot hold the gate. Backup freeze end bypasses the +// gate so it can release the freeze held by begin. Routing these through the +// generic middleware would deadlock their coordination. var operationGateExemptPaths = map[string]bool{ queryEndpointPath: true, sessionPath: true, sessionLoginPath: true, + meetingImportEndpointPath: true, "/api/v1/cli/add-calendar/plan": true, "/api/v1/cli/delete-staged/plan": true, "/api/v1/cli/embeddings/plan": true, diff --git a/internal/api/routes.go b/internal/api/routes.go index 82657b7a0..bc97b5dd9 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -162,7 +162,7 @@ func (s *Server) humaAuthMiddleware(ctx huma.Context, next func(huma.Context)) { } func writeHumaError(ctx huma.Context, status int, code string, message string) { - ctx.SetHeader("Content-Type", "application/json") + ctx.SetHeader("Content-Type", applicationJSONMediaType) ctx.SetStatus(status) _ = json.NewEncoder(ctx.BodyWriter()).Encode(ErrorResponse{ //nolint:errchkjson // best-effort error response write Error: code, @@ -273,6 +273,7 @@ func (s *Server) registerHumaRoutes(api huma.API, apiV1 huma.API) { registerAPIV1RawHumaJSONRoute[MessageListResponse](apiV1, "listMessages", http.MethodGet, "/messages", "List messages", s.handleListMessages) registerAPIV1RawHumaJSONRoute[MessageDetail](apiV1, "getMessage", http.MethodGet, "/messages/{id}", "Get one message", s.handleGetMessage) + s.registerMeetingImportRoute(apiV1) registerAPIV1RawHumaJSONRoute[ConversationResponse](apiV1, "getConversation", http.MethodGet, "/conversations/{id}", "Get a bounded containing conversation", s.handleGetConversation) registerAPIV1RawHumaJSONRoute[AttachmentInfo](apiV1, "getAttachment", http.MethodGet, "/attachments/{id}", "Get attachment metadata", s.handleGetAttachment) registerAPIV1RawHumaBinaryRoute( @@ -838,7 +839,7 @@ func jsonRequestBodyFor[T any](api huma.API) *huma.RequestBody { return &huma.RequestBody{ Required: true, Content: map[string]*huma.MediaType{ - "application/json": {Schema: schemaFor[T](api)}, + applicationJSONMediaType: {Schema: schemaFor[T](api)}, }, } } @@ -852,7 +853,7 @@ func jsonResponsesFor[T any](api huma.API, successStatuses ...int) map[string]*h responses[httpStatusKey(status)] = &huma.Response{ Description: http.StatusText(status), Content: map[string]*huma.MediaType{ - "application/json": {Schema: schemaFor[T](api)}, + applicationJSONMediaType: {Schema: schemaFor[T](api)}, }, } } @@ -869,7 +870,7 @@ func oneOfJSONResponses(api huma.API, responseTypes ...reflect.Type) map[string] httpStatusKey(http.StatusOK): { Description: http.StatusText(http.StatusOK), Content: map[string]*huma.MediaType{ - "application/json": {Schema: &huma.Schema{OneOf: oneOf}}, + applicationJSONMediaType: {Schema: &huma.Schema{OneOf: oneOf}}, }, }, "default": errorResponseFor(api), @@ -908,7 +909,7 @@ func errorResponseFor(api huma.API) *huma.Response { return &huma.Response{ Description: "Error", Content: map[string]*huma.MediaType{ - "application/json": {Schema: schemaFor[ErrorResponse](api)}, + applicationJSONMediaType: {Schema: schemaFor[ErrorResponse](api)}, }, } } diff --git a/internal/api/scheduler_jobs.go b/internal/api/scheduler_jobs.go index fa809fe5f..0e2e64961 100644 --- a/internal/api/scheduler_jobs.go +++ b/internal/api/scheduler_jobs.go @@ -6,9 +6,23 @@ import ( "go.kenn.io/msgvault/internal/circleback" "go.kenn.io/msgvault/internal/gcal" "go.kenn.io/msgvault/internal/granola" + "go.kenn.io/msgvault/internal/meetingimport" "go.kenn.io/msgvault/internal/synctechsms" ) +type sourceScheduleKind uint8 + +const ( + sourceScheduleNonSchedulable sourceScheduleKind = iota + sourceScheduleAccount + sourceScheduleGeneric +) + +type sourceScheduleClassification struct { + kind sourceScheduleKind + jobName string +} + // sourceTypeBeeper mirrors the unexported sourceTypeBeeper constant in // internal/beeper (and cmd/msgvault/cmd/constants.go); it can't be imported // because it isn't exported, so the literal is duplicated here. @@ -26,6 +40,27 @@ const BeeperJobName = sourceTypeBeeper // Slack workspace source. const SlackJobName = sourceTypeSlack +// classifySourceScheduling determines which scheduler, if any, may operate a +// store source. Account scheduling is opt-in so imported or unknown source +// types cannot borrow a scheduled account merely by sharing its identifier. +func classifySourceScheduling(sourceType, identifier string) sourceScheduleClassification { + switch sourceType { + case "", "gmail", "imap", "teams", "discord": + return sourceScheduleClassification{kind: sourceScheduleAccount} + case meetingimport.SourceType: + return sourceScheduleClassification{kind: sourceScheduleNonSchedulable} + default: + jobName, ok := SchedulerJobNameForSource(sourceType, identifier) + if !ok { + return sourceScheduleClassification{kind: sourceScheduleNonSchedulable} + } + return sourceScheduleClassification{ + kind: sourceScheduleGeneric, + jobName: jobName, + } + } +} + // SchedulerJobNameForSource returns the scheduler generic-job name that // drives syncing for a store source of the given type and identifier, and // whether such a job governs this source type at all. It is the single diff --git a/internal/api/server.go b/internal/api/server.go index b21639b8a..f7aedb0f5 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -620,6 +620,17 @@ func serveWithoutWriteDeadline( next.ServeHTTP(w, r) } +func serveMeetingImportWithReadDeadline( + w http.ResponseWriter, + r *http.Request, + next http.Handler, +) { + controller := http.NewResponseController(w) + _ = controller.SetReadDeadline(time.Now().Add(DaemonLongRequestTimeout)) + _ = controller.SetWriteDeadline(time.Time{}) + next.ServeHTTP(w, r) +} + func serveWithProtectiveRequestDeadline( w http.ResponseWriter, r *http.Request, @@ -637,6 +648,12 @@ func serveWithProtectiveRequestDeadline( func (s *Server) timeoutMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && + r.URL.Path == meetingImportEndpointPath && + s.apiRequestAuthorized(r) { + serveMeetingImportWithReadDeadline(w, r, next) + return + } if s.requestUsesCLITimeoutPolicy(r) { if cliRequestNeedsProtectiveCeiling(r) { serveWithProtectiveRequestDeadline(w, r, next) @@ -684,9 +701,9 @@ func cliRequestNeedsProtectiveCeiling(r *http.Request) bool { // requestTimeoutForPath returns the context deadline to impose on a request // and whether one applies at all. POST /api/v1/query gets its own generous -// ceiling; the streaming/long-running CLI operations stay unbounded (they -// report progress and are gated by the operation gate); everything else gets -// the standard per-request timeout. +// ceiling; the streaming/long-running CLI operations and meeting imports stay +// unbounded (they report progress and are gated by the operation gate); +// everything else gets the standard per-request timeout. func (s *Server) requestTimeoutForPath(path string) (time.Duration, bool) { if path == queryEndpointPath { return s.queryTimeout, true @@ -701,6 +718,7 @@ func isLongDaemonRequest(path string) bool { switch path { case "/api/v1/cli/build-cache", "/api/v1/cli/deduplicate/plan", + meetingImportEndpointPath, "/api/v1/cli/rebuild-fts", "/api/v1/cli/repair-encoding", "/api/v1/cli/run", @@ -913,7 +931,7 @@ func (s *Server) handleDaemonShutdown(w http.ResponseWriter, r *http.Request) { return } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", applicationJSONMediaType) w.WriteHeader(http.StatusAccepted) _, _ = w.Write([]byte(`{"status":"shutting_down"}`)) go s.shutdownFunc() diff --git a/internal/api/server_test.go b/internal/api/server_test.go index 1dd0b56b2..6b13f4f99 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -1027,8 +1027,12 @@ func (w *deadlineClearingRecorder) SetWriteDeadline(deadline time.Time) error { } func TestTimeoutMiddlewareDeadlinePolicy(t *testing.T) { + const apiKey = "deadline-policy-test-key" srv := NewServerWithOptions(ServerOptions{ - Config: &config.Config{Server: config.ServerConfig{APIPort: 8080}}, + Config: &config.Config{Server: config.ServerConfig{ + APIPort: 8080, + APIKey: apiKey, + }}, Logger: testLogger(), }) @@ -1040,6 +1044,7 @@ func TestTimeoutMiddlewareDeadlinePolicy(t *testing.T) { marked := httptest.NewRequest(http.MethodGet, "/api/v1/cli/stats", nil) marked.RemoteAddr = "127.0.0.1:4242" marked.Header.Set(apiprotocol.ClientClassHeader, apiprotocol.ClientClassCLI) + marked.Header.Set("X-Api-Key", apiKey) markedDeleteDeduped := httptest.NewRequest( http.MethodPost, "/api/v1/cli/delete-deduped", @@ -1047,13 +1052,19 @@ func TestTimeoutMiddlewareDeadlinePolicy(t *testing.T) { ) markedDeleteDeduped.RemoteAddr = "127.0.0.1:4242" markedDeleteDeduped.Header.Set(apiprotocol.ClientClassHeader, apiprotocol.ClientClassCLI) + markedDeleteDeduped.Header.Set("X-Api-Key", apiKey) + meetingImport := httptest.NewRequest(http.MethodPost, "/api/v1/import/meeting", nil) + meetingImport.Header.Set("X-Api-Key", apiKey) + unauthorizedMeetingImport := httptest.NewRequest(http.MethodPost, "/api/v1/import/meeting", nil) + unauthorizedMeetingImport.Header.Set("X-Api-Key", "invalid") bounded := httptest.NewRequest(http.MethodGet, "/api/v1/cli/stats", nil) tests := []struct { - name string - request *http.Request - wantReadClear bool - wantWriteClear bool + name string + request *http.Request + wantReadClear bool + wantReadDeadline bool + wantWriteClear bool }{ {name: "unmarked long path", request: longPath, wantWriteClear: true}, {name: "marked request", request: marked, wantReadClear: true, wantWriteClear: true}, @@ -1063,6 +1074,17 @@ func TestTimeoutMiddlewareDeadlinePolicy(t *testing.T) { wantReadClear: true, wantWriteClear: true, }, + { + name: "meeting import", + request: meetingImport, + wantReadDeadline: true, + wantWriteClear: true, + }, + { + name: "unauthorized meeting import", + request: unauthorizedMeetingImport, + wantWriteClear: true, + }, {name: "bounded request", request: bounded}, } for _, tt := range tests { @@ -1074,6 +1096,11 @@ func TestTimeoutMiddlewareDeadlinePolicy(t *testing.T) { if tt.wantReadClear { require.Len(recorder.readDeadlines, 1, "read deadline changes") assert.True(recorder.readDeadlines[0].IsZero(), "read deadline cleared, not extended") + } else if tt.wantReadDeadline { + require.Len(recorder.readDeadlines, 1, "read deadline changes") + remaining := time.Until(recorder.readDeadlines[0]) + assert.Greater(remaining, DaemonLongRequestTimeout-time.Second) + assert.LessOrEqual(remaining, DaemonLongRequestTimeout) } else { assert.Empty(recorder.readDeadlines, "request keeps the server read deadline") } diff --git a/internal/api/settings_test.go b/internal/api/settings_test.go index a218dbe21..fb6667f0b 100644 --- a/internal/api/settings_test.go +++ b/internal/api/settings_test.go @@ -463,7 +463,7 @@ func TestSettingsOpenAPIContract(t *testing.T) { for _, status := range []string{"400", "409", "412", "422", "428"} { assert.Contains(patch.Responses, status) } - assert.Equal("1.32.0", doc.Info.Version) + assert.Equal("1.33.0", doc.Info.Version) settingValue := doc.Components.Schemas.Map()["SettingValue"] require.NotNil(settingValue) diff --git a/internal/calsync/calsync.go b/internal/calsync/calsync.go index bb9832928..38dab3441 100644 --- a/internal/calsync/calsync.go +++ b/internal/calsync/calsync.go @@ -159,7 +159,7 @@ func (s *Syncer) RegisterCalendars(ctx context.Context) ([]gcal.Calendar, error) if !s.includeCalendar(cal) { continue } - src, err := s.getOrCreateCalendarSource(cal) + src, err := s.getOrCreateCalendarSource(ctx, cal) if err != nil { return nil, fmt.Errorf("get/create source for %s: %w", cal.ID, err) } @@ -199,7 +199,7 @@ func (s *Syncer) listCalendars(ctx context.Context) ([]gcal.Calendar, error) { // replaying that token under a later unbounded request can skip or corrupt the // traversal. func (s *Syncer) syncCalendarFull(ctx context.Context, cal gcal.Calendar, result *Result) error { - src, err := s.getOrCreateCalendarSource(cal) + src, err := s.getOrCreateCalendarSource(ctx, cal) if err != nil { return fmt.Errorf("get/create source: %w", err) } @@ -475,7 +475,10 @@ func (s *Syncer) sourceIdentifier(cal gcal.Calendar) string { return s.opts.AccountEmail + "/" + cal.ID } -func (s *Syncer) getOrCreateCalendarSource(cal gcal.Calendar) (*store.Source, error) { +func (s *Syncer) getOrCreateCalendarSource( + ctx context.Context, + cal gcal.Calendar, +) (*store.Source, error) { identifier := s.sourceIdentifier(cal) sources, err := s.store.GetSourcesByTypeAndAccount(gcal.SourceType, s.opts.AccountEmail) @@ -483,6 +486,7 @@ func (s *Syncer) getOrCreateCalendarSource(cal gcal.Calendar) (*store.Source, er return nil, fmt.Errorf("find existing calendar sources: %w", err) } + var source *store.Source var migrate *store.Source for _, src := range sources { cfg := parseSourceConfig(src.SyncConfig) @@ -490,21 +494,42 @@ func (s *Syncer) getOrCreateCalendarSource(cal gcal.Calendar) (*store.Source, er continue } if src.Identifier == identifier { - return src, nil + source = src + break } if migrate == nil { migrate = src } } - if migrate != nil { + if source == nil && migrate != nil { if err := s.store.UpdateSourceIdentifier(migrate.ID, identifier); err != nil { return nil, fmt.Errorf("migrate calendar source identifier: %w", err) } migrate.Identifier = identifier - return migrate, nil + source = migrate + } + if source == nil { + source, err = s.store.GetOrCreateSource(gcal.SourceType, identifier) + if err != nil { + return nil, err + } } + if err := s.confirmCalendarSourceIdentity(ctx, source); err != nil { + return nil, err + } + return source, nil +} - return s.store.GetOrCreateSource(gcal.SourceType, identifier) +func (s *Syncer) confirmCalendarSourceIdentity(ctx context.Context, source *store.Source) error { + if err := s.store.AddAccountIdentityContext( + ctx, + source.ID, + s.opts.AccountEmail, + "account-email", + ); err != nil { + return fmt.Errorf("confirm calendar account identity: %w", err) + } + return nil } func (s *Syncer) updateCalendarSourceOAuthApp(sourceID int64, calendarID string) error { diff --git a/internal/calsync/calsync_test.go b/internal/calsync/calsync_test.go index 68809900f..d0afc070e 100644 --- a/internal/calsync/calsync_test.go +++ b/internal/calsync/calsync_test.go @@ -261,6 +261,48 @@ func TestFull_PersistsEventsAsMessages(t *testing.T) { } } +func TestFull_ConfiguredAccountAttributionSurvivesIdentityMutation(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + m := gcal.NewMockAPI() + m.Calendars = []gcal.Calendar{{ + ID: "primary", + AccessRole: "owner", + }} + ev := timedEvent("configured-organizer", "Configured organizer") + ev.Organizer.Self = false + m.FullEvents["primary"] = [][]gcal.Event{{ev}} + m.FullSyncToken["primary"] = "TOKEN1" + + s, st := newSyncer(t, m, Options{}) + _, err := s.Full(context.Background()) + require.NoError(err) + + src := primarySource(t, st) + row, ok := getMsg(t, st, src.ID, ev.ID) + require.True(ok) + assert.True(row.isFromMe, "configured account organizer should be attributed to the account") + + identities, err := st.ListAccountIdentities(src.ID) + require.NoError(err) + require.Len(identities, 1) + assert.Equal(testAccount, identities[0].Address) + assert.Contains(identities[0].SourceSignal, "account-email") + + require.NoError(st.AddAccountIdentity(src.ID, "other@example.com", "manual")) + row, ok = getMsg(t, st, src.ID, ev.ID) + require.True(ok) + assert.True(row.isFromMe, "adding another identity must preserve configured-account attribution") + + removed, err := st.RemoveAccountIdentity(src.ID, "other@example.com") + require.NoError(err) + assert.Equal(int64(1), removed) + row, ok = getMsg(t, st, src.ID, ev.ID) + require.True(ok) + assert.True(row.isFromMe, "removing another identity must preserve configured-account attribution") +} + func TestFull_ClearsBodyWhenEventBodyBecomesEmpty(t *testing.T) { assert := assert.New(t) require := require.New(t) diff --git a/internal/calsync/incremental.go b/internal/calsync/incremental.go index f35373368..7dce0833e 100644 --- a/internal/calsync/incremental.go +++ b/internal/calsync/incremental.go @@ -46,6 +46,10 @@ func (s *Syncer) Incremental(ctx context.Context) (Result, error) { if err := ctx.Err(); err != nil { return result, err } + if err := s.confirmCalendarSourceIdentity(ctx, src); err != nil { + recordErr(fmt.Errorf("confirm account identity for source %d: %w", src.ID, err)) + continue + } cfg := parseSourceConfig(src.SyncConfig) if cfg.CalendarID == "" { continue diff --git a/internal/calsync/persist.go b/internal/calsync/persist.go index d9a83366c..3e2433f2b 100644 --- a/internal/calsync/persist.go +++ b/internal/calsync/persist.go @@ -108,19 +108,23 @@ func (s *Syncer) ingestEvent(sourceID int64, cal gcal.Calendar, ev gcal.Event) ( body := serializeBody(ev) subject := ev.Summary - fromMe := ev.Organizer.Self || (ev.Organizer.Email != "" && strings.EqualFold(ev.Organizer.Email, s.opts.AccountEmail)) + identityFromMe := !ev.Organizer.Self && + ev.Organizer.Email != "" && + strings.EqualFold(ev.Organizer.Email, s.opts.AccountEmail) + fromMe := ev.Organizer.Self || identityFromMe msgID, err := s.store.UpsertMessage(&store.Message{ - ConversationID: convID, - SourceID: sourceID, - SourceMessageID: smid, - MessageType: gcal.MessageTypeCalendarEvent, - SentAt: eventSentAt(ev), - SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, - IsFromMe: fromMe, - Subject: sql.NullString{String: subject, Valid: subject != ""}, - Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, - SizeEstimate: int64(len(body)), + ConversationID: convID, + SourceID: sourceID, + SourceMessageID: smid, + MessageType: gcal.MessageTypeCalendarEvent, + SentAt: eventSentAt(ev), + SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, + IsFromMe: fromMe, + IdentityDerivedIsFromMe: identityFromMe, + Subject: sql.NullString{String: subject, Valid: subject != ""}, + Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, + SizeEstimate: int64(len(body)), }) if err != nil { return 0, fmt.Errorf("upsert message: %w", err) diff --git a/internal/circleback/importer.go b/internal/circleback/importer.go index ade0d6f5f..ecead01ca 100644 --- a/internal/circleback/importer.go +++ b/internal/circleback/importer.go @@ -977,15 +977,16 @@ func (imp *Importer) ingestMeeting( sentAt := m.StartedAt().UTC() message := &store.Message{ - SourceID: sourceID, - SourceMessageID: smid, - MessageType: MessageType, - SentAt: sql.NullTime{Time: sentAt, Valid: !sentAt.IsZero()}, - SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, - IsFromMe: fromMe, - Subject: sql.NullString{String: title, Valid: title != ""}, - Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, - SizeEstimate: int64(len(body)), + SourceID: sourceID, + SourceMessageID: smid, + MessageType: MessageType, + SentAt: sql.NullTime{Time: sentAt, Valid: !sentAt.IsZero()}, + SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, + IsFromMe: fromMe, + IdentityDerivedIsFromMe: fromMe, + Subject: sql.NullString{String: title, Valid: title != ""}, + Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, + SizeEstimate: int64(len(body)), } metaJSON, err := json.Marshal(imp.buildMetadata( diff --git a/internal/granola/importer.go b/internal/granola/importer.go index 2e877ddba..1c8b296e1 100644 --- a/internal/granola/importer.go +++ b/internal/granola/importer.go @@ -328,15 +328,16 @@ func (imp *Importer) ingestNote(sourceID int64, identifier string, accountIdenti sentAt := noteStartTime(n).UTC() message := &store.Message{ - SourceID: sourceID, - SourceMessageID: n.ID, - MessageType: MessageType, - SentAt: sql.NullTime{Time: sentAt, Valid: !sentAt.IsZero()}, - SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, - IsFromMe: fromMe, - Subject: sql.NullString{String: title, Valid: title != ""}, - Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, - SizeEstimate: int64(len(body)), + SourceID: sourceID, + SourceMessageID: n.ID, + MessageType: MessageType, + SentAt: sql.NullTime{Time: sentAt, Valid: !sentAt.IsZero()}, + SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, + IsFromMe: fromMe, + IdentityDerivedIsFromMe: fromMe, + Subject: sql.NullString{String: title, Valid: title != ""}, + Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, + SizeEstimate: int64(len(body)), } metaJSON, err := json.Marshal(buildMetadata(n, identifier, organizerEmail)) diff --git a/internal/meetingimport/decode.go b/internal/meetingimport/decode.go new file mode 100644 index 000000000..1a6db3c41 --- /dev/null +++ b/internal/meetingimport/decode.go @@ -0,0 +1,51 @@ +package meetingimport + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "unicode/utf8" +) + +func DecodeRequest(r io.Reader, maxBytes int64) (Request, error) { + if r == nil { + return Request{}, fmt.Errorf("%w: empty body", ErrMalformedRequest) + } + if maxBytes <= 0 { + return Request{}, ErrRequestTooLarge + } + + body, err := io.ReadAll(io.LimitReader(r, maxBytes+1)) + if err != nil { + return Request{}, fmt.Errorf("%w: read body: %w", ErrMalformedRequest, err) + } + if int64(len(body)) > maxBytes { + return Request{}, ErrRequestTooLarge + } + if len(bytes.TrimSpace(body)) == 0 { + return Request{}, fmt.Errorf("%w: empty body", ErrMalformedRequest) + } + if !utf8.Valid(body) { + return Request{}, fmt.Errorf("%w: request must be valid UTF-8", ErrMalformedRequest) + } + + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + decoder.UseNumber() + var req Request + if err := decoder.Decode(&req); err != nil { + return Request{}, fmt.Errorf("%w: %w", ErrMalformedRequest, err) + } + + var trailing any + err = decoder.Decode(&trailing) + if err == nil { + return Request{}, fmt.Errorf("%w: trailing JSON value", ErrMalformedRequest) + } + if !errors.Is(err, io.EOF) { + return Request{}, fmt.Errorf("%w: trailing data: %w", ErrMalformedRequest, err) + } + return req, nil +} diff --git a/internal/meetingimport/decode_test.go b/internal/meetingimport/decode_test.go new file mode 100644 index 000000000..73c76f370 --- /dev/null +++ b/internal/meetingimport/decode_test.go @@ -0,0 +1,144 @@ +package meetingimport + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const validRequestJSON = `{ + "source": { + "identifier": " local-meetings ", + "display_name": " Local Meetings ", + "account_email": " USER@example.com " + }, + "meeting": { + "external_id": " 42 ", + "title": " Weekly planning ", + "started_at": "2026-07-23T11:00:00-07:00", + "ended_at": "2026-07-23T11:30:00-07:00", + "summary_markdown": "## Summary\n\nReviewed the launch plan.", + "summary_text": "", + "transcript": "", + "transcript_segments": [ + { + "speaker": " Test Speaker ", + "text": " Let's review the launch plan. ", + "offset_seconds": 4 + } + ], + "organizer": { + "name": " Test Organizer ", + "email": " ORGANIZER@example.com " + }, + "attendees": [ + { + "name": " Test Attendee ", + "email": " ATTENDEE@example.com " + } + ], + "metadata": { + "calendar_event_id": "synthetic-event-42", + "nested": {"accepted": true} + } + } +}` + +func TestDecodeRequestAcceptsCompleteStrictRequest(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req, err := DecodeRequest(strings.NewReader(validRequestJSON), MaxRequestBytes) + require.NoError(err) + + assert.Equal(" local-meetings ", req.Source.Identifier) + assert.Equal(" 42 ", req.Meeting.ExternalID) + require.Len(req.Meeting.TranscriptSegments, 1) + require.NotNil(req.Meeting.TranscriptSegments[0].OffsetSeconds) + assert.InDelta(float64(4), *req.Meeting.TranscriptSegments[0].OffsetSeconds, 0) + nested, ok := req.Meeting.Metadata["nested"].(map[string]any) + require.True(ok, "nested metadata object") + assert.Equal(true, nested["accepted"]) +} + +func TestDecodeRequestPreservesLargeNestedMetadataNumbers(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + body := strings.Replace( + validRequestJSON, + `"nested": {"accepted": true}`, + `"nested": {"accepted": true, "large_id": 9007199254740993, "deep": {"another_id": 18446744073709551615}}`, + 1, + ) + + req, err := DecodeRequest(strings.NewReader(body), MaxRequestBytes) + require.NoError(err) + + nested, ok := req.Meeting.Metadata["nested"].(map[string]any) + require.True(ok, "nested metadata object") + largeID, ok := nested["large_id"].(json.Number) + require.True(ok, "large metadata identifier") + assert.Equal("9007199254740993", largeID.String()) + deep, ok := nested["deep"].(map[string]any) + require.True(ok, "deep metadata object") + anotherID, ok := deep["another_id"].(json.Number) + require.True(ok, "deep metadata identifier") + assert.Equal("18446744073709551615", anotherID.String()) +} + +func TestDecodeRequestRejectsMalformedAndTrailingJSON(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "empty", body: ""}, + {name: "malformed", body: `{"source":`}, + {name: "trailing object", body: validRequestJSON + `{}`}, + {name: "trailing scalar", body: validRequestJSON + ` true`}, + {name: "unknown top level", body: strings.Replace(validRequestJSON, `"source":`, `"unknown": true, "source":`, 1)}, + {name: "unknown source", body: strings.Replace(validRequestJSON, `"identifier":`, `"unknown": true, "identifier":`, 1)}, + {name: "unknown meeting", body: strings.Replace(validRequestJSON, `"external_id":`, `"unknown": true, "external_id":`, 1)}, + {name: "unknown segment", body: strings.Replace(validRequestJSON, `"speaker":`, `"unknown": true, "speaker":`, 1)}, + {name: "invalid utf8", body: string([]byte{'{', '"', 0xff, '"', ':', '1', '}'})}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + _, err := DecodeRequest(strings.NewReader(tt.body), MaxRequestBytes) + require.Error(err) + assert.ErrorIs(err, ErrMalformedRequest) + }) + } +} + +func TestDecodeRequestAllowsUnknownProviderMetadata(t *testing.T) { + body := strings.Replace( + validRequestJSON, + `"calendar_event_id": "synthetic-event-42"`, + `"provider_specific_unknown": {"path": ["a", "b"]}`, + 1, + ) + + req, err := DecodeRequest(strings.NewReader(body), MaxRequestBytes) + require.NoError(t, err) + assert.Contains(t, req.Meeting.Metadata, "provider_specific_unknown") +} + +func TestDecodeRequestEnforcesBodyLimit(t *testing.T) { + require := require.New(t) + + _, err := DecodeRequest(strings.NewReader(validRequestJSON), int64(len(validRequestJSON)-1)) + require.Error(err) + require.ErrorIs(err, ErrRequestTooLarge) + + _, err = DecodeRequest(strings.NewReader(`{}`), 0) + require.Error(err) + require.ErrorIs(err, ErrRequestTooLarge) +} diff --git a/internal/meetingimport/format.go b/internal/meetingimport/format.go new file mode 100644 index 000000000..94f22c4c0 --- /dev/null +++ b/internal/meetingimport/format.go @@ -0,0 +1,204 @@ +package meetingimport + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +type Snapshot struct { + SourceIdentifier string + SourceDisplayName string + AccountEmail string + SourceMessageID string + Title string + StartedAt time.Time + Body string + Snippet string + Metadata []byte + Raw []byte + Organizer *MeetingPerson + Attendees []MeetingPerson +} + +type canonicalMeeting struct { + ExternalID string `json:"external_id"` + Title string `json:"title,omitempty"` + StartedAt string `json:"started_at"` + EndedAt string `json:"ended_at,omitempty"` + SummaryMarkdown string `json:"summary_markdown,omitempty"` + SummaryText string `json:"summary_text,omitempty"` + Transcript string `json:"transcript,omitempty"` + TranscriptSegments []TranscriptSegment `json:"transcript_segments,omitempty"` + Organizer *MeetingPerson `json:"organizer,omitempty"` + Attendees []MeetingPerson `json:"attendees,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +type messageMetadata struct { + Platform string `json:"platform"` + ExternalMeetingID string `json:"external_meeting_id"` + SourceIdentifier string `json:"source_identifier"` + StartedAt string `json:"started_at"` + EndedAt string `json:"ended_at,omitempty"` + DurationSeconds int64 `json:"duration_seconds"` + OrganizerEmail string `json:"organizer_email,omitempty"` + AttendeeCount int `json:"attendee_count"` + HasSummary bool `json:"has_summary"` + HasTranscript bool `json:"has_transcript"` + TranscriptSegmentCount int `json:"transcript_segment_count"` + ProviderMetadata map[string]any `json:"provider_metadata,omitempty"` +} + +func BuildSnapshot(req NormalizedRequest) (Snapshot, error) { + meeting := req.Meeting + title := meeting.Title + if title == "" { + title = "Meeting on " + meeting.StartedAt.UTC().Format(time.DateOnly) + } + + body := buildBody(title, meeting) + raw, err := json.Marshal(buildCanonicalMeeting(meeting)) + if err != nil { + return Snapshot{}, fmt.Errorf("marshal canonical meeting: %w", err) + } + metadata, err := json.Marshal(buildMessageMetadata(req)) + if err != nil { + return Snapshot{}, fmt.Errorf("marshal meeting metadata: %w", err) + } + + return Snapshot{ + SourceIdentifier: req.Source.Identifier, + SourceDisplayName: req.Source.DisplayName, + AccountEmail: req.Source.AccountEmail, + SourceMessageID: "meeting:" + meeting.ExternalID, + Title: title, + StartedAt: meeting.StartedAt.UTC(), + Body: body, + Snippet: snippet(body), + Metadata: metadata, + Raw: raw, + Organizer: meeting.Organizer, + Attendees: meeting.Attendees, + }, nil +} + +func buildBody(title string, meeting NormalizedMeeting) string { + var b strings.Builder + writeLine := func(line string) { + if line == "" { + return + } + b.WriteString(line) + b.WriteByte('\n') + } + + writeLine(title) + writeLine(formatWhen(meeting.StartedAt, meeting.EndedAt)) + + names := make([]string, 0, len(meeting.Attendees)) + for _, attendee := range meeting.Attendees { + if attendee.Name != "" { + names = append(names, attendee.Name) + } + } + if len(names) > 0 { + writeLine("Attendees: " + strings.Join(names, ", ")) + } + + summary := meeting.SummaryMarkdown + if summary == "" { + summary = meeting.SummaryText + } + if summary != "" { + b.WriteByte('\n') + writeLine(summary) + } + + if meeting.Transcript != "" { + b.WriteString("\nTranscript:\n") + writeLine(meeting.Transcript) + } else if len(meeting.TranscriptSegments) > 0 { + b.WriteString("\nTranscript:\n") + for _, segment := range meeting.TranscriptSegments { + writeLine(formatSegment(segment)) + } + } + return strings.TrimSpace(b.String()) +} + +func formatWhen(start time.Time, end *time.Time) string { + line := "When: " + start.UTC().Format("2006-01-02 15:04") + if end != nil { + line += " - " + end.UTC().Format("15:04") + } + return line +} + +func formatSegment(segment TranscriptSegment) string { + label := segment.Speaker + ": " + segment.Text + if segment.OffsetSeconds == nil { + return label + } + total := int(*segment.OffsetSeconds) + hours := total / 3600 + minutes := (total % 3600) / 60 + seconds := total % 60 + if hours > 0 { + return fmt.Sprintf("[%d:%02d:%02d] %s", hours, minutes, seconds, label) + } + return fmt.Sprintf("[%02d:%02d] %s", minutes, seconds, label) +} + +func snippet(body string) string { + const maxRunes = 200 + runes := []rune(strings.TrimSpace(body)) + if len(runes) <= maxRunes { + return string(runes) + } + return string(runes[:maxRunes]) +} + +func buildCanonicalMeeting(meeting NormalizedMeeting) canonicalMeeting { + endedAt := "" + if meeting.EndedAt != nil { + endedAt = meeting.EndedAt.UTC().Format(time.RFC3339Nano) + } + return canonicalMeeting{ + ExternalID: meeting.ExternalID, + Title: meeting.Title, + StartedAt: meeting.StartedAt.UTC().Format(time.RFC3339Nano), + EndedAt: endedAt, + SummaryMarkdown: meeting.SummaryMarkdown, + SummaryText: meeting.SummaryText, + Transcript: meeting.Transcript, + TranscriptSegments: meeting.TranscriptSegments, + Organizer: meeting.Organizer, + Attendees: meeting.Attendees, + Metadata: meeting.Metadata, + } +} + +func buildMessageMetadata(req NormalizedRequest) messageMetadata { + meeting := req.Meeting + metadata := messageMetadata{ + Platform: SourceType, + ExternalMeetingID: meeting.ExternalID, + SourceIdentifier: req.Source.Identifier, + StartedAt: meeting.StartedAt.UTC().Format(time.RFC3339Nano), + AttendeeCount: len(meeting.Attendees), + HasSummary: meeting.SummaryMarkdown != "" || meeting.SummaryText != "", + HasTranscript: meeting.Transcript != "" || len(meeting.TranscriptSegments) > 0, + TranscriptSegmentCount: len(meeting.TranscriptSegments), + ProviderMetadata: meeting.Metadata, + } + if meeting.EndedAt != nil { + metadata.EndedAt = meeting.EndedAt.UTC().Format(time.RFC3339Nano) + metadata.DurationSeconds = int64(meeting.EndedAt.Sub(meeting.StartedAt).Seconds()) + } + if meeting.Organizer != nil { + metadata.OrganizerEmail = meeting.Organizer.Email + } + return metadata +} diff --git a/internal/meetingimport/format_test.go b/internal/meetingimport/format_test.go new file mode 100644 index 000000000..c927eccfd --- /dev/null +++ b/internal/meetingimport/format_test.go @@ -0,0 +1,152 @@ +package meetingimport + +import ( + "encoding/json" + "strings" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func normalizedValidRequest(t *testing.T) NormalizedRequest { + t.Helper() + req := decodedValidRequest(t) + normalized, err := req.Normalize() + require.NoError(t, err) + return normalized +} + +func TestBuildSnapshotRendersGranolaCompatibleBody(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + snapshot, err := BuildSnapshot(normalizedValidRequest(t)) + require.NoError(err) + + assert.Equal("meeting:42", snapshot.SourceMessageID) + assert.Equal("Weekly planning", snapshot.Title) + assert.Equal(`Weekly planning +When: 2026-07-23 18:00 - 18:30 +Attendees: Test Attendee + +## Summary + +Reviewed the launch plan. + +Transcript: +[00:04] Test Speaker: Let's review the launch plan.`, snapshot.Body) + assert.Equal(snapshot.Body, snapshot.Snippet) +} + +func TestBuildSnapshotPreservesPlainTranscriptLines(t *testing.T) { + req := normalizedValidRequest(t) + req.Meeting.SummaryMarkdown = "" + req.Meeting.SummaryText = "Plain summary wins when Markdown is empty." + req.Meeting.TranscriptSegments = nil + req.Meeting.Transcript = "Speaker 1: first line\n indented continuation\nSpeaker 2: final line" + + snapshot, err := BuildSnapshot(req) + require.NoError(t, err) + + assert.Contains(t, snapshot.Body, "\n\nPlain summary wins when Markdown is empty.\n\nTranscript:\n") + assert.Contains(t, snapshot.Body, "Speaker 1: first line\n indented continuation\nSpeaker 2: final line") +} + +func TestBuildSnapshotRendersStructuredSpeakerLabelsAndOffsets(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := normalizedValidRequest(t) + fourSeconds := 4.0 + overHour := 3661.9 + req.Meeting.TranscriptSegments = []TranscriptSegment{ + {Speaker: "Speaker 1", Text: "Anonymous speaker.", OffsetSeconds: &fourSeconds}, + {Speaker: "Test Speaker", Text: "Named speaker.", OffsetSeconds: &overHour}, + {Speaker: "Speaker 2", Text: "No timestamp."}, + } + + snapshot, err := BuildSnapshot(req) + require.NoError(err) + + assert.Contains(snapshot.Body, "[00:04] Speaker 1: Anonymous speaker.") + assert.Contains(snapshot.Body, "[1:01:01] Test Speaker: Named speaker.") + assert.Contains(snapshot.Body, "Speaker 2: No timestamp.") +} + +func TestBuildSnapshotUsesDateFallbackAndOptionalFields(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := normalizedValidRequest(t) + req.Meeting.Title = "" + req.Meeting.EndedAt = nil + req.Meeting.Organizer = nil + req.Meeting.Attendees = nil + req.Meeting.SummaryMarkdown = "" + req.Meeting.SummaryText = "Only a summary." + req.Meeting.TranscriptSegments = nil + + snapshot, err := BuildSnapshot(req) + require.NoError(err) + + assert.Equal("Meeting on 2026-07-23", snapshot.Title) + assert.Equal(`Meeting on 2026-07-23 +When: 2026-07-23 18:00 + +Only a summary.`, snapshot.Body) + assert.Nil(snapshot.Organizer) + assert.Empty(snapshot.Attendees) +} + +func TestBuildSnapshotCapsSnippetAtTwoHundredRunes(t *testing.T) { + req := normalizedValidRequest(t) + req.Meeting.Title = strings.Repeat("界", 210) + + snapshot, err := BuildSnapshot(req) + require.NoError(t, err) + + assert.Equal(t, 200, utf8.RuneCountInString(snapshot.Snippet)) + assert.Equal(t, strings.Repeat("界", 200), snapshot.Snippet) +} + +func TestBuildSnapshotStoresCanonicalRawMeetingAndMetadata(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := normalizedValidRequest(t) + + first, err := BuildSnapshot(req) + require.NoError(err) + second, err := BuildSnapshot(req) + require.NoError(err) + assert.Equal(first.Raw, second.Raw) + assert.Equal(first.Metadata, second.Metadata) + + var raw map[string]any + require.NoError(json.Unmarshal(first.Raw, &raw)) + assert.Equal("42", raw["external_id"]) + assert.Equal("2026-07-23T18:00:00Z", raw["started_at"]) + assert.Equal("2026-07-23T18:30:00Z", raw["ended_at"]) + assert.NotContains(raw, "source") + assert.NotContains(raw, "account_email") + rawMetadata, ok := raw["metadata"].(map[string]any) + require.True(ok, "raw metadata object") + assert.Equal("synthetic-event-42", rawMetadata["calendar_event_id"]) + + var metadata map[string]any + require.NoError(json.Unmarshal(first.Metadata, &metadata)) + assert.Equal(SourceType, metadata["platform"]) + assert.Equal("42", metadata["external_meeting_id"]) + assert.Equal("local-meetings", metadata["source_identifier"]) + assert.InDelta(float64(1800), metadata["duration_seconds"], 0) + assert.Equal("organizer@example.com", metadata["organizer_email"]) + assert.InDelta(float64(1), metadata["attendee_count"], 0) + assert.Equal(true, metadata["has_summary"]) + assert.Equal(true, metadata["has_transcript"]) + assert.InDelta(float64(1), metadata["transcript_segment_count"], 0) + providerMetadata, ok := metadata["provider_metadata"].(map[string]any) + require.True(ok, "provider metadata object") + assert.Equal("synthetic-event-42", providerMetadata["calendar_event_id"]) +} diff --git a/internal/meetingimport/importer.go b/internal/meetingimport/importer.go new file mode 100644 index 000000000..743d2139f --- /dev/null +++ b/internal/meetingimport/importer.go @@ -0,0 +1,295 @@ +package meetingimport + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "go.kenn.io/msgvault/internal/meetingidentity" + "go.kenn.io/msgvault/internal/store" +) + +type Status string + +var ErrUnavailable = errors.New("meeting importer is unavailable") + +const ( + StatusCreated Status = "created" + StatusUpdated Status = "updated" +) + +type Result struct { + Status Status + SourceID int64 + MessageID int64 + SourceMessageID string +} + +type Hooks struct { + AfterSourceSetup func() error + RefreshCache func(context.Context, string) error +} + +type Importer struct { + store *store.Store + hooks Hooks +} + +func NewImporter(s *store.Store, hooks Hooks) *Importer { + return &Importer{store: s, hooks: hooks} +} + +func (i *Importer) Import(ctx context.Context, req Request) (result Result, retErr error) { + if i == nil || i.store == nil { + return Result{}, ErrUnavailable + } + if err := ctx.Err(); err != nil { + return Result{}, err + } + + normalized, err := req.Normalize() + if err != nil { + return Result{}, err + } + snapshot, err := BuildSnapshot(normalized) + if err != nil { + return Result{}, err + } + + source, err := i.store.GetOrCreateSource(SourceType, snapshot.SourceIdentifier) + if err != nil { + return Result{}, fmt.Errorf("resolve meeting source: %w", err) + } + result.SourceID = source.ID + result.SourceMessageID = snapshot.SourceMessageID + + displayName := snapshot.SourceDisplayName + if displayName == "" && (!source.DisplayName.Valid || strings.TrimSpace(source.DisplayName.String) == "") { + displayName = snapshot.SourceIdentifier + } + if displayName != "" && (!source.DisplayName.Valid || source.DisplayName.String != displayName) { + if err := i.store.UpdateSourceDisplayNameContext(ctx, source.ID, displayName); err != nil { + return result, fmt.Errorf("update meeting source display name: %w", err) + } + } + if err := i.store.AddAccountIdentityAndRefreshMessageAttributionContext( + ctx, + source.ID, + snapshot.AccountEmail, + "account-email", + snapshot.SourceMessageID, + ); err != nil { + return result, fmt.Errorf("confirm meeting source identity: %w", err) + } + if i.hooks.AfterSourceSetup != nil { + if err := i.hooks.AfterSourceSetup(); err != nil { + return result, fmt.Errorf("run post-source setup: %w", err) + } + } + + syncID, err := i.store.StartSyncContext(ctx, source.ID, SourceType) + if err != nil { + return result, fmt.Errorf("start meeting import sync: %w", err) + } + checkpoint := &store.Checkpoint{} + defer func() { + if retErr == nil { + return + } + if failErr := i.store.FailSyncWithCheckpoint(syncID, retErr.Error(), checkpoint); failErr != nil { + retErr = errors.Join(retErr, fmt.Errorf("record failed meeting import sync: %w", failErr)) + } + }() + + existing, err := i.store.MessageExistsBatch(source.ID, []string{snapshot.SourceMessageID}) + if err != nil { + return result, fmt.Errorf("lookup existing meeting: %w", err) + } + identities, err := meetingidentity.ForSource(i.store, source.ID, snapshot.AccountEmail) + if err != nil { + return result, err + } + + organizerEmail := "" + organizerName := "" + if snapshot.Organizer != nil { + organizerEmail = snapshot.Organizer.Email + organizerName = snapshot.Organizer.Name + } + expectedIsFromMe := organizerEmail != "" && identities.Contains(organizerEmail) + + existingMessageID, existed := existing[snapshot.SourceMessageID] + if existed { + result.Status = StatusUpdated + result.MessageID = existingMessageID + + storedRaw, rawErr := i.store.GetMessageRaw(existingMessageID) + if rawErr == nil && bytes.Equal(storedRaw, snapshot.Raw) { + storedIsFromMe, attributionErr := i.store.GetMessageIsFromMe(existingMessageID) + if attributionErr == nil && storedIsFromMe == expectedIsFromMe { + if err := ctx.Err(); err != nil { + return result, err + } + checkpoint.MessagesProcessed = 1 + if err := i.store.UpdateSyncCheckpointContext(ctx, syncID, checkpoint); err != nil { + return result, fmt.Errorf("checkpoint meeting import sync: %w", err) + } + if err := i.store.RecomputeConversationStatsForMessageContext(ctx, existingMessageID); err != nil { + return result, fmt.Errorf("recompute meeting conversation stats: %w", err) + } + if err := i.store.CompleteSyncContext(ctx, syncID, ""); err != nil { + return result, fmt.Errorf("complete meeting import sync: %w", err) + } + if i.hooks.RefreshCache != nil { + cacheLabel := SourceType + ":" + snapshot.SourceIdentifier + if err := i.hooks.RefreshCache(ctx, cacheLabel); err != nil { + return result, fmt.Errorf("refresh meeting analytics cache: %w", err) + } + } + return result, nil + } + } + // Missing or unreadable canonical data, or mismatched derived + // attribution, is not a match. Continue through the transactional + // persistence path so a valid retry repairs the message; genuine + // storage failures surface from that rewrite. + } + + participants := make( + []store.ParticipantPersistData, + 0, + len(snapshot.Attendees)+1, + ) + hasOrganizer := snapshot.Organizer != nil + if hasOrganizer { + participants = append(participants, store.ParticipantPersistData{ + EmailAddress: organizerEmail, + DisplayName: organizerName, + Domain: emailDomain(organizerEmail), + }) + } + + attendeeNames := make([]string, 0, len(snapshot.Attendees)) + attendeeEmails := make([]string, 0, len(snapshot.Attendees)) + for _, attendee := range snapshot.Attendees { + if err := ctx.Err(); err != nil { + return result, err + } + participants = append(participants, store.ParticipantPersistData{ + EmailAddress: attendee.Email, + DisplayName: attendee.Name, + Domain: emailDomain(attendee.Email), + }) + attendeeNames = append(attendeeNames, attendee.Name) + attendeeEmails = append(attendeeEmails, attendee.Email) + } + + metadata := sql.NullString{String: string(snapshot.Metadata), Valid: true} + messageID, err := i.store.PersistMessageWithParticipantsContext( + ctx, + participants, + func(participantIDs []int64) *store.MessagePersistData { + attendeeOffset := 0 + var senderID int64 + var fromIDs []int64 + var fromNames []string + if hasOrganizer { + senderID = participantIDs[0] + attendeeOffset = 1 + fromIDs = []int64{senderID} + fromNames = []string{organizerName} + } + attendeeIDs := participantIDs[attendeeOffset:] + conversationParticipants := make( + []store.ConversationParticipantRef, + 0, + len(attendeeIDs), + ) + for _, participantID := range attendeeIDs { + conversationParticipants = append( + conversationParticipants, + store.ConversationParticipantRef{ + ParticipantID: participantID, + Role: "member", + }, + ) + } + + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, + SourceMessageID: snapshot.SourceMessageID, + MessageType: MessageType, + SentAt: sql.NullTime{Time: snapshot.StartedAt, Valid: true}, + SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, + IsFromMe: expectedIsFromMe, + IdentityDerivedIsFromMe: expectedIsFromMe, + Subject: sql.NullString{String: snapshot.Title, Valid: snapshot.Title != ""}, + Snippet: sql.NullString{String: snapshot.Snippet, Valid: snapshot.Snippet != ""}, + SizeEstimate: int64(len(snapshot.Body)), + }, + Conversation: &store.ConversationPersistData{ + SourceConversationID: snapshot.SourceMessageID, + ConversationType: ConversationType, + Title: snapshot.Title, + Participants: conversationParticipants, + }, + Metadata: &metadata, + BodyText: sql.NullString{String: snapshot.Body, Valid: snapshot.Body != ""}, + RawMIME: snapshot.Raw, + RawFormat: RawFormat, + Recipients: []store.RecipientSet{ + {Type: "from", ParticipantIDs: fromIDs, DisplayNames: fromNames}, + {Type: "to", ParticipantIDs: attendeeIDs, DisplayNames: attendeeNames}, + }, + PreserveLabels: true, + FTS: &store.FTSDoc{ + Subject: snapshot.Title, + Body: snapshot.Body, + FromAddr: organizerEmail, + ToAddrs: strings.Join(attendeeEmails, " "), + }, + } + }, + ) + if err != nil { + return result, fmt.Errorf("persist meeting: %w", err) + } + result.MessageID = messageID + result.Status = StatusCreated + checkpoint.MessagesProcessed = 1 + checkpoint.MessagesAdded = 1 + if existed { + result.Status = StatusUpdated + checkpoint.MessagesAdded = 0 + checkpoint.MessagesUpdated = 1 + } + + if err := i.store.UpdateSyncCheckpointContext(ctx, syncID, checkpoint); err != nil { + return result, fmt.Errorf("checkpoint meeting import sync: %w", err) + } + if err := i.store.RecomputeConversationStatsForMessageContext(ctx, messageID); err != nil { + return result, fmt.Errorf("recompute meeting conversation stats: %w", err) + } + if err := i.store.CompleteSyncContext(ctx, syncID, ""); err != nil { + return result, fmt.Errorf("complete meeting import sync: %w", err) + } + if i.hooks.RefreshCache != nil { + cacheLabel := SourceType + ":" + snapshot.SourceIdentifier + if err := i.hooks.RefreshCache(ctx, cacheLabel); err != nil { + return result, fmt.Errorf("refresh meeting analytics cache: %w", err) + } + } + return result, nil +} + +func emailDomain(email string) string { + at := strings.LastIndexByte(email, '@') + if at < 0 || at == len(email)-1 { + return "" + } + return strings.ToLower(email[at+1:]) +} diff --git a/internal/meetingimport/importer_test.go b/internal/meetingimport/importer_test.go new file mode 100644 index 000000000..eb0077ea4 --- /dev/null +++ b/internal/meetingimport/importer_test.go @@ -0,0 +1,885 @@ +package meetingimport + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func validImportRequest(t *testing.T) Request { + t.Helper() + return decodedValidRequest(t) +} + +func TestImporterCreatesCanonicalMeetingAndSyncRun(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + sourceHookCalls := 0 + var cacheLabels []string + importer := NewImporter(st, Hooks{ + AfterSourceSetup: func() error { + sourceHookCalls++ + return nil + }, + RefreshCache: func(_ context.Context, label string) error { + cacheLabels = append(cacheLabels, label) + return nil + }, + }) + + result, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + + assert.Equal(StatusCreated, result.Status) + assert.Equal("meeting:42", result.SourceMessageID) + assert.NotZero(result.SourceID) + assert.NotZero(result.MessageID) + assert.Equal(1, sourceHookCalls) + assert.Equal([]string{"meeting_import:local-meetings"}, cacheLabels) + + var sourceType, identifier, displayName string + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT source_type, identifier, display_name + FROM sources WHERE id = ? + `), result.SourceID).Scan(&sourceType, &identifier, &displayName)) + assert.Equal(SourceType, sourceType) + assert.Equal("local-meetings", identifier) + assert.Equal("Local Meetings", displayName) + + identities, err := st.ListAccountIdentities(result.SourceID) + require.NoError(err) + require.Len(identities, 1) + assert.Equal("user@example.com", identities[0].Address) + assert.Contains(identities[0].SourceSignal, "account-email") + + var ( + messageType, sourceMessageID, subject string + isFromMe bool + senderEmail sql.NullString + conversationType, conversationKey string + body, rawFormat, metadataJSON string + messageCount, participantCount int + ) + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT m.message_type, m.source_message_id, m.subject, m.is_from_me, + p.email_address, c.conversation_type, c.source_conversation_id, + mb.body_text, mr.raw_format, m.metadata, + c.message_count, c.participant_count + FROM messages m + JOIN conversations c ON c.id = m.conversation_id + JOIN message_bodies mb ON mb.message_id = m.id + JOIN message_raw mr ON mr.message_id = m.id + LEFT JOIN participants p ON p.id = m.sender_id + WHERE m.id = ? + `), result.MessageID).Scan( + &messageType, &sourceMessageID, &subject, &isFromMe, + &senderEmail, &conversationType, &conversationKey, + &body, &rawFormat, &metadataJSON, + &messageCount, &participantCount, + )) + assert.Equal(MessageType, messageType) + assert.Equal("meeting:42", sourceMessageID) + assert.Equal("Weekly planning", subject) + assert.False(isFromMe) + assert.Equal("organizer@example.com", senderEmail.String) + assert.Equal(ConversationType, conversationType) + assert.Equal("meeting:42", conversationKey) + assert.Contains(body, "[00:04] Test Speaker: Let's review the launch plan.") + assert.Equal(RawFormat, rawFormat) + assert.Equal(1, messageCount) + assert.Equal(1, participantCount) + + var metadata map[string]any + require.NoError(json.Unmarshal([]byte(metadataJSON), &metadata)) + assert.Equal(SourceType, metadata["platform"]) + + raw, err := st.GetMessageRaw(result.MessageID) + require.NoError(err) + assert.NotContains(string(raw), "account_email") + assert.Contains(string(raw), `"external_id":"42"`) + + var fromCount, toCount, memberCount int + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT COUNT(*) FROM message_recipients + WHERE message_id = ? AND recipient_type = 'from' + `), result.MessageID).Scan(&fromCount)) + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT COUNT(*) FROM message_recipients + WHERE message_id = ? AND recipient_type = 'to' + `), result.MessageID).Scan(&toCount)) + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT COUNT(*) FROM conversation_participants cp + JOIN messages m ON m.conversation_id = cp.conversation_id + WHERE m.id = ? + `), result.MessageID).Scan(&memberCount)) + assert.Equal(1, fromCount) + assert.Equal(1, toCount) + assert.Equal(1, memberCount) + + latest, err := st.GetLatestSync(result.SourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Equal(int64(1), latest.MessagesAdded) + assert.Equal(int64(0), latest.MessagesUpdated) +} + +func TestImporterRetriesUpdateSameMessageAndReplacePeople(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + + first, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + + second, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + assert.Equal(StatusUpdated, second.Status) + assert.Equal(first.MessageID, second.MessageID) + + changed := validImportRequest(t) + changed.Source.DisplayName = "Renamed Meetings" + changed.Source.AccountEmail = "organizer@example.com" + changed.Meeting.Title = "Replacement title" + changed.Meeting.SummaryMarkdown = "Replacement summary" + changed.Meeting.TranscriptSegments = nil + changed.Meeting.Transcript = "Speaker 1: replacement transcript" + changed.Meeting.Organizer = nil + changed.Meeting.Attendees = nil + + third, err := importer.Import(context.Background(), changed) + require.NoError(err) + assert.Equal(StatusUpdated, third.Status) + assert.Equal(first.MessageID, third.MessageID) + + var count int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&count)) + assert.Equal(1, count) + + var subject, body, displayName string + var senderID sql.NullInt64 + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT m.subject, mb.body_text, m.sender_id, s.display_name + FROM messages m + JOIN message_bodies mb ON mb.message_id = m.id + JOIN sources s ON s.id = m.source_id + WHERE m.id = ? + `), first.MessageID).Scan(&subject, &body, &senderID, &displayName)) + assert.Equal("Replacement title", subject) + assert.Contains(body, "Replacement summary") + assert.Contains(body, "Speaker 1: replacement transcript") + assert.False(senderID.Valid) + assert.Equal("Renamed Meetings", displayName) + + for _, tableQuery := range []string{ + `SELECT COUNT(*) FROM message_recipients WHERE message_id = ?`, + `SELECT COUNT(*) FROM conversation_participants cp + JOIN messages m ON m.conversation_id = cp.conversation_id WHERE m.id = ?`, + } { + require.NoError(st.DB().QueryRow(st.Rebind(tableQuery), first.MessageID).Scan(&count)) + assert.Equal(0, count) + } + + latest, err := st.GetLatestSync(first.SourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(0), latest.MessagesAdded) + assert.Equal(int64(1), latest.MessagesUpdated) + + identities, err := st.ListAccountIdentities(first.SourceID) + require.NoError(err) + assert.Len(identities, 2) +} + +func TestImporterUnchangedRetryDoesNotPersistOrCountUpdate(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + cacheCalls := 0 + importer := NewImporter(st, Hooks{ + RefreshCache: func(context.Context, string) error { + cacheCalls++ + return nil + }, + }) + + first, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + + sentinel := time.Date(2000, time.January, 2, 3, 4, 5, 0, time.UTC) + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages SET last_modified = ? WHERE id = ? + `), sentinel, first.MessageID) + require.NoError(err) + var watermarkBefore string + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT CAST(last_modified AS TEXT) FROM messages WHERE id = ? + `), first.MessageID).Scan(&watermarkBefore)) + + retry, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + assert.Equal(StatusUpdated, retry.Status) + assert.Equal(first.MessageID, retry.MessageID) + + var watermarkAfter string + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT CAST(last_modified AS TEXT) FROM messages WHERE id = ? + `), first.MessageID).Scan(&watermarkAfter)) + assert.Equal(watermarkBefore, watermarkAfter) + + latest, err := st.GetLatestSync(first.SourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Equal(int64(0), latest.MessagesAdded) + assert.Equal(int64(0), latest.MessagesUpdated) + assert.Equal(2, cacheCalls) +} + +func TestImporterUnchangedRetryRepairsStatsAfterPostPersistFailure(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + testutil.SkipIfPostgres(t, "uses a SQLite trigger to inject a stats failure") + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + _, err := st.DB().Exec(` + CREATE TRIGGER fail_meeting_import_stats + BEFORE UPDATE OF message_count ON conversations + BEGIN + SELECT RAISE(ABORT, 'forced meeting import stats failure'); + END + `) + require.NoError(err) + + _, err = importer.Import(context.Background(), validImportRequest(t)) + require.Error(err) + assert.Contains(err.Error(), "recompute meeting conversation stats") + + var messageID, sourceID int64 + var messageCount, participantCount int + var preview sql.NullString + require.NoError(st.DB().QueryRow(` + SELECT m.id, m.source_id, c.message_count, c.participant_count, + c.last_message_preview + FROM messages m + JOIN conversations c ON c.id = m.conversation_id + WHERE m.source_message_id = 'meeting:42' + `).Scan(&messageID, &sourceID, &messageCount, &participantCount, &preview)) + assert.Zero(messageCount) + assert.Zero(participantCount) + assert.False(preview.Valid) + + _, err = st.DB().Exec(`DROP TRIGGER fail_meeting_import_stats`) + require.NoError(err) + + retry, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + assert.Equal(StatusUpdated, retry.Status) + assert.Equal(messageID, retry.MessageID) + + var messagePreview sql.NullString + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT c.message_count, c.participant_count, c.last_message_preview, + m.snippet + FROM messages m + JOIN conversations c ON c.id = m.conversation_id + WHERE m.id = ? + `), messageID).Scan(&messageCount, &participantCount, &preview, &messagePreview)) + assert.Equal(1, messageCount) + assert.Equal(1, participantCount) + assert.Equal(messagePreview, preview) + + latest, err := st.GetLatestSync(sourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Zero(latest.MessagesAdded) + assert.Zero(latest.MessagesUpdated) +} + +func TestImporterStatsRecomputeDoesNotTouchSiblingConversations(t *testing.T) { + for _, tc := range []struct { + name string + unchanged bool + want Status + }{ + {name: "created", want: StatusCreated}, + {name: "unchanged", unchanged: true, want: StatusUpdated}, + } { + t.Run(tc.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + normalized, err := req.Normalize() + require.NoError(err) + if tc.unchanged { + _, err = importer.Import(context.Background(), req) + require.NoError(err) + } + + source, err := st.GetOrCreateSource(SourceType, normalized.Source.Identifier) + require.NoError(err) + siblingID, err := st.EnsureConversationWithType( + source.ID, + "sibling-meeting", + ConversationType, + "Sibling meeting", + ) + require.NoError(err) + _, err = st.DB().Exec(st.Rebind(` + UPDATE conversations + SET message_count = 71, + participant_count = 72, + last_message_preview = 'sibling sentinel' + WHERE id = ? + `), siblingID) + require.NoError(err) + + result, err := importer.Import(context.Background(), req) + require.NoError(err) + assert.Equal(tc.want, result.Status) + + var messageCount, participantCount int + var preview sql.NullString + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT message_count, participant_count, last_message_preview + FROM conversations + WHERE id = ? + `), siblingID).Scan(&messageCount, &participantCount, &preview)) + assert.Equal(71, messageCount) + assert.Equal(72, participantCount) + assert.Equal(sql.NullString{String: "sibling sentinel", Valid: true}, preview) + }) + } +} + +func TestImporterRetryRepairsUnreadableCanonicalSnapshot(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + + first, err := importer.Import(context.Background(), req) + require.NoError(err) + wantRaw, err := st.GetMessageRaw(first.MessageID) + require.NoError(err) + + _, err = st.DB().Exec(st.Rebind(` + UPDATE message_raw + SET raw_data = ?, compression = 'zlib' + WHERE message_id = ? + `), []byte("not a zlib stream"), first.MessageID) + require.NoError(err) + _, err = st.GetMessageRaw(first.MessageID) + require.Error(err, "corrupt fixture must be unreadable") + + retry, err := importer.Import(context.Background(), req) + require.NoError(err) + assert.Equal(StatusUpdated, retry.Status) + assert.Equal(first.MessageID, retry.MessageID) + + gotRaw, err := st.GetMessageRaw(first.MessageID) + require.NoError(err) + assert.Equal(wantRaw, gotRaw) + + latest, err := st.GetLatestSync(first.SourceID) + require.NoError(err) + assert.Equal(int64(1), latest.MessagesUpdated) +} + +func TestImporterPreservesDisplayNameWhenRetryOmitsIt(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + initial := validImportRequest(t) + initial.Source.DisplayName = "Named Meeting Source" + + created, err := importer.Import(context.Background(), initial) + require.NoError(err) + assert.Equal(StatusCreated, created.Status) + + retry := validImportRequest(t) + retry.Source.DisplayName = "" + updated, err := importer.Import(context.Background(), retry) + require.NoError(err) + assert.Equal(StatusUpdated, updated.Status) + + sources, err := st.ListSources(SourceType) + require.NoError(err) + require.Len(sources, 1) + assert.Equal("Named Meeting Source", sources[0].DisplayName.String) +} + +func TestImporterDefaultsDisplayNameOnFirstImport(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + req.Source.DisplayName = "" + + result, err := importer.Import(context.Background(), req) + require.NoError(err) + assert.Equal(StatusCreated, result.Status) + + sources, err := st.ListSources(SourceType) + require.NoError(err) + require.Len(sources, 1) + assert.Equal("local-meetings", sources[0].DisplayName.String) +} + +func TestImporterScopesExternalIDsBySource(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + firstRequest := validImportRequest(t) + secondRequest := validImportRequest(t) + secondRequest.Source.Identifier = "second-stream" + + first, err := importer.Import(context.Background(), firstRequest) + require.NoError(err) + second, err := importer.Import(context.Background(), secondRequest) + require.NoError(err) + + assert.NotEqual(first.SourceID, second.SourceID) + assert.NotEqual(first.MessageID, second.MessageID) + assert.Equal(first.SourceMessageID, second.SourceMessageID) +} + +func TestImporterMarksOrganizerFromConfirmedAccountAsFromMe(t *testing.T) { + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + req.Source.AccountEmail = "organizer@example.com" + + result, err := importer.Import(context.Background(), req) + require.NoError(t, err) + + var isFromMe bool + require.NoError(t, st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), result.MessageID).Scan(&isFromMe)) + assert.True(t, isFromMe) +} + +func TestImporterAccountEmailOnlyRetryUpdatesAttribution(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + + first, err := importer.Import(context.Background(), req) + require.NoError(err) + + var isFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), first.MessageID).Scan(&isFromMe)) + assert.False(isFromMe) + + req.Source.AccountEmail = req.Meeting.Organizer.Email + retry, err := importer.Import(context.Background(), req) + require.NoError(err) + assert.Equal(StatusUpdated, retry.Status) + assert.Equal(first.MessageID, retry.MessageID) + + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), first.MessageID).Scan(&isFromMe)) + assert.True(isFromMe) + + latest, err := st.GetLatestSync(first.SourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Zero(latest.MessagesAdded) + assert.Equal(int64(1), latest.MessagesUpdated) +} + +func TestImporterNewAccountEmailRefreshesEarlierMeetingAttributionForSource(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + + first, err := importer.Import(context.Background(), req) + require.NoError(err) + + req.Meeting.ExternalID = "43" + second, err := importer.Import(context.Background(), req) + require.NoError(err) + + otherSourceReq := req + otherSourceReq.Source.Identifier = "other-meetings" + otherSourceReq.Meeting.ExternalID = "44" + otherSource, err := importer.Import(context.Background(), otherSourceReq) + require.NoError(err) + + req.Source.AccountEmail = req.Meeting.Organizer.Email + req.Meeting.ExternalID = "45" + confirmation, err := importer.Import(context.Background(), req) + require.NoError(err) + assert.Equal(StatusCreated, confirmation.Status) + + for _, messageID := range []int64{first.MessageID, second.MessageID, confirmation.MessageID} { + var isFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), messageID).Scan(&isFromMe)) + assert.True(isFromMe, "message %d in the confirmed source", messageID) + } + + var otherSourceIsFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), otherSource.MessageID).Scan(&otherSourceIsFromMe)) + assert.False(otherSourceIsFromMe) + + latest, err := st.GetLatestSync(first.SourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Equal(int64(1), latest.MessagesAdded) + assert.Zero(latest.MessagesUpdated) +} + +func TestImporterPreconfirmedAccountEmailKeepsEarlierMeetingAttributionCurrent(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + + first, err := importer.Import(context.Background(), req) + require.NoError(err) + require.NoError(st.AddAccountIdentity( + first.SourceID, + req.Meeting.Organizer.Email, + "manual", + )) + + req.Source.AccountEmail = req.Meeting.Organizer.Email + req.Meeting.ExternalID = "43" + _, err = importer.Import(context.Background(), req) + require.NoError(err) + + var earlierIsFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), first.MessageID).Scan(&earlierIsFromMe)) + assert.True(earlierIsFromMe) +} + +func TestImporterMarksCacheFailureAndSafelyRetries(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + cacheErr := errors.New("synthetic cache failure") + failCache := true + importer := NewImporter(st, Hooks{ + RefreshCache: func(context.Context, string) error { + if failCache { + return cacheErr + } + return nil + }, + }) + + _, err := importer.Import(context.Background(), validImportRequest(t)) + require.ErrorIs(err, cacheErr) + + var messageID, sourceID int64 + require.NoError(st.DB().QueryRow(` + SELECT id, source_id FROM messages WHERE source_message_id = 'meeting:42' + `).Scan(&messageID, &sourceID)) + assert.NotZero(messageID, "message remains durable after cache failure") + + failed, err := st.GetLatestSync(sourceID) + require.NoError(err) + assert.Equal(store.SyncStatusFailed, failed.Status) + assert.Equal(int64(1), failed.MessagesProcessed) + assert.Equal(int64(1), failed.MessagesAdded) + + failCache = false + result, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + assert.Equal(StatusUpdated, result.Status) + assert.Equal(messageID, result.MessageID) + + completed, err := st.GetLatestSync(sourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, completed.Status) + assert.Equal(int64(0), completed.MessagesUpdated) +} + +func TestImporterSourceHookFailureStopsBeforeSync(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + hookErr := errors.New("synthetic migration failure") + importer := NewImporter(st, Hooks{ + AfterSourceSetup: func() error { return hookErr }, + }) + + _, err := importer.Import(context.Background(), validImportRequest(t)) + require.ErrorIs(err, hookErr) + + var count int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM sync_runs`).Scan(&count)) + assert.Equal(0, count) + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&count)) + assert.Equal(0, count) +} + +func TestImporterCancellationStopsBeforeSourceSetup(t *testing.T) { + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := importer.Import(ctx, validImportRequest(t)) + require.ErrorIs(t, err, context.Canceled) + + var count int + require.NoError(t, st.DB().QueryRow(`SELECT COUNT(*) FROM sources WHERE source_type = 'meeting_import'`).Scan(&count)) + assert.Equal(t, 0, count) +} + +func TestImporterCancellationDuringParticipantResolutionRollsBackParticipants(t *testing.T) { + testutil.SkipIfPostgres(t, "uses a SQLite trigger and registered function to pause participant insertion") + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + st.DB().SetMaxOpenConns(1) + importer := NewImporter(st, Hooks{}) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + participantStarted := make(chan struct{}) + conn, err := st.DB().Conn(context.Background()) + require.NoError(err, "get SQLite connection") + err = conn.Raw(func(driverConn any) error { + sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn) + require.True(ok, "driver connection is SQLite") + return sqliteConn.RegisterFunc("wait_for_participant_cancel", func() int { + close(participantStarted) + <-ctx.Done() + return 0 + }, true) + }) + require.NoError(err, "register cancellation function") + require.NoError(conn.Close(), "return SQLite connection to pool") + _, err = st.DB().Exec(` + CREATE TRIGGER wait_before_meeting_participant + BEFORE INSERT ON participants + WHEN NEW.email_address = 'attendee@example.com' + BEGIN + SELECT wait_for_participant_cancel(); + END + `) + require.NoError(err, "create participant cancellation trigger") + + req := validImportRequest(t) + done := make(chan error, 1) + go func() { + _, importErr := importer.Import(ctx, req) + done <- importErr + }() + + select { + case <-participantStarted: + case <-time.After(time.Second): + require.FailNow("meeting import did not reach attendee insertion") + } + cancel() + + select { + case err = <-done: + case <-time.After(time.Second): + require.FailNow("meeting import did not stop after cancellation") + } + require.ErrorIs(err, context.Canceled) + + var participantCount int + require.NoError(st.DB().QueryRow(` + SELECT COUNT(*) + FROM participants + WHERE email_address IN ('organizer@example.com', 'attendee@example.com') + `).Scan(&participantCount)) + assert.Zero( + participantCount, + "organizer and attendee inserts must roll back with canceled message persistence", + ) + var messageCount int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&messageCount)) + assert.Zero(messageCount) +} + +func TestImporterCancellationDuringCheckpointLeavesFailedSync(t *testing.T) { + testutil.SkipIfPostgres(t, "uses a SQLite trigger and registered function to pause the checkpoint") + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + st.DB().SetMaxOpenConns(1) + importer := NewImporter(st, Hooks{}) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + checkpointStarted := make(chan struct{}) + conn, err := st.DB().Conn(context.Background()) + require.NoError(err, "get SQLite connection") + err = conn.Raw(func(driverConn any) error { + sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn) + require.True(ok, "driver connection is SQLite") + return sqliteConn.RegisterFunc("wait_for_checkpoint_cancel", func() int { + close(checkpointStarted) + <-ctx.Done() + return 0 + }, true) + }) + require.NoError(err, "register cancellation function") + require.NoError(conn.Close(), "return SQLite connection to pool") + _, err = st.DB().Exec(` + CREATE TRIGGER wait_before_meeting_checkpoint + BEFORE UPDATE OF messages_processed ON sync_runs + WHEN NEW.messages_processed = 1 AND NEW.status = 'running' + BEGIN + SELECT wait_for_checkpoint_cancel(); + END + `) + require.NoError(err, "create checkpoint cancellation trigger") + + req := validImportRequest(t) + done := make(chan error, 1) + go func() { + _, importErr := importer.Import(ctx, req) + done <- importErr + }() + + select { + case <-checkpointStarted: + case <-time.After(time.Second): + require.FailNow("meeting import did not reach checkpoint") + } + cancel() + + select { + case err = <-done: + case <-time.After(time.Second): + require.FailNow("meeting import did not stop after cancellation") + } + require.ErrorIs(err, context.Canceled) + + var messageCount int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&messageCount)) + assert.Equal(1, messageCount, "canonical message is committed before checkpointing") + + sources, err := st.ListSources(SourceType) + require.NoError(err) + require.Len(sources, 1) + latest, err := st.GetLatestSync(sources[0].ID) + require.NoError(err) + assert.Equal(store.SyncStatusFailed, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Equal(int64(1), latest.MessagesAdded) + assert.Contains(latest.ErrorMessage.String, context.Canceled.Error()) +} + +func TestImporterRawFailureRollsBackCanonicalSnapshot(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + testutil.SkipIfPostgres(t, "uses a SQLite trigger to inject a raw archive failure") + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + require.True(st.FTS5Available()) + _, err := st.DB().Exec(` + CREATE TRIGGER fail_meeting_import_raw + BEFORE INSERT ON message_raw + WHEN NEW.raw_format = 'meeting_json' + BEGIN + SELECT RAISE(ABORT, 'forced meeting import raw failure'); + END + `) + require.NoError(err) + + _, err = importer.Import(context.Background(), validImportRequest(t)) + require.Error(err) + assert.Contains(err.Error(), "persist meeting") + + for table, want := range map[string]int{ + "messages": 0, + "conversations": 0, + "message_bodies": 0, + "message_raw": 0, + "message_recipients": 0, + "messages_fts": 0, + } { + var got int + require.NoError(st.DB().QueryRow("SELECT COUNT(*) FROM "+table).Scan(&got), table) + assert.Equal(want, got, table) + } + + sources, err := st.ListSources(SourceType) + require.NoError(err) + require.Len(sources, 1) + latest, err := st.GetLatestSync(sources[0].ID) + require.NoError(err) + assert.Equal(store.SyncStatusFailed, latest.Status) + assert.Contains(latest.ErrorMessage.String, "persist meeting") +} + +func TestImporterIndexesSubjectBodyAndAddresses(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + testutil.SkipIfPostgres(t, "asserts against the SQLite FTS5 virtual table") + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + + result, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + + var subject, body, fromAddr, toAddrs string + require.NoError(st.DB().QueryRow(` + SELECT subject, body, from_addr, to_addr + FROM messages_fts WHERE rowid = ? + `, result.MessageID).Scan(&subject, &body, &fromAddr, &toAddrs)) + assert.Equal("Weekly planning", subject) + assert.Contains(body, "launch plan") + assert.Equal("organizer@example.com", fromAddr) + assert.Equal("attendee@example.com", strings.TrimSpace(toAddrs)) +} diff --git a/internal/meetingimport/models.go b/internal/meetingimport/models.go new file mode 100644 index 000000000..ef36019f7 --- /dev/null +++ b/internal/meetingimport/models.go @@ -0,0 +1,302 @@ +// Package meetingimport validates and stores provider-neutral meeting +// transcripts submitted through the msgvault HTTP API. +package meetingimport + +import ( + "errors" + "fmt" + "math" + "net/mail" + "strings" + "time" + "unicode/utf8" +) + +const ( + SourceType = "meeting_import" + ConversationType = "meeting" + MessageType = "meeting_transcript" + RawFormat = "meeting_json" + MaxRequestBytes = int64(16 << 20) + + maxSourceIdentifierChars = 128 + maxSourceDisplayNameChars = 256 + maxExternalIDChars = 256 + maxTitleChars = 4096 +) + +var ( + ErrMalformedRequest = errors.New("malformed meeting import request") + ErrRequestTooLarge = errors.New("meeting import request too large") + ErrValidation = errors.New("meeting import validation failed") +) + +type MeetingImportRequest struct { + Source Source `json:"source"` + Meeting Meeting `json:"meeting"` +} + +// Request is kept as a concise internal alias for the meeting import wire +// contract. The named type gives generated API clients an unambiguous schema. +type Request = MeetingImportRequest + +type Source struct { + Identifier string `json:"identifier" maxLength:"128"` + DisplayName string `json:"display_name,omitempty" maxLength:"256"` + AccountEmail string `json:"account_email" format:"email"` +} + +type Meeting struct { + ExternalID string `json:"external_id" maxLength:"256"` + Title string `json:"title,omitempty" maxLength:"4096"` + StartedAt string `json:"started_at" format:"date-time"` + EndedAt string `json:"ended_at,omitempty" format:"date-time"` + SummaryMarkdown string `json:"summary_markdown,omitempty"` + SummaryText string `json:"summary_text,omitempty"` + Transcript string `json:"transcript,omitempty"` + TranscriptSegments []TranscriptSegment `json:"transcript_segments,omitempty"` + Organizer *MeetingPerson `json:"organizer,omitempty"` + Attendees []MeetingPerson `json:"attendees,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +type MeetingPerson struct { + Name string `json:"name,omitempty"` + Email string `json:"email" format:"email"` +} + +type TranscriptSegment struct { + Speaker string `json:"speaker"` + Text string `json:"text"` + OffsetSeconds *float64 `json:"offset_seconds,omitempty" minimum:"0"` +} + +type NormalizedRequest struct { + Source Source + Meeting NormalizedMeeting +} + +type NormalizedMeeting struct { + ExternalID string + Title string + StartedAt time.Time + EndedAt *time.Time + SummaryMarkdown string + SummaryText string + Transcript string + TranscriptSegments []TranscriptSegment + Organizer *MeetingPerson + Attendees []MeetingPerson + Metadata map[string]any +} + +func (r Request) Normalize() (NormalizedRequest, error) { + source, err := normalizeSource(r.Source) + if err != nil { + return NormalizedRequest{}, err + } + meeting, err := normalizeMeeting(r.Meeting) + if err != nil { + return NormalizedRequest{}, err + } + return NormalizedRequest{Source: source, Meeting: meeting}, nil +} + +func normalizeSource(source Source) (Source, error) { + source.Identifier = strings.TrimSpace(source.Identifier) + if err := validateBoundedRequired("source.identifier", source.Identifier, maxSourceIdentifierChars); err != nil { + return Source{}, err + } + + source.DisplayName = strings.TrimSpace(source.DisplayName) + if err := validateBoundedOptional("source.display_name", source.DisplayName, maxSourceDisplayNameChars); err != nil { + return Source{}, err + } + + accountEmail, err := normalizeEmail("source.account_email", source.AccountEmail) + if err != nil { + return Source{}, err + } + source.AccountEmail = accountEmail + return source, nil +} + +func normalizeMeeting(meeting Meeting) (NormalizedMeeting, error) { + meeting.ExternalID = strings.TrimSpace(meeting.ExternalID) + if err := validateBoundedRequired("meeting.external_id", meeting.ExternalID, maxExternalIDChars); err != nil { + return NormalizedMeeting{}, err + } + meeting.Title = strings.TrimSpace(meeting.Title) + if err := validateBoundedOptional("meeting.title", meeting.Title, maxTitleChars); err != nil { + return NormalizedMeeting{}, err + } + + startedAt, err := parseTimestamp("meeting.started_at", meeting.StartedAt) + if err != nil { + return NormalizedMeeting{}, err + } + var endedAt *time.Time + if strings.TrimSpace(meeting.EndedAt) != "" { + parsed, parseErr := parseTimestamp("meeting.ended_at", meeting.EndedAt) + if parseErr != nil { + return NormalizedMeeting{}, parseErr + } + if parsed.Before(startedAt) { + return NormalizedMeeting{}, validationError("meeting.ended_at must not be before meeting.started_at") + } + endedAt = &parsed + } + + summaryMarkdown := strings.TrimSpace(meeting.SummaryMarkdown) + summaryText := strings.TrimSpace(meeting.SummaryText) + transcript := strings.TrimSpace(meeting.Transcript) + segments, err := normalizeSegments(meeting.TranscriptSegments) + if err != nil { + return NormalizedMeeting{}, err + } + if transcript != "" && len(segments) > 0 { + return NormalizedMeeting{}, validationError("meeting.transcript and meeting.transcript_segments are mutually exclusive") + } + if summaryMarkdown == "" && summaryText == "" && transcript == "" && len(segments) == 0 { + return NormalizedMeeting{}, validationError("meeting requires a summary or transcript") + } + + var organizer *MeetingPerson + if meeting.Organizer != nil { + normalizedOrganizer, normalizeErr := normalizePerson("meeting.organizer", *meeting.Organizer) + if normalizeErr != nil { + return NormalizedMeeting{}, normalizeErr + } + organizer = &normalizedOrganizer + } + attendees, err := normalizeAttendees(meeting.Attendees) + if err != nil { + return NormalizedMeeting{}, err + } + + return NormalizedMeeting{ + ExternalID: meeting.ExternalID, + Title: meeting.Title, + StartedAt: startedAt, + EndedAt: endedAt, + SummaryMarkdown: summaryMarkdown, + SummaryText: summaryText, + Transcript: transcript, + TranscriptSegments: segments, + Organizer: organizer, + Attendees: attendees, + Metadata: meeting.Metadata, + }, nil +} + +func normalizeSegments(segments []TranscriptSegment) ([]TranscriptSegment, error) { + if len(segments) == 0 { + return nil, nil + } + out := make([]TranscriptSegment, len(segments)) + var previousOffset float64 + havePreviousOffset := false + for idx, segment := range segments { + segment.Speaker = strings.TrimSpace(segment.Speaker) + if segment.Speaker == "" { + return nil, validationError("meeting.transcript_segments[%d].speaker is required", idx) + } + segment.Text = strings.TrimSpace(segment.Text) + if segment.Text == "" { + return nil, validationError("meeting.transcript_segments[%d].text is required", idx) + } + if segment.OffsetSeconds != nil { + offset := *segment.OffsetSeconds + if math.IsNaN(offset) || math.IsInf(offset, 0) || offset < 0 { + return nil, validationError("meeting.transcript_segments[%d].offset_seconds must be finite and non-negative", idx) + } + if havePreviousOffset && offset < previousOffset { + return nil, validationError("meeting.transcript_segments offsets must be non-decreasing") + } + previousOffset = offset + havePreviousOffset = true + } + out[idx] = segment + } + return out, nil +} + +func normalizePerson(field string, person MeetingPerson) (MeetingPerson, error) { + email, err := normalizeEmail(field+".email", person.Email) + if err != nil { + return MeetingPerson{}, err + } + return MeetingPerson{ + Name: strings.TrimSpace(person.Name), + Email: email, + }, nil +} + +func normalizeAttendees(attendees []MeetingPerson) ([]MeetingPerson, error) { + out := make([]MeetingPerson, 0, len(attendees)) + seen := make(map[string]struct{}, len(attendees)) + for idx := range attendees { + person, err := normalizePerson( + fmt.Sprintf("meeting.attendees[%d]", idx), + attendees[idx], + ) + if err != nil { + return nil, err + } + key := strings.ToLower(person.Email) + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + out = append(out, person) + } + return out, nil +} + +func normalizeEmail(field, value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "", validationError("%s is required", field) + } + if !utf8.ValidString(value) { + return "", validationError("%s must be valid UTF-8", field) + } + parsed, err := mail.ParseAddress(value) + if err != nil || parsed.Name != "" || !strings.EqualFold(parsed.Address, value) { + return "", validationError("%s must be one email address without a display name", field) + } + return strings.ToLower(parsed.Address), nil +} + +func parseTimestamp(field, value string) (time.Time, error) { + value = strings.TrimSpace(value) + if value == "" { + return time.Time{}, validationError("%s is required", field) + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return time.Time{}, validationError("%s must be an RFC3339 timestamp with an explicit offset", field) + } + return parsed.UTC(), nil +} + +func validateBoundedRequired(field, value string, maxChars int) error { + if value == "" { + return validationError("%s is required", field) + } + return validateBoundedOptional(field, value, maxChars) +} + +func validateBoundedOptional(field, value string, maxChars int) error { + if !utf8.ValidString(value) { + return validationError("%s must be valid UTF-8", field) + } + if utf8.RuneCountInString(value) > maxChars { + return validationError("%s must be at most %d characters", field, maxChars) + } + return nil +} + +func validationError(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrValidation, fmt.Sprintf(format, args...)) +} diff --git a/internal/meetingimport/models_test.go b/internal/meetingimport/models_test.go new file mode 100644 index 000000000..34304999d --- /dev/null +++ b/internal/meetingimport/models_test.go @@ -0,0 +1,181 @@ +package meetingimport + +import ( + "math" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func decodedValidRequest(t *testing.T) Request { + t.Helper() + req, err := DecodeRequest(strings.NewReader(validRequestJSON), MaxRequestBytes) + require.NoError(t, err) + return req +} + +func TestRequestNormalizeCanonicalizesValues(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := decodedValidRequest(t) + req.Meeting.Attendees = append(req.Meeting.Attendees, + MeetingPerson{Name: "Duplicate", Email: "attendee@EXAMPLE.com"}, + ) + + got, err := req.Normalize() + require.NoError(err) + + assert.Equal("local-meetings", got.Source.Identifier) + assert.Equal("Local Meetings", got.Source.DisplayName) + assert.Equal("user@example.com", got.Source.AccountEmail) + assert.Equal("42", got.Meeting.ExternalID) + assert.Equal("Weekly planning", got.Meeting.Title) + assert.Equal(time.Date(2026, 7, 23, 18, 0, 0, 0, time.UTC), got.Meeting.StartedAt) + require.NotNil(got.Meeting.EndedAt) + assert.Equal(time.Date(2026, 7, 23, 18, 30, 0, 0, time.UTC), *got.Meeting.EndedAt) + require.NotNil(got.Meeting.Organizer) + assert.Equal(MeetingPerson{Name: "Test Organizer", Email: "organizer@example.com"}, *got.Meeting.Organizer) + assert.Equal([]MeetingPerson{{Name: "Test Attendee", Email: "attendee@example.com"}}, got.Meeting.Attendees) + assert.Equal("Test Speaker", got.Meeting.TranscriptSegments[0].Speaker) + assert.Equal("Let's review the launch plan.", got.Meeting.TranscriptSegments[0].Text) +} + +func TestRequestNormalizeValidatesRequiredAndBoundedFields(t *testing.T) { + tests := []struct { + name string + mutate func(*Request) + }{ + {name: "source identifier required", mutate: func(r *Request) { r.Source.Identifier = " " }}, + {name: "source identifier character limit", mutate: func(r *Request) { r.Source.Identifier = strings.Repeat("é", 129) }}, + {name: "display name character limit", mutate: func(r *Request) { r.Source.DisplayName = strings.Repeat("é", 257) }}, + {name: "account email required", mutate: func(r *Request) { r.Source.AccountEmail = "" }}, + {name: "account email invalid", mutate: func(r *Request) { r.Source.AccountEmail = "not-an-email" }}, + {name: "account display address rejected", mutate: func(r *Request) { r.Source.AccountEmail = "User " }}, + {name: "external id required", mutate: func(r *Request) { r.Meeting.ExternalID = "" }}, + {name: "external id character limit", mutate: func(r *Request) { r.Meeting.ExternalID = strings.Repeat("é", 257) }}, + {name: "title character limit", mutate: func(r *Request) { r.Meeting.Title = strings.Repeat("é", 4097) }}, + {name: "started at required", mutate: func(r *Request) { r.Meeting.StartedAt = "" }}, + {name: "started at timezone required", mutate: func(r *Request) { r.Meeting.StartedAt = "2026-07-23T18:00:00" }}, + {name: "started at malformed", mutate: func(r *Request) { r.Meeting.StartedAt = "later" }}, + {name: "ended at timezone required", mutate: func(r *Request) { r.Meeting.EndedAt = "2026-07-23T18:30:00" }}, + {name: "ended before start", mutate: func(r *Request) { r.Meeting.EndedAt = "2026-07-23T10:59:59-07:00" }}, + {name: "organizer email required", mutate: func(r *Request) { r.Meeting.Organizer.Email = "" }}, + {name: "organizer email invalid", mutate: func(r *Request) { r.Meeting.Organizer.Email = "bad" }}, + {name: "attendee email required", mutate: func(r *Request) { r.Meeting.Attendees[0].Email = "" }}, + {name: "attendee email invalid", mutate: func(r *Request) { r.Meeting.Attendees[0].Email = "bad" }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := decodedValidRequest(t) + tt.mutate(&req) + _, err := req.Normalize() + require.Error(t, err) + assert.ErrorIs(t, err, ErrValidation) + }) + } +} + +func TestRequestNormalizeAcceptsMultibyteValuesAtCharacterLimits(t *testing.T) { + req := validImportRequest(t) + req.Source.Identifier = strings.Repeat("é", 128) + req.Source.DisplayName = strings.Repeat("é", 256) + req.Meeting.ExternalID = strings.Repeat("é", 256) + req.Meeting.Title = strings.Repeat("é", 4096) + + _, err := req.Normalize() + require.NoError(t, err) +} + +func TestRequestNormalizeValidatesMeetingContent(t *testing.T) { + tests := []struct { + name string + mutate func(*Request) + }{ + { + name: "summary and transcripts empty", + mutate: func(r *Request) { + r.Meeting.SummaryMarkdown = "" + r.Meeting.SummaryText = "" + r.Meeting.Transcript = "" + r.Meeting.TranscriptSegments = nil + }, + }, + { + name: "plain and structured transcript conflict", + mutate: func(r *Request) { r.Meeting.Transcript = "Speaker 1: duplicate" }, + }, + { + name: "segment speaker required", + mutate: func(r *Request) { r.Meeting.TranscriptSegments[0].Speaker = " " }, + }, + { + name: "segment text required", + mutate: func(r *Request) { r.Meeting.TranscriptSegments[0].Text = " " }, + }, + { + name: "negative segment offset", + mutate: func(r *Request) { + value := -1.0 + r.Meeting.TranscriptSegments[0].OffsetSeconds = &value + }, + }, + { + name: "nan segment offset", + mutate: func(r *Request) { + value := math.NaN() + r.Meeting.TranscriptSegments[0].OffsetSeconds = &value + }, + }, + { + name: "infinite segment offset", + mutate: func(r *Request) { + value := math.Inf(1) + r.Meeting.TranscriptSegments[0].OffsetSeconds = &value + }, + }, + { + name: "decreasing segment offsets", + mutate: func(r *Request) { + later := 8.0 + earlier := 7.0 + r.Meeting.TranscriptSegments = append(r.Meeting.TranscriptSegments, + TranscriptSegment{Speaker: "Speaker 2", Text: "Second", OffsetSeconds: &later}, + TranscriptSegment{Speaker: "Speaker 1", Text: "Third", OffsetSeconds: &earlier}, + ) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := decodedValidRequest(t) + tt.mutate(&req) + _, err := req.Normalize() + require.Error(t, err) + assert.ErrorIs(t, err, ErrValidation) + }) + } +} + +func TestRequestNormalizeAllowsEqualTimesAndPlainTranscript(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := decodedValidRequest(t) + req.Meeting.EndedAt = req.Meeting.StartedAt + req.Meeting.TranscriptSegments = nil + req.Meeting.Transcript = "\nSpeaker 1: hello\nSpeaker 2: hi\n" + + got, err := req.Normalize() + require.NoError(err) + + assert.Equal("Speaker 1: hello\nSpeaker 2: hi", got.Meeting.Transcript) + assert.Empty(got.Meeting.TranscriptSegments) + require.NotNil(got.Meeting.EndedAt) + assert.True(got.Meeting.EndedAt.Equal(got.Meeting.StartedAt)) +} diff --git a/internal/store/account_identities.go b/internal/store/account_identities.go index 1b1ae8303..85289d93d 100644 --- a/internal/store/account_identities.go +++ b/internal/store/account_identities.go @@ -116,13 +116,13 @@ func looksLikeEmail(addr string) bool { // column accommodates email, phone E.164, and synthetic identifiers like // chat handles where case can be significant). // -// Confirming a brand new (source_id, address) pair changes which -// participants are owners for the source, so it bumps both the identity -// revision (the owner_participants cache dataset depends on it) and the -// account-identity revision (the message-baked is_from_me flag depends on -// it and can only be repaired by a full cache rebuild). Merging a new -// signal into an already-confirmed address does not change that mapping, -// so it leaves both revisions untouched. +// Confirming a brand new (source_id, address) pair changes which participants +// are owners for the source. The same transaction repairs is_from_me in the +// primary store and bumps both the identity revision (the owner_participants +// cache dataset depends on it) and the account-identity revision (Parquet +// shards bake the flag and need a full cache rebuild). Merging a new signal +// into an already-confirmed address does not change that mapping, so it leaves +// attribution and both revisions untouched. // // Concurrency: the read-modify-write runs inside a transaction that first // takes lockIdentityMutationTx's write lock, mirroring LinkParticipants so @@ -141,6 +141,48 @@ func (s *Store) AddAccountIdentityContext( ctx context.Context, sourceID int64, address, signal string, +) error { + return s.addAccountIdentityContext( + ctx, + sourceID, + address, + signal, + func(ctx context.Context, tx *loggedTx) error { + return refreshSourceMessageAttributionContext(ctx, tx, sourceID, "") + }, + ) +} + +// AddAccountIdentityAndRefreshMessageAttributionContext confirms an identity +// and, only when the identity is brand new, marks matching earlier messages in +// the source as sent by the account. Identity creation and attribution repair +// run in the same transaction. excludeSourceMessageID keeps the meeting +// currently being retried on its normal persistence path. +func (s *Store) AddAccountIdentityAndRefreshMessageAttributionContext( + ctx context.Context, + sourceID int64, + address, signal, excludeSourceMessageID string, +) error { + return s.addAccountIdentityContext( + ctx, + sourceID, + address, + signal, + func(ctx context.Context, tx *loggedTx) error { + return refreshSourceMessageAttributionContext( + ctx, tx, sourceID, excludeSourceMessageID, + ) + }, + ) +} + +type accountIdentityInsertHook func(context.Context, *loggedTx) error + +func (s *Store) addAccountIdentityContext( + ctx context.Context, + sourceID int64, + address, signal string, + onInsert accountIdentityInsertHook, ) error { addr := strings.TrimSpace(address) if addr == "" { @@ -156,7 +198,7 @@ func (s *Store) AddAccountIdentityContext( if err := ctx.Err(); err != nil { return err } - err := s.addAccountIdentityOnce(ctx, sourceID, addr, signal, match) + err := s.addAccountIdentityOnce(ctx, sourceID, addr, signal, match, onInsert) if err == nil { return nil } @@ -171,7 +213,11 @@ func (s *Store) AddAccountIdentityContext( // transaction. The caller's retry loop handles unique-violation // (concurrent INSERT race) and busy/snapshot errors (SQLite). func (s *Store) addAccountIdentityOnce( - ctx context.Context, sourceID int64, addr, signal string, match identifierMatch, + ctx context.Context, + sourceID int64, + addr, signal string, + match identifierMatch, + onInsert accountIdentityInsertHook, ) error { return s.withTxContext(ctx, func(tx *loggedTx) error { if err := s.lockIdentityMutationTxContext(ctx, tx); err != nil { @@ -201,6 +247,11 @@ func (s *Store) addAccountIdentityOnce( if err := s.bumpAccountIdentityRevisionContext(ctx, tx); err != nil { return err } + if onInsert != nil { + if err := onInsert(ctx, tx); err != nil { + return err + } + } case err != nil: return fmt.Errorf("read existing source_signal: %w", err) default: @@ -334,7 +385,10 @@ func (s *Store) RemoveAccountIdentityContext( if _, err := s.bumpIdentityRevisionContext(ctx, tx); err != nil { return err } - return s.bumpAccountIdentityRevisionContext(ctx, tx) + if err := s.bumpAccountIdentityRevisionContext(ctx, tx); err != nil { + return err + } + return refreshSourceMessageAttributionContext(ctx, tx, sourceID, "") }) if err != nil { return 0, err diff --git a/internal/store/account_identities_test.go b/internal/store/account_identities_test.go index 212061956..4ea1f6d2f 100644 --- a/internal/store/account_identities_test.go +++ b/internal/store/account_identities_test.go @@ -1,14 +1,176 @@ package store_test import ( + "database/sql" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/testutil" "go.kenn.io/msgvault/internal/testutil/storetest" ) +func TestRemoveAccountIdentityRecomputesOnlyIdentityDerivedAttribution(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := storetest.New(t) + st := f.Store + + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + identityDerived := f.NewMessage().WithSourceMessageID("identity-derived").Build() + identityDerived.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + identityDerivedID, err := st.UpsertMessage(identityDerived) + require.NoError(err, "persist identity-derived candidate") + + sourceNative := f.NewMessage(). + WithSourceMessageID("source-native"). + WithIsFromMe(true). + Build() + sourceNative.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + sourceNativeID, err := st.UpsertMessage(sourceNative) + require.NoError(err, "persist source-native message") + _, err = st.DB().Exec( + st.Rebind(`UPDATE messages SET source_is_from_me = NULL WHERE id = ?`), + sourceNativeID, + ) + require.NoError(err, "simulate legacy message without attribution provenance") + _, err = st.DB().Exec( + st.Rebind(`DELETE FROM applied_migrations WHERE name = ?`), + "message_attribution_provenance_v2", + ) + require.NoError(err, "reset attribution migration sentinel") + require.NoError(st.InitSchema(), "initialize legacy attribution provenance") + + require.NoError(st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual")) + afterAdd, err := st.GetMessageIsFromMe(identityDerivedID) + require.NoError(err, "read identity-derived attribution after add") + assert.True(afterAdd) + + removed, err := st.RemoveAccountIdentity(f.Source.ID, "owner@example.com") + require.NoError(err, "RemoveAccountIdentity") + require.Equal(int64(1), removed) + + afterRemove, err := st.GetMessageIsFromMe(identityDerivedID) + require.NoError(err, "read identity-derived attribution after remove") + assert.False(afterRemove, "removing the last matching identity must clear derived attribution") + nativeAfterRemove, err := st.GetMessageIsFromMe(sourceNativeID) + require.NoError(err, "read source-native attribution after remove") + assert.True(nativeAfterRemove, "identity removal must preserve source-native attribution") +} + +func TestAddAccountIdentityUpdatesOnlyChangedMessageAttribution(t *testing.T) { + testutil.SkipIfPostgres(t, "SQLite audit trigger measures updated message rows") + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + ownerID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + otherID := f.EnsureParticipant("other@example.com", "Other", "example.com") + + ownerMessage := f.NewMessage().WithSourceMessageID("owner-message").Build() + ownerMessage.SenderID = sql.NullInt64{Int64: ownerID, Valid: true} + ownerMessageID, err := st.UpsertMessage(ownerMessage) + require.NoError(err, "persist matching message") + + otherMessage := f.NewMessage().WithSourceMessageID("other-message").Build() + otherMessage.SenderID = sql.NullInt64{Int64: otherID, Valid: true} + otherMessageID, err := st.UpsertMessage(otherMessage) + require.NoError(err, "persist unrelated message") + + _, err = st.DB().Exec(` + DROP TRIGGER IF EXISTS trg_messages_last_modified; + CREATE TABLE message_update_audit (message_id INTEGER NOT NULL); + CREATE TRIGGER audit_message_update + AFTER UPDATE ON messages + BEGIN + INSERT INTO message_update_audit (message_id) VALUES (NEW.id); + END; + `) + require.NoError(err, "install message update audit") + + require.NoError( + st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual"), + "add matching identity", + ) + + var ownerUpdates, otherUpdates int + require.NoError(st.DB().QueryRow( + `SELECT COUNT(*) FROM message_update_audit WHERE message_id = ?`, + ownerMessageID, + ).Scan(&ownerUpdates)) + require.NoError(st.DB().QueryRow( + `SELECT COUNT(*) FROM message_update_audit WHERE message_id = ?`, + otherMessageID, + ).Scan(&otherUpdates)) + assert.Equal(1, ownerUpdates, "matching attribution must be updated") + assert.Zero(otherUpdates, "unchanged attribution must not rewrite the message") +} + +func TestUpsertMessageDerivesAttributionFromConfirmedIdentity(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + require.NoError( + st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual"), + "confirm identity before ingestion", + ) + + message := f.NewMessage().WithSourceMessageID("import-after-confirmation").Build() + message.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist message") + + var isFromMe, sourceIsFromMe, identityIsFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me, source_is_from_me, identity_is_from_me + FROM messages + WHERE id = ? + `), messageID).Scan(&isFromMe, &sourceIsFromMe, &identityIsFromMe)) + assert.True(isFromMe, "confirmed sender must be attributed during initial persistence") + assert.False(sourceIsFromMe, "incoming message did not carry source-native attribution") + assert.True(identityIsFromMe, "confirmed sender attribution must retain identity provenance") +} + +func TestUpsertMessagePreservesRepairedIdentityAttribution(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + message := f.NewMessage().WithSourceMessageID("resynced-message").Build() + message.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist unattributed message") + + require.NoError( + st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual"), + "confirm identity and repair existing message", + ) + isFromMe, err := st.GetMessageIsFromMe(messageID) + require.NoError(err, "read repaired attribution") + require.True(isFromMe, "identity confirmation must repair the existing message") + + message.Subject = sql.NullString{String: "resynced subject", Valid: true} + _, err = st.UpsertMessage(message) + require.NoError(err, "re-upsert message without caller-derived attribution") + + var sourceIsFromMe, identityIsFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me, source_is_from_me, identity_is_from_me + FROM messages + WHERE id = ? + `), messageID).Scan(&isFromMe, &sourceIsFromMe, &identityIsFromMe)) + assert.True(isFromMe, "re-sync must preserve attribution from the confirmed sender") + assert.False(sourceIsFromMe, "re-sync did not introduce source-native attribution") + assert.True(identityIsFromMe, "re-sync must preserve identity-derived provenance") +} + func TestAddAndListAccountIdentities(t *testing.T) { require := require.New(t) assert := assert.New(t) diff --git a/internal/store/attribution_provenance_migration_test.go b/internal/store/attribution_provenance_migration_test.go new file mode 100644 index 000000000..094a6e9bc --- /dev/null +++ b/internal/store/attribution_provenance_migration_test.go @@ -0,0 +1,325 @@ +package store_test + +import ( + "database/sql" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/gcal" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil/storetest" +) + +func TestInitSchemaBackfillsLegacyIdentityDerivedAttribution(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + + type migratedMessage struct { + sourceID int64 + messageID int64 + } + migrated := make([]migratedMessage, 0, 2) + for _, sourceType := range []string{"granola", "circleback"} { + src, err := st.GetOrCreateSource(sourceType, sourceType+"-account") + require.NoError(err, "create %s source", sourceType) + convID, err := st.EnsureConversationWithType( + src.ID, + sourceType+"-conversation", + "meeting", + "Legacy meeting", + ) + require.NoError(err, "create %s conversation", sourceType) + messageID, err := st.UpsertMessage(&store.Message{ + SourceID: src.ID, + ConversationID: convID, + SourceMessageID: sourceType + "-message", + MessageType: "meeting", + SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + IsFromMe: true, + }) + require.NoError(err, "persist legacy %s message", sourceType) + require.NoError( + st.AddAccountIdentity(src.ID, "owner@example.com", "account-identifier"), + "add %s identity", + sourceType, + ) + migrated = append(migrated, migratedMessage{sourceID: src.ID, messageID: messageID}) + } + + nativeMessage := f.NewMessage(). + WithSourceMessageID("source-native-control"). + WithIsFromMe(true). + Build() + nativeMessage.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + nativeMessageID, err := st.UpsertMessage(nativeMessage) + require.NoError(err, "persist source-native control") + + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages + SET source_is_from_me = NULL, identity_is_from_me = FALSE + WHERE id IN (?, ?, ?) + `), migrated[0].messageID, migrated[1].messageID, nativeMessageID) + require.NoError(err, "simulate rows written before attribution provenance") + _, err = st.DB().Exec(st.Rebind(` + DELETE FROM applied_migrations + WHERE name = ? + `), "message_attribution_provenance_v2") + require.NoError(err, "reset attribution migration sentinel") + _, err = st.DB().Exec(` + INSERT INTO applied_migrations (name) + VALUES ('message_attribution_provenance_v1') + ON CONFLICT (name) DO NOTHING + `) + require.NoError(err, "record prior attribution migration") + + require.NoError(st.InitSchema(), "run production schema migration") + + for _, message := range migrated { + var sourceDerived sql.NullBool + var identityDerived bool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me, identity_is_from_me + FROM messages + WHERE id = ? + `), message.messageID).Scan(&sourceDerived, &identityDerived), + "read migrated provenance for message %d", + message.messageID, + ) + assert.True(sourceDerived.Valid, "migration must assign explicit source provenance") + assert.False(sourceDerived.Bool, "legacy meeting attribution is not source-native") + assert.True(identityDerived, "legacy meeting attribution must remain identity-derived") + + removed, removeErr := st.RemoveAccountIdentity(message.sourceID, "owner@example.com") + require.NoError(removeErr, "remove migrated identity") + require.Equal(int64(1), removed) + isFromMe, attributionErr := st.GetMessageIsFromMe(message.messageID) + require.NoError(attributionErr, "read attribution after identity removal") + assert.False(isFromMe, "identity removal must clear migrated attribution") + } + + var nativeSourceDerived sql.NullBool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me + FROM messages + WHERE id = ? + `), nativeMessageID).Scan(&nativeSourceDerived), + "read source-native control provenance", + ) + assert.True(nativeSourceDerived.Valid, "migration must initialize source-native provenance") + assert.True(nativeSourceDerived.Bool, "migration must preserve source-native attribution") +} + +func TestInitSchemaClearsLegacyAttributionWithoutRemainingIdentity(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + senderID := f.EnsureParticipant("former-owner@example.com", "Former Owner", "example.com") + + src, err := st.GetOrCreateSource("granola", "removed-identity-account") + require.NoError(err, "create granola source") + convID, err := st.EnsureConversationWithType( + src.ID, + "removed-identity-conversation", + "meeting", + "Legacy meeting", + ) + require.NoError(err, "create granola conversation") + messageID, err := st.UpsertMessage(&store.Message{ + SourceID: src.ID, + ConversationID: convID, + SourceMessageID: "removed-identity-message", + MessageType: "meeting", + SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + IsFromMe: true, + }) + require.NoError(err, "persist stale legacy attribution") + + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages + SET source_is_from_me = NULL, identity_is_from_me = FALSE + WHERE id = ? + `), messageID) + require.NoError(err, "simulate message written before attribution provenance") + _, err = st.DB().Exec(st.Rebind(` + DELETE FROM applied_migrations + WHERE name = ? + `), "message_attribution_provenance_v2") + require.NoError(err, "reset attribution migration sentinel") + + require.NoError(st.InitSchema(), "run production schema migration") + + var sourceDerived, identityDerived, isFromMe bool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me, identity_is_from_me, is_from_me + FROM messages + WHERE id = ? + `), messageID).Scan(&sourceDerived, &identityDerived, &isFromMe), + "read reconciled attribution", + ) + assert.False(sourceDerived, "legacy Granola attribution is not source-native") + assert.False(identityDerived, "removed identity must not remain as provenance") + assert.False(isFromMe, "migration must clear attribution whose identity was already removed") +} + +func TestInitSchemaReconcilesConfirmedIdentityForEverySource(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + require.NoError( + st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual"), + "confirm Gmail identity", + ) + message := f.NewMessage().WithSourceMessageID("legacy-gmail-identity").Build() + message.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist matching Gmail message") + + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages + SET is_from_me = FALSE, + source_is_from_me = NULL, + identity_is_from_me = FALSE + WHERE id = ? + `), messageID) + require.NoError(err, "simulate stale legacy Gmail attribution") + _, err = st.DB().Exec(st.Rebind(` + DELETE FROM applied_migrations + WHERE name = ? + `), "message_attribution_provenance_v2") + require.NoError(err, "reset attribution migration sentinel") + + revisionBefore, err := st.AccountIdentityRevision() + require.NoError(err, "read account identity revision before migration") + require.NoError(st.InitSchema(), "run production schema migration") + + var sourceDerived, identityDerived, isFromMe bool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me, identity_is_from_me, is_from_me + FROM messages + WHERE id = ? + `), messageID).Scan(&sourceDerived, &identityDerived, &isFromMe), + "read reconciled Gmail attribution", + ) + assert.False(sourceDerived, "incoming Gmail message was not source-native") + assert.True(identityDerived, "confirmed identity must be reconciled for Gmail") + assert.True(isFromMe, "effective attribution must include the confirmed identity") + + revisionAfter, err := st.AccountIdentityRevision() + require.NoError(err, "read account identity revision after migration") + assert.Equal( + revisionBefore+1, + revisionAfter, + "provenance migration must invalidate caches atomically", + ) +} + +func TestInitSchemaBackfillsLegacyCalendarAttributionProvenance(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + src, err := st.GetOrCreateSource(gcal.SourceType, "owner@example.com/primary") + require.NoError(err, "create calendar source") + require.NoError( + st.AddAccountIdentity(src.ID, "owner@example.com", "account-email"), + "confirm configured calendar account", + ) + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + + persistLegacyEvent := func(sourceMessageID, raw string) int64 { + convID, ensureErr := st.EnsureConversationWithType( + src.ID, + sourceMessageID+"-conversation", + gcal.ConversationType, + "Legacy calendar event", + ) + require.NoError(ensureErr, "create %s conversation", sourceMessageID) + messageID, upsertErr := st.UpsertMessage(&store.Message{ + SourceID: src.ID, + ConversationID: convID, + SourceMessageID: sourceMessageID, + MessageType: gcal.MessageTypeCalendarEvent, + SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + IsFromMe: true, + }) + require.NoError(upsertErr, "persist legacy %s message", sourceMessageID) + require.NoError( + st.UpsertMessageRawWithFormat(messageID, []byte(raw), gcal.RawFormat), + "persist legacy %s raw event", + sourceMessageID, + ) + return messageID + } + + identityMessageID := persistLegacyEvent( + "configured-account-organizer", + `{"organizer":{"email":"owner@example.com"}}`, + ) + nativeMessageID := persistLegacyEvent( + "google-self-organizer", + `{"organizer":{"email":"owner@example.com","self":true}}`, + ) + + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages + SET source_is_from_me = NULL, + identity_is_from_me = FALSE, + is_from_me = TRUE + WHERE id IN (?, ?) + `), identityMessageID, nativeMessageID) + require.NoError(err, "simulate calendar rows written before attribution provenance") + _, err = st.DB().Exec(st.Rebind(` + DELETE FROM applied_migrations + WHERE name = ? + `), "message_attribution_provenance_v2") + require.NoError(err, "reset attribution migration sentinel") + + require.NoError(st.InitSchema(), "run production schema migration") + + var identitySourceDerived, identityDerived bool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me, identity_is_from_me + FROM messages + WHERE id = ? + `), identityMessageID).Scan(&identitySourceDerived, &identityDerived), + "read configured-account provenance", + ) + assert.False(identitySourceDerived, "configured-account match is not source-native") + assert.True(identityDerived, "configured-account match must remain identity-derived") + + var nativeSourceDerived bool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me + FROM messages + WHERE id = ? + `), nativeMessageID).Scan(&nativeSourceDerived), + "read Organizer.Self provenance", + ) + assert.True(nativeSourceDerived, "Organizer.Self must remain source-native") + + removed, err := st.RemoveAccountIdentity(src.ID, "owner@example.com") + require.NoError(err, "remove configured calendar identity") + require.Equal(int64(1), removed) + + identityIsFromMe, err := st.GetMessageIsFromMe(identityMessageID) + require.NoError(err, "read configured-account attribution after identity removal") + assert.False(identityIsFromMe, "identity removal must clear configured-account attribution") + nativeIsFromMe, err := st.GetMessageIsFromMe(nativeMessageID) + require.NoError(err, "read Organizer.Self attribution after identity removal") + assert.True(nativeIsFromMe, "identity removal must preserve Organizer.Self attribution") +} diff --git a/internal/store/dialect_pg.go b/internal/store/dialect_pg.go index b4050027b..67640ed45 100644 --- a/internal/store/dialect_pg.go +++ b/internal/store/dialect_pg.go @@ -338,6 +338,8 @@ func (d *PostgreSQLDialect) LegacyColumnMigrations() []ColumnMigration { {`ALTER TABLE participants ADD COLUMN IF NOT EXISTS phone_number TEXT`, "phone_number"}, {`ALTER TABLE participants ADD COLUMN IF NOT EXISTS canonical_id TEXT`, "canonical_id"}, {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS sender_id BIGINT REFERENCES participants(id)`, "sender_id"}, + {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS source_is_from_me BOOLEAN`, "source_is_from_me"}, + {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS identity_is_from_me BOOLEAN NOT NULL DEFAULT FALSE`, "identity_is_from_me"}, {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS message_type TEXT NOT NULL DEFAULT 'email'`, "message_type"}, {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_count INTEGER DEFAULT 0`, "attachment_count"}, {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS deleted_from_source_at TIMESTAMPTZ`, "deleted_from_source_at"}, diff --git a/internal/store/dialect_sqlite.go b/internal/store/dialect_sqlite.go index b0809e818..04d4eb584 100644 --- a/internal/store/dialect_sqlite.go +++ b/internal/store/dialect_sqlite.go @@ -274,6 +274,8 @@ func (d *SQLiteDialect) LegacyColumnMigrations() []ColumnMigration { {`ALTER TABLE participants ADD COLUMN phone_number TEXT`, "phone_number"}, {`ALTER TABLE participants ADD COLUMN canonical_id TEXT`, "canonical_id"}, {`ALTER TABLE messages ADD COLUMN sender_id INTEGER REFERENCES participants(id)`, "sender_id"}, + {`ALTER TABLE messages ADD COLUMN source_is_from_me BOOLEAN`, "source_is_from_me"}, + {`ALTER TABLE messages ADD COLUMN identity_is_from_me BOOLEAN NOT NULL DEFAULT FALSE`, "identity_is_from_me"}, {`ALTER TABLE messages ADD COLUMN message_type TEXT NOT NULL DEFAULT 'email'`, "message_type"}, {`ALTER TABLE messages ADD COLUMN attachment_count INTEGER DEFAULT 0`, "attachment_count"}, {`ALTER TABLE messages ADD COLUMN deleted_from_source_at DATETIME`, "deleted_from_source_at"}, diff --git a/internal/store/messages.go b/internal/store/messages.go index 3dd94ecfa..a36028a5c 100644 --- a/internal/store/messages.go +++ b/internal/store/messages.go @@ -25,6 +25,24 @@ type querier interface { QueryRow(query string, args ...any) *sql.Row } +type contextStatementQuerier interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +type boundQuerier struct { + ctx context.Context + q contextStatementQuerier +} + +func (q boundQuerier) Exec(query string, args ...any) (sql.Result, error) { + return q.q.ExecContext(q.ctx, query, args...) +} + +func (q boundQuerier) QueryRow(query string, args ...any) *sql.Row { + return q.q.QueryRowContext(q.ctx, query, args...) +} + type contextQuerier interface { ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) } @@ -37,6 +55,14 @@ type RecipientSet struct { DisplayNames []string } +// ParticipantPersistData describes one email participant to resolve inside a +// message persistence transaction. +type ParticipantPersistData struct { + EmailAddress string + DisplayName string + Domain string +} + // MessagePersistData bundles everything needed to atomically // persist a message and its related rows in a single transaction. type MessagePersistData struct { @@ -76,13 +102,16 @@ type Message struct { InternalDate sql.NullTime SenderID sql.NullInt64 IsFromMe bool - Subject sql.NullString - Snippet sql.NullString - SizeEstimate int64 - HasAttachments bool - AttachmentCount int - DeletedAt sql.NullTime - ArchivedAt time.Time + // IdentityDerivedIsFromMe reports that IsFromMe came from a confirmed + // account identity rather than a source-native sent-by-me signal. + IdentityDerivedIsFromMe bool + Subject sql.NullString + Snippet sql.NullString + SizeEstimate int64 + HasAttachments bool + AttachmentCount int + DeletedAt sql.NullTime + ArchivedAt time.Time } // MessageMetadataRecord is the archive identity and optional provider metadata @@ -584,13 +613,47 @@ func (s *Store) EnsureConversation(sourceID int64, sourceConversationID, title s // upsertMessageSQL returns the message upsert SQL with dialect-specific timestamp. func upsertMessageSQL(now string) string { return fmt.Sprintf(` + WITH attribution AS ( + SELECT + CAST(? AS BOOLEAN) AS source_is_from_me, + ( + CAST(? AS BOOLEAN) + OR EXISTS ( + SELECT 1 + FROM account_identities ai + JOIN participants p ON p.id = ? + WHERE ai.source_id = ? + AND p.email_address IS NOT NULL + AND LOWER(p.email_address) = LOWER(ai.address) + ) + OR EXISTS ( + SELECT 1 + FROM account_identities ai + JOIN participant_identifiers pi ON pi.participant_id = ? + WHERE ai.source_id = ? + AND ( + (pi.identifier_type = 'email' + AND LOWER(pi.identifier_value) = LOWER(ai.address)) + OR (pi.identifier_type <> 'email' + AND pi.identifier_value = ai.address) + ) + ) + ) AS identity_is_from_me + ) INSERT INTO messages ( conversation_id, source_id, source_message_id, rfc822_message_id, message_type, - sent_at, received_at, internal_date, sender_id, is_from_me, + sent_at, received_at, internal_date, sender_id, + is_from_me, source_is_from_me, identity_is_from_me, subject, snippet, size_estimate, has_attachments, attachment_count, archived_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, %s) + ) + SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, + (source_is_from_me OR identity_is_from_me), + source_is_from_me, identity_is_from_me, + ?, ?, ?, ?, ?, %s + FROM attribution + WHERE TRUE ON CONFLICT(source_id, source_message_id) DO UPDATE SET embed_gen = CASE WHEN COALESCE(messages.subject, '') <> COALESCE(excluded.subject, '') THEN NULL @@ -603,6 +666,8 @@ func upsertMessageSQL(now string) string { internal_date = excluded.internal_date, sender_id = excluded.sender_id, is_from_me = excluded.is_from_me, + source_is_from_me = excluded.source_is_from_me, + identity_is_from_me = excluded.identity_is_from_me, subject = excluded.subject, snippet = excluded.snippet, size_estimate = excluded.size_estimate, @@ -617,10 +682,15 @@ func (s *Store) UpsertMessage(msg *Message) (int64, error) { func upsertMessageWith(q querier, d Dialect, msg *Message) (int64, error) { sql := upsertMessageSQL(d.Now()) + sourceIsFromMe := msg.IsFromMe && !msg.IdentityDerivedIsFromMe + identityIsFromMe := msg.IsFromMe && msg.IdentityDerivedIsFromMe args := []any{ + sourceIsFromMe, identityIsFromMe, + msg.SenderID, msg.SourceID, + msg.SenderID, msg.SourceID, msg.ConversationID, msg.SourceID, msg.SourceMessageID, msg.RFC822MessageID, msg.MessageType, - msg.SentAt, msg.ReceivedAt, msg.InternalDate, msg.SenderID, msg.IsFromMe, + msg.SentAt, msg.ReceivedAt, msg.InternalDate, msg.SenderID, msg.Subject, msg.Snippet, msg.SizeEstimate, msg.HasAttachments, msg.AttachmentCount, } @@ -779,85 +849,257 @@ func (s *Store) GetMessageRaw(messageID int64) ([]byte, error) { return compressed, nil } +// GetMessageIsFromMe returns the baked account-attribution flag for a message. +func (s *Store) GetMessageIsFromMe(messageID int64) (bool, error) { + var isFromMe bool + err := s.db.QueryRow(` + SELECT COALESCE(is_from_me, FALSE) + FROM messages + WHERE id = ? + `, messageID).Scan(&isFromMe) + return isFromMe, err +} + +const messageIdentityAttributionMatch = `( + EXISTS ( + SELECT 1 + FROM account_identities ai + JOIN participants p ON p.id = messages.sender_id + WHERE ai.source_id = messages.source_id + AND p.email_address IS NOT NULL + AND LOWER(p.email_address) = LOWER(ai.address) + ) + OR EXISTS ( + SELECT 1 + FROM account_identities ai + JOIN participant_identifiers pi ON pi.participant_id = messages.sender_id + WHERE ai.source_id = messages.source_id + AND ( + (pi.identifier_type = 'email' AND LOWER(pi.identifier_value) = LOWER(ai.address)) + OR (pi.identifier_type <> 'email' AND pi.identifier_value = ai.address) + ) + ) +)` + +const messageSourceAttribution = `COALESCE(source_is_from_me, FALSE)` + +func refreshSourceMessageAttributionContext( + ctx context.Context, + q contextQuerier, + sourceID int64, + excludeSourceMessageID string, +) error { + // InitSchema assigns source provenance to every legacy row once. Runtime + // identity changes therefore only need to update the derived and effective + // values, and the change predicate avoids firing last_modified triggers for + // messages whose attribution already agrees with the current identity set. + _, err := q.ExecContext(ctx, fmt.Sprintf(` + UPDATE messages + SET identity_is_from_me = %[2]s, + is_from_me = (%[1]s OR %[2]s) + WHERE source_id = ? + AND (? = '' OR source_message_id <> ?) + AND ( + identity_is_from_me <> %[2]s + OR is_from_me IS NULL + OR is_from_me <> (%[1]s OR %[2]s) + ) + `, messageSourceAttribution, messageIdentityAttributionMatch), + sourceID, excludeSourceMessageID, excludeSourceMessageID) + if err != nil { + return fmt.Errorf("refresh source message attribution: %w", err) + } + return nil +} + +func refreshParticipantMessageAttributionContext( + ctx context.Context, + q contextQuerier, + participantIDs ...int64, +) error { + ids := make([]int64, 0, len(participantIDs)) + for _, participantID := range participantIDs { + if participantID != 0 { + ids = append(ids, participantID) + } + } + if len(ids) == 0 { + return nil + } + + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",") + args := make([]any, len(ids)) + for i, participantID := range ids { + args[i] = participantID + } + _, err := q.ExecContext(ctx, fmt.Sprintf(` + UPDATE messages + SET identity_is_from_me = %[2]s, + is_from_me = (%[1]s OR %[2]s) + WHERE sender_id IN (`+placeholders+`) + AND ( + identity_is_from_me <> %[2]s + OR is_from_me IS NULL + OR is_from_me <> (%[1]s OR %[2]s) + ) + `, messageSourceAttribution, messageIdentityAttributionMatch), args...) + if err != nil { + return fmt.Errorf("refresh participant message attribution: %w", err) + } + return nil +} + // PersistMessage atomically stores a message and its requested related // snapshots in one transaction. Existing email callers persist body, raw MIME, // recipients, and labels; non-email callers can additionally include // conversation state, metadata, provider raw data, and FTS. func (s *Store) PersistMessage(data *MessagePersistData) (int64, error) { + return s.PersistMessageContext(context.Background(), data) +} + +// PersistMessageContext is the request-aware form of PersistMessage. Every +// statement in the transaction observes ctx, and cancellation rolls the full +// message snapshot back. +func (s *Store) PersistMessageContext(ctx context.Context, data *MessagePersistData) (int64, error) { if data == nil || data.Message == nil { return 0, errors.New("persist message requires a message") } + return s.persistMessageWithParticipantsContext(ctx, nil, func([]int64) *MessagePersistData { + return data + }) +} + +// PersistMessageWithParticipantsContext resolves participants and persists the +// message snapshot in one request-aware transaction. build receives IDs in the +// same order as participants and must return the snapshot that references them. +func (s *Store) PersistMessageWithParticipantsContext( + ctx context.Context, + participants []ParticipantPersistData, + build func(participantIDs []int64) *MessagePersistData, +) (int64, error) { + if build == nil { + return 0, errors.New("persist message requires a participant builder") + } + return s.persistMessageWithParticipantsContext(ctx, participants, build) +} + +func (s *Store) persistMessageWithParticipantsContext( + ctx context.Context, + participants []ParticipantPersistData, + build func(participantIDs []int64) *MessagePersistData, +) (int64, error) { var messageID int64 - err := s.withTx(func(tx *loggedTx) error { - message := data.Message - if data.Conversation != nil { - conversationID, err := ensureConversationWithType( - tx, s.dialect, data.Message.SourceID, - data.Conversation.SourceConversationID, - data.Conversation.ConversationType, - data.Conversation.Title, + err := s.withTxContext(ctx, func(tx *loggedTx) error { + q := boundQuerier{ctx: ctx, q: tx} + participantIDs := make([]int64, len(participants)) + for idx, participant := range participants { + if err := ctx.Err(); err != nil { + return err + } + participantID, err := ensureParticipantWith( + q, + s.dialect, + participant.EmailAddress, + participant.DisplayName, + participant.Domain, ) if err != nil { - return fmt.Errorf("ensure conversation: %w", err) - } - if err := replaceConversationParticipantsTx( - tx, s.dialect, conversationID, data.Conversation.Participants, - ); err != nil { - return fmt.Errorf("replace conversation participants: %w", err) + return fmt.Errorf("ensure participant %d: %w", idx, err) } - messageCopy := *data.Message - messageCopy.ConversationID = conversationID - message = &messageCopy + participantIDs[idx] = participantID } - id, err := upsertMessageWith(tx, s.dialect, message) + if err := ctx.Err(); err != nil { + return err + } + data := build(participantIDs) + id, err := s.persistMessageWith(ctx, q, data) if err != nil { - return fmt.Errorf("upsert message: %w", err) + return err } messageID = id - if data.Metadata != nil { - if err := setMessageMetadataWith(tx, s.dialect, messageID, *data.Metadata); err != nil { - return fmt.Errorf("set metadata: %w", err) - } - } + return nil + }) + return messageID, err +} - if err := upsertMessageBody( - tx, s.dialect, s.fts5Available, messageID, data.BodyText, data.BodyHTML, +func (s *Store) persistMessageWith( + ctx context.Context, + q querier, + data *MessagePersistData, +) (int64, error) { + if data == nil || data.Message == nil { + return 0, errors.New("persist message requires a message") + } + message := data.Message + if data.Conversation != nil { + conversationID, err := ensureConversationWithType( + q, s.dialect, data.Message.SourceID, + data.Conversation.SourceConversationID, + data.Conversation.ConversationType, + data.Conversation.Title, + ) + if err != nil { + return 0, fmt.Errorf("ensure conversation: %w", err) + } + if err := replaceConversationParticipantsTx( + q, s.dialect, conversationID, data.Conversation.Participants, ); err != nil { - return fmt.Errorf("upsert body: %w", err) + return 0, fmt.Errorf("replace conversation participants: %w", err) } + messageCopy := *data.Message + messageCopy.ConversationID = conversationID + message = &messageCopy + } - if len(data.RawMIME) > 0 { - rawFormat := data.RawFormat - if rawFormat == "" { - rawFormat = "mime" - } - if err := upsertMessageRawWithFormat(tx, messageID, data.RawMIME, rawFormat); err != nil { - return fmt.Errorf("upsert raw: %w", err) - } + messageID, err := upsertMessageWith(q, s.dialect, message) + if err != nil { + return 0, fmt.Errorf("upsert message: %w", err) + } + if data.Metadata != nil { + if err := setMessageMetadataWith(q, s.dialect, messageID, *data.Metadata); err != nil { + return 0, fmt.Errorf("set metadata: %w", err) } + } - for _, rs := range data.Recipients { - if err := replaceMessageRecipientsTx(tx, messageID, rs); err != nil { - return fmt.Errorf("store %s recipients: %w", rs.Type, err) - } + if err := upsertMessageBody( + q, s.dialect, s.fts5Available, messageID, data.BodyText, data.BodyHTML, + ); err != nil { + return 0, fmt.Errorf("upsert body: %w", err) + } + + if len(data.RawMIME) > 0 { + if err := ctx.Err(); err != nil { + return 0, err + } + rawFormat := data.RawFormat + if rawFormat == "" { + rawFormat = "mime" + } + if err := upsertMessageRawWithFormat(q, messageID, data.RawMIME, rawFormat); err != nil { + return 0, fmt.Errorf("upsert raw: %w", err) } + } - if !data.PreserveLabels { - if err := replaceMessageLabelsTx(tx, messageID, data.LabelIDs); err != nil { - return fmt.Errorf("store labels: %w", err) - } + for _, rs := range data.Recipients { + if err := replaceMessageRecipientsTx(q, messageID, rs); err != nil { + return 0, fmt.Errorf("store %s recipients: %w", rs.Type, err) } - if data.FTS != nil && s.fts5Available { - fts := *data.FTS - fts.MessageID = messageID - if err := s.dialect.FTSUpsert(tx, fts); err != nil { - return fmt.Errorf("upsert fts: %w", err) - } + } + + if !data.PreserveLabels { + if err := replaceMessageLabelsTx(q, messageID, data.LabelIDs); err != nil { + return 0, fmt.Errorf("store labels: %w", err) } - return nil - }) - return messageID, err + } + if data.FTS != nil && s.fts5Available { + fts := *data.FTS + fts.MessageID = messageID + if err := s.dialect.FTSUpsert(q, fts); err != nil { + return 0, fmt.Errorf("upsert fts: %w", err) + } + } + return messageID, nil } // Participant represents a person in the participants table. @@ -876,6 +1118,32 @@ type Participant struct { // name and domain are left untouched on conflict to preserve any // hand-edited values. func (s *Store) EnsureParticipant(email, displayName, domain string) (int64, error) { + return s.EnsureParticipantContext(context.Background(), email, displayName, domain) +} + +// EnsureParticipantContext is the request-aware form of EnsureParticipant. +func (s *Store) EnsureParticipantContext( + ctx context.Context, + email, + displayName, + domain string, +) (int64, error) { + return ensureParticipantWith( + boundQuerier{ctx: ctx, q: s.db}, + s.dialect, + email, + displayName, + domain, + ) +} + +func ensureParticipantWith( + q querier, + dialect Dialect, + email, + displayName, + domain string, +) (int64, error) { // ON CONFLICT must mirror the partial unique index on // participants(email_address) WHERE email_address IS NOT NULL — both // PG and SQLite require the WHERE clause on the conflict target to @@ -883,13 +1151,13 @@ func (s *Store) EnsureParticipant(email, displayName, domain string) (int64, err // the same column) makes RETURNING fire for both INSERT and the // existing-row case, giving us the id either way. var id int64 - err := s.db.QueryRow(fmt.Sprintf(` + err := q.QueryRow(fmt.Sprintf(` INSERT INTO participants (email_address, display_name, domain, created_at, updated_at) VALUES (?, ?, ?, %s, %s) ON CONFLICT (email_address) WHERE email_address IS NOT NULL DO UPDATE SET email_address = EXCLUDED.email_address RETURNING id - `, s.dialect.Now(), s.dialect.Now()), email, displayName, domain).Scan(&id) + `, dialect.Now(), dialect.Now()), email, displayName, domain).Scan(&id) if err != nil { return 0, err } @@ -957,7 +1225,7 @@ func (s *Store) ReplaceMessageRecipients(messageID int64, recipientType string, }) } -func replaceMessageRecipientsTx(tx *loggedTx, messageID int64, rs RecipientSet) error { +func replaceMessageRecipientsTx(tx querier, messageID int64, rs RecipientSet) error { _, err := tx.Exec(` DELETE FROM message_recipients WHERE message_id = ? AND recipient_type = ? `, messageID, rs.Type) @@ -1240,7 +1508,7 @@ func (s *Store) ReplaceMessageLabels(messageID int64, labelIDs []int64) error { }) } -func replaceMessageLabelsTx(tx *loggedTx, messageID int64, labelIDs []int64) error { +func replaceMessageLabelsTx(tx querier, messageID int64, labelIDs []int64) error { _, err := tx.Exec(` DELETE FROM message_labels WHERE message_id = ? `, messageID) @@ -1847,7 +2115,31 @@ func (s *Store) backfillFTSBatchContext( // last_message_at, and last_message_preview from the current table state. // Safe to call multiple times — always produces the same result (idempotent). func (s *Store) RecomputeConversationStats(sourceID int64) error { - _, err := s.db.Exec(` + return s.recomputeConversationStats("source_id = ?", sourceID) +} + +// RecomputeConversationStatsForMessage updates the denormalized stats only for +// the conversation containing messageID. +func (s *Store) RecomputeConversationStatsForMessage(messageID int64) error { + return s.RecomputeConversationStatsForMessageContext(context.Background(), messageID) +} + +// RecomputeConversationStatsForMessageContext is the request-aware form of +// RecomputeConversationStatsForMessage. +func (s *Store) RecomputeConversationStatsForMessageContext(ctx context.Context, messageID int64) error { + return s.recomputeConversationStatsContext( + ctx, + "id = (SELECT conversation_id FROM messages WHERE id = ?)", + messageID, + ) +} + +func (s *Store) recomputeConversationStats(whereClause string, arg any) error { + return s.recomputeConversationStatsContext(context.Background(), whereClause, arg) +} + +func (s *Store) recomputeConversationStatsContext(ctx context.Context, whereClause string, arg any) error { + _, err := s.db.ExecContext(ctx, fmt.Sprintf(` UPDATE conversations SET message_count = ( SELECT COUNT(*) FROM messages @@ -1868,8 +2160,8 @@ func (s *Store) RecomputeConversationStats(sourceID int64) error { ORDER BY COALESCE(sent_at, received_at, internal_date) DESC, id DESC LIMIT 1 ) - WHERE source_id = ? - `, sourceID) + WHERE %s + `, whereClause), arg) if err != nil { return fmt.Errorf("recompute conversation stats: %w", err) } @@ -2186,6 +2478,13 @@ func (s *Store) MergeParticipants(oldID, newID int64) error { if _, err := tx.Exec(`UPDATE participant_identifiers SET participant_id = ? WHERE participant_id = ?`, newID, oldID); err != nil { return err } + // Sender and identifier repoints can add or remove identity evidence. + // Repair the primary-store provenance before committing the merge. + if err := refreshParticipantMessageAttributionContext( + context.Background(), tx, newID, + ); err != nil { + return err + } // Repoint (and, if needed, restructure) any link edges referencing // oldID before the delete below drops them via ON DELETE CASCADE. if err := s.rewriteLinksForMerge(tx, oldID, newID); err != nil { @@ -2200,10 +2499,9 @@ func (s *Store) MergeParticipants(oldID, newID int64) error { if _, err := s.bumpIdentityRevision(tx); err != nil { return err } - // Also bump the account-identity revision: the merge repoints - // messages.sender_id, so a merge involving the sender of any - // message with a baked is_from_me leaves that flag stale in the - // message Parquet shards, which only a full rebuild re-derives. + // Also bump the account-identity revision: the primary rows were + // repaired above, but existing message Parquet shards still bake the + // pre-merge attribution and require a full rebuild. if err := s.bumpAccountIdentityRevision(tx); err != nil { return err } @@ -2237,12 +2535,13 @@ func (s *Store) ParticipantByIdentifier(identifierType, identifierValue string) // search values and the participant_identifiers Parquet export), and the // derived-dataset refresh repairs that drift cheaply. When the changed // identifier additionally matches a confirmed account-identity address, it is -// owner evidence: the analytics cache bakes it into owner_participants, the -// per-row is_owner flags of the relationship activity index, and the -// export-derived is_from_me flag in message shards — so the mutation also -// bumps the identity and account-identity revisions the way MergeParticipants -// does, forcing the full rebuild that re-derives committed shards. No-op -// calls (the common importer re-run) bump nothing. +// owner evidence: the mutation repairs affected messages in the primary store, +// while the analytics cache bakes it into owner_participants, the per-row +// is_owner flags of the relationship activity index, and the export-derived +// is_from_me flag in message shards. It therefore also bumps the identity and +// account-identity revisions the way MergeParticipants does, forcing the full +// rebuild that re-derives committed shards. No-op calls (the common importer +// re-run) bump nothing. func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, identifierValue string) error { identifierType = strings.TrimSpace(identifierType) identifierValue = strings.TrimSpace(identifierValue) @@ -2252,8 +2551,10 @@ func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, id return s.withTx(func(tx *loggedTx) error { // Fast path first, read-only: importer re-runs hit the no-op case // constantly, and it must not take any write lock. - noop, err := participantIdentifierIsNoopTx(tx, participantID, identifierType, identifierValue) - if err != nil || noop { + existingParticipantID, exists, err := participantIdentifierTargetTx( + tx, identifierType, identifierValue, + ) + if err != nil || (exists && existingParticipantID == participantID) { return err } // The write path may bump the identity revision below (owner @@ -2266,8 +2567,10 @@ func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, id if err := s.lockIdentityMutationTx(tx); err != nil { return err } - noop, err = participantIdentifierIsNoopTx(tx, participantID, identifierType, identifierValue) - if err != nil || noop { + existingParticipantID, exists, err = participantIdentifierTargetTx( + tx, identifierType, identifierValue, + ) + if err != nil || (exists && existingParticipantID == participantID) { return err } if _, err := tx.Exec(` @@ -2293,6 +2596,11 @@ func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, id if !ownerEvidence { return nil } + if err := refreshParticipantMessageAttributionContext( + context.Background(), tx, existingParticipantID, participantID, + ); err != nil { + return err + } if _, err := s.bumpIdentityRevision(tx); err != nil { return err } @@ -2300,20 +2608,22 @@ func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, id }) } -// participantIdentifierIsNoopTx reports whether (identifierType, -// identifierValue) already points at participantID, without taking any lock. -func participantIdentifierIsNoopTx( - tx *loggedTx, participantID int64, identifierType, identifierValue string, -) (bool, error) { +// participantIdentifierTargetTx returns the participant currently owning an +// identifier, if any, without taking any lock. +func participantIdentifierTargetTx( + tx *loggedTx, + identifierType, + identifierValue string, +) (int64, bool, error) { var existingID int64 err := tx.QueryRow(` SELECT participant_id FROM participant_identifiers WHERE identifier_type = ? AND identifier_value = ? `, identifierType, identifierValue).Scan(&existingID) if err != nil && !errors.Is(err, sql.ErrNoRows) { - return false, fmt.Errorf("lookup participant identifier: %w", err) + return 0, false, fmt.Errorf("lookup participant identifier: %w", err) } - return err == nil && existingID == participantID, nil + return existingID, err == nil, nil } func (s *Store) EnsureParticipantByIdentifier(identifierType, identifierValue, displayName string) (int64, error) { @@ -2719,7 +3029,7 @@ func (s *Store) ReplaceConversationParticipants(conversationID int64, participan }) } -func replaceConversationParticipantsTx(tx *loggedTx, dialect Dialect, conversationID int64, participants []ConversationParticipantRef) error { +func replaceConversationParticipantsTx(tx querier, dialect Dialect, conversationID int64, participants []ConversationParticipantRef) error { if _, err := tx.Exec(`DELETE FROM conversation_participants WHERE conversation_id = ?`, conversationID); err != nil { return err } diff --git a/internal/store/migrate_init_schema_ledger_test.go b/internal/store/migrate_init_schema_ledger_test.go index dff65a903..8074d1b76 100644 --- a/internal/store/migrate_init_schema_ledger_test.go +++ b/internal/store/migrate_init_schema_ledger_test.go @@ -9,13 +9,13 @@ import ( "github.com/stretchr/testify/require" ) -// TestInitSchema_OneShotMigrationsGatedOnLedger verifies the two data +// TestInitSchema_OneShotMigrationsGatedOnLedger verifies the data // migrations InitSchema used to re-verify on every start (the attachments -// dedupe and the messages.last_modified backfill — a full messages-table -// scan, the dominant daemon-startup cost on a large archive) are gated on -// the applied_migrations ledger: +// dedupe, attribution provenance reconciliation, and the +// messages.last_modified backfill — full messages-table scans on a large +// archive) are gated on the applied_migrations ledger: // -// 1. a fresh InitSchema runs them once and records both sentinels, +// 1. a fresh InitSchema runs them once and records all sentinels, // 2. a later InitSchema with the sentinel present skips the work, // 3. clearing the sentinel makes the next InitSchema run it again. // @@ -32,6 +32,7 @@ func TestInitSchema_OneShotMigrationsGatedOnLedger(t *testing.T) { for _, name := range []string{ migrationAttachmentsContentHashUnique, + migrationMessageAttributionProvenance, migrationMessagesLastModifiedBackfill, } { applied, err := st.IsMigrationApplied(name) diff --git a/internal/store/migrate_legacy_identity.go b/internal/store/migrate_legacy_identity.go index 11d079a60..a8ae2972c 100644 --- a/internal/store/migrate_legacy_identity.go +++ b/internal/store/migrate_legacy_identity.go @@ -146,6 +146,7 @@ func (s *Store) MigrateLegacyIdentityConfigContext( if !SourceTypeUsesEmailIdentity(src.SourceType) { continue } + insertedForSource := false for _, addr := range normalized { // Comparison rule (email-shaped → case-insensitive; // everything else → case-sensitive) is shared with @@ -173,6 +174,7 @@ func (s *Store) MigrateLegacyIdentityConfigContext( // this source, exactly like AddAccountIdentity's insert // branch — see the matching comment there. insertedAny = true + insertedForSource = true case qerr != nil: return fmt.Errorf("read existing identity (source=%d, addr=%s): %w", src.ID, addr, qerr) default: @@ -190,6 +192,11 @@ func (s *Store) MigrateLegacyIdentityConfigContext( } } } + if insertedForSource { + if err := refreshSourceMessageAttributionContext(ctx, tx, src.ID, ""); err != nil { + return fmt.Errorf("refresh migrated identity attribution (source=%d): %w", src.ID, err) + } + } } // A daemon that started before this migration ran must not keep diff --git a/internal/store/migrate_legacy_identity_test.go b/internal/store/migrate_legacy_identity_test.go index 9b0b8d4f1..5eab39f15 100644 --- a/internal/store/migrate_legacy_identity_test.go +++ b/internal/store/migrate_legacy_identity_test.go @@ -1,6 +1,7 @@ package store_test import ( + "database/sql" "testing" "github.com/stretchr/testify/assert" @@ -155,6 +156,33 @@ func TestMigrateLegacyIdentityConfig_BumpsRevisionsOnlyWhenItInserts(t *testing. "a no-op re-run must not bump the account identity revision") } +func TestMigrateLegacyIdentityConfigRefreshesExistingMessages(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + message := f.NewMessage().WithSourceMessageID("legacy-config-owner").Build() + message.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist message before identity migration") + + applied, _, _, _, err := st.MigrateLegacyIdentityConfig([]string{"owner@example.com"}) //nolint:dogsled // 5-return migration; test needs only applied+err + require.NoError(err, "MigrateLegacyIdentityConfig") + require.True(applied, "migration should insert the configured identity") + + var sourceDerived, identityDerived, isFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me, identity_is_from_me, is_from_me + FROM messages + WHERE id = ? + `), messageID).Scan(&sourceDerived, &identityDerived, &isFromMe)) + assert.False(sourceDerived, "incoming Gmail message was not source-native") + assert.True(identityDerived, "migration must immediately repair identity provenance") + assert.True(isFromMe, "migration must immediately repair effective attribution") +} + func TestMigrateLegacyIdentityConfig_MergesExistingSignal(t *testing.T) { require := require.New(t) f := storetest.New(t) diff --git a/internal/store/migrate_phone_unique.go b/internal/store/migrate_phone_unique.go index 165b91872..d85fdf83c 100644 --- a/internal/store/migrate_phone_unique.go +++ b/internal/store/migrate_phone_unique.go @@ -250,10 +250,9 @@ func (s *Store) mergeParticipant(ctx context.Context, tx *loggedTx, winner, lose if _, err := s.bumpIdentityRevision(tx); err != nil { return fmt.Errorf("bump identity revision (loser=%d, winner=%d): %w", loser, winner, err) } - // Also bump the account-identity revision: the merge repoints - // messages.sender_id, so a merge involving the sender of any message - // with a baked is_from_me leaves that flag stale in the message - // Parquet shards, which only a full rebuild re-derives. + // Also bump the account-identity revision: the primary rows are repaired + // after the survivor metadata is finalized below, but existing message + // Parquet shards still require a full rebuild. if err := s.bumpAccountIdentityRevision(tx); err != nil { return fmt.Errorf("bump account identity revision (loser=%d, winner=%d): %w", loser, winner, err) } @@ -291,6 +290,17 @@ func (s *Store) mergeParticipant(ctx context.Context, tx *loggedTx, winner, lose ); err != nil { return fmt.Errorf("coalesce metadata onto winner (winner=%d, loser=%d): %w", winner, loser, err) } + // The sender repoint plus the survivor's final email/identifiers can add or + // remove identity evidence. Repair primary-store provenance atomically with + // the legacy merge. + if err := refreshParticipantMessageAttributionContext(ctx, tx, winner); err != nil { + return fmt.Errorf( + "refresh message attribution (winner=%d, loser=%d): %w", + winner, + loser, + err, + ) + } // (8) Finally drop the loser. participant_identifiers cascades; the // other FKs are already cleared by the repoints above. diff --git a/internal/store/migrate_phone_unique_test.go b/internal/store/migrate_phone_unique_test.go index 8aba8daed..70ae44e9c 100644 --- a/internal/store/migrate_phone_unique_test.go +++ b/internal/store/migrate_phone_unique_test.go @@ -82,6 +82,17 @@ func TestEnsureParticipantsPhoneUniqueIndex_LegacyNonUnique(t *testing.T) { // Make sure the legacy schema actually permitted the duplicate. require.NotEqual(winner, loser, "seeded participants must have distinct ids") + require.NoError( + st.AddAccountIdentity(source.ID, "+15555551234", "manual"), + "confirm duplicated phone as source identity", + ) + _, err = st.db.Exec(` + INSERT INTO participant_identifiers ( + participant_id, identifier_type, identifier_value, is_primary + ) + VALUES (?, 'phone', '+15555551234', TRUE) + `, winner) + require.NoError(err, "seed winner owner identifier") // Attach FK references to BOTH participants so we can prove the // repoint+dedupe logic runs end-to-end: @@ -183,6 +194,9 @@ func TestEnsureParticipantsPhoneUniqueIndex_LegacyNonUnique(t *testing.T) { "read msg-C sender") assert.True(msgCSender.Valid, "msg-C sender = %+v, want winner %d", msgCSender, winner) assert.Equal(winner, msgCSender.Int64, "msg-C sender") + msgCIsFromMe, err := st.GetMessageIsFromMe(msgC) + require.NoError(err, "read msg-C attribution") + assert.True(msgCIsFromMe, "sender repoint must refresh identity-derived attribution") // 6) The index is now UNIQUE. Verify via sqlite_master. var sqlDef string diff --git a/internal/store/migrations.go b/internal/store/migrations.go index f551d049b..bf40a3548 100644 --- a/internal/store/migrations.go +++ b/internal/store/migrations.go @@ -1,8 +1,13 @@ package store import ( + "bytes" + "compress/zlib" "context" + "database/sql" + "encoding/json" "fmt" + "io" ) // One-time data migrations run by InitSchema and gated on the @@ -13,9 +18,181 @@ import ( const ( migrationAttachmentsContentHashUnique = "attachments_content_hash_unique_index" migrationMessagesLastModifiedBackfill = "messages_last_modified_backfill" + migrationMessageAttributionProvenance = "message_attribution_provenance_v2" migrationArchiveIdentity = "archive_identity_v1" ) +func backfillLegacyMessageAttributionProvenance( + ctx context.Context, + tx *loggedTx, +) error { + if err := backfillLegacyCalendarAttribution(ctx, tx); err != nil { + return err + } + + _, err := tx.ExecContext(ctx, ` + UPDATE messages + SET source_is_from_me = FALSE, + identity_is_from_me = COALESCE(is_from_me, FALSE) + WHERE source_is_from_me IS NULL + AND source_id IN ( + SELECT id + FROM sources + WHERE source_type IN ('granola', 'circleback') + ) + `) + if err != nil { + return fmt.Errorf("backfill message attribution provenance: %w", err) + } + + _, err = tx.ExecContext(ctx, ` + UPDATE messages + SET source_is_from_me = COALESCE(is_from_me, FALSE), + identity_is_from_me = FALSE + WHERE source_is_from_me IS NULL + `) + if err != nil { + return fmt.Errorf("initialize source-native message attribution provenance: %w", err) + } + + rows, err := tx.QueryContext(ctx, ` + SELECT id + FROM sources + WHERE source_type IN ('granola', 'circleback') + OR EXISTS ( + SELECT 1 + FROM account_identities ai + WHERE ai.source_id = sources.id + ) + `) + if err != nil { + return fmt.Errorf("list identity-derived attribution sources: %w", err) + } + var sourceIDs []int64 + for rows.Next() { + var sourceID int64 + if err := rows.Scan(&sourceID); err != nil { + _ = rows.Close() + return fmt.Errorf("scan identity-derived attribution source: %w", err) + } + sourceIDs = append(sourceIDs, sourceID) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return fmt.Errorf("iterate identity-derived attribution sources: %w", err) + } + if err := rows.Close(); err != nil { + return fmt.Errorf("close identity-derived attribution sources: %w", err) + } + + for _, sourceID := range sourceIDs { + if err := refreshSourceMessageAttributionContext(ctx, tx, sourceID, ""); err != nil { + return fmt.Errorf("reconcile source %d attribution: %w", sourceID, err) + } + } + return nil +} + +func backfillLegacyCalendarAttribution( + ctx context.Context, + tx *loggedTx, +) error { + var lastMessageID int64 + for { + if err := ctx.Err(); err != nil { + return err + } + + rows, err := tx.QueryContext(ctx, ` + SELECT m.id, mr.raw_data, mr.compression + FROM messages m + JOIN sources s ON s.id = m.source_id + JOIN message_raw mr ON mr.message_id = m.id + WHERE m.source_is_from_me IS NULL + AND COALESCE(m.is_from_me, FALSE) = TRUE + AND s.source_type = 'gcal' + AND mr.raw_format = 'gcal_json' + AND m.id > ? + ORDER BY m.id + LIMIT 500 + `, lastMessageID) + if err != nil { + return fmt.Errorf("list legacy calendar attribution: %w", err) + } + + var ( + batchSize int + messageIDs []int64 + ) + for rows.Next() { + var ( + messageID int64 + rawData []byte + compression sql.NullString + ) + if err := rows.Scan(&messageID, &rawData, &compression); err != nil { + _ = rows.Close() + return fmt.Errorf("scan legacy calendar attribution: %w", err) + } + lastMessageID = messageID + batchSize++ + organizerSelf, ok := legacyCalendarOrganizerSelf(rawData, compression) + if ok && !organizerSelf { + messageIDs = append(messageIDs, messageID) + } + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return fmt.Errorf("iterate legacy calendar attribution: %w", err) + } + if err := rows.Close(); err != nil { + return fmt.Errorf("close legacy calendar attribution: %w", err) + } + + if err := execInChunks( + tx, + messageIDs, + nil, + `UPDATE messages + SET source_is_from_me = FALSE, + identity_is_from_me = COALESCE(is_from_me, FALSE) + WHERE id IN (%s)`, + ); err != nil { + return fmt.Errorf("backfill calendar attribution provenance: %w", err) + } + if batchSize < 500 { + return nil + } + } +} + +func legacyCalendarOrganizerSelf( + rawData []byte, + compression sql.NullString, +) (bool, bool) { + if compression.Valid && compression.String == "zlib" { + reader, err := zlib.NewReader(bytes.NewReader(rawData)) + if err != nil { + return false, false + } + defer func() { _ = reader.Close() }() + rawData, err = io.ReadAll(reader) + if err != nil { + return false, false + } + } + + var event struct { + Organizer *struct { + Self bool `json:"self"` + } `json:"organizer"` + } + if err := json.Unmarshal(rawData, &event); err != nil || event.Organizer == nil { + return false, false + } + return event.Organizer.Self, true +} + // IsMigrationApplied reports whether the named one-time data migration // has already run. func (s *Store) IsMigrationApplied(name string) (bool, error) { diff --git a/internal/store/participant_identifiers_test.go b/internal/store/participant_identifiers_test.go index d45ad1ca5..1923c6074 100644 --- a/internal/store/participant_identifiers_test.go +++ b/internal/store/participant_identifiers_test.go @@ -1,6 +1,7 @@ package store_test import ( + "database/sql" "testing" "github.com/stretchr/testify/assert" @@ -78,6 +79,51 @@ func TestSetParticipantIdentifierRepointOwnerEvidenceBumpsRevisions(t *testing.T assert.Equal(second, id, "identifier must point at the new participant") } +func TestSetParticipantIdentifierRepointRefreshesSenderAttribution(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + require.NoError(st.AddAccountIdentity(f.Source.ID, "+15550100", "manual"), "confirm owner identity") + first := f.EnsureParticipant("first@example.com", "First", "example.com") + second := f.EnsureParticipant("second@example.com", "Second", "example.com") + require.NoError(st.SetParticipantIdentifier(first, "phone", "+15550100"), "seed owner evidence") + + persistMessage := func(sourceMessageID string, senderID int64) int64 { + message := f.NewMessage().WithSourceMessageID(sourceMessageID).Build() + message.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist %s", sourceMessageID) + return messageID + } + firstMessageID := persistMessage("identifier-first", first) + secondMessageID := persistMessage("identifier-second", second) + + require.NoError(st.SetParticipantIdentifier(second, "phone", "+15550100"), "repoint owner evidence") + + firstIsFromMe, err := st.GetMessageIsFromMe(firstMessageID) + require.NoError(err, "read former owner attribution") + assert.False(firstIsFromMe, "reassignment must clear attribution from the former identifier owner") + secondIsFromMe, err := st.GetMessageIsFromMe(secondMessageID) + require.NoError(err, "read new owner attribution") + assert.True(secondIsFromMe, "reassignment must attribute messages from the new identifier owner") + + var firstIdentityDerived, secondIdentityDerived bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT identity_is_from_me + FROM messages + WHERE id = ? + `), firstMessageID).Scan(&firstIdentityDerived)) + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT identity_is_from_me + FROM messages + WHERE id = ? + `), secondMessageID).Scan(&secondIdentityDerived)) + assert.False(firstIdentityDerived, "former identifier owner provenance") + assert.True(secondIdentityDerived, "new identifier owner provenance") +} + // TestSetParticipantIdentifierNonOwnerEvidenceBumpsOnlyIdentifierRevision // verifies plain alternate identifiers (the common importer case) advance // only the participant-identifier revision — the identity directory bakes diff --git a/internal/store/participant_links_test.go b/internal/store/participant_links_test.go index edf7973e6..e1605aa6f 100644 --- a/internal/store/participant_links_test.go +++ b/internal/store/participant_links_test.go @@ -1,6 +1,7 @@ package store_test import ( + "database/sql" "errors" "fmt" "sync" @@ -400,6 +401,34 @@ func TestMergeParticipantsWithoutLinksStillBumpsRevisionButRewritesNoLinks(t *te assert.Equal(0, edgeCount, "merge without pre-existing links must not create any link edge") } +func TestMergeParticipantsRefreshesSenderAttribution(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + require.NoError( + st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual"), + "confirm owner identity", + ) + owner := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + survivor := f.EnsureParticipant("other@example.com", "Other", "example.com") + message := f.NewMessage().WithSourceMessageID("merge-attribution").Build() + message.SenderID = sql.NullInt64{Int64: owner, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist owner-attributed message") + + isFromMe, err := st.GetMessageIsFromMe(messageID) + require.NoError(err, "read attribution before merge") + require.True(isFromMe, "confirmed owner sender must begin attributed") + + require.NoError(st.MergeParticipants(owner, survivor), "merge owner into unmatched survivor") + + isFromMe, err = st.GetMessageIsFromMe(messageID) + require.NoError(err, "read attribution after merge") + assert.False(isFromMe, "merge must clear attribution after sender no longer matches") +} + // TestLinkParticipants_ConcurrentDisjointClusters covers the race that // lockIdentityMutationTx serializes against: two LinkParticipants calls // that each try to connect two already-linked, previously-disjoint diff --git a/internal/store/schema.sql b/internal/store/schema.sql index 527e56554..1c0f0b20c 100644 --- a/internal/store/schema.sql +++ b/internal/store/schema.sql @@ -162,6 +162,8 @@ CREATE TABLE IF NOT EXISTS messages ( -- Sender sender_id INTEGER REFERENCES participants(id), is_from_me BOOLEAN DEFAULT FALSE, + source_is_from_me BOOLEAN, + identity_is_from_me BOOLEAN NOT NULL DEFAULT FALSE, -- Content subject TEXT, -- email subject, NULL for chat diff --git a/internal/store/schema_pg.sql b/internal/store/schema_pg.sql index 9872d5fd4..d44d748b6 100644 --- a/internal/store/schema_pg.sql +++ b/internal/store/schema_pg.sql @@ -143,6 +143,8 @@ CREATE TABLE IF NOT EXISTS messages ( sender_id BIGINT REFERENCES participants(id), is_from_me BOOLEAN DEFAULT FALSE, + source_is_from_me BOOLEAN, + identity_is_from_me BOOLEAN NOT NULL DEFAULT FALSE, subject TEXT, snippet TEXT, diff --git a/internal/store/store.go b/internal/store/store.go index 67e3f28ca..f663f7748 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -515,10 +515,19 @@ func (s *Store) withTxContext(ctx context.Context, fn func(tx *loggedTx) error) } return err } + if err := ctx.Err(); err != nil { + _ = tx.Rollback() + return err + } if err := tx.Commit(); err != nil { slog.Warn("sql tx commit failed", "error", err.Error(), "duration_ms", time.Since(start).Milliseconds()) + if errors.Is(err, sql.ErrTxDone) { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + } return err } // A tx crossing the slow threshold is a diagnostic, not a problem — @@ -664,7 +673,9 @@ type chunkInsert struct { // parameter limit (999). valueBuilder generates the VALUES placeholders and // args for each chunk of row indices. Rebinding to the dialect's placeholder // form happens inside tx.Exec (loggedTx wraps the dialect's Rebind). -func insertInChunks(tx *loggedTx, c chunkInsert, valueBuilder func(start, end int) ([]string, []any)) error { +func insertInChunks(tx interface { + Exec(query string, args ...any) (sql.Result, error) +}, c chunkInsert, valueBuilder func(start, end int) ([]string, []any)) error { // SQLite default SQLITE_MAX_VARIABLE_NUMBER is 999 // Leave some margin for safety const maxParams = 900 @@ -832,6 +843,53 @@ func (s *Store) InitSchema() error { } } + // Initialize explicit attribution provenance for every legacy message once + // under the maintenance timeout escape hatch. Granola and Circleback + // historically derived is_from_me from confirmed organizer identities. + // Google Calendar combined that signal with Organizer.Self, so its archived + // event payload separates source-native ownership from identity-derived + // ownership. Other providers' existing values are source-native. Runtime + // identity mutations can then update only rows whose derived attribution + // actually changes instead of rewriting an entire source to initialize NULL + // provenance. + attributionMigrated, err := s.IsMigrationApplied(migrationMessageAttributionProvenance) + if err != nil { + return err + } + if !attributionMigrated { + if err := s.runMaintenance( + context.Background(), + func(ctx context.Context, tx *loggedTx) error { + if err := backfillLegacyMessageAttributionProvenance(ctx, tx); err != nil { + return err + } + + // Published message shards used to trust the effective + // is_from_me value. The provenance migration changes cache + // inputs even when no identity is added or removed, so advance + // the account-identity revision in the same transaction as the + // repaired rows. Empty archives have no stale shards to + // invalidate. + var hasMessages bool + if err := tx.QueryRowContext( + ctx, + `SELECT EXISTS (SELECT 1 FROM messages)`, + ).Scan(&hasMessages); err != nil { + return fmt.Errorf("check attribution migration cache impact: %w", err) + } + if !hasMessages { + return nil + } + return s.bumpAccountIdentityRevisionContext(ctx, tx) + }, + ); err != nil { + return err + } + if err := s.MarkMigrationApplied(migrationMessageAttributionProvenance); err != nil { + return err + } + } + // Partial expression indexes for live-message listing and date filtering. // The first is a covering index for the ListMessages page // (GET /api/v1/messages). diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 4d59a8950..32ef38769 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -1,11 +1,13 @@ package store_test import ( + "context" "database/sql" "fmt" "testing" "time" + "github.com/mattn/go-sqlite3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/msgvault/internal/mime" @@ -1590,6 +1592,73 @@ func TestStore_PersistMessage_Atomicity(t *testing.T) { assert.Empty(t, existing, "message should not exist after failed PersistMessage") } +func TestStore_PersistMessageContext_CancellationRollsBack(t *testing.T) { + testutil.SkipIfPostgres(t, "uses a SQLite trigger and registered function to pause persistence") + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + f.Store.DB().SetMaxOpenConns(1) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + persistStarted := make(chan struct{}) + conn, err := f.Store.DB().Conn(context.Background()) + require.NoError(err, "get SQLite connection") + err = conn.Raw(func(driverConn any) error { + sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn) + require.True(ok, "driver connection is SQLite") + return sqliteConn.RegisterFunc("wait_for_persist_cancel", func() int { + close(persistStarted) + <-ctx.Done() + return 0 + }, true) + }) + require.NoError(err, "register cancellation function") + require.NoError(conn.Close(), "return SQLite connection to pool") + _, err = f.Store.DB().Exec(` + CREATE TRIGGER wait_before_meeting_raw_insert + BEFORE INSERT ON message_raw + WHEN NEW.raw_format = 'meeting_json' + BEGIN + SELECT wait_for_persist_cancel(); + END + `) + require.NoError(err, "create cancellation trigger") + + msg := storetest.NewMessage(f.Source.ID, f.ConvID). + WithSourceMessageID("persist-cancel"). + WithSubject("Canceled persistence"). + Build() + done := make(chan error, 1) + go func() { + _, persistErr := f.Store.PersistMessageContext(ctx, &store.MessagePersistData{ + Message: msg, + BodyText: sql.NullString{String: "must roll back", Valid: true}, + RawMIME: []byte(`{"meeting":"cancel"}`), + RawFormat: "meeting_json", + }) + done <- persistErr + }() + + select { + case <-persistStarted: + case <-time.After(time.Second): + require.FailNow("message persistence did not reach cancellation trigger") + } + cancel() + + select { + case err = <-done: + case <-time.After(time.Second): + require.FailNow("message persistence did not stop after cancellation") + } + require.ErrorIs(err, context.Canceled) + + existing, err := f.Store.MessageExistsBatch(f.Source.ID, []string{"persist-cancel"}) + require.NoError(err, "lookup canceled message") + assert.Empty(existing, "canceled message transaction must roll back") +} + func TestStore_OAuthAppColumn(t *testing.T) { require := require.New(t) assert := assert.New(t) diff --git a/internal/store/subset.go b/internal/store/subset.go index 0f79f50ff..fc508f9d9 100644 --- a/internal/store/subset.go +++ b/internal/store/subset.go @@ -22,6 +22,18 @@ type CopyResult struct { Elapsed time.Duration } +const messageCopyColumns = ` + id, conversation_id, source_id, source_message_id, + rfc822_message_id, message_type, + sent_at, received_at, read_at, delivered_at, internal_date, + sender_id, is_from_me, source_is_from_me, identity_is_from_me, + subject, snippet, reply_to_message_id, thread_position, + is_read, is_delivered, is_sent, is_edited, is_forwarded, + size_estimate, has_attachments, attachment_count, + deleted_at, deleted_from_source_at, delete_batch_id, + archived_at, indexing_version, last_modified, metadata, embed_gen +` + // CopySubset copies rowCount most recent messages (and all referenced // data) from srcDBPath into a new database in dstDir. The destination // schema is initialized using the embedded store schema. @@ -417,7 +429,9 @@ func copyData(tx *sql.Tx, rowCount int, includeIdentity bool) (*CopyResult, erro } if _, err := tx.Exec(` - INSERT INTO messages SELECT * FROM src.messages + INSERT INTO messages (` + messageCopyColumns + `) + SELECT ` + messageCopyColumns + ` + FROM src.messages WHERE id IN (SELECT id FROM selected_messages)`); err != nil { return nil, fmt.Errorf("copy messages: %w", err) } diff --git a/internal/store/subset_test.go b/internal/store/subset_test.go index de354522c..d72308e11 100644 --- a/internal/store/subset_test.go +++ b/internal/store/subset_test.go @@ -185,6 +185,72 @@ func TestCopySubset_Basic(t *testing.T) { assert.False(hasViolation, "foreign key violations found in destination database") } +func TestCopySubset_UpgradedMessageColumnOrder(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + srcDir := t.TempDir() + dstDir := filepath.Join(t.TempDir(), "dst") + srcDB := createTestSourceDB(t, srcDir, 1) + + st, err := Open(srcDB) + require.NoError(err, "open source for upgrade") + _, err = st.DB().Exec(` + ALTER TABLE messages DROP COLUMN identity_is_from_me; + ALTER TABLE messages DROP COLUMN source_is_from_me; + DELETE FROM applied_migrations + WHERE name = 'message_attribution_provenance_v2'; + `) + require.NoError(err, "simulate pre-attribution schema") + require.NoError(st.InitSchema(), "upgrade source schema") + _, err = st.DB().Exec(` + UPDATE messages + SET is_from_me = TRUE, + source_is_from_me = FALSE, + identity_is_from_me = TRUE, + metadata = '{"schema":"upgraded"}', + embed_gen = 7 + WHERE id = 1 + `) + require.NoError(err, "seed upgraded message columns") + require.NoError(st.Close(), "close upgraded source") + + result, err := CopySubset(srcDB, dstDir, 1, false) + require.NoError(err, "CopySubset from upgraded schema") + assert.Equal(int64(1), result.Messages) + + db, err := sql.Open("sqlite3", filepath.Join(dstDir, "msgvault.db")) + require.NoError(err, "open copied database") + defer func() { _ = db.Close() }() + + var sourceMessageID, messageType, subject, metadata string + var isFromMe, sourceIsFromMe, identityIsFromMe bool + var embedGen int64 + require.NoError(db.QueryRow(` + SELECT source_message_id, message_type, subject, + is_from_me, source_is_from_me, identity_is_from_me, + metadata, embed_gen + FROM messages + WHERE id = 1 + `).Scan( + &sourceMessageID, + &messageType, + &subject, + &isFromMe, + &sourceIsFromMe, + &identityIsFromMe, + &metadata, + &embedGen, + )) + assert.Equal("msg_1", sourceMessageID) + assert.Equal("email", messageType) + assert.Equal("Subject B", subject) + assert.True(isFromMe) + assert.False(sourceIsFromMe) + assert.True(identityIsFromMe) + assert.JSONEq(`{"schema":"upgraded"}`, metadata) + assert.Equal(int64(7), embedGen) +} + func TestCopySubset_AllRows(t *testing.T) { srcDir := t.TempDir() dstDir := filepath.Join(t.TempDir(), "dst") diff --git a/internal/store/sync.go b/internal/store/sync.go index 7c2c1797e..5f9604dfb 100644 --- a/internal/store/sync.go +++ b/internal/store/sync.go @@ -16,6 +16,8 @@ const ( SyncRunItemStatusError = "error" SyncRunItemStatusSkipped = "skipped" + + manualTransactionCleanupTimeout = 5 * time.Second ) // ErrSyncRunNotFound is returned by the sync-run getters (GetActiveSync, @@ -186,7 +188,11 @@ type SourceImportItem struct { // source via SELECT ... FOR UPDATE before doing the read-modify-write // on sync_runs. func (s *Store) StartSync(sourceID int64, syncType string) (int64, error) { - ctx := context.Background() + return s.StartSyncContext(context.Background(), sourceID, syncType) +} + +// StartSyncContext is the request-aware form of StartSync. +func (s *Store) StartSyncContext(ctx context.Context, sourceID int64, syncType string) (int64, error) { const maxAttempts = 5 for range maxAttempts { id, err := s.startSyncOnce(ctx, sourceID) @@ -213,7 +219,12 @@ func (s *Store) startSyncOnce(ctx context.Context, sourceID int64) (retID int64, committed := false defer func() { if !committed { - _, _ = conn.ExecContext(ctx, "ROLLBACK") + rollbackCtx, rollbackCancel := context.WithTimeout( + context.WithoutCancel(ctx), + manualTransactionCleanupTimeout, + ) + defer rollbackCancel() + _, _ = conn.ExecContext(rollbackCtx, "ROLLBACK") } }() @@ -267,7 +278,13 @@ func (s *Store) startSyncOnce(ctx context.Context, sourceID int64) (retID int64, // UpdateSyncCheckpoint saves progress for resumption. func (s *Store) UpdateSyncCheckpoint(syncID int64, cp *Checkpoint) error { - _, err := s.db.Exec(` + return s.UpdateSyncCheckpointContext(context.Background(), syncID, cp) +} + +// UpdateSyncCheckpointContext is the request-aware form of +// UpdateSyncCheckpoint. +func (s *Store) UpdateSyncCheckpointContext(ctx context.Context, syncID int64, cp *Checkpoint) error { + _, err := s.db.ExecContext(ctx, ` UPDATE sync_runs SET cursor_before = ?, messages_processed = ?, @@ -364,7 +381,12 @@ func (s *Store) ListSyncRunItems(syncRunID int64, status string, limit int) ([]S // CompleteSync marks a sync as successfully completed. func (s *Store) CompleteSync(syncID int64, finalHistoryID string) error { - _, err := s.db.Exec(fmt.Sprintf(` + return s.CompleteSyncContext(context.Background(), syncID, finalHistoryID) +} + +// CompleteSyncContext is the request-aware form of CompleteSync. +func (s *Store) CompleteSyncContext(ctx context.Context, syncID int64, finalHistoryID string) error { + _, err := s.db.ExecContext(ctx, fmt.Sprintf(` UPDATE sync_runs SET status = 'completed', completed_at = %s, diff --git a/internal/store/sync_context_test.go b/internal/store/sync_context_test.go new file mode 100644 index 000000000..13bff3d21 --- /dev/null +++ b/internal/store/sync_context_test.go @@ -0,0 +1,115 @@ +package store + +import ( + "context" + "database/sql" + "database/sql/driver" + "io" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type canceledStartSyncConnector struct { + cancel context.CancelFunc + rollbackCtx chan error +} + +func (c *canceledStartSyncConnector) Connect(context.Context) (driver.Conn, error) { + return &canceledStartSyncConn{cancel: c.cancel, rollbackCtx: c.rollbackCtx}, nil +} + +func (c *canceledStartSyncConnector) Driver() driver.Driver { + return &canceledStartSyncDriver{connector: c} +} + +type canceledStartSyncDriver struct { + connector *canceledStartSyncConnector +} + +func (d *canceledStartSyncDriver) Open(string) (driver.Conn, error) { + return d.connector.Connect(context.Background()) +} + +type canceledStartSyncConn struct { + cancel context.CancelFunc + rollbackCtx chan error +} + +func (c *canceledStartSyncConn) Prepare(string) (driver.Stmt, error) { + return nil, driver.ErrSkip +} + +func (c *canceledStartSyncConn) Close() error { return nil } + +func (c *canceledStartSyncConn) Begin() (driver.Tx, error) { + return nil, driver.ErrSkip +} + +func (c *canceledStartSyncConn) ExecContext( + ctx context.Context, + query string, + _ []driver.NamedValue, +) (driver.Result, error) { + if strings.TrimSpace(query) == "ROLLBACK" { + c.rollbackCtx <- ctx.Err() + } + return driver.RowsAffected(1), nil +} + +func (c *canceledStartSyncConn) QueryContext( + _ context.Context, + query string, + _ []driver.NamedValue, +) (driver.Rows, error) { + if strings.Contains(query, "INSERT INTO sync_runs") { + c.cancel() + return nil, context.Canceled + } + return &singleInt64Rows{value: 1}, nil +} + +type singleInt64Rows struct { + value int64 + read bool +} + +func (r *singleInt64Rows) Columns() []string { return []string{"id"} } +func (r *singleInt64Rows) Close() error { return nil } +func (r *singleInt64Rows) Next(dest []driver.Value) error { + if r.read { + return io.EOF + } + r.read = true + dest[0] = r.value + return nil +} + +func TestStartSyncContextRollbackOutlivesRequestCancellation(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + rollbackCtx := make(chan error, 1) + db := sql.OpenDB(&canceledStartSyncConnector{ + cancel: cancel, + rollbackCtx: rollbackCtx, + }) + t.Cleanup(func() { + require.NoError(db.Close()) + }) + st := &Store{ + db: newLoggedDB(db, identityRebind), + dialect: &SQLiteDialect{}, + } + + _, err := st.StartSyncContext(ctx, 1, "gmail") + require.ErrorIs(err, context.Canceled) + select { + case rollbackErr := <-rollbackCtx: + require.NoError(rollbackErr, "rollback must use a context independent of request cancellation") + case <-time.After(time.Second): + require.FailNow("manual transaction rollback was not attempted") + } +} diff --git a/internal/tui/meeting_mode_test.go b/internal/tui/meeting_mode_test.go index 44ca0ff63..f734c92b6 100644 --- a/internal/tui/meeting_mode_test.go +++ b/internal/tui/meeting_mode_test.go @@ -84,17 +84,31 @@ func TestMeetingAccountsExcludeUnrelatedSources(t *testing.T) { query.AccountInfo{ID: 2, SourceType: meetingSourceGranola, Identifier: "work-notes"}, query.AccountInfo{ID: 3, SourceType: meetingSourceCircleback, Identifier: "team-meetings"}, query.AccountInfo{ID: 4, SourceType: "teams", Identifier: "team-chat"}, + query.AccountInfo{ID: 5, SourceType: meetingSourceImported, Identifier: "local-meetings"}, ).Build() accounts := model.meetingAccounts() - require.Len(t, accounts, 2) - assert.Equal(t, []string{"work-notes", "team-meetings"}, []string{ + require.Len(t, accounts, 3) + assert.Equal(t, []string{"work-notes", "team-meetings", "local-meetings"}, []string{ accounts[0].Identifier, accounts[1].Identifier, + accounts[2].Identifier, }) } +func TestMeetingImportedSourceLabelUsesDisplayNameAndFallbacks(t *testing.T) { + model := NewBuilder().WithAccounts( + query.AccountInfo{ID: 5, SourceType: meetingSourceImported, Identifier: "local-meetings", DisplayName: "Imported Interviews"}, + query.AccountInfo{ID: 6, SourceType: meetingSourceImported, Identifier: "second-stream"}, + query.AccountInfo{ID: 7, SourceType: meetingSourceImported}, + ).Build() + + assert.Equal(t, "Imported Interviews", model.meetingSourceLabel(5)) + assert.Equal(t, "second-stream", model.meetingSourceLabel(6)) + assert.Equal(t, "Imported", model.meetingSourceLabel(7)) +} + func TestMeetingAccountSelectorUsesMeetingSources(t *testing.T) { assert := assert.New(t) selectedID := int64(3) diff --git a/internal/tui/meeting_state.go b/internal/tui/meeting_state.go index 1d017c83f..f4a4be12f 100644 --- a/internal/tui/meeting_state.go +++ b/internal/tui/meeting_state.go @@ -12,6 +12,7 @@ const meetingMessageType = "meeting_transcript" const ( meetingSourceGranola = "granola" meetingSourceCircleback = "circleback" + meetingSourceImported = "meeting_import" ) type meetingViewLevel int @@ -76,7 +77,7 @@ func (m Model) meetingAccounts() []query.AccountInfo { accounts := make([]query.AccountInfo, 0, len(m.accounts)) for _, account := range m.accounts { switch strings.ToLower(strings.TrimSpace(account.SourceType)) { - case meetingSourceGranola, meetingSourceCircleback: + case meetingSourceGranola, meetingSourceCircleback, meetingSourceImported: accounts = append(accounts, account) } } diff --git a/internal/tui/meeting_view.go b/internal/tui/meeting_view.go index 739db9330..34722f1b5 100644 --- a/internal/tui/meeting_view.go +++ b/internal/tui/meeting_view.go @@ -60,6 +60,14 @@ func (m Model) meetingSourceLabel(sourceID int64) string { return "Granola" case meetingSourceCircleback: return "Circleback" + case meetingSourceImported: + if account.DisplayName != "" { + return textutil.SanitizeTerminal(account.DisplayName) + } + if account.Identifier != "" { + return textutil.SanitizeTerminal(account.Identifier) + } + return "Imported" } if account.DisplayName != "" { return textutil.SanitizeTerminal(account.DisplayName) @@ -84,9 +92,9 @@ func meetingColumnWidths(width int) (date, title, organizer, source int) { func (m Model) meetingListView() string { if len(m.meetingState.messages) == 0 && !m.loading && m.err == nil && !m.meetingState.searchActive && m.meetingState.searchQuery == "" { - message := "No meetings found. Sync Granola or Circleback to import meetings." + message := "No meetings found. Sync a provider or import meetings." if len(m.meetingAccounts()) == 0 { - message = "No meeting sources configured. Add Granola or Circleback to begin." + message = "No meeting sources configured. Add a provider or import meetings to begin." } content := m.fillScreen(m.styles.normalRow.Render(padRight(message, m.width))) if m.modal != modalNone { diff --git a/internal/tui/meeting_view_test.go b/internal/tui/meeting_view_test.go index 111531165..3654752c2 100644 --- a/internal/tui/meeting_view_test.go +++ b/internal/tui/meeting_view_test.go @@ -62,8 +62,32 @@ func TestMeetingEmptyStateGuidesSourceSetup(t *testing.T) { view := stripANSI(model.renderView()) assert.Contains(t, view, "No meeting sources configured") - assert.Contains(t, view, "Granola") - assert.Contains(t, view, "Circleback") + assert.Contains(t, view, "provider") + assert.Contains(t, view, "import") +} + +func TestMeetingListViewShowsImportedSourceDisplayName(t *testing.T) { + model := NewBuilder().WithAccounts( + query.AccountInfo{ + ID: 5, + SourceType: meetingSourceImported, + Identifier: "local-meetings", + DisplayName: "Imported Interviews", + }, + ).WithSize(120, 24).Build() + model.mode = modeMeetings + model.loading = false + model.meetingState.messages = []query.MessageSummary{{ + ID: 10, + SourceID: 5, + Subject: "Synthetic interview", + SentAt: time.Date(2026, 7, 23, 18, 0, 0, 0, time.UTC), + MessageType: meetingMessageType, + }} + + view := stripANSI(model.renderView()) + + assert.Contains(t, view, "Imported ...") } func TestMeetingListViewShowsSearchInput(t *testing.T) { diff --git a/pkg/client/client.go b/pkg/client/client.go index 28c8ca7dd..b01c94927 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -222,6 +222,30 @@ func (c *Client) AddAccount( return nil, fmt.Errorf("add account: %w", err) } +// ImportMeeting accepts both documented success statuses. The generated +// convenience method treats only 201 as success even though an idempotent +// update returns 200. +func (c *Client) ImportMeeting( + ctx context.Context, + options *generated.ImportMeetingRequestOptions, + reqEditors ...runtime.RequestEditorFn, +) (*generated.ImportMeetingResponseJSON, error) { + resp, err := c.ImportMeetingWithResponse(ctx, options, reqEditors...) + if err != nil { + return nil, err + } + if resp.JSON201 != nil { + return resp.JSON201, nil + } + if resp.JSON200 != nil { + return resp.JSON200, nil + } + err = runtime.NewClientAPIError( + fmt.Errorf("unexpected status code: %d", resp.StatusCode), + runtime.WithStatusCode(resp.StatusCode)) + return nil, fmt.Errorf("import meeting: %w", err) +} + // StageDeletion accepts both documented success statuses. The generated // convenience method treats only 201 as success even though the daemon // returns 200 for dry-run staging requests. diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 04de81a4e..005bee0ed 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -336,6 +336,41 @@ func TestCreatePersonAcceptsIdempotentOK(t *testing.T) { assert.Equal(int64(2), got.Revision, "revision") } +func TestImportMeetingAcceptsIdempotentOK(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(http.MethodPost, r.Method, "method") + assert.Equal("/api/v1/import/meeting", r.URL.Path, "path") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte( + `{"status":"updated","source_id":3,"message_id":901,"source_message_id":"meeting:synthetic-meeting-42"}`, + )) + })) + t.Cleanup(server.Close) + + c, err := New(server.URL) + require.NoError(err, "New") + + got, err := c.ImportMeeting(context.Background(), &generated.ImportMeetingRequestOptions{ + Body: &generated.ImportMeetingBody{ + Source: generated.Source{ + Identifier: "local-meetings", + AccountEmail: "user@example.com", + }, + Meeting: generated.Meeting{ + ExternalID: "synthetic-meeting-42", + StartedAt: "2026-07-23T18:00:00Z", + }, + }, + }) + require.NoError(err, "ImportMeeting update") + + assert.Equal(generated.MeetingImportResponseStatusUpdated, got.Status, "status") + assert.Equal(int64(901), got.MessageID, "message id") +} + func TestStageDeletionAcceptsDryRunOK(t *testing.T) { assert := assert.New(t) require := require.New(t) diff --git a/pkg/client/generated/client.go b/pkg/client/generated/client.go index 6cb984d4b..b7ad7035d 100644 --- a/pkg/client/generated/client.go +++ b/pkg/client/generated/client.go @@ -303,6 +303,10 @@ type ClientInterface interface { UnlinkIdentityParticipants(ctx context.Context, options *UnlinkIdentityParticipantsRequestOptions, reqEditors ...runtime.RequestEditorFn) (*UnlinkIdentityParticipantsResponse, error) UnlinkIdentityParticipantsWithResponse(ctx context.Context, options *UnlinkIdentityParticipantsRequestOptions, reqEditors ...runtime.RequestEditorFn) (*UnlinkIdentityParticipantsResp, error) + // ImportMeeting Import one meeting + ImportMeeting(ctx context.Context, options *ImportMeetingRequestOptions, reqEditors ...runtime.RequestEditorFn) (*ImportMeetingResponseJSON, error) + ImportMeetingWithResponse(ctx context.Context, options *ImportMeetingRequestOptions, reqEditors ...runtime.RequestEditorFn) (*ImportMeetingResp, error) + // SearchIntegrationTasks Search tasks in the configured project SearchIntegrationTasks(ctx context.Context, options *SearchIntegrationTasksRequestOptions, reqEditors ...runtime.RequestEditorFn) (*SearchIntegrationTasksResponse, error) SearchIntegrationTasksWithResponse(ctx context.Context, options *SearchIntegrationTasksRequestOptions, reqEditors ...runtime.RequestEditorFn) (*SearchIntegrationTasksResp, error) @@ -4643,6 +4647,70 @@ func (c *Client) UnlinkIdentityParticipants(ctx context.Context, options *Unlink return responseParser(ctx, resp) } +// ImportMeeting Import one meeting +func (c *Client) ImportMeeting(ctx context.Context, options *ImportMeetingRequestOptions, reqEditors ...runtime.RequestEditorFn) (*ImportMeetingResponseJSON, error) { + var err error + reqParams := runtime.RequestOptionsParameters{ + RequestURL: c.apiClient.GetBaseURL() + "/api/v1/import/meeting", + Method: "POST", + Options: options, + ContentType: "application/json", + } + + req, err := c.apiClient.CreateRequest(ctx, reqParams, reqEditors...) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + + responseParser := func(ctx context.Context, resp *runtime.Response) (*ImportMeetingResponseJSON, error) { + bodyBytes := resp.Content + if resp.StatusCode != 201 { + target := new(ImportMeetingErrorResponse) + // Handle empty error response body gracefully - skip unmarshal if no content + if len(bodyBytes) > 0 { + if err = json.Unmarshal(bodyBytes, target); err != nil { + return nil, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "ImportMeetingErrorResponse", + Body: bodyBytes, + Err: err, + } + } + } + // Return error with (possibly empty) target + if errTarget, ok := any(*target).(error); ok { + return nil, runtime.NewClientAPIError(errTarget, runtime.WithStatusCode(resp.StatusCode)) + } + return nil, runtime.NewClientAPIError(fmt.Errorf("API error (status %d): %v", resp.StatusCode, *target), + runtime.WithStatusCode(resp.StatusCode)) + } + target := new(ImportMeetingResponseJSON) + // Handle empty response body gracefully + if len(bodyBytes) == 0 { + return target, nil + } + if err = json.Unmarshal(bodyBytes, target); err != nil { + return nil, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "ImportMeetingResponseJSON", + Body: bodyBytes, + Err: err, + } + } + return target, nil + } + + resp, err := c.apiClient.ExecuteRequest(ctx, req, "/api/v1/import/meeting") + if err != nil { + return nil, fmt.Errorf("error executing request: %w", err) + } + return responseParser(ctx, resp) +} + // SearchIntegrationTasks Search tasks in the configured project func (c *Client) SearchIntegrationTasks(ctx context.Context, options *SearchIntegrationTasksRequestOptions, reqEditors ...runtime.RequestEditorFn) (*SearchIntegrationTasksResponse, error) { var err error diff --git a/pkg/client/generated/client_options.go b/pkg/client/generated/client_options.go index 9798c6dbe..5a714cacb 100644 --- a/pkg/client/generated/client_options.go +++ b/pkg/client/generated/client_options.go @@ -2533,6 +2533,50 @@ func (o *UnlinkIdentityParticipantsRequestOptions) GetHeader() (map[string]strin return nil, nil } +// ImportMeetingRequestOptions is the options needed to make a request to ImportMeeting. +type ImportMeetingRequestOptions struct { + Body *ImportMeetingBody +} + +// Validate validates all the fields in the options. +// Use it if fields validation was not run. +func (o *ImportMeetingRequestOptions) Validate() error { + var errors runtime.ValidationErrors + + if o.Body != nil { + if v, ok := any(o.Body).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Body", err) + } + } + } + if len(errors) == 0 { + return nil + } + + return errors +} + +// GetPathParams returns the path params as a map. +func (o *ImportMeetingRequestOptions) GetPathParams() (map[string]any, error) { + return nil, nil +} + +// GetQuery returns the query params as a map. +func (o *ImportMeetingRequestOptions) GetQuery() (map[string]any, error) { + return nil, nil +} + +// GetBody returns the payload in any type that can be marshalled to JSON by the client. +func (o *ImportMeetingRequestOptions) GetBody() any { + return o.Body +} + +// GetHeader returns the headers as a map. +func (o *ImportMeetingRequestOptions) GetHeader() (map[string]string, error) { + return nil, nil +} + // SearchIntegrationTasksRequestOptions is the options needed to make a request to SearchIntegrationTasks. type SearchIntegrationTasksRequestOptions struct { Query *SearchIntegrationTasksQuery diff --git a/pkg/client/generated/client_with_response.go b/pkg/client/generated/client_with_response.go index 885308ebc..21e9d4178 100644 --- a/pkg/client/generated/client_with_response.go +++ b/pkg/client/generated/client_with_response.go @@ -4885,6 +4885,72 @@ func (c *Client) UnlinkIdentityParticipantsWithResponse(ctx context.Context, opt } } +// ImportMeeting Import one meeting +func (c *Client) ImportMeetingWithResponse(ctx context.Context, options *ImportMeetingRequestOptions, reqEditors ...runtime.RequestEditorFn) (*ImportMeetingResp, error) { + var err error + reqParams := runtime.RequestOptionsParameters{ + RequestURL: c.apiClient.GetBaseURL() + "/api/v1/import/meeting", + Method: "POST", + Options: options, + ContentType: "application/json", + } + + req, err := c.apiClient.CreateRequest(ctx, reqParams, reqEditors...) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + + resp, err := c.apiClient.ExecuteRequest(ctx, req, "/api/v1/import/meeting") + if err != nil { + return nil, fmt.Errorf("error executing request: %w", err) + } + + out := &ImportMeetingResp{ + HTTPResponse: resp.Raw, + Body: resp.Content, + StatusCode: resp.StatusCode, + } + + switch resp.StatusCode { + case 200: + out.JSON200 = new(ImportMeetingResponse) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON200); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "ImportMeetingResponse", + Body: bodyBytes, + Err: err, + } + } + } + return out, nil + case 201: + out.JSON201 = new(ImportMeetingResponseJSON) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON201); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "ImportMeetingResponseJSON", + Body: bodyBytes, + Err: err, + } + } + } + return out, nil + case 500: + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + default: + return out, runtime.NewClientAPIError(fmt.Errorf("unexpected status code: %d", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + } +} + // SearchIntegrationTasks Search tasks in the configured project func (c *Client) SearchIntegrationTasksWithResponse(ctx context.Context, options *SearchIntegrationTasksRequestOptions, reqEditors ...runtime.RequestEditorFn) (*SearchIntegrationTasksResp, error) { var err error diff --git a/pkg/client/generated/enums.go b/pkg/client/generated/enums.go index 314ef88bb..b2e98a8ed 100644 --- a/pkg/client/generated/enums.go +++ b/pkg/client/generated/enums.go @@ -333,6 +333,23 @@ func (i IdentitySearchSortField) Validate() error { } } +type MeetingImportResponseStatus string + +const ( + MeetingImportResponseStatusCreated MeetingImportResponseStatus = "created" + MeetingImportResponseStatusUpdated MeetingImportResponseStatus = "updated" +) + +// Validate checks if the MeetingImportResponseStatus value is valid +func (m MeetingImportResponseStatus) Validate() error { + switch m { + case MeetingImportResponseStatusCreated, MeetingImportResponseStatusUpdated: + return nil + default: + return runtime.NewValidationErrorsFromString("Enum", fmt.Sprintf("must be a valid MeetingImportResponseStatus value, got: %v", m)) + } +} + type RemoveResultCacheState string const ( diff --git a/pkg/client/generated/payloads.go b/pkg/client/generated/payloads.go index 8d72fd20f..a0fc72624 100644 --- a/pkg/client/generated/payloads.go +++ b/pkg/client/generated/payloads.go @@ -68,6 +68,8 @@ type LinkIdentityParticipantsBody = IdentityLinkRequest type UnlinkIdentityParticipantsBody = IdentityLinkRequest +type ImportMeetingBody = MeetingImportRequest + type CreateOrLinkMessageTaskBody = TaskLinkMutationRequest type SearchPeopleBody = IdentitySearchHTTPRequest diff --git a/pkg/client/generated/responses.go b/pkg/client/generated/responses.go index f9186692d..e936626cb 100644 --- a/pkg/client/generated/responses.go +++ b/pkg/client/generated/responses.go @@ -841,6 +841,12 @@ type UnlinkIdentityParticipantsResponse = IdentityLinkResponse type UnlinkIdentityParticipantsErrorResponse = ErrorResponse +type ImportMeetingResponse = MeetingImportResponse + +type ImportMeetingResponseJSON = MeetingImportResponse + +type ImportMeetingErrorResponse = ErrorResponse + type SearchIntegrationTasksResponse = TaskSearchResponse type SearchIntegrationTasksErrorResponse = ErrorResponse @@ -1950,6 +1956,14 @@ type UnlinkIdentityParticipantsResp struct { JSON200 *UnlinkIdentityParticipantsResponse } +type ImportMeetingResp struct { + HTTPResponse *http.Response + Body []byte + StatusCode int + JSON200 *ImportMeetingResponse + JSON201 *ImportMeetingResponseJSON +} + type SearchIntegrationTasksResp struct { HTTPResponse *http.Response Body []byte diff --git a/pkg/client/generated/types.go b/pkg/client/generated/types.go index 495d660af..e6617ec62 100644 --- a/pkg/client/generated/types.go +++ b/pkg/client/generated/types.go @@ -2441,6 +2441,115 @@ type MatchSummary struct { StrongestExcerpt *string `json:"strongest_excerpt,omitempty"` } +type Meeting struct { + Attendees []MeetingPerson `json:"attendees,omitempty"` + EndedAt *string `json:"ended_at,omitempty"` + ExternalID string `json:"external_id" validate:"required,max=256"` + Metadata map[string]any `json:"metadata,omitempty"` + Organizer *MeetingPerson `json:"organizer,omitempty"` + StartedAt string `json:"started_at" validate:"required"` + SummaryMarkdown *string `json:"summary_markdown,omitempty"` + SummaryText *string `json:"summary_text,omitempty"` + Title *string `json:"title,omitempty" validate:"omitempty,max=4096"` + Transcript *string `json:"transcript,omitempty"` + TranscriptSegments []TranscriptSegment `json:"transcript_segments,omitempty"` +} + +func (m Meeting) Validate() error { + var errors runtime.ValidationErrors + for i, item := range m.Attendees { + if v, ok := any(item).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append(fmt.Sprintf("Attendees[%d]", i), err) + } + } + } + if err := typesValidator.Var(m.ExternalID, "required,max=256"); err != nil { + errors = errors.Append("ExternalID", err) + } + if m.Organizer != nil { + if v, ok := any(m.Organizer).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Organizer", err) + } + } + } + if err := typesValidator.Var(m.StartedAt, "required"); err != nil { + errors = errors.Append("StartedAt", err) + } + if m.Title != nil { + if err := typesValidator.Var(m.Title, "omitempty,max=4096"); err != nil { + errors = errors.Append("Title", err) + } + } + for i, item := range m.TranscriptSegments { + if v, ok := any(item).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append(fmt.Sprintf("TranscriptSegments[%d]", i), err) + } + } + } + if len(errors) == 0 { + return nil + } + return errors +} + +type MeetingImportRequest struct { + Meeting Meeting `json:"meeting"` + Source Source `json:"source"` +} + +func (m MeetingImportRequest) Validate() error { + var errors runtime.ValidationErrors + if v, ok := any(m.Meeting).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Meeting", err) + } + } + if v, ok := any(m.Source).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Source", err) + } + } + if len(errors) == 0 { + return nil + } + return errors +} + +type MeetingImportResponse struct { + MessageID int64 `json:"message_id"` + SourceID int64 `json:"source_id"` + SourceMessageID string `json:"source_message_id" validate:"required"` + Status MeetingImportResponseStatus `json:"status" validate:"required"` +} + +func (m MeetingImportResponse) Validate() error { + var errors runtime.ValidationErrors + if err := typesValidator.Var(m.SourceMessageID, "required"); err != nil { + errors = errors.Append("SourceMessageID", err) + } + if v, ok := any(m.Status).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Status", err) + } + } + if len(errors) == 0 { + return nil + } + return errors +} + +type MeetingPerson struct { + Email string `json:"email" validate:"required"` + Name *string `json:"name,omitempty"` +} + +func (m MeetingPerson) Validate() error { + return runtime.ConvertValidatorError(typesValidator.Struct(m)) +} + type MessageDetail struct { Attachments []AttachmentInfo `json:"attachments,omitempty" validate:"required"` Bcc []string `json:"bcc,omitempty"` @@ -3557,6 +3666,16 @@ func (s SimilarSearchResponse) Validate() error { return errors } +type Source struct { + AccountEmail string `json:"account_email" validate:"required"` + DisplayName *string `json:"display_name,omitempty" validate:"omitempty,max=256"` + Identifier string `json:"identifier" validate:"required,max=128"` +} + +func (s Source) Validate() error { + return runtime.ConvertValidatorError(typesValidator.Struct(s)) +} + type SourceCount struct { Count int64 `json:"count"` SourceType string `json:"source_type" validate:"required"` @@ -4139,6 +4258,16 @@ type TotalStatsResponse struct { TotalSize int64 `json:"total_size"` } +type TranscriptSegment struct { + OffsetSeconds *float64 `json:"offset_seconds,omitempty" validate:"omitempty,gte=0"` + Speaker string `json:"speaker" validate:"required"` + Text string `json:"text" validate:"required"` +} + +func (t TranscriptSegment) Validate() error { + return runtime.ConvertValidatorError(typesValidator.Struct(t)) +} + type UpdateRequest struct { DisplayName string `json:"display_name" validate:"required"` Email string `json:"email" validate:"required"` diff --git a/pkg/client/openapi.yaml b/pkg/client/openapi.yaml index c9f3c55b8..b4e856564 100644 --- a/pkg/client/openapi.yaml +++ b/pkg/client/openapi.yaml @@ -2543,6 +2543,96 @@ components: strongest_excerpt: type: string type: object + Meeting: + additionalProperties: false + properties: + attendees: + items: + $ref: "#/components/schemas/MeetingPerson" + nullable: true + type: array + ended_at: + format: date-time + type: string + x-go-type: string + external_id: + maxLength: 256 + type: string + metadata: + additionalProperties: + x-go-type: any + type: object + organizer: + $ref: "#/components/schemas/MeetingPerson" + started_at: + format: date-time + type: string + x-go-type: string + summary_markdown: + type: string + summary_text: + type: string + title: + maxLength: 4096 + type: string + transcript: + type: string + transcript_segments: + items: + $ref: "#/components/schemas/TranscriptSegment" + nullable: true + type: array + required: + - external_id + - started_at + type: object + MeetingImportRequest: + additionalProperties: false + properties: + meeting: + $ref: "#/components/schemas/Meeting" + source: + $ref: "#/components/schemas/Source" + required: + - source + - meeting + type: object + MeetingImportResponse: + properties: + message_id: + format: int64 + type: integer + source_id: + format: int64 + type: integer + source_message_id: + type: string + status: + enum: + - created + - updated + type: string + x-enum-names: + - MeetingImportResponseStatusCreated + - MeetingImportResponseStatusUpdated + required: + - status + - source_id + - message_id + - source_message_id + type: object + MeetingPerson: + additionalProperties: false + properties: + email: + format: email + type: string + x-go-type: string + name: + type: string + required: + - email + type: object MessageDetail: properties: attachments: @@ -3651,6 +3741,23 @@ components: - generation - messages type: object + Source: + additionalProperties: false + properties: + account_email: + format: email + type: string + x-go-type: string + display_name: + maxLength: 256 + type: string + identifier: + maxLength: 128 + type: string + required: + - identifier + - account_email + type: object SourceCount: properties: count: @@ -4314,6 +4421,21 @@ components: - label_count - account_count type: object + TranscriptSegment: + additionalProperties: false + properties: + offset_seconds: + format: double + minimum: 0 + type: number + speaker: + type: string + text: + type: string + required: + - speaker + - text + type: object UpdateRequest: additionalProperties: false properties: @@ -4351,7 +4473,7 @@ components: type: apiKey info: title: msgvault API - version: 1.32.0 + version: 1.33.0 openapi: 3.0.3 paths: /api/ping: @@ -7069,6 +7191,39 @@ paths: summary: Remove a link edge between two participants tags: - API + /api/v1/import/meeting: + post: + operationId: importMeeting + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportResponse" + description: OK + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportResponse" + description: Created + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Import one meeting + tags: + - API /api/v1/integrations/tasks/search: get: operationId: searchIntegrationTasks diff --git a/web/src/lib/api/generated/schema.d.ts b/web/src/lib/api/generated/schema.d.ts index 68a8123cf..deefeca80 100644 --- a/web/src/lib/api/generated/schema.d.ts +++ b/web/src/lib/api/generated/schema.d.ts @@ -1032,6 +1032,23 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/import/meeting": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Import one meeting */ + post: operations["importMeeting"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/integrations/tasks/search": { parameters: { query?: never; @@ -2875,6 +2892,51 @@ export interface components { } & { [key: string]: unknown; }; + Meeting: { + attendees?: components["schemas"]["MeetingPerson"][] | null; + /** Format: date-time */ + ended_at?: string; + external_id: string; + metadata?: { + [key: string]: unknown; + }; + organizer?: components["schemas"]["MeetingPerson"]; + /** Format: date-time */ + started_at: string; + summary_markdown?: string; + summary_text?: string; + title?: string; + transcript?: string; + transcript_segments?: components["schemas"]["TranscriptSegment"][] | null; + } & (({ + summary_markdown: string; + } | { + summary_text: string; + } | { + transcript: string; + } | { + transcript_segments: unknown[]; + }) & unknown); + MeetingImportRequest: { + meeting: components["schemas"]["Meeting"]; + source: components["schemas"]["Source"]; + }; + MeetingImportResponse: { + /** Format: int64 */ + message_id: number; + /** Format: int64 */ + source_id: number; + source_message_id: string; + /** @enum {string} */ + status: "created" | "updated"; + } & { + [key: string]: unknown; + }; + MeetingPerson: { + /** Format: email */ + email: string; + name?: string; + }; MessageDetail: { attachments: components["schemas"]["AttachmentInfo"][] | null; bcc?: string[] | null; @@ -3371,6 +3433,12 @@ export interface components { } & { [key: string]: unknown; }; + Source: { + /** Format: email */ + account_email: string; + display_name?: string; + identifier: string; + }; SourceCount: { /** Format: int64 */ count: number; @@ -3685,6 +3753,12 @@ export interface components { } & { [key: string]: unknown; }; + TranscriptSegment: { + /** Format: double */ + offset_seconds?: number; + speaker: string; + text: string; + }; UpdateRequest: { display_name: string; email: string; @@ -6895,6 +6969,48 @@ export interface operations { }; }; }; + importMeeting: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["MeetingImportRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MeetingImportResponse"]; + }; + }; + /** @description Created */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MeetingImportResponse"]; + }; + }; + /** @description Error */ + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + }; + }; searchIntegrationTasks: { parameters: { query: { diff --git a/web/src/lib/components/sources/SourcesWorkspace.svelte b/web/src/lib/components/sources/SourcesWorkspace.svelte index 0668e873e..2aff03f26 100644 --- a/web/src/lib/components/sources/SourcesWorkspace.svelte +++ b/web/src/lib/components/sources/SourcesWorkspace.svelte @@ -74,6 +74,10 @@ return source.sync_unavailable_reason === 'sync_already_running'; } + function isOnDemandSource(source: Source): boolean { + return source.source_type === 'meeting_import'; + } + // allowIdle bypasses the active-sync gate below for error-path retries: // a status-load failure must be able to reschedule itself even when no // source is actively syncing and no accepted run is being awaited, or an @@ -243,6 +247,8 @@ {#if source.scheduled} Scheduled · {source.schedule ?? 'schedule unavailable'} {#if source.next_sync_at}Next {source.next_sync_at}{/if} + {:else if isOnDemandSource(source)} + On demand · imported through the API {:else}Not scheduled{/if} {#if source.scheduler_last_error}Scheduler: {source.scheduler_last_error}{/if} @@ -275,6 +281,8 @@
{#if source.can_sync}