diff --git a/api/openapi.yaml b/api/openapi.yaml index 18554c1b6..3bf94ea0c 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -2626,6 +2626,134 @@ components: strongest_excerpt: type: string type: object + Meeting: + additionalProperties: false + allOf: + - anyOf: + - properties: + summary_markdown: + minLength: 1 + type: string + required: + - summary_markdown + type: object + - properties: + summary_text: + minLength: 1 + type: string + required: + - summary_text + type: object + - properties: + transcript: + minLength: 1 + type: string + required: + - transcript + type: object + - properties: + transcript_segments: + minItems: 1 + type: array + required: + - transcript_segments + type: object + - not: + properties: + transcript: + minLength: 1 + type: string + transcript_segments: + minItems: 1 + type: array + required: + - transcript + - transcript_segments + type: object + properties: + attendees: + items: + $ref: "#/components/schemas/MeetingPerson" + type: + - array + - "null" + ended_at: + format: date-time + type: string + external_id: + maxLength: 256 + type: string + metadata: + additionalProperties: {} + type: object + organizer: + $ref: "#/components/schemas/MeetingPerson" + started_at: + format: date-time + type: string + summary_markdown: + type: string + summary_text: + type: string + title: + maxLength: 4096 + type: string + transcript: + type: string + transcript_segments: + items: + $ref: "#/components/schemas/TranscriptSegment" + type: + - array + - "null" + required: + - external_id + - started_at + type: object + MeetingImportRequest: + additionalProperties: false + properties: + meeting: + $ref: "#/components/schemas/Meeting" + source: + $ref: "#/components/schemas/Source" + required: + - source + - meeting + type: object + MeetingImportResponse: + additionalProperties: true + properties: + message_id: + format: int64 + type: integer + source_id: + format: int64 + type: integer + source_message_id: + type: string + status: + enum: + - created + - updated + type: string + required: + - status + - source_id + - message_id + - source_message_id + type: object + MeetingPerson: + additionalProperties: false + properties: + email: + format: email + type: string + name: + type: string + required: + - email + type: object MessageDetail: additionalProperties: true properties: @@ -3798,6 +3926,22 @@ components: - generation - messages type: object + Source: + additionalProperties: false + properties: + account_email: + format: email + type: string + display_name: + maxLength: 256 + type: string + identifier: + maxLength: 128 + type: string + required: + - identifier + - account_email + type: object SourceCount: additionalProperties: true properties: @@ -4506,6 +4650,21 @@ components: - label_count - account_count type: object + TranscriptSegment: + additionalProperties: false + properties: + offset_seconds: + format: double + minimum: 0 + type: number + speaker: + type: string + text: + type: string + required: + - speaker + - text + type: object UpdateRequest: additionalProperties: false properties: @@ -4545,7 +4704,7 @@ components: type: apiKey info: title: msgvault API - version: 1.32.0 + version: 1.33.0 openapi: 3.1.0 paths: /api/ping: @@ -7263,6 +7422,39 @@ paths: summary: Remove a link edge between two participants tags: - API + /api/v1/import/meeting: + post: + operationId: importMeeting + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportResponse" + description: OK + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportResponse" + description: Created + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Import one meeting + tags: + - API /api/v1/integrations/tasks/search: get: operationId: searchIntegrationTasks diff --git a/cmd/msgvault/cmd/build_cache.go b/cmd/msgvault/cmd/build_cache.go index c3a7a09c8..1e96f4b78 100644 --- a/cmd/msgvault/cmd/build_cache.go +++ b/cmd/msgvault/cmd/build_cache.go @@ -735,6 +735,18 @@ func buildCacheLocked( return nil, fmt.Errorf("inspect attachment MIME schema: %w", err) } sourceSnapshot.hasAttachmentMIME = attachmentMIMEColumnCount > 0 + var messageSourceAttributionColumnCount int + if err := sourceSnapshot.QueryRow(` + SELECT COUNT(*) FROM pragma_table_info('messages') + WHERE name = 'source_is_from_me' + `).Scan(&messageSourceAttributionColumnCount); err != nil { + return nil, fmt.Errorf("inspect message attribution schema: %w", err) + } + messageSourceAttribution := "COALESCE(m.is_from_me, FALSE)" + if messageSourceAttributionColumnCount > 0 { + messageSourceAttribution = "COALESCE(m.source_is_from_me, FALSE)" + } + sourceSnapshot.hasMessageSourceAttribution = messageSourceAttributionColumnCount > 0 if err := sourceSnapshot.Prepare(); err != nil { return nil, err } @@ -1067,7 +1079,7 @@ func buildCacheLocked( m.deleted_from_source_at, m.sender_id, COALESCE(TRY_CAST(m.message_type AS VARCHAR), '') as message_type, - (COALESCE(m.is_from_me, FALSE) OR EXISTS ( + (%s OR EXISTS ( SELECT 1 FROM sqlite_db.account_identities ai JOIN sqlite_db.participants sp ON sp.id = m.sender_id WHERE ai.source_id = m.source_id @@ -1090,7 +1102,7 @@ func buildCacheLocked( OVERWRITE_OR_IGNORE, COMPRESSION 'zstd' ) - `, idFilter, escapedMessagesDir)); err != nil { + `, messageSourceAttribution, idFilter, escapedMessagesDir)); err != nil { return nil, fmt.Errorf("export messages: %w", err) } @@ -1126,7 +1138,7 @@ func buildCacheLocked( m.deleted_from_source_at, m.sender_id, COALESCE(TRY_CAST(m.message_type AS VARCHAR), '') as message_type, - (COALESCE(m.is_from_me, FALSE) OR EXISTS ( + (%s OR EXISTS ( SELECT 1 FROM sqlite_db.account_identities ai JOIN sqlite_db.participants sp ON sp.id = m.sender_id WHERE ai.source_id = m.source_id @@ -1143,7 +1155,7 @@ func buildCacheLocked( FROM sqlite_db.messages m WHERE 1 = 0 ) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd') - `, escapedEmptyShard)); err != nil { + `, messageSourceAttribution, escapedEmptyShard)); err != nil { return nil, fmt.Errorf("export empty messages shard: %w", err) } } @@ -1448,12 +1460,13 @@ func writeCacheStatsLine(w io.Writer, format string, args ...any) error { // transaction directly. The fallback exports every table from one go-sqlite3 // transaction before DuckDB reads the resulting static CSV files. type cacheSourceSnapshot struct { - duckDB *sql.DB - duckTx *sql.Tx - sqliteDB *sql.DB - sqliteTx *sql.Tx - tmpDir string - hasAttachmentMIME bool + duckDB *sql.DB + duckTx *sql.Tx + sqliteDB *sql.DB + sqliteTx *sql.Tx + tmpDir string + hasAttachmentMIME bool + hasMessageSourceAttribution bool } type cacheSnapshotTable struct { @@ -1574,6 +1587,14 @@ func (s *cacheSourceSnapshot) tables() []cacheSnapshotTable { if s.hasAttachmentMIME { attachmentQuery = "SELECT id, message_id, size, filename, mime_type FROM attachments" } + messageColumns := "id, source_id, source_message_id, conversation_id, subject, snippet, sent_at, size_estimate, has_attachments, attachment_count, deleted_from_source_at, deleted_at, sender_id, message_type, is_from_me" + messageTypes := "types={'id': 'BIGINT', 'source_id': 'BIGINT', 'source_message_id': 'VARCHAR', 'conversation_id': 'BIGINT', 'subject': 'VARCHAR', 'snippet': 'VARCHAR', 'sent_at': 'TIMESTAMP', 'size_estimate': 'BIGINT', 'has_attachments': 'BOOLEAN', 'attachment_count': 'INTEGER', 'deleted_from_source_at': 'TIMESTAMP', 'deleted_at': 'TIMESTAMP', 'sender_id': 'BIGINT', 'message_type': 'VARCHAR', 'is_from_me': 'BOOLEAN'" + if s.hasMessageSourceAttribution { + messageColumns += ", source_is_from_me" + messageTypes += ", 'source_is_from_me': 'BOOLEAN'" + } + messageTypes += "}" + // Every column carries an explicit type so the schema DuckDB sees never // depends on the data: read_csv_auto sniffs empty or all-NULL columns as // VARCHAR, which breaks downstream SQL that binds these columns against @@ -1583,8 +1604,7 @@ func (s *cacheSourceSnapshot) tables() []cacheSnapshotTable { // `deleted_at IS NULL` filter on this path the same way it does // on the sqlite_scanner path; otherwise DuckDB binds against a // CSV view that lacks the column and the export fails on Windows. - {tableMessages, "SELECT id, source_id, source_message_id, conversation_id, subject, snippet, sent_at, size_estimate, has_attachments, attachment_count, deleted_from_source_at, deleted_at, sender_id, message_type, is_from_me FROM messages WHERE sent_at IS NOT NULL", - "types={'id': 'BIGINT', 'source_id': 'BIGINT', 'source_message_id': 'VARCHAR', 'conversation_id': 'BIGINT', 'subject': 'VARCHAR', 'snippet': 'VARCHAR', 'sent_at': 'TIMESTAMP', 'size_estimate': 'BIGINT', 'has_attachments': 'BOOLEAN', 'attachment_count': 'INTEGER', 'deleted_from_source_at': 'TIMESTAMP', 'deleted_at': 'TIMESTAMP', 'sender_id': 'BIGINT', 'message_type': 'VARCHAR', 'is_from_me': 'BOOLEAN'}"}, + {tableMessages, "SELECT " + messageColumns + " FROM messages WHERE sent_at IS NOT NULL", messageTypes}, {"message_recipients", "SELECT message_id, participant_id, recipient_type, display_name FROM message_recipients", "types={'message_id': 'BIGINT', 'participant_id': 'BIGINT', 'recipient_type': 'VARCHAR', 'display_name': 'VARCHAR'}"}, {"message_labels", "SELECT message_id, label_id FROM message_labels", diff --git a/cmd/msgvault/cmd/build_cache_identity_test.go b/cmd/msgvault/cmd/build_cache_identity_test.go index 6fae598b5..fd1a804b8 100644 --- a/cmd/msgvault/cmd/build_cache_identity_test.go +++ b/cmd/msgvault/cmd/build_cache_identity_test.go @@ -129,6 +129,14 @@ func TestBuildCache_DerivesIsFromMeAndIdentityDatasets(t *testing.T) { }) require.NoError(err) require.NoError(st.ReplaceMessageRecipients(controlMsgID, "from", []int64{otherParticipantID}, []string{""})) + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages + SET is_from_me = TRUE, + source_is_from_me = FALSE, + identity_is_from_me = TRUE + WHERE id = ? + `), controlMsgID) + require.NoError(err, "simulate stale persisted effective attribution") require.NoError(st.Close()) diff --git a/cmd/msgvault/cmd/build_cache_test.go b/cmd/msgvault/cmd/build_cache_test.go index 1d03edd81..fc0602f86 100644 --- a/cmd/msgvault/cmd/build_cache_test.go +++ b/cmd/msgvault/cmd/build_cache_test.go @@ -2491,6 +2491,42 @@ func TestBuildCacheCSVSnapshotFallback(t *testing.T) { assert.Positive(t, result.ExportedCount) }) + t.Run("message attribution provenance", func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + tmpDir := setupTestSQLite(t) + dbPath := filepath.Join(tmpDir, "test.db") + analyticsDir := filepath.Join(tmpDir, "analytics") + + db, err := sql.Open("sqlite3", dbPath) + require.NoError(err) + _, err = db.Exec(` + ALTER TABLE messages ADD COLUMN source_is_from_me BOOLEAN DEFAULT FALSE; + UPDATE messages + SET is_from_me = FALSE, source_is_from_me = TRUE + WHERE id = 1; + `) + require.NoError(err) + require.NoError(db.Close()) + + result, err := buildCache(dbPath, analyticsDir, true) + require.NoError(err) + assert.Positive(result.ExportedCount) + + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { require.NoError(duckDB.Close()) }() + + var isFromMe bool + require.NoError(duckDB.QueryRow( + `SELECT is_from_me + FROM read_parquet(?, hive_partitioning=true) + WHERE id = 1`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet"), + ).Scan(&isFromMe)) + assert.True(isFromMe, "CSV fallback must export source-native attribution") + }) + t.Run("empty", func(t *testing.T) { tmpDir := setupTestSQLiteEmpty(t) result, err := buildCache(filepath.Join(tmpDir, "test.db"), filepath.Join(tmpDir, "analytics"), true) diff --git a/cmd/msgvault/cmd/meeting_import_e2e_test.go b/cmd/msgvault/cmd/meeting_import_e2e_test.go new file mode 100644 index 000000000..f71e38c43 --- /dev/null +++ b/cmd/msgvault/cmd/meeting_import_e2e_test.go @@ -0,0 +1,123 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/meetingimport" + "go.kenn.io/msgvault/internal/testutil" +) + +const meetingImportE2EBody = `{ + "source": { + "identifier": "local-meetings", + "display_name": "Local Meetings", + "account_email": "user@example.com" + }, + "meeting": { + "external_id": "42", + "title": "Weekly planning", + "started_at": "2026-07-23T18:00:00Z", + "summary_text": "Initial summary.", + "transcript": "Speaker 1: initial transcript", + "organizer": {"name": "Test Organizer", "email": "organizer@example.com"}, + "attendees": [{"name": "Test Attendee", "email": "attendee@example.com"}] + } +}` + +func postMeetingImport( + t *testing.T, + srv *api.Server, + body string, +) api.MeetingImportResponse { + t.Helper() + req := httptest.NewRequest( + http.MethodPost, + "/api/v1/import/meeting", + bytes.NewBufferString(body), + ) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, req) + require.Contains(t, []int{http.StatusOK, http.StatusCreated}, resp.Code, "body: %s", resp.Body.String()) + var decoded api.MeetingImportResponse + require.NoError(t, json.NewDecoder(resp.Body).Decode(&decoded)) + return decoded +} + +func TestMeetingImportAPIToStoreUpdatesCanonicalMessage(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + adapter := &storeAPIAdapter{ + store: st, + meetingImporter: meetingimport.NewImporter(st, meetingimport.Hooks{ + AfterSourceSetup: func() error { return nil }, + RefreshCache: func(_ context.Context, _ string) error { return nil }, + }), + } + srv := api.NewServer( + &config.Config{Server: config.ServerConfig{}}, + adapter, + nil, + slog.New(slog.DiscardHandler), + ) + + created := postMeetingImport(t, srv, meetingImportE2EBody) + assert.Equal(meetingimport.StatusCreated, created.Status) + + getReq := httptest.NewRequest( + http.MethodGet, + fmt.Sprintf("/api/v1/messages/%d", created.MessageID), + nil, + ) + getResp := httptest.NewRecorder() + srv.Router().ServeHTTP(getResp, getReq) + require.Equal(http.StatusOK, getResp.Code, "body: %s", getResp.Body.String()) + var initial api.MessageDetail + require.NoError(json.NewDecoder(getResp.Body).Decode(&initial)) + assert.Equal("Weekly planning", initial.Subject) + assert.Contains(initial.Body, "Initial summary.") + assert.Contains(initial.Body, "Speaker 1: initial transcript") + assert.Contains(initial.To, "Test Attendee ") + + replacement := strings.ReplaceAll(meetingImportE2EBody, "Weekly planning", "Replacement title") + replacement = strings.ReplaceAll(replacement, "Initial summary.", "Replacement summary.") + replacement = strings.ReplaceAll(replacement, "Speaker 1: initial transcript", "Speaker 2: replacement transcript") + replacement = strings.Replace( + replacement, + `"attendees": [{"name": "Test Attendee", "email": "attendee@example.com"}]`, + `"attendees": []`, + 1, + ) + updated := postMeetingImport(t, srv, replacement) + assert.Equal(meetingimport.StatusUpdated, updated.Status) + assert.Equal(created.MessageID, updated.MessageID) + + getReq = httptest.NewRequest( + http.MethodGet, + fmt.Sprintf("/api/v1/messages/%d", updated.MessageID), + nil, + ) + getResp = httptest.NewRecorder() + srv.Router().ServeHTTP(getResp, getReq) + require.Equal(http.StatusOK, getResp.Code, "body: %s", getResp.Body.String()) + var current api.MessageDetail + require.NoError(json.NewDecoder(getResp.Body).Decode(¤t)) + assert.Equal("Replacement title", current.Subject) + assert.Contains(current.Body, "Replacement summary.") + assert.Contains(current.Body, "Speaker 2: replacement transcript") + assert.Empty(current.To) +} diff --git a/cmd/msgvault/cmd/serve.go b/cmd/msgvault/cmd/serve.go index d4b0b7c86..ff2e712bc 100644 --- a/cmd/msgvault/cmd/serve.go +++ b/cmd/msgvault/cmd/serve.go @@ -24,6 +24,7 @@ import ( "go.kenn.io/msgvault/internal/discord" "go.kenn.io/msgvault/internal/gmail" "go.kenn.io/msgvault/internal/granola" + "go.kenn.io/msgvault/internal/meetingimport" "go.kenn.io/msgvault/internal/microsoft" "go.kenn.io/msgvault/internal/oauth" "go.kenn.io/msgvault/internal/query" @@ -432,7 +433,18 @@ func runServe(cmd *cobra.Command, args []string) error { sched.Start() // Create adapters for the API interfaces - storeAdapter := &storeAPIAdapter{store: s, attachmentMaintenance: attachmentMaint, analyticsDir: cfg.AnalyticsDir()} + meetingImporter := meetingimport.NewImporter(s, meetingimport.Hooks{ + AfterSourceSetup: func() error { + return runPostSourceCreateMigrations(s) + }, + RefreshCache: rebuildCacheAfterScheduledSync, + }) + storeAdapter := &storeAPIAdapter{ + store: s, + attachmentMaintenance: attachmentMaint, + meetingImporter: meetingImporter, + analyticsDir: cfg.AnalyticsDir(), + } schedAdapter := &schedulerAdapter{scheduler: sched} // Create and start API server @@ -835,6 +847,7 @@ func newDaemonIdleTracker(c *config.Config, stop context.CancelFunc) *api.IdleTr type storeAPIAdapter struct { store *store.Store attachmentMaintenance *attachmentMaintenance + meetingImporter *meetingimport.Importer // analyticsDir is the daemon's Parquet analytics cache directory, used // to read the revision committed by the derived-refresh child. analyticsDir string @@ -842,6 +855,7 @@ type storeAPIAdapter struct { var _ api.MessageStore = (*storeAPIAdapter)(nil) var _ api.CtxMessageStore = (*storeAPIAdapter)(nil) +var _ api.MeetingImporter = (*storeAPIAdapter)(nil) var _ api.SourceStatusStore = (*storeAPIAdapter)(nil) var _ api.CLIStore = (*storeAPIAdapter)(nil) var _ api.ContextCLIStore = (*storeAPIAdapter)(nil) @@ -883,6 +897,16 @@ func (a *storeAPIAdapter) GetStats() (*api.StoreStats, error) { return a.store.GetStats() } +func (a *storeAPIAdapter) ImportMeeting( + ctx context.Context, + req meetingimport.Request, +) (meetingimport.Result, error) { + if a == nil || a.meetingImporter == nil { + return meetingimport.Result{}, meetingimport.ErrUnavailable + } + return a.meetingImporter.Import(ctx, req) +} + func (a *storeAPIAdapter) GetStatsContext(ctx context.Context) (*api.StoreStats, error) { return a.store.GetStatsContext(ctx) } diff --git a/cmd/msgvault/cmd/store_adapter_test.go b/cmd/msgvault/cmd/store_adapter_test.go index 6b38d6f8b..f846e7dc2 100644 --- a/cmd/msgvault/cmd/store_adapter_test.go +++ b/cmd/msgvault/cmd/store_adapter_test.go @@ -14,6 +14,7 @@ import ( "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/apiprotocol" "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/meetingimport" "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" @@ -30,6 +31,7 @@ func TestStoreAPIAdapterImplementsCtxMessageStore(t *testing.T) { } var _ api.CtxMessageStore = (*storeAPIAdapter)(nil) +var _ api.MeetingImporter = (*storeAPIAdapter)(nil) type scopedStatsProductionAdapter struct { *storeAPIAdapter @@ -382,6 +384,33 @@ func TestStoreAPIAdapterContextReadsHonorCancellation(t *testing.T) { require.ErrorIs(err, context.Canceled, "GetMessagesSummariesByIDsContext must honor a cancelled context") } +func TestStoreAPIAdapterMeetingImport(t *testing.T) { + require := require.New(t) + + st := testutil.NewTestStore(t) + req, err := meetingimport.DecodeRequest(strings.NewReader(`{ + "source": { + "identifier": "local-meetings", + "account_email": "user@example.com" + }, + "meeting": { + "external_id": "42", + "started_at": "2026-07-23T18:00:00Z", + "transcript": "Speaker 1: synthetic transcript" + } + }`), meetingimport.MaxRequestBytes) + require.NoError(err) + adapter := &storeAPIAdapter{ + store: st, + meetingImporter: meetingimport.NewImporter(st, meetingimport.Hooks{}), + } + + result, err := adapter.ImportMeeting(context.Background(), req) + require.NoError(err) + require.Equal(meetingimport.StatusCreated, result.Status) + require.NotZero(result.MessageID) +} + var _ api.ConversationWindowStore = (*storeAPIAdapter)(nil) // TestStoreAPIAdapterImplementsConversationWindowStore is a compile-time diff --git a/docs/api-server.md b/docs/api-server.md index cc8dba9e5..cbe3251a0 100644 --- a/docs/api-server.md +++ b/docs/api-server.md @@ -579,6 +579,53 @@ with a run-level error. --- +### Import a meeting {#post-apiv1importmeeting} + +**Endpoint:** `POST /api/v1/import/meeting` + +Import one provider-neutral meeting without configuring a provider account. +The authenticated, on-demand endpoint accepts at most 16 MiB of JSON and is +idempotent on `source.identifier` plus `meeting.external_id`: the first import +returns `201` / `created`, and retries return `200` / `updated`. + +```bash +curl http://localhost:8080/api/v1/import/meeting \ + -H "Authorization: Bearer your-secret-key" \ + -H "Content-Type: application/json" \ + --data '{ + "source": { + "identifier": "local-meetings", + "display_name": "Local Meetings", + "account_email": "you@example.com" + }, + "meeting": { + "external_id": "planning-2026-07-29", + "title": "Planning", + "started_at": "2026-07-29T09:00:00-04:00", + "summary_text": "Reviewed the launch plan.", + "organizer": {"name": "You", "email": "you@example.com"}, + "attendees": [{"name": "Teammate", "email": "teammate@example.com"}] + } + }' +``` + +```json +{ + "status": "created", + "source_id": 12, + "message_id": 901, + "source_message_id": "meeting:planning-2026-07-29" +} +``` + +Timestamps must be RFC 3339 values with explicit offsets. A meeting must +contain at least one non-empty `summary_markdown`, `summary_text`, `transcript`, +or `transcript_segments` value; plain and segmented transcripts are mutually +exclusive. Segment offsets must be finite, non-negative, and non-decreasing. +Unknown fields are rejected except within `meeting.metadata`. + +--- + ### OAuth token exchange {#post-apiv1authtokenemail} **Endpoint:** `POST /api/v1/auth/token/{email}` @@ -749,7 +796,7 @@ explore contract is in the generated OpenAPI document (`/openapi.json`). { "action": "open_in_source", "reason": "trusted_source_link_unavailable" } ], "action_targets": [], - "operation_token": "3q2fF0kaVYlIuXQ8yYb-KzGH5mo2vNc1", + "operation_token": "", "expires_at": "2026-07-06T15:35:00Z" } ``` diff --git a/docs/usage/meetings.md b/docs/usage/meetings.md index d62e4e722..c55be3fb4 100644 --- a/docs/usage/meetings.md +++ b/docs/usage/meetings.md @@ -37,14 +37,52 @@ source. Add other confirmed aliases with the identity command: msgvault identity add work you+meetings@example.com ``` -When the primary email or aliases change, run a full provider sync to repair -`is_from_me` on meetings already in the archive: +Adding a new confirmed identity immediately repairs `is_from_me` on matching +messages already stored for that source. No provider resync is required. + +## Import from any meeting source + +The provider-neutral import API archives one meeting at a time and requires no +`[[granola]]`, `[[circleback]]`, or other provider configuration. Configure an +API key, start `msgvault serve`, then send authenticated JSON to +`POST /api/v1/import/meeting`: ```bash -msgvault sync-granola work --full -msgvault sync-circleback work --full +curl http://localhost:8080/api/v1/import/meeting \ + -H "Authorization: Bearer your-secret-key" \ + -H "Content-Type: application/json" \ + --data '{ + "source": { + "identifier": "local-meetings", + "display_name": "Local Meetings", + "account_email": "you@example.com" + }, + "meeting": { + "external_id": "weekly-planning-42", + "title": "Weekly planning", + "started_at": "2026-07-29T09:00:00-04:00", + "summary_markdown": "## Decisions\n\nShip the new importer.", + "transcript_segments": [ + {"speaker": "Alex", "text": "Let's ship it.", "offset_seconds": 4} + ] + } + }' ``` +Choose a stable `source.identifier` for the upstream dataset and preserve the +upstream meeting ID as `meeting.external_id`. That pair is the idempotency key: +the first import returns `201` with status `created`; unchanged retries and +replacements return `200` with status `updated` without creating duplicates. +`source.account_email` identifies you for sender attribution and becomes a +confirmed identity for the whole source. + +Each meeting needs at least one summary, a plain transcript, or segmented +transcript. Plain and segmented transcripts are mutually exclusive. Timestamps +use RFC 3339 with an explicit offset, request bodies are limited to 16 MiB, and +provider-specific fields belong under `meeting.metadata`. These sources are +on-demand: import through the API again to add or update meetings rather than +using **Sync now** or a scheduler. + ## Granola ### Prerequisites diff --git a/internal/api/explore.go b/internal/api/explore.go index 19a4010c5..a300106a1 100644 --- a/internal/api/explore.go +++ b/internal/api/explore.go @@ -237,7 +237,7 @@ func registerExploreRoute[Req any, Resp any](api huma.API, operationID, path, su op.Responses[httpStatusKey(http.StatusServiceUnavailable)] = &huma.Response{ Description: http.StatusText(http.StatusServiceUnavailable), Content: map[string]*huma.MediaType{ - "application/json": {Schema: &huma.Schema{AnyOf: []*huma.Schema{ + applicationJSONMediaType: {Schema: &huma.Schema{AnyOf: []*huma.Schema{ schemaFor[ExploreCacheUnavailableResponse](api), schemaFor[ErrorResponse](api), }}}, diff --git a/internal/api/handlers.go b/internal/api/handlers.go index fc1f9f89a..716a126ec 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -366,7 +366,7 @@ type scoreBreakdown struct { // writeJSON writes a JSON response. func writeJSON(w http.ResponseWriter, status int, data any) { - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", applicationJSONMediaType) w.WriteHeader(status) // Headers already sent; if Encode fails mid-stream (broken pipe, // non-serializable value) there's no meaningful recovery beyond @@ -1364,17 +1364,13 @@ func (s *Server) sourceStatus(statusStore SourceStatusStore, source *store.Sourc if err := s.hydrateSyncRunStatus(statusStore, status.ActiveSync); err != nil { return SourceStatus{}, err } + scheduling := classifySourceScheduling(source.SourceType, source.Identifier) schedulerRunning := false if s.scheduler != nil { - // Dispatch by source type first: generic-job source types - // (synctech-sms, gcal, granola, circleback, beeper) are never - // governed by the account scheduler, even if their identifier - // happens to collide with a scheduled account email. Only fall - // back to the account-scheduler path for source types that don't - // map to a generic job, so gmail/imap behavior stays byte-identical. - if _, ok := SchedulerJobNameForSource(source.SourceType, source.Identifier); ok { - schedulerRunning = s.applyGenericJobStatus(&status, source) - } else { + switch scheduling.kind { + case sourceScheduleGeneric: + schedulerRunning = s.applyGenericJobStatus(&status, scheduling.jobName) + case sourceScheduleAccount: status.Scheduled = s.scheduler.IsScheduled(source.Identifier) for _, scheduled := range s.scheduler.Status() { if scheduled.Email != source.Identifier { @@ -1388,9 +1384,12 @@ func (s *Server) sourceStatus(statusStore SourceStatusStore, source *store.Sourc } break } + case sourceScheduleNonSchedulable: } } switch { + case scheduling.kind == sourceScheduleNonSchedulable: + status.SyncUnavailableReason = "source_not_schedulable" case status.ActiveSync != nil || schedulerRunning: status.SyncUnavailableReason = "sync_already_running" case s.scheduler == nil: @@ -1427,11 +1426,7 @@ func (s *Server) sourceStatus(statusStore SourceStatusStore, source *store.Sourc // SchedulerJobNameForSource) when source's type is driven by one of those // jobs rather than the account scheduler. It reports whether that job is // currently running (false if no matching job exists). -func (s *Server) applyGenericJobStatus(status *SourceStatus, source *store.Source) bool { - jobName, ok := SchedulerJobNameForSource(source.SourceType, source.Identifier) - if !ok { - return false - } +func (s *Server) applyGenericJobStatus(status *SourceStatus, jobName string) bool { for _, job := range s.scheduler.JobStatus() { if job.Name != jobName { continue @@ -1527,45 +1522,44 @@ func (s *Server) handleTriggerSync(w http.ResponseWriter, r *http.Request) { } sourceType := r.URL.Query().Get("source_type") - // Generic (non-account) sources — synctech-sms, gcal, granola, circleback, - // beeper — are driven by named generic scheduler jobs. The caller passes - // the source type explicitly so this dispatches authoritatively, without - // scanning the store for a matching identifier (which could otherwise - // collide with a scheduled account email and trigger the wrong sync). - if jobName, ok := SchedulerJobNameForSource(sourceType, account); ok { - if !s.scheduler.IsJobScheduled(jobName) { + scheduling := classifySourceScheduling(sourceType, account) + switch scheduling.kind { + case sourceScheduleNonSchedulable: + writeError(w, http.StatusBadRequest, "source_not_schedulable", "Source type cannot be scheduled: "+sourceType) + return + case sourceScheduleGeneric: + if !s.scheduler.IsJobScheduled(scheduling.jobName) { writeError(w, http.StatusNotFound, "not_found", "Account is not scheduled: "+account) return } - if err := s.scheduler.StartJob(jobName); err != nil { + if err := s.scheduler.StartJob(scheduling.jobName); err != nil { s.logger.Error("failed to trigger generic sync job", - "job", jobName, "identifier", account, "error", err) + "job", scheduling.jobName, "identifier", account, "error", err) writeError(w, http.StatusConflict, "sync_error", err.Error()) return } - s.logger.Info("generic sync triggered via API", "job", jobName, "identifier", account) + s.logger.Info("generic sync triggered via API", "job", scheduling.jobName, "identifier", account) writeJSON(w, http.StatusAccepted, StatusMessageResponse{ Status: "accepted", Message: "Sync started for " + account, }) return + case sourceScheduleAccount: + if !s.scheduler.IsScheduled(account) { + writeError(w, http.StatusNotFound, "not_found", "Account is not scheduled: "+account) + return + } + if err := s.scheduler.TriggerSync(account); err != nil { + s.logger.Error("failed to trigger sync", "account", account, "error", err) + writeError(w, http.StatusConflict, "sync_error", err.Error()) + return + } + s.logger.Info("sync triggered via API", "account", account) + writeJSON(w, http.StatusAccepted, StatusMessageResponse{ + Status: "accepted", + Message: "Sync started for " + account, + }) } - - // Account-scheduler sources (gmail, imap, ...) are keyed by email. - if !s.scheduler.IsScheduled(account) { - writeError(w, http.StatusNotFound, "not_found", "Account is not scheduled: "+account) - return - } - if err := s.scheduler.TriggerSync(account); err != nil { - s.logger.Error("failed to trigger sync", "account", account, "error", err) - writeError(w, http.StatusConflict, "sync_error", err.Error()) - return - } - s.logger.Info("sync triggered via API", "account", account) - writeJSON(w, http.StatusAccepted, StatusMessageResponse{ - Status: "accepted", - Message: "Sync started for " + account, - }) } // handleSchedulerStatus returns the scheduler status. diff --git a/internal/api/handlers_test.go b/internal/api/handlers_test.go index fdcd67690..b3a3ed904 100644 --- a/internal/api/handlers_test.go +++ b/internal/api/handlers_test.go @@ -38,6 +38,7 @@ import ( "go.kenn.io/msgvault/internal/deletion" "go.kenn.io/msgvault/internal/gcal" "go.kenn.io/msgvault/internal/granola" + "go.kenn.io/msgvault/internal/meetingimport" "go.kenn.io/msgvault/internal/opserr" "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/query/querytest" @@ -3614,6 +3615,119 @@ func TestHandleSourceStatusGenericIdentifierCollidesWithScheduledAccount(t *test assert.True(got.CanSync, "generic job is scheduled and not running") } +func TestMeetingImportIdentifierCollisionDoesNotEnableAccountSync(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + const collidingIdentifier = "shared@example.com" + + st := testutil.NewTestStore(t) + sched := newMockScheduler() + sched.scheduled[collidingIdentifier] = true + sched.statuses = []AccountStatus{{ + Email: collidingIdentifier, + Schedule: "0 3 * * *", + }} + _, err := st.GetOrCreateSource(meetingimport.SourceType, collidingIdentifier) + require.NoError(err, "GetOrCreateSource meeting import") + srv := NewServer( + &config.Config{Server: config.ServerConfig{APIPort: 8080}}, + st, + sched, + testLogger(), + ) + + statusReq := httptest.NewRequest( + http.MethodGet, + "/api/v1/sources/status?source_type="+meetingimport.SourceType, + nil, + ) + statusResp := httptest.NewRecorder() + srv.Router().ServeHTTP(statusResp, statusReq) + + require.Equal(http.StatusOK, statusResp.Code, statusResp.Body.String()) + var status SourceStatusResponse + require.NoError(json.NewDecoder(statusResp.Body).Decode(&status), "decode response") + require.Len(status.Sources, 1, "sources") + assert.False(status.Sources[0].Scheduled, "meeting imports are not scheduler jobs") + assert.False(status.Sources[0].CanSync, "meeting imports cannot be synced") + assert.Equal("source_not_schedulable", status.Sources[0].SyncUnavailableReason) + + triggerResp := servePOSTTestRequest( + srv, + "/api/v1/sync/"+collidingIdentifier+"?source_type="+meetingimport.SourceType, + ) + + assert.Equal(http.StatusBadRequest, triggerResp.Code, triggerResp.Body.String()) + assert.Empty(sched.startedJobs, "must not start a generic job") + assert.Empty(sched.triggeredJobs, "must not trigger the colliding account") +} + +func TestAccountScheduledSourceTypesSupportStatusAndTrigger(t *testing.T) { + cases := []struct { + name string + sourceType string + identifier string + }{ + {"teams", "teams", "alice@example.com"}, + {"discord", "discord", "113456789012345678"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + sched := newMockScheduler() + sched.scheduled[tc.identifier] = true + sched.statuses = []AccountStatus{{ + Email: tc.identifier, + Schedule: "0 */6 * * *", + }} + var triggeredIdentifier string + sched.triggerFn = func(identifier string) error { + triggeredIdentifier = identifier + return nil + } + + source, err := st.GetOrCreateSource(tc.sourceType, tc.identifier) + require.NoError(err, "GetOrCreateSource") + srv := NewServer( + &config.Config{Server: config.ServerConfig{APIPort: 8080}}, + st, + sched, + testLogger(), + ) + + statusReq := httptest.NewRequest( + http.MethodGet, + "/api/v1/sources/status?source_type="+tc.sourceType, + nil, + ) + statusResp := httptest.NewRecorder() + srv.Router().ServeHTTP(statusResp, statusReq) + + require.Equal(http.StatusOK, statusResp.Code, statusResp.Body.String()) + var status SourceStatusResponse + require.NoError(json.NewDecoder(statusResp.Body).Decode(&status), "decode response") + require.Len(status.Sources, 1, "sources") + assert.Equal(source.ID, status.Sources[0].ID, "source ID") + assert.True(status.Sources[0].Scheduled, "Scheduled") + assert.Equal("0 */6 * * *", status.Sources[0].Schedule, "Schedule") + assert.True(status.Sources[0].CanSync, "CanSync") + assert.Empty(status.Sources[0].SyncUnavailableReason, "SyncUnavailableReason") + + triggerResp := servePOSTTestRequest( + srv, + "/api/v1/sync/"+tc.identifier+"?source_type="+tc.sourceType, + ) + + require.Equal(http.StatusAccepted, triggerResp.Code, triggerResp.Body.String()) + assert.Equal(tc.identifier, triggeredIdentifier, "triggered account identifier") + assert.Empty(sched.startedJobs, "must not start a generic scheduler job") + }) + } +} + // TestSchedulerJobNameForSource covers every generic-job source type plus an // account-scheduler type (gmail), which must report ok=false since it's // governed by the account scheduler, not a generic job. diff --git a/internal/api/json_body.go b/internal/api/json_body.go index 60028d046..cebaa5942 100644 --- a/internal/api/json_body.go +++ b/internal/api/json_body.go @@ -10,6 +10,8 @@ import ( "github.com/danielgtaylor/huma/v2" ) +const applicationJSONMediaType = "application/json" + // operationDeclaresJSONRequestBody reports whether a raw-route operation // documents an application/json request body, which is how every JSON // mutation route in this package registers (jsonRequestBodyFor). It keys the @@ -19,7 +21,7 @@ func operationDeclaresJSONRequestBody(op *huma.Operation) bool { if op.RequestBody == nil { return false } - _, ok := op.RequestBody.Content["application/json"] + _, ok := op.RequestBody.Content[applicationJSONMediaType] return ok } @@ -43,7 +45,7 @@ func enforceJSONRequestMediaType(next http.HandlerFunc) http.HandlerFunc { func hasJSONContentType(r *http.Request) bool { mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) - return err == nil && mediaType == "application/json" + return err == nil && mediaType == applicationJSONMediaType } // requireSingleJSONValue verifies no second JSON value follows the one dec diff --git a/internal/api/meeting_import.go b/internal/api/meeting_import.go new file mode 100644 index 000000000..5fd0ea85a --- /dev/null +++ b/internal/api/meeting_import.go @@ -0,0 +1,187 @@ +package api + +import ( + "context" + "errors" + "mime" + "net/http" + + "github.com/danielgtaylor/huma/v2" + "go.kenn.io/msgvault/internal/meetingimport" +) + +const meetingImportEndpointPath = "/api/v1/import/meeting" + +type MeetingImporter interface { + ImportMeeting(ctx context.Context, req meetingimport.Request) (meetingimport.Result, error) +} + +type MeetingImportResponse struct { + Status meetingimport.Status `json:"status" enum:"created,updated"` + SourceID int64 `json:"source_id"` + MessageID int64 `json:"message_id"` + SourceMessageID string `json:"source_message_id"` +} + +func (s *Server) registerMeetingImportRoute(api huma.API) { + op := rawAPIV1Operation( + "importMeeting", + http.MethodPost, + "/import/meeting", + "Import one meeting", + ) + op.RequestBody = jsonRequestBodyFor[meetingimport.Request](api) + hardenMeetingImportSchemas(api.OpenAPI()) + op.Responses = jsonResponsesFor[MeetingImportResponse]( + api, + http.StatusOK, + http.StatusCreated, + ) + op.Errors = []int{ + http.StatusBadRequest, + http.StatusUnauthorized, + http.StatusRequestEntityTooLarge, + http.StatusUnsupportedMediaType, + http.StatusUnprocessableEntity, + http.StatusInternalServerError, + http.StatusServiceUnavailable, + } + registerRawHumaRoute(api, op, s.handleMeetingImport) +} + +func hardenMeetingImportSchemas(doc *huma.OpenAPI) { + if doc == nil || doc.Components == nil || doc.Components.Schemas == nil { + return + } + meeting := doc.Components.Schemas.Map()["Meeting"] + if meeting == nil { + return + } + one := 1 + contentRequired := []*huma.Schema{ + { + Type: huma.TypeObject, + Required: []string{"summary_markdown"}, + Properties: map[string]*huma.Schema{ + "summary_markdown": {Type: huma.TypeString, MinLength: &one}, + }, + }, + { + Type: huma.TypeObject, + Required: []string{"summary_text"}, + Properties: map[string]*huma.Schema{ + "summary_text": {Type: huma.TypeString, MinLength: &one}, + }, + }, + { + Type: huma.TypeObject, + Required: []string{"transcript"}, + Properties: map[string]*huma.Schema{ + "transcript": {Type: huma.TypeString, MinLength: &one}, + }, + }, + { + Type: huma.TypeObject, + Required: []string{"transcript_segments"}, + Properties: map[string]*huma.Schema{ + "transcript_segments": {Type: huma.TypeArray, MinItems: &one}, + }, + }, + } + transcriptsExclusive := &huma.Schema{ + Type: huma.TypeObject, + Required: []string{"transcript", "transcript_segments"}, + Properties: map[string]*huma.Schema{ + "transcript": {Type: huma.TypeString, MinLength: &one}, + "transcript_segments": {Type: huma.TypeArray, MinItems: &one}, + }, + } + meeting.AllOf = []*huma.Schema{ + {AnyOf: contentRequired}, + {Not: transcriptsExclusive}, + } +} + +func (s *Server) handleMeetingImport(w http.ResponseWriter, r *http.Request) { + mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != applicationJSONMediaType { + writeError(w, http.StatusUnsupportedMediaType, "unsupported_media_type", + "Content-Type must be application/json") + return + } + + req, err := meetingimport.DecodeRequest(r.Body, meetingimport.MaxRequestBytes) + switch { + case errors.Is(err, meetingimport.ErrRequestTooLarge): + writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", + "Meeting import request exceeds 16 MiB") + return + case errors.Is(err, meetingimport.ErrMalformedRequest): + writeError(w, http.StatusBadRequest, "bad_request", "Invalid meeting import JSON") + return + case err != nil: + writeError(w, http.StatusBadRequest, "bad_request", "Invalid meeting import request") + return + } + if _, err := req.Normalize(); err != nil { + writeError(w, http.StatusUnprocessableEntity, "validation_failed", + "Meeting import request failed validation") + return + } + + importer, ok := s.store.(MeetingImporter) + if !ok || importer == nil { + writeError(w, http.StatusServiceUnavailable, "service_unavailable", + "Meeting import is unavailable") + return + } + + gateCtx, cancel := context.WithTimeout(r.Context(), operationGateWaitLimit) + defer cancel() + done, ok := s.beginLabeledOperationGateWork( + gateCtx, + operationGateLabelFromPath(meetingImportEndpointPath), + ) + if !ok { + writeOperationGateBusy(w, s.operationGate) + return + } + defer done() + + result, err := importer.ImportMeeting(r.Context(), req) + if err != nil { + if s.writeIfContextError(w, err) { + return + } + if errors.Is(err, meetingimport.ErrUnavailable) { + writeError(w, http.StatusServiceUnavailable, "service_unavailable", + "Meeting import is unavailable") + return + } + if errors.Is(err, meetingimport.ErrValidation) { + writeError(w, http.StatusUnprocessableEntity, "validation_failed", + "Meeting import request failed validation") + return + } + if s.logger != nil { + s.logger.Error("meeting import failed", + "source", req.Source.Identifier, + "external_id", req.Meeting.ExternalID, + "error_class", "internal") + } + writeError(w, http.StatusInternalServerError, "internal_error", + "Meeting import failed") + return + } + + status := http.StatusOK + if result.Status == meetingimport.StatusCreated { + status = http.StatusCreated + } + writeJSON(w, status, MeetingImportResponse{ + Status: result.Status, + SourceID: result.SourceID, + MessageID: result.MessageID, + SourceMessageID: result.SourceMessageID, + }) +} diff --git a/internal/api/meeting_import_test.go b/internal/api/meeting_import_test.go new file mode 100644 index 000000000..eab6f91b9 --- /dev/null +++ b/internal/api/meeting_import_test.go @@ -0,0 +1,489 @@ +package api + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/meetingimport" +) + +const ( + meetingImportTestAPIKey = "synthetic-api-key-for-tests" + validMeetingImportBody = `{ + "source": { + "identifier": "local-meetings", + "display_name": "Local Meetings", + "account_email": "user@example.com" + }, + "meeting": { + "external_id": "42", + "title": "Weekly planning", + "started_at": "2026-07-23T18:00:00Z", + "ended_at": "2026-07-23T18:30:00Z", + "summary_markdown": "## Summary\n\nReviewed the launch plan.", + "transcript_segments": [ + {"speaker": "Test Speaker", "text": "Review the launch plan.", "offset_seconds": 4} + ], + "organizer": {"name": "Test Organizer", "email": "organizer@example.com"}, + "attendees": [{"name": "Test Attendee", "email": "attendee@example.com"}], + "metadata": {"provider_key": "synthetic-value"} + } + }` +) + +type fakeMeetingImportStore struct { + *mockStore + + result meetingimport.Result + err error + calls int + req meetingimport.Request +} + +func (s *fakeMeetingImportStore) ImportMeeting( + _ context.Context, + req meetingimport.Request, +) (meetingimport.Result, error) { + s.calls++ + s.req = req + return s.result, s.err +} + +func newMeetingImportTestServer(t *testing.T, importer MeetingImporter) *Server { + t.Helper() + base := &mockStore{stats: &StoreStats{}} + var store MessageStore = base + if importer != nil { + typed, ok := importer.(MessageStore) + require.True(t, ok, "test importer must implement MessageStore") + store = typed + } + return NewServer( + &config.Config{Server: config.ServerConfig{APIKey: meetingImportTestAPIKey}}, + store, + nil, + testLogger(), + ) +} + +func meetingImportRequest(body string) *http.Request { + req := httptest.NewRequest(http.MethodPost, "/api/v1/import/meeting", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", meetingImportTestAPIKey) + return req +} + +func TestMeetingImportRequiresAuthentication(t *testing.T) { + store := &fakeMeetingImportStore{ + mockStore: &mockStore{stats: &StoreStats{}}, + result: meetingimport.Result{Status: meetingimport.StatusCreated}, + } + srv := newMeetingImportTestServer(t, store) + + for _, key := range []string{"", "wrong-key"} { + req := meetingImportRequest(validMeetingImportBody) + if key == "" { + req.Header.Del("X-Api-Key") + } else { + req.Header.Set("X-Api-Key", key) + } + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, req) + + assert.Equal(t, http.StatusUnauthorized, resp.Code, "key=%q body=%s", key, resp.Body.String()) + } + assert.Equal(t, 0, store.calls) +} + +func TestMeetingImportReturnsCreatedAndUpdated(t *testing.T) { + tests := []struct { + name string + status meetingimport.Status + wantStatus int + }{ + {name: "created", status: meetingimport.StatusCreated, wantStatus: http.StatusCreated}, + {name: "updated", status: meetingimport.StatusUpdated, wantStatus: http.StatusOK}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + store := &fakeMeetingImportStore{ + mockStore: &mockStore{stats: &StoreStats{}}, + result: meetingimport.Result{ + Status: tt.status, + SourceID: 3, + MessageID: 901, + SourceMessageID: "meeting:42", + }, + } + srv := newMeetingImportTestServer(t, store) + req := meetingImportRequest(validMeetingImportBody) + req.Header.Set("Content-Type", "application/json; charset=utf-8") + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, req) + + require.Equal(tt.wantStatus, resp.Code, "body: %s", resp.Body.String()) + assert.Equal(1, store.calls) + assert.Equal("42", store.req.Meeting.ExternalID) + var body MeetingImportResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal(tt.status, body.Status) + assert.Equal(int64(3), body.SourceID) + assert.Equal(int64(901), body.MessageID) + assert.Equal("meeting:42", body.SourceMessageID) + }) + } +} + +func TestMeetingImportRejectsInvalidRequests(t *testing.T) { + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := newMeetingImportTestServer(t, store) + + unknown := strings.Replace(validMeetingImportBody, `"external_id":`, `"unknown": true, "external_id":`, 1) + noContent := strings.Replace( + validMeetingImportBody, + `"summary_markdown": "## Summary\n\nReviewed the launch plan.",`, + `"summary_markdown": "",`, + 1, + ) + noContent = strings.Replace(noContent, + `"transcript_segments": [ + {"speaker": "Test Speaker", "text": "Review the launch plan.", "offset_seconds": 4} + ],`, + `"transcript_segments": [],`, + 1, + ) + + tests := []struct { + name string + body string + mediaType string + wantCode int + wantError string + }{ + {name: "malformed", body: `{"source":`, mediaType: "application/json", wantCode: http.StatusBadRequest, wantError: "bad_request"}, + {name: "trailing", body: validMeetingImportBody + `{}`, mediaType: "application/json", wantCode: http.StatusBadRequest, wantError: "bad_request"}, + {name: "unknown field", body: unknown, mediaType: "application/json", wantCode: http.StatusBadRequest, wantError: "bad_request"}, + {name: "semantic validation", body: noContent, mediaType: "application/json", wantCode: http.StatusUnprocessableEntity, wantError: "validation_failed"}, + {name: "wrong media type", body: validMeetingImportBody, mediaType: "text/plain", wantCode: http.StatusUnsupportedMediaType, wantError: "unsupported_media_type"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := meetingImportRequest(tt.body) + req.Header.Set("Content-Type", tt.mediaType) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, req) + + require.Equal(tt.wantCode, resp.Code, "body: %s", resp.Body.String()) + var body ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal(tt.wantError, body.Error) + assert.NotContains(body.Message, "Review the launch plan") + }) + } + assert.Equal(t, 0, store.calls) +} + +func TestMeetingImportRejectsOversizedBody(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := newMeetingImportTestServer(t, store) + req := meetingImportRequest(strings.Repeat("x", int(meetingimport.MaxRequestBytes)+1)) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, req) + + require.Equal(http.StatusRequestEntityTooLarge, resp.Code, "body: %s", resp.Body.String()) + var body ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal("request_too_large", body.Error) + assert.Equal(0, store.calls) +} + +func TestMeetingImportReturnsUnavailableWithoutCapability(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + srv := newMeetingImportTestServer(t, nil) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, meetingImportRequest(validMeetingImportBody)) + + require.Equal(http.StatusServiceUnavailable, resp.Code, "body: %s", resp.Body.String()) + var body ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal("service_unavailable", body.Error) +} + +func TestMeetingImportSanitizesInternalErrors(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + store := &fakeMeetingImportStore{ + mockStore: &mockStore{stats: &StoreStats{}}, + err: errors.New("database failed near secret transcript content"), + } + srv := newMeetingImportTestServer(t, store) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, meetingImportRequest(validMeetingImportBody)) + + require.Equal(http.StatusInternalServerError, resp.Code, "body: %s", resp.Body.String()) + var body ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal("internal_error", body.Error) + assert.NotContains(body.Message, "secret transcript content") +} + +func TestMeetingImportOpenAPIDocument(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := newMeetingImportTestServer(t, store) + req := httptest.NewRequest(http.MethodGet, "/openapi.json", nil) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, req) + + require.Equal(http.StatusOK, resp.Code) + var doc map[string]any + require.NoError(json.NewDecoder(resp.Body).Decode(&doc)) + paths, ok := doc["paths"].(map[string]any) + require.True(ok, "paths object") + path, ok := paths["/api/v1/import/meeting"].(map[string]any) + require.True(ok, "meeting import path object") + post, ok := path["post"].(map[string]any) + require.True(ok, "meeting import operation object") + assert.Equal("importMeeting", post["operationId"]) + assert.NotEmpty(post["security"]) + responses, ok := post["responses"].(map[string]any) + require.True(ok, "responses object") + assert.Contains(responses, "200") + assert.Contains(responses, "201") + + components, ok := doc["components"].(map[string]any) + require.True(ok, "components object") + schemas, ok := components["schemas"].(map[string]any) + require.True(ok, "schemas object") + meeting, ok := schemas["Meeting"].(map[string]any) + require.True(ok, "meeting component") + allOf, ok := meeting["allOf"].([]any) + require.True(ok, "meeting allOf constraints") + assert.Len(allOf, 2) + response, ok := schemas["MeetingImportResponse"].(map[string]any) + require.True(ok, "meeting response component") + properties, ok := response["properties"].(map[string]any) + require.True(ok, "meeting response properties") + status, ok := properties["status"].(map[string]any) + require.True(ok, "meeting response status") + assert.Equal([]any{"created", "updated"}, status["enum"]) +} + +func TestMeetingImportBodyLimitDoesNotReadPastBoundary(t *testing.T) { + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := newMeetingImportTestServer(t, store) + body := bytes.NewReader(bytes.Repeat([]byte("x"), int(meetingimport.MaxRequestBytes)+1)) + req := httptest.NewRequest(http.MethodPost, "/api/v1/import/meeting", body) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", meetingImportTestAPIKey) + resp := httptest.NewRecorder() + + srv.Router().ServeHTTP(resp, req) + + assert.Equal(t, http.StatusRequestEntityTooLarge, resp.Code) +} + +func TestMeetingImportReadsBodyBeforeWaitingOnOperationGate(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + const ordinaryReadTimeout = 100 * time.Millisecond + + gate := NewSerialOperationGate() + releaseGate, ok := gate.BeginWork() + require.True(ok, "hold operation gate") + releaseGateOnce := sync.OnceFunc(releaseGate) + defer releaseGateOnce() + + store := &fakeMeetingImportStore{ + mockStore: &mockStore{stats: &StoreStats{}}, + result: meetingimport.Result{ + Status: meetingimport.StatusCreated, + SourceID: 3, + MessageID: 901, + SourceMessageID: "meeting:42", + }, + } + srv := NewServerWithOptions(ServerOptions{ + Config: &config.Config{Server: config.ServerConfig{ + APIKey: meetingImportTestAPIKey, + }}, + Store: store, + Logger: testLogger(), + OperationGate: gate, + }) + srv.readTimeout = ordinaryReadTimeout + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(err, "listen") + serveErr := make(chan error, 1) + go func() { + serveErr <- srv.StartOnListener(listener) + }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(srv.Shutdown(ctx), "shutdown") + require.ErrorIs(<-serveErr, http.ErrServerClosed, "serve result") + }) + + conn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(err, "dial server") + defer func() { _ = conn.Close() }() + require.NoError(conn.SetDeadline(time.Now().Add(5*time.Second)), "bound test connection") + + body := []byte(validMeetingImportBody) + _, err = fmt.Fprintf(conn, + "POST /api/v1/import/meeting HTTP/1.1\r\n"+ + "Host: %s\r\n"+ + "Content-Type: application/json\r\n"+ + "X-Api-Key: %s\r\n"+ + "Content-Length: %d\r\n\r\n"+ + "%s", + listener.Addr().String(), + meetingImportTestAPIKey, + len(body), + body[:len(body)-1], + ) + require.NoError(err, "write headers and partial body") + + time.Sleep(2 * ordinaryReadTimeout) + assert.False(gate.HasRequestWaiters(), "partial upload must not hold or queue on the mutation gate") + _, err = conn.Write(body[len(body)-1:]) + require.NoError(err, "finish body after ordinary server read timeout") + require.Eventually(gate.HasRequestWaiters, time.Second, time.Millisecond, + "validated meeting import waits on operation gate") + releaseGateOnce() + + resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: http.MethodPost}) + require.NoError(err, "read meeting import response") + defer func() { _ = resp.Body.Close() }() + assert.Equal(http.StatusCreated, resp.StatusCode) + assert.Equal(1, store.calls) +} + +func TestMeetingImportValidatesBodyBeforeWaitingOnOperationGate(t *testing.T) { + require := require.New(t) + + oldLimit := operationGateWaitLimit + operationGateWaitLimit = 20 * time.Millisecond + t.Cleanup(func() { operationGateWaitLimit = oldLimit }) + + gate := NewSerialOperationGate() + releaseGate, ok := gate.BeginWork() + require.True(ok, "hold operation gate") + defer releaseGate() + + srv := NewServerWithOptions(ServerOptions{ + Config: &config.Config{Server: config.ServerConfig{ + APIKey: meetingImportTestAPIKey, + }}, + Store: &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}}, + Logger: testLogger(), + OperationGate: gate, + }) + + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, meetingImportRequest(`{}`)) + + require.Equal(http.StatusUnprocessableEntity, resp.Code, "body: %s", resp.Body.String()) + require.False(gate.HasRequestWaiters(), "invalid body must not queue on the mutation gate") +} + +func TestMeetingImportReturnsBusyWithinBoundAfterValidation(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + oldLimit := operationGateWaitLimit + operationGateWaitLimit = 20 * time.Millisecond + t.Cleanup(func() { operationGateWaitLimit = oldLimit }) + + gate := NewSerialOperationGate() + releaseGate, ok := gate.BeginLabeledWorkContext(context.Background(), "scheduled source sync") + require.True(ok, "hold operation gate") + defer releaseGate() + + store := &fakeMeetingImportStore{mockStore: &mockStore{stats: &StoreStats{}}} + srv := NewServerWithOptions(ServerOptions{ + Config: &config.Config{Server: config.ServerConfig{ + APIKey: meetingImportTestAPIKey, + }}, + Store: store, + Logger: testLogger(), + OperationGate: gate, + }) + + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, meetingImportRequest(validMeetingImportBody)) + + require.Equal(http.StatusServiceUnavailable, resp.Code, "body: %s", resp.Body.String()) + var body ErrorResponse + require.NoError(json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal("operation_in_progress", body.Error) + assert.Contains(body.Message, "scheduled source sync") + assert.Equal(0, store.calls) +} + +func TestMeetingImportUnavailableDoesNotWaitOnOperationGate(t *testing.T) { + require := require.New(t) + + oldLimit := operationGateWaitLimit + operationGateWaitLimit = 20 * time.Millisecond + t.Cleanup(func() { operationGateWaitLimit = oldLimit }) + + gate := NewSerialOperationGate() + releaseGate, ok := gate.BeginWork() + require.True(ok, "hold operation gate") + defer releaseGate() + + srv := NewServerWithOptions(ServerOptions{ + Config: &config.Config{Server: config.ServerConfig{ + APIKey: meetingImportTestAPIKey, + }}, + Store: &mockStore{stats: &StoreStats{}}, + Logger: testLogger(), + OperationGate: gate, + }) + + resp := httptest.NewRecorder() + srv.Router().ServeHTTP(resp, meetingImportRequest(validMeetingImportBody)) + + require.Equal(http.StatusServiceUnavailable, resp.Code, "body: %s", resp.Body.String()) + require.Contains(resp.Body.String(), `"error":"service_unavailable"`) + require.False(gate.HasRequestWaiters(), "unavailable importer must not queue on the mutation gate") +} diff --git a/internal/api/middleware.go b/internal/api/middleware.go index d1579f960..8eba3d6e7 100644 --- a/internal/api/middleware.go +++ b/internal/api/middleware.go @@ -295,7 +295,7 @@ func RateLimitMiddleware( ip := clientIP(r) if !limiter.Allow(ip) { - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", applicationJSONMediaType) w.Header().Set("Retry-After", "1") w.WriteHeader(http.StatusTooManyRequests) _, _ = w.Write([]byte(`{"error":"rate_limit_exceeded","message":"Too many requests. Please slow down."}`)) diff --git a/internal/api/openapi.go b/internal/api/openapi.go index e5e831d39..dd3c9df14 100644 --- a/internal/api/openapi.go +++ b/internal/api/openapi.go @@ -152,7 +152,10 @@ import ( // profiles, update the display-name override and delete a profile with // revision-tag optimistic concurrency, and surface the covering profile on // the /people/{id} analytical detail. -const APISchemaVersion = "1.32.0" +// 1.33.0 adds provider-neutral single-meeting ingestion with strict request +// schemas and idempotent create/update responses. +// Additive (minor bump): the major-version compatibility gate stays at 1. +const APISchemaVersion = "1.33.0" // OpenAPIDocument builds the API schema from the same Huma route registration // used by the daemon. It binds no socket and needs no database. @@ -434,6 +437,36 @@ func applyClientCodegenExtensions(doc *huma.OpenAPI) { return } schemas := doc.Components.Schemas.Map() + const emailProperty = "email" + if meeting := schemas["Meeting"]; meeting != nil { + // The Go client generator treats composed object schemas as union + // wrappers. Runtime validation and the public schema retain these + // cross-field rules; the generated request keeps its useful struct shape. + meeting.AllOf = nil + for _, property := range []string{"started_at", "ended_at"} { + if timestamp := meeting.Properties[property]; timestamp != nil { + setCodegenGoType(timestamp, "string") + } + } + } + for schemaName, property := range map[string]string{ + "MeetingPerson": emailProperty, + "Source": "account_email", + } { + if schema := schemas[schemaName]; schema != nil { + if email := schema.Properties[property]; email != nil { + setCodegenGoType(email, "string") + } + } + } + queryResult := schemas["QueryResult"] + if queryResult != nil && queryResult.Properties != nil { + rows := queryResult.Properties["rows"] + if rows != nil && rows.Items != nil && rows.Items.Items != nil { + setCodegenGoType(rows.Items.Items, "any") + } + } + for _, schemaName := range []string{"FileSearchRow", "FileMetadataResponse"} { if schema := schemas[schemaName]; schema != nil { for _, property := range []string{"filename", "mime_type"} { @@ -478,6 +511,12 @@ func applyClientCodegenExtensions(doc *huma.OpenAPI) { "ExploreGroupDimensionSource", "ExploreGroupDimensionParticipant", "ExploreGroupDimensionDomain", "ExploreGroupDimensionMessageType", "ExploreGroupDimensionKind", "ExploreGroupDimensionYear", "ExploreGroupDimensionMonth", }) + if response := schemas["MeetingImportResponse"]; response != nil { + setEnumNames(response.Properties["status"], []any{ + "MeetingImportResponseStatusCreated", + "MeetingImportResponseStatusUpdated", + }) + } for schemaName, properties := range map[string]map[string][]any{ "ExploreCacheUnavailableResponse": { "readiness": {"ExploreCacheUnavailableResponseReadinessAbsent", "ExploreCacheUnavailableResponseReadinessInterrupted", "ExploreCacheUnavailableResponseReadinessStaleSchema", "ExploreCacheUnavailableResponseReadinessDrifted"}, @@ -517,19 +556,26 @@ func applyClientCodegenExtensions(doc *huma.OpenAPI) { setEnumNames(schema.Properties[propertyName], enumNames) } } - queryResult := schemas["QueryResult"] - if queryResult == nil || queryResult.Properties == nil { + meeting := schemas["Meeting"] + if meeting == nil || meeting.Properties == nil { return } - rows := queryResult.Properties["rows"] - if rows == nil || rows.Items == nil || rows.Items.Items == nil { + metadata := meeting.Properties["metadata"] + if metadata == nil { return } - cell := rows.Items.Items - if cell.Extensions == nil { - cell.Extensions = map[string]any{} + values, ok := metadata.AdditionalProperties.(*huma.Schema) + if !ok { + return + } + setCodegenGoType(values, "any") +} + +func setCodegenGoType(schema *huma.Schema, goType string) { + if schema.Extensions == nil { + schema.Extensions = map[string]any{} } - cell.Extensions["x-go-type"] = "any" + schema.Extensions["x-go-type"] = goType } func replaceStrictResponseAdditionalProperties(doc *huma.OpenAPI, replacement any) { diff --git a/internal/api/openapi_test.go b/internal/api/openapi_test.go index c5421af8d..6060ce64b 100644 --- a/internal/api/openapi_test.go +++ b/internal/api/openapi_test.go @@ -181,6 +181,109 @@ func TestOpenAPIFastSearchDocumentsSourceIDs(t *testing.T) { assert.Fail("source_ids query parameter is not documented for fastSearch") } +func TestOpenAPIMeetingImportContract(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + assert.Equal("1.33.0", APISchemaVersion, "meeting import is an additive schema release") + + doc := OpenAPIDocument() + path := doc.Paths["/api/v1/import/meeting"] + require.NotNil(path, "meeting import path") + op := path.Post + require.NotNil(op, "meeting import operation") + assert.Equal("importMeeting", op.OperationID) + require.Len(op.Security, 1, "API-key security requirement") + _, secured := op.Security[0]["apiKey"] + assert.True(secured, "apiKey security requirement") + + require.NotNil(op.RequestBody, "request body") + assert.True(op.RequestBody.Required, "request body is required") + requestMedia := op.RequestBody.Content["application/json"] + require.NotNil(requestMedia, "JSON request media type") + require.NotNil(requestMedia.Schema, "request schema") + assert.Equal("#/components/schemas/MeetingImportRequest", requestMedia.Schema.Ref) + + schemas := doc.Components.Schemas.Map() + requestSchema := schemas["MeetingImportRequest"] + require.NotNil(requestSchema, "request component") + requestAdditionalProperties, ok := requestSchema.AdditionalProperties.(bool) + require.True(ok, "request additionalProperties is boolean") + assert.False(requestAdditionalProperties, "request rejects unknown fields") + assert.ElementsMatch([]string{"source", "meeting"}, requestSchema.Required) + + for _, name := range []string{"Source", "Meeting", "MeetingPerson", "TranscriptSegment"} { + schema := schemas[name] + require.NotNil(schema, "%s component", name) + additionalProperties, ok := schema.AdditionalProperties.(bool) + require.True(ok, "%s additionalProperties is boolean", name) + assert.False(additionalProperties, "%s rejects unknown fields", name) + } + assert.ElementsMatch( + []string{"external_id", "started_at"}, + schemas["Meeting"].Required, + ) + source := schemas["Source"] + require.NotNil(source.Properties["identifier"].MaxLength) + assert.Equal(128, *source.Properties["identifier"].MaxLength) + require.NotNil(source.Properties["display_name"].MaxLength) + assert.Equal(256, *source.Properties["display_name"].MaxLength) + assert.Equal("email", source.Properties["account_email"].Format) + + meeting := schemas["Meeting"] + require.NotNil(meeting.Properties["external_id"].MaxLength) + assert.Equal(256, *meeting.Properties["external_id"].MaxLength) + require.NotNil(meeting.Properties["title"].MaxLength) + assert.Equal(4096, *meeting.Properties["title"].MaxLength) + assert.Equal("date-time", meeting.Properties["started_at"].Format) + assert.Equal("date-time", meeting.Properties["ended_at"].Format) + require.Len(meeting.AllOf, 2, "meeting cross-field constraints") + require.Len(meeting.AllOf[0].AnyOf, 4, "meeting requires a non-empty summary or transcript") + assert.ElementsMatch( + []string{"summary_markdown", "summary_text", "transcript", "transcript_segments"}, + []string{ + meeting.AllOf[0].AnyOf[0].Required[0], + meeting.AllOf[0].AnyOf[1].Required[0], + meeting.AllOf[0].AnyOf[2].Required[0], + meeting.AllOf[0].AnyOf[3].Required[0], + }, + ) + for idx, property := range []string{"summary_markdown", "summary_text", "transcript"} { + require.NotNil(meeting.AllOf[0].AnyOf[idx].Properties[property].MinLength) + assert.Equal(1, *meeting.AllOf[0].AnyOf[idx].Properties[property].MinLength) + } + require.NotNil(meeting.AllOf[0].AnyOf[3].Properties["transcript_segments"].MinItems) + assert.Equal(1, *meeting.AllOf[0].AnyOf[3].Properties["transcript_segments"].MinItems) + require.NotNil(meeting.AllOf[1].Not, "plain and segmented transcripts are mutually exclusive") + assert.ElementsMatch( + []string{"transcript", "transcript_segments"}, + meeting.AllOf[1].Not.Required, + ) + + assert.Equal("email", schemas["MeetingPerson"].Properties["email"].Format) + offset := schemas["TranscriptSegment"].Properties["offset_seconds"] + require.NotNil(offset.Minimum) + assert.Zero(*offset.Minimum) + + metadata := schemas["Meeting"].Properties["metadata"] + require.NotNil(metadata, "metadata schema") + _, extensible := metadata.AdditionalProperties.(*huma.Schema) + assert.True(extensible, "metadata accepts provider-specific values") + + responseSchema := schemas["MeetingImportResponse"] + require.NotNil(responseSchema, "meeting import response component") + assert.Equal([]any{"created", "updated"}, responseSchema.Properties["status"].Enum) + + for _, status := range []string{"200", "201"} { + response := op.Responses[status] + require.NotNil(response, "response %s", status) + media := response.Content["application/json"] + require.NotNil(media, "response %s JSON media type", status) + require.NotNil(media.Schema, "response %s schema", status) + assert.Equal("#/components/schemas/MeetingImportResponse", media.Schema.Ref) + } +} + func TestOpenAPIBinaryRoutesDocumentJSONErrors(t *testing.T) { doc := OpenAPIDocument() routes := map[string]struct { @@ -459,6 +562,31 @@ func TestOpenAPIClientArtifactUpToDate(t *testing.T) { } } +func TestOpenAPIGeneratedMeetingImportClient(t *testing.T) { + assertGeneratedFileContains(t, "client.go", + "ImportMeeting(ctx context.Context, options *ImportMeetingRequestOptions") + assertGeneratedFileContains(t, "client_options.go", + "type ImportMeetingRequestOptions struct") + assertGeneratedFileContains(t, "payloads.go", + "type ImportMeetingBody = MeetingImportRequest") + assertGeneratedFileContains(t, "responses.go", + "type ImportMeetingResp struct") + assertGeneratedFileContains(t, "types.go", + "type MeetingImportResponse struct") + assertGeneratedFileContains(t, "types.go", + "Metadata map[string]any") + assertGeneratedFileContains(t, "enums.go", + `MeetingImportResponseStatusUpdated MeetingImportResponseStatus = "updated"`) +} + +func assertGeneratedFileContains(t *testing.T, name, expected string) { + t.Helper() + content, err := os.ReadFile(filepath.Join(openAPIClientGeneratedDir, name)) + require.NoError(t, err, "read generated client file %s", name) + assert.Contains(t, string(content), expected, + "%s is missing the meeting import contract; run `make api-generate`", name) +} + func normalizeGeneratedArtifact(src []byte) string { return strings.ReplaceAll(string(src), "\r\n", "\n") } diff --git a/internal/api/operation_gate.go b/internal/api/operation_gate.go index fd11deebf..4a0e0bfa7 100644 --- a/internal/api/operation_gate.go +++ b/internal/api/operation_gate.go @@ -293,20 +293,21 @@ func writeOperationGateBusy(w http.ResponseWriter, gate OperationGate) { writeError(w, http.StatusServiceUnavailable, "operation_in_progress", message) } -// operationGateExemptPaths are non-GET endpoints that do not mutate the -// archive: they must not queue behind long archive operations. Most only read; +// operationGateExemptPaths bypass the generic mutation gate. Most only read; // the session endpoints mutate process-local authentication state. Verify is // NOT exempt: its subprocess opens the store read-write and runs schema // init/migrations. // -// The backup freeze endpoints are exempt for a different reason: begin -// acquires the operation gate itself (see beginLabeledOperationGateWork in -// handleBackupFreezeBegin), so routing them through the generic middleware -// gate as well would deadlock begin against its own acquisition. +// Backup freeze begin and meeting import coordinate the gate in their handlers. +// Meeting import first reads and validates its bounded request body so a slow +// authenticated upload cannot hold the gate. Backup freeze end bypasses the +// gate so it can release the freeze held by begin. Routing these through the +// generic middleware would deadlock their coordination. var operationGateExemptPaths = map[string]bool{ queryEndpointPath: true, sessionPath: true, sessionLoginPath: true, + meetingImportEndpointPath: true, "/api/v1/cli/add-calendar/plan": true, "/api/v1/cli/delete-staged/plan": true, "/api/v1/cli/embeddings/plan": true, diff --git a/internal/api/routes.go b/internal/api/routes.go index 82657b7a0..bc97b5dd9 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -162,7 +162,7 @@ func (s *Server) humaAuthMiddleware(ctx huma.Context, next func(huma.Context)) { } func writeHumaError(ctx huma.Context, status int, code string, message string) { - ctx.SetHeader("Content-Type", "application/json") + ctx.SetHeader("Content-Type", applicationJSONMediaType) ctx.SetStatus(status) _ = json.NewEncoder(ctx.BodyWriter()).Encode(ErrorResponse{ //nolint:errchkjson // best-effort error response write Error: code, @@ -273,6 +273,7 @@ func (s *Server) registerHumaRoutes(api huma.API, apiV1 huma.API) { registerAPIV1RawHumaJSONRoute[MessageListResponse](apiV1, "listMessages", http.MethodGet, "/messages", "List messages", s.handleListMessages) registerAPIV1RawHumaJSONRoute[MessageDetail](apiV1, "getMessage", http.MethodGet, "/messages/{id}", "Get one message", s.handleGetMessage) + s.registerMeetingImportRoute(apiV1) registerAPIV1RawHumaJSONRoute[ConversationResponse](apiV1, "getConversation", http.MethodGet, "/conversations/{id}", "Get a bounded containing conversation", s.handleGetConversation) registerAPIV1RawHumaJSONRoute[AttachmentInfo](apiV1, "getAttachment", http.MethodGet, "/attachments/{id}", "Get attachment metadata", s.handleGetAttachment) registerAPIV1RawHumaBinaryRoute( @@ -838,7 +839,7 @@ func jsonRequestBodyFor[T any](api huma.API) *huma.RequestBody { return &huma.RequestBody{ Required: true, Content: map[string]*huma.MediaType{ - "application/json": {Schema: schemaFor[T](api)}, + applicationJSONMediaType: {Schema: schemaFor[T](api)}, }, } } @@ -852,7 +853,7 @@ func jsonResponsesFor[T any](api huma.API, successStatuses ...int) map[string]*h responses[httpStatusKey(status)] = &huma.Response{ Description: http.StatusText(status), Content: map[string]*huma.MediaType{ - "application/json": {Schema: schemaFor[T](api)}, + applicationJSONMediaType: {Schema: schemaFor[T](api)}, }, } } @@ -869,7 +870,7 @@ func oneOfJSONResponses(api huma.API, responseTypes ...reflect.Type) map[string] httpStatusKey(http.StatusOK): { Description: http.StatusText(http.StatusOK), Content: map[string]*huma.MediaType{ - "application/json": {Schema: &huma.Schema{OneOf: oneOf}}, + applicationJSONMediaType: {Schema: &huma.Schema{OneOf: oneOf}}, }, }, "default": errorResponseFor(api), @@ -908,7 +909,7 @@ func errorResponseFor(api huma.API) *huma.Response { return &huma.Response{ Description: "Error", Content: map[string]*huma.MediaType{ - "application/json": {Schema: schemaFor[ErrorResponse](api)}, + applicationJSONMediaType: {Schema: schemaFor[ErrorResponse](api)}, }, } } diff --git a/internal/api/scheduler_jobs.go b/internal/api/scheduler_jobs.go index fa809fe5f..0e2e64961 100644 --- a/internal/api/scheduler_jobs.go +++ b/internal/api/scheduler_jobs.go @@ -6,9 +6,23 @@ import ( "go.kenn.io/msgvault/internal/circleback" "go.kenn.io/msgvault/internal/gcal" "go.kenn.io/msgvault/internal/granola" + "go.kenn.io/msgvault/internal/meetingimport" "go.kenn.io/msgvault/internal/synctechsms" ) +type sourceScheduleKind uint8 + +const ( + sourceScheduleNonSchedulable sourceScheduleKind = iota + sourceScheduleAccount + sourceScheduleGeneric +) + +type sourceScheduleClassification struct { + kind sourceScheduleKind + jobName string +} + // sourceTypeBeeper mirrors the unexported sourceTypeBeeper constant in // internal/beeper (and cmd/msgvault/cmd/constants.go); it can't be imported // because it isn't exported, so the literal is duplicated here. @@ -26,6 +40,27 @@ const BeeperJobName = sourceTypeBeeper // Slack workspace source. const SlackJobName = sourceTypeSlack +// classifySourceScheduling determines which scheduler, if any, may operate a +// store source. Account scheduling is opt-in so imported or unknown source +// types cannot borrow a scheduled account merely by sharing its identifier. +func classifySourceScheduling(sourceType, identifier string) sourceScheduleClassification { + switch sourceType { + case "", "gmail", "imap", "teams", "discord": + return sourceScheduleClassification{kind: sourceScheduleAccount} + case meetingimport.SourceType: + return sourceScheduleClassification{kind: sourceScheduleNonSchedulable} + default: + jobName, ok := SchedulerJobNameForSource(sourceType, identifier) + if !ok { + return sourceScheduleClassification{kind: sourceScheduleNonSchedulable} + } + return sourceScheduleClassification{ + kind: sourceScheduleGeneric, + jobName: jobName, + } + } +} + // SchedulerJobNameForSource returns the scheduler generic-job name that // drives syncing for a store source of the given type and identifier, and // whether such a job governs this source type at all. It is the single diff --git a/internal/api/server.go b/internal/api/server.go index b21639b8a..f7aedb0f5 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -620,6 +620,17 @@ func serveWithoutWriteDeadline( next.ServeHTTP(w, r) } +func serveMeetingImportWithReadDeadline( + w http.ResponseWriter, + r *http.Request, + next http.Handler, +) { + controller := http.NewResponseController(w) + _ = controller.SetReadDeadline(time.Now().Add(DaemonLongRequestTimeout)) + _ = controller.SetWriteDeadline(time.Time{}) + next.ServeHTTP(w, r) +} + func serveWithProtectiveRequestDeadline( w http.ResponseWriter, r *http.Request, @@ -637,6 +648,12 @@ func serveWithProtectiveRequestDeadline( func (s *Server) timeoutMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && + r.URL.Path == meetingImportEndpointPath && + s.apiRequestAuthorized(r) { + serveMeetingImportWithReadDeadline(w, r, next) + return + } if s.requestUsesCLITimeoutPolicy(r) { if cliRequestNeedsProtectiveCeiling(r) { serveWithProtectiveRequestDeadline(w, r, next) @@ -684,9 +701,9 @@ func cliRequestNeedsProtectiveCeiling(r *http.Request) bool { // requestTimeoutForPath returns the context deadline to impose on a request // and whether one applies at all. POST /api/v1/query gets its own generous -// ceiling; the streaming/long-running CLI operations stay unbounded (they -// report progress and are gated by the operation gate); everything else gets -// the standard per-request timeout. +// ceiling; the streaming/long-running CLI operations and meeting imports stay +// unbounded (they report progress and are gated by the operation gate); +// everything else gets the standard per-request timeout. func (s *Server) requestTimeoutForPath(path string) (time.Duration, bool) { if path == queryEndpointPath { return s.queryTimeout, true @@ -701,6 +718,7 @@ func isLongDaemonRequest(path string) bool { switch path { case "/api/v1/cli/build-cache", "/api/v1/cli/deduplicate/plan", + meetingImportEndpointPath, "/api/v1/cli/rebuild-fts", "/api/v1/cli/repair-encoding", "/api/v1/cli/run", @@ -913,7 +931,7 @@ func (s *Server) handleDaemonShutdown(w http.ResponseWriter, r *http.Request) { return } - w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Type", applicationJSONMediaType) w.WriteHeader(http.StatusAccepted) _, _ = w.Write([]byte(`{"status":"shutting_down"}`)) go s.shutdownFunc() diff --git a/internal/api/server_test.go b/internal/api/server_test.go index 1dd0b56b2..6b13f4f99 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -1027,8 +1027,12 @@ func (w *deadlineClearingRecorder) SetWriteDeadline(deadline time.Time) error { } func TestTimeoutMiddlewareDeadlinePolicy(t *testing.T) { + const apiKey = "deadline-policy-test-key" srv := NewServerWithOptions(ServerOptions{ - Config: &config.Config{Server: config.ServerConfig{APIPort: 8080}}, + Config: &config.Config{Server: config.ServerConfig{ + APIPort: 8080, + APIKey: apiKey, + }}, Logger: testLogger(), }) @@ -1040,6 +1044,7 @@ func TestTimeoutMiddlewareDeadlinePolicy(t *testing.T) { marked := httptest.NewRequest(http.MethodGet, "/api/v1/cli/stats", nil) marked.RemoteAddr = "127.0.0.1:4242" marked.Header.Set(apiprotocol.ClientClassHeader, apiprotocol.ClientClassCLI) + marked.Header.Set("X-Api-Key", apiKey) markedDeleteDeduped := httptest.NewRequest( http.MethodPost, "/api/v1/cli/delete-deduped", @@ -1047,13 +1052,19 @@ func TestTimeoutMiddlewareDeadlinePolicy(t *testing.T) { ) markedDeleteDeduped.RemoteAddr = "127.0.0.1:4242" markedDeleteDeduped.Header.Set(apiprotocol.ClientClassHeader, apiprotocol.ClientClassCLI) + markedDeleteDeduped.Header.Set("X-Api-Key", apiKey) + meetingImport := httptest.NewRequest(http.MethodPost, "/api/v1/import/meeting", nil) + meetingImport.Header.Set("X-Api-Key", apiKey) + unauthorizedMeetingImport := httptest.NewRequest(http.MethodPost, "/api/v1/import/meeting", nil) + unauthorizedMeetingImport.Header.Set("X-Api-Key", "invalid") bounded := httptest.NewRequest(http.MethodGet, "/api/v1/cli/stats", nil) tests := []struct { - name string - request *http.Request - wantReadClear bool - wantWriteClear bool + name string + request *http.Request + wantReadClear bool + wantReadDeadline bool + wantWriteClear bool }{ {name: "unmarked long path", request: longPath, wantWriteClear: true}, {name: "marked request", request: marked, wantReadClear: true, wantWriteClear: true}, @@ -1063,6 +1074,17 @@ func TestTimeoutMiddlewareDeadlinePolicy(t *testing.T) { wantReadClear: true, wantWriteClear: true, }, + { + name: "meeting import", + request: meetingImport, + wantReadDeadline: true, + wantWriteClear: true, + }, + { + name: "unauthorized meeting import", + request: unauthorizedMeetingImport, + wantWriteClear: true, + }, {name: "bounded request", request: bounded}, } for _, tt := range tests { @@ -1074,6 +1096,11 @@ func TestTimeoutMiddlewareDeadlinePolicy(t *testing.T) { if tt.wantReadClear { require.Len(recorder.readDeadlines, 1, "read deadline changes") assert.True(recorder.readDeadlines[0].IsZero(), "read deadline cleared, not extended") + } else if tt.wantReadDeadline { + require.Len(recorder.readDeadlines, 1, "read deadline changes") + remaining := time.Until(recorder.readDeadlines[0]) + assert.Greater(remaining, DaemonLongRequestTimeout-time.Second) + assert.LessOrEqual(remaining, DaemonLongRequestTimeout) } else { assert.Empty(recorder.readDeadlines, "request keeps the server read deadline") } diff --git a/internal/api/settings_test.go b/internal/api/settings_test.go index a218dbe21..fb6667f0b 100644 --- a/internal/api/settings_test.go +++ b/internal/api/settings_test.go @@ -463,7 +463,7 @@ func TestSettingsOpenAPIContract(t *testing.T) { for _, status := range []string{"400", "409", "412", "422", "428"} { assert.Contains(patch.Responses, status) } - assert.Equal("1.32.0", doc.Info.Version) + assert.Equal("1.33.0", doc.Info.Version) settingValue := doc.Components.Schemas.Map()["SettingValue"] require.NotNil(settingValue) diff --git a/internal/calsync/calsync.go b/internal/calsync/calsync.go index bb9832928..38dab3441 100644 --- a/internal/calsync/calsync.go +++ b/internal/calsync/calsync.go @@ -159,7 +159,7 @@ func (s *Syncer) RegisterCalendars(ctx context.Context) ([]gcal.Calendar, error) if !s.includeCalendar(cal) { continue } - src, err := s.getOrCreateCalendarSource(cal) + src, err := s.getOrCreateCalendarSource(ctx, cal) if err != nil { return nil, fmt.Errorf("get/create source for %s: %w", cal.ID, err) } @@ -199,7 +199,7 @@ func (s *Syncer) listCalendars(ctx context.Context) ([]gcal.Calendar, error) { // replaying that token under a later unbounded request can skip or corrupt the // traversal. func (s *Syncer) syncCalendarFull(ctx context.Context, cal gcal.Calendar, result *Result) error { - src, err := s.getOrCreateCalendarSource(cal) + src, err := s.getOrCreateCalendarSource(ctx, cal) if err != nil { return fmt.Errorf("get/create source: %w", err) } @@ -475,7 +475,10 @@ func (s *Syncer) sourceIdentifier(cal gcal.Calendar) string { return s.opts.AccountEmail + "/" + cal.ID } -func (s *Syncer) getOrCreateCalendarSource(cal gcal.Calendar) (*store.Source, error) { +func (s *Syncer) getOrCreateCalendarSource( + ctx context.Context, + cal gcal.Calendar, +) (*store.Source, error) { identifier := s.sourceIdentifier(cal) sources, err := s.store.GetSourcesByTypeAndAccount(gcal.SourceType, s.opts.AccountEmail) @@ -483,6 +486,7 @@ func (s *Syncer) getOrCreateCalendarSource(cal gcal.Calendar) (*store.Source, er return nil, fmt.Errorf("find existing calendar sources: %w", err) } + var source *store.Source var migrate *store.Source for _, src := range sources { cfg := parseSourceConfig(src.SyncConfig) @@ -490,21 +494,42 @@ func (s *Syncer) getOrCreateCalendarSource(cal gcal.Calendar) (*store.Source, er continue } if src.Identifier == identifier { - return src, nil + source = src + break } if migrate == nil { migrate = src } } - if migrate != nil { + if source == nil && migrate != nil { if err := s.store.UpdateSourceIdentifier(migrate.ID, identifier); err != nil { return nil, fmt.Errorf("migrate calendar source identifier: %w", err) } migrate.Identifier = identifier - return migrate, nil + source = migrate + } + if source == nil { + source, err = s.store.GetOrCreateSource(gcal.SourceType, identifier) + if err != nil { + return nil, err + } } + if err := s.confirmCalendarSourceIdentity(ctx, source); err != nil { + return nil, err + } + return source, nil +} - return s.store.GetOrCreateSource(gcal.SourceType, identifier) +func (s *Syncer) confirmCalendarSourceIdentity(ctx context.Context, source *store.Source) error { + if err := s.store.AddAccountIdentityContext( + ctx, + source.ID, + s.opts.AccountEmail, + "account-email", + ); err != nil { + return fmt.Errorf("confirm calendar account identity: %w", err) + } + return nil } func (s *Syncer) updateCalendarSourceOAuthApp(sourceID int64, calendarID string) error { diff --git a/internal/calsync/calsync_test.go b/internal/calsync/calsync_test.go index 68809900f..d0afc070e 100644 --- a/internal/calsync/calsync_test.go +++ b/internal/calsync/calsync_test.go @@ -261,6 +261,48 @@ func TestFull_PersistsEventsAsMessages(t *testing.T) { } } +func TestFull_ConfiguredAccountAttributionSurvivesIdentityMutation(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + m := gcal.NewMockAPI() + m.Calendars = []gcal.Calendar{{ + ID: "primary", + AccessRole: "owner", + }} + ev := timedEvent("configured-organizer", "Configured organizer") + ev.Organizer.Self = false + m.FullEvents["primary"] = [][]gcal.Event{{ev}} + m.FullSyncToken["primary"] = "TOKEN1" + + s, st := newSyncer(t, m, Options{}) + _, err := s.Full(context.Background()) + require.NoError(err) + + src := primarySource(t, st) + row, ok := getMsg(t, st, src.ID, ev.ID) + require.True(ok) + assert.True(row.isFromMe, "configured account organizer should be attributed to the account") + + identities, err := st.ListAccountIdentities(src.ID) + require.NoError(err) + require.Len(identities, 1) + assert.Equal(testAccount, identities[0].Address) + assert.Contains(identities[0].SourceSignal, "account-email") + + require.NoError(st.AddAccountIdentity(src.ID, "other@example.com", "manual")) + row, ok = getMsg(t, st, src.ID, ev.ID) + require.True(ok) + assert.True(row.isFromMe, "adding another identity must preserve configured-account attribution") + + removed, err := st.RemoveAccountIdentity(src.ID, "other@example.com") + require.NoError(err) + assert.Equal(int64(1), removed) + row, ok = getMsg(t, st, src.ID, ev.ID) + require.True(ok) + assert.True(row.isFromMe, "removing another identity must preserve configured-account attribution") +} + func TestFull_ClearsBodyWhenEventBodyBecomesEmpty(t *testing.T) { assert := assert.New(t) require := require.New(t) diff --git a/internal/calsync/incremental.go b/internal/calsync/incremental.go index f35373368..7dce0833e 100644 --- a/internal/calsync/incremental.go +++ b/internal/calsync/incremental.go @@ -46,6 +46,10 @@ func (s *Syncer) Incremental(ctx context.Context) (Result, error) { if err := ctx.Err(); err != nil { return result, err } + if err := s.confirmCalendarSourceIdentity(ctx, src); err != nil { + recordErr(fmt.Errorf("confirm account identity for source %d: %w", src.ID, err)) + continue + } cfg := parseSourceConfig(src.SyncConfig) if cfg.CalendarID == "" { continue diff --git a/internal/calsync/persist.go b/internal/calsync/persist.go index d9a83366c..3e2433f2b 100644 --- a/internal/calsync/persist.go +++ b/internal/calsync/persist.go @@ -108,19 +108,23 @@ func (s *Syncer) ingestEvent(sourceID int64, cal gcal.Calendar, ev gcal.Event) ( body := serializeBody(ev) subject := ev.Summary - fromMe := ev.Organizer.Self || (ev.Organizer.Email != "" && strings.EqualFold(ev.Organizer.Email, s.opts.AccountEmail)) + identityFromMe := !ev.Organizer.Self && + ev.Organizer.Email != "" && + strings.EqualFold(ev.Organizer.Email, s.opts.AccountEmail) + fromMe := ev.Organizer.Self || identityFromMe msgID, err := s.store.UpsertMessage(&store.Message{ - ConversationID: convID, - SourceID: sourceID, - SourceMessageID: smid, - MessageType: gcal.MessageTypeCalendarEvent, - SentAt: eventSentAt(ev), - SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, - IsFromMe: fromMe, - Subject: sql.NullString{String: subject, Valid: subject != ""}, - Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, - SizeEstimate: int64(len(body)), + ConversationID: convID, + SourceID: sourceID, + SourceMessageID: smid, + MessageType: gcal.MessageTypeCalendarEvent, + SentAt: eventSentAt(ev), + SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, + IsFromMe: fromMe, + IdentityDerivedIsFromMe: identityFromMe, + Subject: sql.NullString{String: subject, Valid: subject != ""}, + Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, + SizeEstimate: int64(len(body)), }) if err != nil { return 0, fmt.Errorf("upsert message: %w", err) diff --git a/internal/circleback/importer.go b/internal/circleback/importer.go index ade0d6f5f..ecead01ca 100644 --- a/internal/circleback/importer.go +++ b/internal/circleback/importer.go @@ -977,15 +977,16 @@ func (imp *Importer) ingestMeeting( sentAt := m.StartedAt().UTC() message := &store.Message{ - SourceID: sourceID, - SourceMessageID: smid, - MessageType: MessageType, - SentAt: sql.NullTime{Time: sentAt, Valid: !sentAt.IsZero()}, - SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, - IsFromMe: fromMe, - Subject: sql.NullString{String: title, Valid: title != ""}, - Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, - SizeEstimate: int64(len(body)), + SourceID: sourceID, + SourceMessageID: smid, + MessageType: MessageType, + SentAt: sql.NullTime{Time: sentAt, Valid: !sentAt.IsZero()}, + SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, + IsFromMe: fromMe, + IdentityDerivedIsFromMe: fromMe, + Subject: sql.NullString{String: title, Valid: title != ""}, + Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, + SizeEstimate: int64(len(body)), } metaJSON, err := json.Marshal(imp.buildMetadata( diff --git a/internal/granola/importer.go b/internal/granola/importer.go index 2e877ddba..1c8b296e1 100644 --- a/internal/granola/importer.go +++ b/internal/granola/importer.go @@ -328,15 +328,16 @@ func (imp *Importer) ingestNote(sourceID int64, identifier string, accountIdenti sentAt := noteStartTime(n).UTC() message := &store.Message{ - SourceID: sourceID, - SourceMessageID: n.ID, - MessageType: MessageType, - SentAt: sql.NullTime{Time: sentAt, Valid: !sentAt.IsZero()}, - SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, - IsFromMe: fromMe, - Subject: sql.NullString{String: title, Valid: title != ""}, - Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, - SizeEstimate: int64(len(body)), + SourceID: sourceID, + SourceMessageID: n.ID, + MessageType: MessageType, + SentAt: sql.NullTime{Time: sentAt, Valid: !sentAt.IsZero()}, + SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, + IsFromMe: fromMe, + IdentityDerivedIsFromMe: fromMe, + Subject: sql.NullString{String: title, Valid: title != ""}, + Snippet: sql.NullString{String: snippet(body), Valid: body != ""}, + SizeEstimate: int64(len(body)), } metaJSON, err := json.Marshal(buildMetadata(n, identifier, organizerEmail)) diff --git a/internal/meetingimport/decode.go b/internal/meetingimport/decode.go new file mode 100644 index 000000000..1a6db3c41 --- /dev/null +++ b/internal/meetingimport/decode.go @@ -0,0 +1,51 @@ +package meetingimport + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "unicode/utf8" +) + +func DecodeRequest(r io.Reader, maxBytes int64) (Request, error) { + if r == nil { + return Request{}, fmt.Errorf("%w: empty body", ErrMalformedRequest) + } + if maxBytes <= 0 { + return Request{}, ErrRequestTooLarge + } + + body, err := io.ReadAll(io.LimitReader(r, maxBytes+1)) + if err != nil { + return Request{}, fmt.Errorf("%w: read body: %w", ErrMalformedRequest, err) + } + if int64(len(body)) > maxBytes { + return Request{}, ErrRequestTooLarge + } + if len(bytes.TrimSpace(body)) == 0 { + return Request{}, fmt.Errorf("%w: empty body", ErrMalformedRequest) + } + if !utf8.Valid(body) { + return Request{}, fmt.Errorf("%w: request must be valid UTF-8", ErrMalformedRequest) + } + + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + decoder.UseNumber() + var req Request + if err := decoder.Decode(&req); err != nil { + return Request{}, fmt.Errorf("%w: %w", ErrMalformedRequest, err) + } + + var trailing any + err = decoder.Decode(&trailing) + if err == nil { + return Request{}, fmt.Errorf("%w: trailing JSON value", ErrMalformedRequest) + } + if !errors.Is(err, io.EOF) { + return Request{}, fmt.Errorf("%w: trailing data: %w", ErrMalformedRequest, err) + } + return req, nil +} diff --git a/internal/meetingimport/decode_test.go b/internal/meetingimport/decode_test.go new file mode 100644 index 000000000..73c76f370 --- /dev/null +++ b/internal/meetingimport/decode_test.go @@ -0,0 +1,144 @@ +package meetingimport + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const validRequestJSON = `{ + "source": { + "identifier": " local-meetings ", + "display_name": " Local Meetings ", + "account_email": " USER@example.com " + }, + "meeting": { + "external_id": " 42 ", + "title": " Weekly planning ", + "started_at": "2026-07-23T11:00:00-07:00", + "ended_at": "2026-07-23T11:30:00-07:00", + "summary_markdown": "## Summary\n\nReviewed the launch plan.", + "summary_text": "", + "transcript": "", + "transcript_segments": [ + { + "speaker": " Test Speaker ", + "text": " Let's review the launch plan. ", + "offset_seconds": 4 + } + ], + "organizer": { + "name": " Test Organizer ", + "email": " ORGANIZER@example.com " + }, + "attendees": [ + { + "name": " Test Attendee ", + "email": " ATTENDEE@example.com " + } + ], + "metadata": { + "calendar_event_id": "synthetic-event-42", + "nested": {"accepted": true} + } + } +}` + +func TestDecodeRequestAcceptsCompleteStrictRequest(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req, err := DecodeRequest(strings.NewReader(validRequestJSON), MaxRequestBytes) + require.NoError(err) + + assert.Equal(" local-meetings ", req.Source.Identifier) + assert.Equal(" 42 ", req.Meeting.ExternalID) + require.Len(req.Meeting.TranscriptSegments, 1) + require.NotNil(req.Meeting.TranscriptSegments[0].OffsetSeconds) + assert.InDelta(float64(4), *req.Meeting.TranscriptSegments[0].OffsetSeconds, 0) + nested, ok := req.Meeting.Metadata["nested"].(map[string]any) + require.True(ok, "nested metadata object") + assert.Equal(true, nested["accepted"]) +} + +func TestDecodeRequestPreservesLargeNestedMetadataNumbers(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + body := strings.Replace( + validRequestJSON, + `"nested": {"accepted": true}`, + `"nested": {"accepted": true, "large_id": 9007199254740993, "deep": {"another_id": 18446744073709551615}}`, + 1, + ) + + req, err := DecodeRequest(strings.NewReader(body), MaxRequestBytes) + require.NoError(err) + + nested, ok := req.Meeting.Metadata["nested"].(map[string]any) + require.True(ok, "nested metadata object") + largeID, ok := nested["large_id"].(json.Number) + require.True(ok, "large metadata identifier") + assert.Equal("9007199254740993", largeID.String()) + deep, ok := nested["deep"].(map[string]any) + require.True(ok, "deep metadata object") + anotherID, ok := deep["another_id"].(json.Number) + require.True(ok, "deep metadata identifier") + assert.Equal("18446744073709551615", anotherID.String()) +} + +func TestDecodeRequestRejectsMalformedAndTrailingJSON(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "empty", body: ""}, + {name: "malformed", body: `{"source":`}, + {name: "trailing object", body: validRequestJSON + `{}`}, + {name: "trailing scalar", body: validRequestJSON + ` true`}, + {name: "unknown top level", body: strings.Replace(validRequestJSON, `"source":`, `"unknown": true, "source":`, 1)}, + {name: "unknown source", body: strings.Replace(validRequestJSON, `"identifier":`, `"unknown": true, "identifier":`, 1)}, + {name: "unknown meeting", body: strings.Replace(validRequestJSON, `"external_id":`, `"unknown": true, "external_id":`, 1)}, + {name: "unknown segment", body: strings.Replace(validRequestJSON, `"speaker":`, `"unknown": true, "speaker":`, 1)}, + {name: "invalid utf8", body: string([]byte{'{', '"', 0xff, '"', ':', '1', '}'})}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + _, err := DecodeRequest(strings.NewReader(tt.body), MaxRequestBytes) + require.Error(err) + assert.ErrorIs(err, ErrMalformedRequest) + }) + } +} + +func TestDecodeRequestAllowsUnknownProviderMetadata(t *testing.T) { + body := strings.Replace( + validRequestJSON, + `"calendar_event_id": "synthetic-event-42"`, + `"provider_specific_unknown": {"path": ["a", "b"]}`, + 1, + ) + + req, err := DecodeRequest(strings.NewReader(body), MaxRequestBytes) + require.NoError(t, err) + assert.Contains(t, req.Meeting.Metadata, "provider_specific_unknown") +} + +func TestDecodeRequestEnforcesBodyLimit(t *testing.T) { + require := require.New(t) + + _, err := DecodeRequest(strings.NewReader(validRequestJSON), int64(len(validRequestJSON)-1)) + require.Error(err) + require.ErrorIs(err, ErrRequestTooLarge) + + _, err = DecodeRequest(strings.NewReader(`{}`), 0) + require.Error(err) + require.ErrorIs(err, ErrRequestTooLarge) +} diff --git a/internal/meetingimport/format.go b/internal/meetingimport/format.go new file mode 100644 index 000000000..94f22c4c0 --- /dev/null +++ b/internal/meetingimport/format.go @@ -0,0 +1,204 @@ +package meetingimport + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +type Snapshot struct { + SourceIdentifier string + SourceDisplayName string + AccountEmail string + SourceMessageID string + Title string + StartedAt time.Time + Body string + Snippet string + Metadata []byte + Raw []byte + Organizer *MeetingPerson + Attendees []MeetingPerson +} + +type canonicalMeeting struct { + ExternalID string `json:"external_id"` + Title string `json:"title,omitempty"` + StartedAt string `json:"started_at"` + EndedAt string `json:"ended_at,omitempty"` + SummaryMarkdown string `json:"summary_markdown,omitempty"` + SummaryText string `json:"summary_text,omitempty"` + Transcript string `json:"transcript,omitempty"` + TranscriptSegments []TranscriptSegment `json:"transcript_segments,omitempty"` + Organizer *MeetingPerson `json:"organizer,omitempty"` + Attendees []MeetingPerson `json:"attendees,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +type messageMetadata struct { + Platform string `json:"platform"` + ExternalMeetingID string `json:"external_meeting_id"` + SourceIdentifier string `json:"source_identifier"` + StartedAt string `json:"started_at"` + EndedAt string `json:"ended_at,omitempty"` + DurationSeconds int64 `json:"duration_seconds"` + OrganizerEmail string `json:"organizer_email,omitempty"` + AttendeeCount int `json:"attendee_count"` + HasSummary bool `json:"has_summary"` + HasTranscript bool `json:"has_transcript"` + TranscriptSegmentCount int `json:"transcript_segment_count"` + ProviderMetadata map[string]any `json:"provider_metadata,omitempty"` +} + +func BuildSnapshot(req NormalizedRequest) (Snapshot, error) { + meeting := req.Meeting + title := meeting.Title + if title == "" { + title = "Meeting on " + meeting.StartedAt.UTC().Format(time.DateOnly) + } + + body := buildBody(title, meeting) + raw, err := json.Marshal(buildCanonicalMeeting(meeting)) + if err != nil { + return Snapshot{}, fmt.Errorf("marshal canonical meeting: %w", err) + } + metadata, err := json.Marshal(buildMessageMetadata(req)) + if err != nil { + return Snapshot{}, fmt.Errorf("marshal meeting metadata: %w", err) + } + + return Snapshot{ + SourceIdentifier: req.Source.Identifier, + SourceDisplayName: req.Source.DisplayName, + AccountEmail: req.Source.AccountEmail, + SourceMessageID: "meeting:" + meeting.ExternalID, + Title: title, + StartedAt: meeting.StartedAt.UTC(), + Body: body, + Snippet: snippet(body), + Metadata: metadata, + Raw: raw, + Organizer: meeting.Organizer, + Attendees: meeting.Attendees, + }, nil +} + +func buildBody(title string, meeting NormalizedMeeting) string { + var b strings.Builder + writeLine := func(line string) { + if line == "" { + return + } + b.WriteString(line) + b.WriteByte('\n') + } + + writeLine(title) + writeLine(formatWhen(meeting.StartedAt, meeting.EndedAt)) + + names := make([]string, 0, len(meeting.Attendees)) + for _, attendee := range meeting.Attendees { + if attendee.Name != "" { + names = append(names, attendee.Name) + } + } + if len(names) > 0 { + writeLine("Attendees: " + strings.Join(names, ", ")) + } + + summary := meeting.SummaryMarkdown + if summary == "" { + summary = meeting.SummaryText + } + if summary != "" { + b.WriteByte('\n') + writeLine(summary) + } + + if meeting.Transcript != "" { + b.WriteString("\nTranscript:\n") + writeLine(meeting.Transcript) + } else if len(meeting.TranscriptSegments) > 0 { + b.WriteString("\nTranscript:\n") + for _, segment := range meeting.TranscriptSegments { + writeLine(formatSegment(segment)) + } + } + return strings.TrimSpace(b.String()) +} + +func formatWhen(start time.Time, end *time.Time) string { + line := "When: " + start.UTC().Format("2006-01-02 15:04") + if end != nil { + line += " - " + end.UTC().Format("15:04") + } + return line +} + +func formatSegment(segment TranscriptSegment) string { + label := segment.Speaker + ": " + segment.Text + if segment.OffsetSeconds == nil { + return label + } + total := int(*segment.OffsetSeconds) + hours := total / 3600 + minutes := (total % 3600) / 60 + seconds := total % 60 + if hours > 0 { + return fmt.Sprintf("[%d:%02d:%02d] %s", hours, minutes, seconds, label) + } + return fmt.Sprintf("[%02d:%02d] %s", minutes, seconds, label) +} + +func snippet(body string) string { + const maxRunes = 200 + runes := []rune(strings.TrimSpace(body)) + if len(runes) <= maxRunes { + return string(runes) + } + return string(runes[:maxRunes]) +} + +func buildCanonicalMeeting(meeting NormalizedMeeting) canonicalMeeting { + endedAt := "" + if meeting.EndedAt != nil { + endedAt = meeting.EndedAt.UTC().Format(time.RFC3339Nano) + } + return canonicalMeeting{ + ExternalID: meeting.ExternalID, + Title: meeting.Title, + StartedAt: meeting.StartedAt.UTC().Format(time.RFC3339Nano), + EndedAt: endedAt, + SummaryMarkdown: meeting.SummaryMarkdown, + SummaryText: meeting.SummaryText, + Transcript: meeting.Transcript, + TranscriptSegments: meeting.TranscriptSegments, + Organizer: meeting.Organizer, + Attendees: meeting.Attendees, + Metadata: meeting.Metadata, + } +} + +func buildMessageMetadata(req NormalizedRequest) messageMetadata { + meeting := req.Meeting + metadata := messageMetadata{ + Platform: SourceType, + ExternalMeetingID: meeting.ExternalID, + SourceIdentifier: req.Source.Identifier, + StartedAt: meeting.StartedAt.UTC().Format(time.RFC3339Nano), + AttendeeCount: len(meeting.Attendees), + HasSummary: meeting.SummaryMarkdown != "" || meeting.SummaryText != "", + HasTranscript: meeting.Transcript != "" || len(meeting.TranscriptSegments) > 0, + TranscriptSegmentCount: len(meeting.TranscriptSegments), + ProviderMetadata: meeting.Metadata, + } + if meeting.EndedAt != nil { + metadata.EndedAt = meeting.EndedAt.UTC().Format(time.RFC3339Nano) + metadata.DurationSeconds = int64(meeting.EndedAt.Sub(meeting.StartedAt).Seconds()) + } + if meeting.Organizer != nil { + metadata.OrganizerEmail = meeting.Organizer.Email + } + return metadata +} diff --git a/internal/meetingimport/format_test.go b/internal/meetingimport/format_test.go new file mode 100644 index 000000000..c927eccfd --- /dev/null +++ b/internal/meetingimport/format_test.go @@ -0,0 +1,152 @@ +package meetingimport + +import ( + "encoding/json" + "strings" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func normalizedValidRequest(t *testing.T) NormalizedRequest { + t.Helper() + req := decodedValidRequest(t) + normalized, err := req.Normalize() + require.NoError(t, err) + return normalized +} + +func TestBuildSnapshotRendersGranolaCompatibleBody(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + snapshot, err := BuildSnapshot(normalizedValidRequest(t)) + require.NoError(err) + + assert.Equal("meeting:42", snapshot.SourceMessageID) + assert.Equal("Weekly planning", snapshot.Title) + assert.Equal(`Weekly planning +When: 2026-07-23 18:00 - 18:30 +Attendees: Test Attendee + +## Summary + +Reviewed the launch plan. + +Transcript: +[00:04] Test Speaker: Let's review the launch plan.`, snapshot.Body) + assert.Equal(snapshot.Body, snapshot.Snippet) +} + +func TestBuildSnapshotPreservesPlainTranscriptLines(t *testing.T) { + req := normalizedValidRequest(t) + req.Meeting.SummaryMarkdown = "" + req.Meeting.SummaryText = "Plain summary wins when Markdown is empty." + req.Meeting.TranscriptSegments = nil + req.Meeting.Transcript = "Speaker 1: first line\n indented continuation\nSpeaker 2: final line" + + snapshot, err := BuildSnapshot(req) + require.NoError(t, err) + + assert.Contains(t, snapshot.Body, "\n\nPlain summary wins when Markdown is empty.\n\nTranscript:\n") + assert.Contains(t, snapshot.Body, "Speaker 1: first line\n indented continuation\nSpeaker 2: final line") +} + +func TestBuildSnapshotRendersStructuredSpeakerLabelsAndOffsets(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := normalizedValidRequest(t) + fourSeconds := 4.0 + overHour := 3661.9 + req.Meeting.TranscriptSegments = []TranscriptSegment{ + {Speaker: "Speaker 1", Text: "Anonymous speaker.", OffsetSeconds: &fourSeconds}, + {Speaker: "Test Speaker", Text: "Named speaker.", OffsetSeconds: &overHour}, + {Speaker: "Speaker 2", Text: "No timestamp."}, + } + + snapshot, err := BuildSnapshot(req) + require.NoError(err) + + assert.Contains(snapshot.Body, "[00:04] Speaker 1: Anonymous speaker.") + assert.Contains(snapshot.Body, "[1:01:01] Test Speaker: Named speaker.") + assert.Contains(snapshot.Body, "Speaker 2: No timestamp.") +} + +func TestBuildSnapshotUsesDateFallbackAndOptionalFields(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := normalizedValidRequest(t) + req.Meeting.Title = "" + req.Meeting.EndedAt = nil + req.Meeting.Organizer = nil + req.Meeting.Attendees = nil + req.Meeting.SummaryMarkdown = "" + req.Meeting.SummaryText = "Only a summary." + req.Meeting.TranscriptSegments = nil + + snapshot, err := BuildSnapshot(req) + require.NoError(err) + + assert.Equal("Meeting on 2026-07-23", snapshot.Title) + assert.Equal(`Meeting on 2026-07-23 +When: 2026-07-23 18:00 + +Only a summary.`, snapshot.Body) + assert.Nil(snapshot.Organizer) + assert.Empty(snapshot.Attendees) +} + +func TestBuildSnapshotCapsSnippetAtTwoHundredRunes(t *testing.T) { + req := normalizedValidRequest(t) + req.Meeting.Title = strings.Repeat("界", 210) + + snapshot, err := BuildSnapshot(req) + require.NoError(t, err) + + assert.Equal(t, 200, utf8.RuneCountInString(snapshot.Snippet)) + assert.Equal(t, strings.Repeat("界", 200), snapshot.Snippet) +} + +func TestBuildSnapshotStoresCanonicalRawMeetingAndMetadata(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := normalizedValidRequest(t) + + first, err := BuildSnapshot(req) + require.NoError(err) + second, err := BuildSnapshot(req) + require.NoError(err) + assert.Equal(first.Raw, second.Raw) + assert.Equal(first.Metadata, second.Metadata) + + var raw map[string]any + require.NoError(json.Unmarshal(first.Raw, &raw)) + assert.Equal("42", raw["external_id"]) + assert.Equal("2026-07-23T18:00:00Z", raw["started_at"]) + assert.Equal("2026-07-23T18:30:00Z", raw["ended_at"]) + assert.NotContains(raw, "source") + assert.NotContains(raw, "account_email") + rawMetadata, ok := raw["metadata"].(map[string]any) + require.True(ok, "raw metadata object") + assert.Equal("synthetic-event-42", rawMetadata["calendar_event_id"]) + + var metadata map[string]any + require.NoError(json.Unmarshal(first.Metadata, &metadata)) + assert.Equal(SourceType, metadata["platform"]) + assert.Equal("42", metadata["external_meeting_id"]) + assert.Equal("local-meetings", metadata["source_identifier"]) + assert.InDelta(float64(1800), metadata["duration_seconds"], 0) + assert.Equal("organizer@example.com", metadata["organizer_email"]) + assert.InDelta(float64(1), metadata["attendee_count"], 0) + assert.Equal(true, metadata["has_summary"]) + assert.Equal(true, metadata["has_transcript"]) + assert.InDelta(float64(1), metadata["transcript_segment_count"], 0) + providerMetadata, ok := metadata["provider_metadata"].(map[string]any) + require.True(ok, "provider metadata object") + assert.Equal("synthetic-event-42", providerMetadata["calendar_event_id"]) +} diff --git a/internal/meetingimport/importer.go b/internal/meetingimport/importer.go new file mode 100644 index 000000000..743d2139f --- /dev/null +++ b/internal/meetingimport/importer.go @@ -0,0 +1,295 @@ +package meetingimport + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "strings" + + "go.kenn.io/msgvault/internal/meetingidentity" + "go.kenn.io/msgvault/internal/store" +) + +type Status string + +var ErrUnavailable = errors.New("meeting importer is unavailable") + +const ( + StatusCreated Status = "created" + StatusUpdated Status = "updated" +) + +type Result struct { + Status Status + SourceID int64 + MessageID int64 + SourceMessageID string +} + +type Hooks struct { + AfterSourceSetup func() error + RefreshCache func(context.Context, string) error +} + +type Importer struct { + store *store.Store + hooks Hooks +} + +func NewImporter(s *store.Store, hooks Hooks) *Importer { + return &Importer{store: s, hooks: hooks} +} + +func (i *Importer) Import(ctx context.Context, req Request) (result Result, retErr error) { + if i == nil || i.store == nil { + return Result{}, ErrUnavailable + } + if err := ctx.Err(); err != nil { + return Result{}, err + } + + normalized, err := req.Normalize() + if err != nil { + return Result{}, err + } + snapshot, err := BuildSnapshot(normalized) + if err != nil { + return Result{}, err + } + + source, err := i.store.GetOrCreateSource(SourceType, snapshot.SourceIdentifier) + if err != nil { + return Result{}, fmt.Errorf("resolve meeting source: %w", err) + } + result.SourceID = source.ID + result.SourceMessageID = snapshot.SourceMessageID + + displayName := snapshot.SourceDisplayName + if displayName == "" && (!source.DisplayName.Valid || strings.TrimSpace(source.DisplayName.String) == "") { + displayName = snapshot.SourceIdentifier + } + if displayName != "" && (!source.DisplayName.Valid || source.DisplayName.String != displayName) { + if err := i.store.UpdateSourceDisplayNameContext(ctx, source.ID, displayName); err != nil { + return result, fmt.Errorf("update meeting source display name: %w", err) + } + } + if err := i.store.AddAccountIdentityAndRefreshMessageAttributionContext( + ctx, + source.ID, + snapshot.AccountEmail, + "account-email", + snapshot.SourceMessageID, + ); err != nil { + return result, fmt.Errorf("confirm meeting source identity: %w", err) + } + if i.hooks.AfterSourceSetup != nil { + if err := i.hooks.AfterSourceSetup(); err != nil { + return result, fmt.Errorf("run post-source setup: %w", err) + } + } + + syncID, err := i.store.StartSyncContext(ctx, source.ID, SourceType) + if err != nil { + return result, fmt.Errorf("start meeting import sync: %w", err) + } + checkpoint := &store.Checkpoint{} + defer func() { + if retErr == nil { + return + } + if failErr := i.store.FailSyncWithCheckpoint(syncID, retErr.Error(), checkpoint); failErr != nil { + retErr = errors.Join(retErr, fmt.Errorf("record failed meeting import sync: %w", failErr)) + } + }() + + existing, err := i.store.MessageExistsBatch(source.ID, []string{snapshot.SourceMessageID}) + if err != nil { + return result, fmt.Errorf("lookup existing meeting: %w", err) + } + identities, err := meetingidentity.ForSource(i.store, source.ID, snapshot.AccountEmail) + if err != nil { + return result, err + } + + organizerEmail := "" + organizerName := "" + if snapshot.Organizer != nil { + organizerEmail = snapshot.Organizer.Email + organizerName = snapshot.Organizer.Name + } + expectedIsFromMe := organizerEmail != "" && identities.Contains(organizerEmail) + + existingMessageID, existed := existing[snapshot.SourceMessageID] + if existed { + result.Status = StatusUpdated + result.MessageID = existingMessageID + + storedRaw, rawErr := i.store.GetMessageRaw(existingMessageID) + if rawErr == nil && bytes.Equal(storedRaw, snapshot.Raw) { + storedIsFromMe, attributionErr := i.store.GetMessageIsFromMe(existingMessageID) + if attributionErr == nil && storedIsFromMe == expectedIsFromMe { + if err := ctx.Err(); err != nil { + return result, err + } + checkpoint.MessagesProcessed = 1 + if err := i.store.UpdateSyncCheckpointContext(ctx, syncID, checkpoint); err != nil { + return result, fmt.Errorf("checkpoint meeting import sync: %w", err) + } + if err := i.store.RecomputeConversationStatsForMessageContext(ctx, existingMessageID); err != nil { + return result, fmt.Errorf("recompute meeting conversation stats: %w", err) + } + if err := i.store.CompleteSyncContext(ctx, syncID, ""); err != nil { + return result, fmt.Errorf("complete meeting import sync: %w", err) + } + if i.hooks.RefreshCache != nil { + cacheLabel := SourceType + ":" + snapshot.SourceIdentifier + if err := i.hooks.RefreshCache(ctx, cacheLabel); err != nil { + return result, fmt.Errorf("refresh meeting analytics cache: %w", err) + } + } + return result, nil + } + } + // Missing or unreadable canonical data, or mismatched derived + // attribution, is not a match. Continue through the transactional + // persistence path so a valid retry repairs the message; genuine + // storage failures surface from that rewrite. + } + + participants := make( + []store.ParticipantPersistData, + 0, + len(snapshot.Attendees)+1, + ) + hasOrganizer := snapshot.Organizer != nil + if hasOrganizer { + participants = append(participants, store.ParticipantPersistData{ + EmailAddress: organizerEmail, + DisplayName: organizerName, + Domain: emailDomain(organizerEmail), + }) + } + + attendeeNames := make([]string, 0, len(snapshot.Attendees)) + attendeeEmails := make([]string, 0, len(snapshot.Attendees)) + for _, attendee := range snapshot.Attendees { + if err := ctx.Err(); err != nil { + return result, err + } + participants = append(participants, store.ParticipantPersistData{ + EmailAddress: attendee.Email, + DisplayName: attendee.Name, + Domain: emailDomain(attendee.Email), + }) + attendeeNames = append(attendeeNames, attendee.Name) + attendeeEmails = append(attendeeEmails, attendee.Email) + } + + metadata := sql.NullString{String: string(snapshot.Metadata), Valid: true} + messageID, err := i.store.PersistMessageWithParticipantsContext( + ctx, + participants, + func(participantIDs []int64) *store.MessagePersistData { + attendeeOffset := 0 + var senderID int64 + var fromIDs []int64 + var fromNames []string + if hasOrganizer { + senderID = participantIDs[0] + attendeeOffset = 1 + fromIDs = []int64{senderID} + fromNames = []string{organizerName} + } + attendeeIDs := participantIDs[attendeeOffset:] + conversationParticipants := make( + []store.ConversationParticipantRef, + 0, + len(attendeeIDs), + ) + for _, participantID := range attendeeIDs { + conversationParticipants = append( + conversationParticipants, + store.ConversationParticipantRef{ + ParticipantID: participantID, + Role: "member", + }, + ) + } + + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, + SourceMessageID: snapshot.SourceMessageID, + MessageType: MessageType, + SentAt: sql.NullTime{Time: snapshot.StartedAt, Valid: true}, + SenderID: sql.NullInt64{Int64: senderID, Valid: senderID != 0}, + IsFromMe: expectedIsFromMe, + IdentityDerivedIsFromMe: expectedIsFromMe, + Subject: sql.NullString{String: snapshot.Title, Valid: snapshot.Title != ""}, + Snippet: sql.NullString{String: snapshot.Snippet, Valid: snapshot.Snippet != ""}, + SizeEstimate: int64(len(snapshot.Body)), + }, + Conversation: &store.ConversationPersistData{ + SourceConversationID: snapshot.SourceMessageID, + ConversationType: ConversationType, + Title: snapshot.Title, + Participants: conversationParticipants, + }, + Metadata: &metadata, + BodyText: sql.NullString{String: snapshot.Body, Valid: snapshot.Body != ""}, + RawMIME: snapshot.Raw, + RawFormat: RawFormat, + Recipients: []store.RecipientSet{ + {Type: "from", ParticipantIDs: fromIDs, DisplayNames: fromNames}, + {Type: "to", ParticipantIDs: attendeeIDs, DisplayNames: attendeeNames}, + }, + PreserveLabels: true, + FTS: &store.FTSDoc{ + Subject: snapshot.Title, + Body: snapshot.Body, + FromAddr: organizerEmail, + ToAddrs: strings.Join(attendeeEmails, " "), + }, + } + }, + ) + if err != nil { + return result, fmt.Errorf("persist meeting: %w", err) + } + result.MessageID = messageID + result.Status = StatusCreated + checkpoint.MessagesProcessed = 1 + checkpoint.MessagesAdded = 1 + if existed { + result.Status = StatusUpdated + checkpoint.MessagesAdded = 0 + checkpoint.MessagesUpdated = 1 + } + + if err := i.store.UpdateSyncCheckpointContext(ctx, syncID, checkpoint); err != nil { + return result, fmt.Errorf("checkpoint meeting import sync: %w", err) + } + if err := i.store.RecomputeConversationStatsForMessageContext(ctx, messageID); err != nil { + return result, fmt.Errorf("recompute meeting conversation stats: %w", err) + } + if err := i.store.CompleteSyncContext(ctx, syncID, ""); err != nil { + return result, fmt.Errorf("complete meeting import sync: %w", err) + } + if i.hooks.RefreshCache != nil { + cacheLabel := SourceType + ":" + snapshot.SourceIdentifier + if err := i.hooks.RefreshCache(ctx, cacheLabel); err != nil { + return result, fmt.Errorf("refresh meeting analytics cache: %w", err) + } + } + return result, nil +} + +func emailDomain(email string) string { + at := strings.LastIndexByte(email, '@') + if at < 0 || at == len(email)-1 { + return "" + } + return strings.ToLower(email[at+1:]) +} diff --git a/internal/meetingimport/importer_test.go b/internal/meetingimport/importer_test.go new file mode 100644 index 000000000..eb0077ea4 --- /dev/null +++ b/internal/meetingimport/importer_test.go @@ -0,0 +1,885 @@ +package meetingimport + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func validImportRequest(t *testing.T) Request { + t.Helper() + return decodedValidRequest(t) +} + +func TestImporterCreatesCanonicalMeetingAndSyncRun(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + sourceHookCalls := 0 + var cacheLabels []string + importer := NewImporter(st, Hooks{ + AfterSourceSetup: func() error { + sourceHookCalls++ + return nil + }, + RefreshCache: func(_ context.Context, label string) error { + cacheLabels = append(cacheLabels, label) + return nil + }, + }) + + result, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + + assert.Equal(StatusCreated, result.Status) + assert.Equal("meeting:42", result.SourceMessageID) + assert.NotZero(result.SourceID) + assert.NotZero(result.MessageID) + assert.Equal(1, sourceHookCalls) + assert.Equal([]string{"meeting_import:local-meetings"}, cacheLabels) + + var sourceType, identifier, displayName string + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT source_type, identifier, display_name + FROM sources WHERE id = ? + `), result.SourceID).Scan(&sourceType, &identifier, &displayName)) + assert.Equal(SourceType, sourceType) + assert.Equal("local-meetings", identifier) + assert.Equal("Local Meetings", displayName) + + identities, err := st.ListAccountIdentities(result.SourceID) + require.NoError(err) + require.Len(identities, 1) + assert.Equal("user@example.com", identities[0].Address) + assert.Contains(identities[0].SourceSignal, "account-email") + + var ( + messageType, sourceMessageID, subject string + isFromMe bool + senderEmail sql.NullString + conversationType, conversationKey string + body, rawFormat, metadataJSON string + messageCount, participantCount int + ) + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT m.message_type, m.source_message_id, m.subject, m.is_from_me, + p.email_address, c.conversation_type, c.source_conversation_id, + mb.body_text, mr.raw_format, m.metadata, + c.message_count, c.participant_count + FROM messages m + JOIN conversations c ON c.id = m.conversation_id + JOIN message_bodies mb ON mb.message_id = m.id + JOIN message_raw mr ON mr.message_id = m.id + LEFT JOIN participants p ON p.id = m.sender_id + WHERE m.id = ? + `), result.MessageID).Scan( + &messageType, &sourceMessageID, &subject, &isFromMe, + &senderEmail, &conversationType, &conversationKey, + &body, &rawFormat, &metadataJSON, + &messageCount, &participantCount, + )) + assert.Equal(MessageType, messageType) + assert.Equal("meeting:42", sourceMessageID) + assert.Equal("Weekly planning", subject) + assert.False(isFromMe) + assert.Equal("organizer@example.com", senderEmail.String) + assert.Equal(ConversationType, conversationType) + assert.Equal("meeting:42", conversationKey) + assert.Contains(body, "[00:04] Test Speaker: Let's review the launch plan.") + assert.Equal(RawFormat, rawFormat) + assert.Equal(1, messageCount) + assert.Equal(1, participantCount) + + var metadata map[string]any + require.NoError(json.Unmarshal([]byte(metadataJSON), &metadata)) + assert.Equal(SourceType, metadata["platform"]) + + raw, err := st.GetMessageRaw(result.MessageID) + require.NoError(err) + assert.NotContains(string(raw), "account_email") + assert.Contains(string(raw), `"external_id":"42"`) + + var fromCount, toCount, memberCount int + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT COUNT(*) FROM message_recipients + WHERE message_id = ? AND recipient_type = 'from' + `), result.MessageID).Scan(&fromCount)) + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT COUNT(*) FROM message_recipients + WHERE message_id = ? AND recipient_type = 'to' + `), result.MessageID).Scan(&toCount)) + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT COUNT(*) FROM conversation_participants cp + JOIN messages m ON m.conversation_id = cp.conversation_id + WHERE m.id = ? + `), result.MessageID).Scan(&memberCount)) + assert.Equal(1, fromCount) + assert.Equal(1, toCount) + assert.Equal(1, memberCount) + + latest, err := st.GetLatestSync(result.SourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Equal(int64(1), latest.MessagesAdded) + assert.Equal(int64(0), latest.MessagesUpdated) +} + +func TestImporterRetriesUpdateSameMessageAndReplacePeople(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + + first, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + + second, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + assert.Equal(StatusUpdated, second.Status) + assert.Equal(first.MessageID, second.MessageID) + + changed := validImportRequest(t) + changed.Source.DisplayName = "Renamed Meetings" + changed.Source.AccountEmail = "organizer@example.com" + changed.Meeting.Title = "Replacement title" + changed.Meeting.SummaryMarkdown = "Replacement summary" + changed.Meeting.TranscriptSegments = nil + changed.Meeting.Transcript = "Speaker 1: replacement transcript" + changed.Meeting.Organizer = nil + changed.Meeting.Attendees = nil + + third, err := importer.Import(context.Background(), changed) + require.NoError(err) + assert.Equal(StatusUpdated, third.Status) + assert.Equal(first.MessageID, third.MessageID) + + var count int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&count)) + assert.Equal(1, count) + + var subject, body, displayName string + var senderID sql.NullInt64 + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT m.subject, mb.body_text, m.sender_id, s.display_name + FROM messages m + JOIN message_bodies mb ON mb.message_id = m.id + JOIN sources s ON s.id = m.source_id + WHERE m.id = ? + `), first.MessageID).Scan(&subject, &body, &senderID, &displayName)) + assert.Equal("Replacement title", subject) + assert.Contains(body, "Replacement summary") + assert.Contains(body, "Speaker 1: replacement transcript") + assert.False(senderID.Valid) + assert.Equal("Renamed Meetings", displayName) + + for _, tableQuery := range []string{ + `SELECT COUNT(*) FROM message_recipients WHERE message_id = ?`, + `SELECT COUNT(*) FROM conversation_participants cp + JOIN messages m ON m.conversation_id = cp.conversation_id WHERE m.id = ?`, + } { + require.NoError(st.DB().QueryRow(st.Rebind(tableQuery), first.MessageID).Scan(&count)) + assert.Equal(0, count) + } + + latest, err := st.GetLatestSync(first.SourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(0), latest.MessagesAdded) + assert.Equal(int64(1), latest.MessagesUpdated) + + identities, err := st.ListAccountIdentities(first.SourceID) + require.NoError(err) + assert.Len(identities, 2) +} + +func TestImporterUnchangedRetryDoesNotPersistOrCountUpdate(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + cacheCalls := 0 + importer := NewImporter(st, Hooks{ + RefreshCache: func(context.Context, string) error { + cacheCalls++ + return nil + }, + }) + + first, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + + sentinel := time.Date(2000, time.January, 2, 3, 4, 5, 0, time.UTC) + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages SET last_modified = ? WHERE id = ? + `), sentinel, first.MessageID) + require.NoError(err) + var watermarkBefore string + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT CAST(last_modified AS TEXT) FROM messages WHERE id = ? + `), first.MessageID).Scan(&watermarkBefore)) + + retry, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + assert.Equal(StatusUpdated, retry.Status) + assert.Equal(first.MessageID, retry.MessageID) + + var watermarkAfter string + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT CAST(last_modified AS TEXT) FROM messages WHERE id = ? + `), first.MessageID).Scan(&watermarkAfter)) + assert.Equal(watermarkBefore, watermarkAfter) + + latest, err := st.GetLatestSync(first.SourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Equal(int64(0), latest.MessagesAdded) + assert.Equal(int64(0), latest.MessagesUpdated) + assert.Equal(2, cacheCalls) +} + +func TestImporterUnchangedRetryRepairsStatsAfterPostPersistFailure(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + testutil.SkipIfPostgres(t, "uses a SQLite trigger to inject a stats failure") + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + _, err := st.DB().Exec(` + CREATE TRIGGER fail_meeting_import_stats + BEFORE UPDATE OF message_count ON conversations + BEGIN + SELECT RAISE(ABORT, 'forced meeting import stats failure'); + END + `) + require.NoError(err) + + _, err = importer.Import(context.Background(), validImportRequest(t)) + require.Error(err) + assert.Contains(err.Error(), "recompute meeting conversation stats") + + var messageID, sourceID int64 + var messageCount, participantCount int + var preview sql.NullString + require.NoError(st.DB().QueryRow(` + SELECT m.id, m.source_id, c.message_count, c.participant_count, + c.last_message_preview + FROM messages m + JOIN conversations c ON c.id = m.conversation_id + WHERE m.source_message_id = 'meeting:42' + `).Scan(&messageID, &sourceID, &messageCount, &participantCount, &preview)) + assert.Zero(messageCount) + assert.Zero(participantCount) + assert.False(preview.Valid) + + _, err = st.DB().Exec(`DROP TRIGGER fail_meeting_import_stats`) + require.NoError(err) + + retry, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + assert.Equal(StatusUpdated, retry.Status) + assert.Equal(messageID, retry.MessageID) + + var messagePreview sql.NullString + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT c.message_count, c.participant_count, c.last_message_preview, + m.snippet + FROM messages m + JOIN conversations c ON c.id = m.conversation_id + WHERE m.id = ? + `), messageID).Scan(&messageCount, &participantCount, &preview, &messagePreview)) + assert.Equal(1, messageCount) + assert.Equal(1, participantCount) + assert.Equal(messagePreview, preview) + + latest, err := st.GetLatestSync(sourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Zero(latest.MessagesAdded) + assert.Zero(latest.MessagesUpdated) +} + +func TestImporterStatsRecomputeDoesNotTouchSiblingConversations(t *testing.T) { + for _, tc := range []struct { + name string + unchanged bool + want Status + }{ + {name: "created", want: StatusCreated}, + {name: "unchanged", unchanged: true, want: StatusUpdated}, + } { + t.Run(tc.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + normalized, err := req.Normalize() + require.NoError(err) + if tc.unchanged { + _, err = importer.Import(context.Background(), req) + require.NoError(err) + } + + source, err := st.GetOrCreateSource(SourceType, normalized.Source.Identifier) + require.NoError(err) + siblingID, err := st.EnsureConversationWithType( + source.ID, + "sibling-meeting", + ConversationType, + "Sibling meeting", + ) + require.NoError(err) + _, err = st.DB().Exec(st.Rebind(` + UPDATE conversations + SET message_count = 71, + participant_count = 72, + last_message_preview = 'sibling sentinel' + WHERE id = ? + `), siblingID) + require.NoError(err) + + result, err := importer.Import(context.Background(), req) + require.NoError(err) + assert.Equal(tc.want, result.Status) + + var messageCount, participantCount int + var preview sql.NullString + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT message_count, participant_count, last_message_preview + FROM conversations + WHERE id = ? + `), siblingID).Scan(&messageCount, &participantCount, &preview)) + assert.Equal(71, messageCount) + assert.Equal(72, participantCount) + assert.Equal(sql.NullString{String: "sibling sentinel", Valid: true}, preview) + }) + } +} + +func TestImporterRetryRepairsUnreadableCanonicalSnapshot(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + + first, err := importer.Import(context.Background(), req) + require.NoError(err) + wantRaw, err := st.GetMessageRaw(first.MessageID) + require.NoError(err) + + _, err = st.DB().Exec(st.Rebind(` + UPDATE message_raw + SET raw_data = ?, compression = 'zlib' + WHERE message_id = ? + `), []byte("not a zlib stream"), first.MessageID) + require.NoError(err) + _, err = st.GetMessageRaw(first.MessageID) + require.Error(err, "corrupt fixture must be unreadable") + + retry, err := importer.Import(context.Background(), req) + require.NoError(err) + assert.Equal(StatusUpdated, retry.Status) + assert.Equal(first.MessageID, retry.MessageID) + + gotRaw, err := st.GetMessageRaw(first.MessageID) + require.NoError(err) + assert.Equal(wantRaw, gotRaw) + + latest, err := st.GetLatestSync(first.SourceID) + require.NoError(err) + assert.Equal(int64(1), latest.MessagesUpdated) +} + +func TestImporterPreservesDisplayNameWhenRetryOmitsIt(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + initial := validImportRequest(t) + initial.Source.DisplayName = "Named Meeting Source" + + created, err := importer.Import(context.Background(), initial) + require.NoError(err) + assert.Equal(StatusCreated, created.Status) + + retry := validImportRequest(t) + retry.Source.DisplayName = "" + updated, err := importer.Import(context.Background(), retry) + require.NoError(err) + assert.Equal(StatusUpdated, updated.Status) + + sources, err := st.ListSources(SourceType) + require.NoError(err) + require.Len(sources, 1) + assert.Equal("Named Meeting Source", sources[0].DisplayName.String) +} + +func TestImporterDefaultsDisplayNameOnFirstImport(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + req.Source.DisplayName = "" + + result, err := importer.Import(context.Background(), req) + require.NoError(err) + assert.Equal(StatusCreated, result.Status) + + sources, err := st.ListSources(SourceType) + require.NoError(err) + require.Len(sources, 1) + assert.Equal("local-meetings", sources[0].DisplayName.String) +} + +func TestImporterScopesExternalIDsBySource(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + firstRequest := validImportRequest(t) + secondRequest := validImportRequest(t) + secondRequest.Source.Identifier = "second-stream" + + first, err := importer.Import(context.Background(), firstRequest) + require.NoError(err) + second, err := importer.Import(context.Background(), secondRequest) + require.NoError(err) + + assert.NotEqual(first.SourceID, second.SourceID) + assert.NotEqual(first.MessageID, second.MessageID) + assert.Equal(first.SourceMessageID, second.SourceMessageID) +} + +func TestImporterMarksOrganizerFromConfirmedAccountAsFromMe(t *testing.T) { + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + req.Source.AccountEmail = "organizer@example.com" + + result, err := importer.Import(context.Background(), req) + require.NoError(t, err) + + var isFromMe bool + require.NoError(t, st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), result.MessageID).Scan(&isFromMe)) + assert.True(t, isFromMe) +} + +func TestImporterAccountEmailOnlyRetryUpdatesAttribution(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + + first, err := importer.Import(context.Background(), req) + require.NoError(err) + + var isFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), first.MessageID).Scan(&isFromMe)) + assert.False(isFromMe) + + req.Source.AccountEmail = req.Meeting.Organizer.Email + retry, err := importer.Import(context.Background(), req) + require.NoError(err) + assert.Equal(StatusUpdated, retry.Status) + assert.Equal(first.MessageID, retry.MessageID) + + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), first.MessageID).Scan(&isFromMe)) + assert.True(isFromMe) + + latest, err := st.GetLatestSync(first.SourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Zero(latest.MessagesAdded) + assert.Equal(int64(1), latest.MessagesUpdated) +} + +func TestImporterNewAccountEmailRefreshesEarlierMeetingAttributionForSource(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + + first, err := importer.Import(context.Background(), req) + require.NoError(err) + + req.Meeting.ExternalID = "43" + second, err := importer.Import(context.Background(), req) + require.NoError(err) + + otherSourceReq := req + otherSourceReq.Source.Identifier = "other-meetings" + otherSourceReq.Meeting.ExternalID = "44" + otherSource, err := importer.Import(context.Background(), otherSourceReq) + require.NoError(err) + + req.Source.AccountEmail = req.Meeting.Organizer.Email + req.Meeting.ExternalID = "45" + confirmation, err := importer.Import(context.Background(), req) + require.NoError(err) + assert.Equal(StatusCreated, confirmation.Status) + + for _, messageID := range []int64{first.MessageID, second.MessageID, confirmation.MessageID} { + var isFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), messageID).Scan(&isFromMe)) + assert.True(isFromMe, "message %d in the confirmed source", messageID) + } + + var otherSourceIsFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), otherSource.MessageID).Scan(&otherSourceIsFromMe)) + assert.False(otherSourceIsFromMe) + + latest, err := st.GetLatestSync(first.SourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Equal(int64(1), latest.MessagesAdded) + assert.Zero(latest.MessagesUpdated) +} + +func TestImporterPreconfirmedAccountEmailKeepsEarlierMeetingAttributionCurrent(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + req := validImportRequest(t) + + first, err := importer.Import(context.Background(), req) + require.NoError(err) + require.NoError(st.AddAccountIdentity( + first.SourceID, + req.Meeting.Organizer.Email, + "manual", + )) + + req.Source.AccountEmail = req.Meeting.Organizer.Email + req.Meeting.ExternalID = "43" + _, err = importer.Import(context.Background(), req) + require.NoError(err) + + var earlierIsFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me FROM messages WHERE id = ? + `), first.MessageID).Scan(&earlierIsFromMe)) + assert.True(earlierIsFromMe) +} + +func TestImporterMarksCacheFailureAndSafelyRetries(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + cacheErr := errors.New("synthetic cache failure") + failCache := true + importer := NewImporter(st, Hooks{ + RefreshCache: func(context.Context, string) error { + if failCache { + return cacheErr + } + return nil + }, + }) + + _, err := importer.Import(context.Background(), validImportRequest(t)) + require.ErrorIs(err, cacheErr) + + var messageID, sourceID int64 + require.NoError(st.DB().QueryRow(` + SELECT id, source_id FROM messages WHERE source_message_id = 'meeting:42' + `).Scan(&messageID, &sourceID)) + assert.NotZero(messageID, "message remains durable after cache failure") + + failed, err := st.GetLatestSync(sourceID) + require.NoError(err) + assert.Equal(store.SyncStatusFailed, failed.Status) + assert.Equal(int64(1), failed.MessagesProcessed) + assert.Equal(int64(1), failed.MessagesAdded) + + failCache = false + result, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + assert.Equal(StatusUpdated, result.Status) + assert.Equal(messageID, result.MessageID) + + completed, err := st.GetLatestSync(sourceID) + require.NoError(err) + assert.Equal(store.SyncStatusCompleted, completed.Status) + assert.Equal(int64(0), completed.MessagesUpdated) +} + +func TestImporterSourceHookFailureStopsBeforeSync(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + st := testutil.NewTestStore(t) + hookErr := errors.New("synthetic migration failure") + importer := NewImporter(st, Hooks{ + AfterSourceSetup: func() error { return hookErr }, + }) + + _, err := importer.Import(context.Background(), validImportRequest(t)) + require.ErrorIs(err, hookErr) + + var count int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM sync_runs`).Scan(&count)) + assert.Equal(0, count) + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&count)) + assert.Equal(0, count) +} + +func TestImporterCancellationStopsBeforeSourceSetup(t *testing.T) { + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := importer.Import(ctx, validImportRequest(t)) + require.ErrorIs(t, err, context.Canceled) + + var count int + require.NoError(t, st.DB().QueryRow(`SELECT COUNT(*) FROM sources WHERE source_type = 'meeting_import'`).Scan(&count)) + assert.Equal(t, 0, count) +} + +func TestImporterCancellationDuringParticipantResolutionRollsBackParticipants(t *testing.T) { + testutil.SkipIfPostgres(t, "uses a SQLite trigger and registered function to pause participant insertion") + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + st.DB().SetMaxOpenConns(1) + importer := NewImporter(st, Hooks{}) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + participantStarted := make(chan struct{}) + conn, err := st.DB().Conn(context.Background()) + require.NoError(err, "get SQLite connection") + err = conn.Raw(func(driverConn any) error { + sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn) + require.True(ok, "driver connection is SQLite") + return sqliteConn.RegisterFunc("wait_for_participant_cancel", func() int { + close(participantStarted) + <-ctx.Done() + return 0 + }, true) + }) + require.NoError(err, "register cancellation function") + require.NoError(conn.Close(), "return SQLite connection to pool") + _, err = st.DB().Exec(` + CREATE TRIGGER wait_before_meeting_participant + BEFORE INSERT ON participants + WHEN NEW.email_address = 'attendee@example.com' + BEGIN + SELECT wait_for_participant_cancel(); + END + `) + require.NoError(err, "create participant cancellation trigger") + + req := validImportRequest(t) + done := make(chan error, 1) + go func() { + _, importErr := importer.Import(ctx, req) + done <- importErr + }() + + select { + case <-participantStarted: + case <-time.After(time.Second): + require.FailNow("meeting import did not reach attendee insertion") + } + cancel() + + select { + case err = <-done: + case <-time.After(time.Second): + require.FailNow("meeting import did not stop after cancellation") + } + require.ErrorIs(err, context.Canceled) + + var participantCount int + require.NoError(st.DB().QueryRow(` + SELECT COUNT(*) + FROM participants + WHERE email_address IN ('organizer@example.com', 'attendee@example.com') + `).Scan(&participantCount)) + assert.Zero( + participantCount, + "organizer and attendee inserts must roll back with canceled message persistence", + ) + var messageCount int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&messageCount)) + assert.Zero(messageCount) +} + +func TestImporterCancellationDuringCheckpointLeavesFailedSync(t *testing.T) { + testutil.SkipIfPostgres(t, "uses a SQLite trigger and registered function to pause the checkpoint") + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + st.DB().SetMaxOpenConns(1) + importer := NewImporter(st, Hooks{}) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + checkpointStarted := make(chan struct{}) + conn, err := st.DB().Conn(context.Background()) + require.NoError(err, "get SQLite connection") + err = conn.Raw(func(driverConn any) error { + sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn) + require.True(ok, "driver connection is SQLite") + return sqliteConn.RegisterFunc("wait_for_checkpoint_cancel", func() int { + close(checkpointStarted) + <-ctx.Done() + return 0 + }, true) + }) + require.NoError(err, "register cancellation function") + require.NoError(conn.Close(), "return SQLite connection to pool") + _, err = st.DB().Exec(` + CREATE TRIGGER wait_before_meeting_checkpoint + BEFORE UPDATE OF messages_processed ON sync_runs + WHEN NEW.messages_processed = 1 AND NEW.status = 'running' + BEGIN + SELECT wait_for_checkpoint_cancel(); + END + `) + require.NoError(err, "create checkpoint cancellation trigger") + + req := validImportRequest(t) + done := make(chan error, 1) + go func() { + _, importErr := importer.Import(ctx, req) + done <- importErr + }() + + select { + case <-checkpointStarted: + case <-time.After(time.Second): + require.FailNow("meeting import did not reach checkpoint") + } + cancel() + + select { + case err = <-done: + case <-time.After(time.Second): + require.FailNow("meeting import did not stop after cancellation") + } + require.ErrorIs(err, context.Canceled) + + var messageCount int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&messageCount)) + assert.Equal(1, messageCount, "canonical message is committed before checkpointing") + + sources, err := st.ListSources(SourceType) + require.NoError(err) + require.Len(sources, 1) + latest, err := st.GetLatestSync(sources[0].ID) + require.NoError(err) + assert.Equal(store.SyncStatusFailed, latest.Status) + assert.Equal(int64(1), latest.MessagesProcessed) + assert.Equal(int64(1), latest.MessagesAdded) + assert.Contains(latest.ErrorMessage.String, context.Canceled.Error()) +} + +func TestImporterRawFailureRollsBackCanonicalSnapshot(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + testutil.SkipIfPostgres(t, "uses a SQLite trigger to inject a raw archive failure") + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + require.True(st.FTS5Available()) + _, err := st.DB().Exec(` + CREATE TRIGGER fail_meeting_import_raw + BEFORE INSERT ON message_raw + WHEN NEW.raw_format = 'meeting_json' + BEGIN + SELECT RAISE(ABORT, 'forced meeting import raw failure'); + END + `) + require.NoError(err) + + _, err = importer.Import(context.Background(), validImportRequest(t)) + require.Error(err) + assert.Contains(err.Error(), "persist meeting") + + for table, want := range map[string]int{ + "messages": 0, + "conversations": 0, + "message_bodies": 0, + "message_raw": 0, + "message_recipients": 0, + "messages_fts": 0, + } { + var got int + require.NoError(st.DB().QueryRow("SELECT COUNT(*) FROM "+table).Scan(&got), table) + assert.Equal(want, got, table) + } + + sources, err := st.ListSources(SourceType) + require.NoError(err) + require.Len(sources, 1) + latest, err := st.GetLatestSync(sources[0].ID) + require.NoError(err) + assert.Equal(store.SyncStatusFailed, latest.Status) + assert.Contains(latest.ErrorMessage.String, "persist meeting") +} + +func TestImporterIndexesSubjectBodyAndAddresses(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + testutil.SkipIfPostgres(t, "asserts against the SQLite FTS5 virtual table") + st := testutil.NewTestStore(t) + importer := NewImporter(st, Hooks{}) + + result, err := importer.Import(context.Background(), validImportRequest(t)) + require.NoError(err) + + var subject, body, fromAddr, toAddrs string + require.NoError(st.DB().QueryRow(` + SELECT subject, body, from_addr, to_addr + FROM messages_fts WHERE rowid = ? + `, result.MessageID).Scan(&subject, &body, &fromAddr, &toAddrs)) + assert.Equal("Weekly planning", subject) + assert.Contains(body, "launch plan") + assert.Equal("organizer@example.com", fromAddr) + assert.Equal("attendee@example.com", strings.TrimSpace(toAddrs)) +} diff --git a/internal/meetingimport/models.go b/internal/meetingimport/models.go new file mode 100644 index 000000000..ef36019f7 --- /dev/null +++ b/internal/meetingimport/models.go @@ -0,0 +1,302 @@ +// Package meetingimport validates and stores provider-neutral meeting +// transcripts submitted through the msgvault HTTP API. +package meetingimport + +import ( + "errors" + "fmt" + "math" + "net/mail" + "strings" + "time" + "unicode/utf8" +) + +const ( + SourceType = "meeting_import" + ConversationType = "meeting" + MessageType = "meeting_transcript" + RawFormat = "meeting_json" + MaxRequestBytes = int64(16 << 20) + + maxSourceIdentifierChars = 128 + maxSourceDisplayNameChars = 256 + maxExternalIDChars = 256 + maxTitleChars = 4096 +) + +var ( + ErrMalformedRequest = errors.New("malformed meeting import request") + ErrRequestTooLarge = errors.New("meeting import request too large") + ErrValidation = errors.New("meeting import validation failed") +) + +type MeetingImportRequest struct { + Source Source `json:"source"` + Meeting Meeting `json:"meeting"` +} + +// Request is kept as a concise internal alias for the meeting import wire +// contract. The named type gives generated API clients an unambiguous schema. +type Request = MeetingImportRequest + +type Source struct { + Identifier string `json:"identifier" maxLength:"128"` + DisplayName string `json:"display_name,omitempty" maxLength:"256"` + AccountEmail string `json:"account_email" format:"email"` +} + +type Meeting struct { + ExternalID string `json:"external_id" maxLength:"256"` + Title string `json:"title,omitempty" maxLength:"4096"` + StartedAt string `json:"started_at" format:"date-time"` + EndedAt string `json:"ended_at,omitempty" format:"date-time"` + SummaryMarkdown string `json:"summary_markdown,omitempty"` + SummaryText string `json:"summary_text,omitempty"` + Transcript string `json:"transcript,omitempty"` + TranscriptSegments []TranscriptSegment `json:"transcript_segments,omitempty"` + Organizer *MeetingPerson `json:"organizer,omitempty"` + Attendees []MeetingPerson `json:"attendees,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` +} + +type MeetingPerson struct { + Name string `json:"name,omitempty"` + Email string `json:"email" format:"email"` +} + +type TranscriptSegment struct { + Speaker string `json:"speaker"` + Text string `json:"text"` + OffsetSeconds *float64 `json:"offset_seconds,omitempty" minimum:"0"` +} + +type NormalizedRequest struct { + Source Source + Meeting NormalizedMeeting +} + +type NormalizedMeeting struct { + ExternalID string + Title string + StartedAt time.Time + EndedAt *time.Time + SummaryMarkdown string + SummaryText string + Transcript string + TranscriptSegments []TranscriptSegment + Organizer *MeetingPerson + Attendees []MeetingPerson + Metadata map[string]any +} + +func (r Request) Normalize() (NormalizedRequest, error) { + source, err := normalizeSource(r.Source) + if err != nil { + return NormalizedRequest{}, err + } + meeting, err := normalizeMeeting(r.Meeting) + if err != nil { + return NormalizedRequest{}, err + } + return NormalizedRequest{Source: source, Meeting: meeting}, nil +} + +func normalizeSource(source Source) (Source, error) { + source.Identifier = strings.TrimSpace(source.Identifier) + if err := validateBoundedRequired("source.identifier", source.Identifier, maxSourceIdentifierChars); err != nil { + return Source{}, err + } + + source.DisplayName = strings.TrimSpace(source.DisplayName) + if err := validateBoundedOptional("source.display_name", source.DisplayName, maxSourceDisplayNameChars); err != nil { + return Source{}, err + } + + accountEmail, err := normalizeEmail("source.account_email", source.AccountEmail) + if err != nil { + return Source{}, err + } + source.AccountEmail = accountEmail + return source, nil +} + +func normalizeMeeting(meeting Meeting) (NormalizedMeeting, error) { + meeting.ExternalID = strings.TrimSpace(meeting.ExternalID) + if err := validateBoundedRequired("meeting.external_id", meeting.ExternalID, maxExternalIDChars); err != nil { + return NormalizedMeeting{}, err + } + meeting.Title = strings.TrimSpace(meeting.Title) + if err := validateBoundedOptional("meeting.title", meeting.Title, maxTitleChars); err != nil { + return NormalizedMeeting{}, err + } + + startedAt, err := parseTimestamp("meeting.started_at", meeting.StartedAt) + if err != nil { + return NormalizedMeeting{}, err + } + var endedAt *time.Time + if strings.TrimSpace(meeting.EndedAt) != "" { + parsed, parseErr := parseTimestamp("meeting.ended_at", meeting.EndedAt) + if parseErr != nil { + return NormalizedMeeting{}, parseErr + } + if parsed.Before(startedAt) { + return NormalizedMeeting{}, validationError("meeting.ended_at must not be before meeting.started_at") + } + endedAt = &parsed + } + + summaryMarkdown := strings.TrimSpace(meeting.SummaryMarkdown) + summaryText := strings.TrimSpace(meeting.SummaryText) + transcript := strings.TrimSpace(meeting.Transcript) + segments, err := normalizeSegments(meeting.TranscriptSegments) + if err != nil { + return NormalizedMeeting{}, err + } + if transcript != "" && len(segments) > 0 { + return NormalizedMeeting{}, validationError("meeting.transcript and meeting.transcript_segments are mutually exclusive") + } + if summaryMarkdown == "" && summaryText == "" && transcript == "" && len(segments) == 0 { + return NormalizedMeeting{}, validationError("meeting requires a summary or transcript") + } + + var organizer *MeetingPerson + if meeting.Organizer != nil { + normalizedOrganizer, normalizeErr := normalizePerson("meeting.organizer", *meeting.Organizer) + if normalizeErr != nil { + return NormalizedMeeting{}, normalizeErr + } + organizer = &normalizedOrganizer + } + attendees, err := normalizeAttendees(meeting.Attendees) + if err != nil { + return NormalizedMeeting{}, err + } + + return NormalizedMeeting{ + ExternalID: meeting.ExternalID, + Title: meeting.Title, + StartedAt: startedAt, + EndedAt: endedAt, + SummaryMarkdown: summaryMarkdown, + SummaryText: summaryText, + Transcript: transcript, + TranscriptSegments: segments, + Organizer: organizer, + Attendees: attendees, + Metadata: meeting.Metadata, + }, nil +} + +func normalizeSegments(segments []TranscriptSegment) ([]TranscriptSegment, error) { + if len(segments) == 0 { + return nil, nil + } + out := make([]TranscriptSegment, len(segments)) + var previousOffset float64 + havePreviousOffset := false + for idx, segment := range segments { + segment.Speaker = strings.TrimSpace(segment.Speaker) + if segment.Speaker == "" { + return nil, validationError("meeting.transcript_segments[%d].speaker is required", idx) + } + segment.Text = strings.TrimSpace(segment.Text) + if segment.Text == "" { + return nil, validationError("meeting.transcript_segments[%d].text is required", idx) + } + if segment.OffsetSeconds != nil { + offset := *segment.OffsetSeconds + if math.IsNaN(offset) || math.IsInf(offset, 0) || offset < 0 { + return nil, validationError("meeting.transcript_segments[%d].offset_seconds must be finite and non-negative", idx) + } + if havePreviousOffset && offset < previousOffset { + return nil, validationError("meeting.transcript_segments offsets must be non-decreasing") + } + previousOffset = offset + havePreviousOffset = true + } + out[idx] = segment + } + return out, nil +} + +func normalizePerson(field string, person MeetingPerson) (MeetingPerson, error) { + email, err := normalizeEmail(field+".email", person.Email) + if err != nil { + return MeetingPerson{}, err + } + return MeetingPerson{ + Name: strings.TrimSpace(person.Name), + Email: email, + }, nil +} + +func normalizeAttendees(attendees []MeetingPerson) ([]MeetingPerson, error) { + out := make([]MeetingPerson, 0, len(attendees)) + seen := make(map[string]struct{}, len(attendees)) + for idx := range attendees { + person, err := normalizePerson( + fmt.Sprintf("meeting.attendees[%d]", idx), + attendees[idx], + ) + if err != nil { + return nil, err + } + key := strings.ToLower(person.Email) + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + out = append(out, person) + } + return out, nil +} + +func normalizeEmail(field, value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "", validationError("%s is required", field) + } + if !utf8.ValidString(value) { + return "", validationError("%s must be valid UTF-8", field) + } + parsed, err := mail.ParseAddress(value) + if err != nil || parsed.Name != "" || !strings.EqualFold(parsed.Address, value) { + return "", validationError("%s must be one email address without a display name", field) + } + return strings.ToLower(parsed.Address), nil +} + +func parseTimestamp(field, value string) (time.Time, error) { + value = strings.TrimSpace(value) + if value == "" { + return time.Time{}, validationError("%s is required", field) + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return time.Time{}, validationError("%s must be an RFC3339 timestamp with an explicit offset", field) + } + return parsed.UTC(), nil +} + +func validateBoundedRequired(field, value string, maxChars int) error { + if value == "" { + return validationError("%s is required", field) + } + return validateBoundedOptional(field, value, maxChars) +} + +func validateBoundedOptional(field, value string, maxChars int) error { + if !utf8.ValidString(value) { + return validationError("%s must be valid UTF-8", field) + } + if utf8.RuneCountInString(value) > maxChars { + return validationError("%s must be at most %d characters", field, maxChars) + } + return nil +} + +func validationError(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrValidation, fmt.Sprintf(format, args...)) +} diff --git a/internal/meetingimport/models_test.go b/internal/meetingimport/models_test.go new file mode 100644 index 000000000..34304999d --- /dev/null +++ b/internal/meetingimport/models_test.go @@ -0,0 +1,181 @@ +package meetingimport + +import ( + "math" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func decodedValidRequest(t *testing.T) Request { + t.Helper() + req, err := DecodeRequest(strings.NewReader(validRequestJSON), MaxRequestBytes) + require.NoError(t, err) + return req +} + +func TestRequestNormalizeCanonicalizesValues(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := decodedValidRequest(t) + req.Meeting.Attendees = append(req.Meeting.Attendees, + MeetingPerson{Name: "Duplicate", Email: "attendee@EXAMPLE.com"}, + ) + + got, err := req.Normalize() + require.NoError(err) + + assert.Equal("local-meetings", got.Source.Identifier) + assert.Equal("Local Meetings", got.Source.DisplayName) + assert.Equal("user@example.com", got.Source.AccountEmail) + assert.Equal("42", got.Meeting.ExternalID) + assert.Equal("Weekly planning", got.Meeting.Title) + assert.Equal(time.Date(2026, 7, 23, 18, 0, 0, 0, time.UTC), got.Meeting.StartedAt) + require.NotNil(got.Meeting.EndedAt) + assert.Equal(time.Date(2026, 7, 23, 18, 30, 0, 0, time.UTC), *got.Meeting.EndedAt) + require.NotNil(got.Meeting.Organizer) + assert.Equal(MeetingPerson{Name: "Test Organizer", Email: "organizer@example.com"}, *got.Meeting.Organizer) + assert.Equal([]MeetingPerson{{Name: "Test Attendee", Email: "attendee@example.com"}}, got.Meeting.Attendees) + assert.Equal("Test Speaker", got.Meeting.TranscriptSegments[0].Speaker) + assert.Equal("Let's review the launch plan.", got.Meeting.TranscriptSegments[0].Text) +} + +func TestRequestNormalizeValidatesRequiredAndBoundedFields(t *testing.T) { + tests := []struct { + name string + mutate func(*Request) + }{ + {name: "source identifier required", mutate: func(r *Request) { r.Source.Identifier = " " }}, + {name: "source identifier character limit", mutate: func(r *Request) { r.Source.Identifier = strings.Repeat("é", 129) }}, + {name: "display name character limit", mutate: func(r *Request) { r.Source.DisplayName = strings.Repeat("é", 257) }}, + {name: "account email required", mutate: func(r *Request) { r.Source.AccountEmail = "" }}, + {name: "account email invalid", mutate: func(r *Request) { r.Source.AccountEmail = "not-an-email" }}, + {name: "account display address rejected", mutate: func(r *Request) { r.Source.AccountEmail = "User " }}, + {name: "external id required", mutate: func(r *Request) { r.Meeting.ExternalID = "" }}, + {name: "external id character limit", mutate: func(r *Request) { r.Meeting.ExternalID = strings.Repeat("é", 257) }}, + {name: "title character limit", mutate: func(r *Request) { r.Meeting.Title = strings.Repeat("é", 4097) }}, + {name: "started at required", mutate: func(r *Request) { r.Meeting.StartedAt = "" }}, + {name: "started at timezone required", mutate: func(r *Request) { r.Meeting.StartedAt = "2026-07-23T18:00:00" }}, + {name: "started at malformed", mutate: func(r *Request) { r.Meeting.StartedAt = "later" }}, + {name: "ended at timezone required", mutate: func(r *Request) { r.Meeting.EndedAt = "2026-07-23T18:30:00" }}, + {name: "ended before start", mutate: func(r *Request) { r.Meeting.EndedAt = "2026-07-23T10:59:59-07:00" }}, + {name: "organizer email required", mutate: func(r *Request) { r.Meeting.Organizer.Email = "" }}, + {name: "organizer email invalid", mutate: func(r *Request) { r.Meeting.Organizer.Email = "bad" }}, + {name: "attendee email required", mutate: func(r *Request) { r.Meeting.Attendees[0].Email = "" }}, + {name: "attendee email invalid", mutate: func(r *Request) { r.Meeting.Attendees[0].Email = "bad" }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := decodedValidRequest(t) + tt.mutate(&req) + _, err := req.Normalize() + require.Error(t, err) + assert.ErrorIs(t, err, ErrValidation) + }) + } +} + +func TestRequestNormalizeAcceptsMultibyteValuesAtCharacterLimits(t *testing.T) { + req := validImportRequest(t) + req.Source.Identifier = strings.Repeat("é", 128) + req.Source.DisplayName = strings.Repeat("é", 256) + req.Meeting.ExternalID = strings.Repeat("é", 256) + req.Meeting.Title = strings.Repeat("é", 4096) + + _, err := req.Normalize() + require.NoError(t, err) +} + +func TestRequestNormalizeValidatesMeetingContent(t *testing.T) { + tests := []struct { + name string + mutate func(*Request) + }{ + { + name: "summary and transcripts empty", + mutate: func(r *Request) { + r.Meeting.SummaryMarkdown = "" + r.Meeting.SummaryText = "" + r.Meeting.Transcript = "" + r.Meeting.TranscriptSegments = nil + }, + }, + { + name: "plain and structured transcript conflict", + mutate: func(r *Request) { r.Meeting.Transcript = "Speaker 1: duplicate" }, + }, + { + name: "segment speaker required", + mutate: func(r *Request) { r.Meeting.TranscriptSegments[0].Speaker = " " }, + }, + { + name: "segment text required", + mutate: func(r *Request) { r.Meeting.TranscriptSegments[0].Text = " " }, + }, + { + name: "negative segment offset", + mutate: func(r *Request) { + value := -1.0 + r.Meeting.TranscriptSegments[0].OffsetSeconds = &value + }, + }, + { + name: "nan segment offset", + mutate: func(r *Request) { + value := math.NaN() + r.Meeting.TranscriptSegments[0].OffsetSeconds = &value + }, + }, + { + name: "infinite segment offset", + mutate: func(r *Request) { + value := math.Inf(1) + r.Meeting.TranscriptSegments[0].OffsetSeconds = &value + }, + }, + { + name: "decreasing segment offsets", + mutate: func(r *Request) { + later := 8.0 + earlier := 7.0 + r.Meeting.TranscriptSegments = append(r.Meeting.TranscriptSegments, + TranscriptSegment{Speaker: "Speaker 2", Text: "Second", OffsetSeconds: &later}, + TranscriptSegment{Speaker: "Speaker 1", Text: "Third", OffsetSeconds: &earlier}, + ) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := decodedValidRequest(t) + tt.mutate(&req) + _, err := req.Normalize() + require.Error(t, err) + assert.ErrorIs(t, err, ErrValidation) + }) + } +} + +func TestRequestNormalizeAllowsEqualTimesAndPlainTranscript(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + req := decodedValidRequest(t) + req.Meeting.EndedAt = req.Meeting.StartedAt + req.Meeting.TranscriptSegments = nil + req.Meeting.Transcript = "\nSpeaker 1: hello\nSpeaker 2: hi\n" + + got, err := req.Normalize() + require.NoError(err) + + assert.Equal("Speaker 1: hello\nSpeaker 2: hi", got.Meeting.Transcript) + assert.Empty(got.Meeting.TranscriptSegments) + require.NotNil(got.Meeting.EndedAt) + assert.True(got.Meeting.EndedAt.Equal(got.Meeting.StartedAt)) +} diff --git a/internal/store/account_identities.go b/internal/store/account_identities.go index 1b1ae8303..85289d93d 100644 --- a/internal/store/account_identities.go +++ b/internal/store/account_identities.go @@ -116,13 +116,13 @@ func looksLikeEmail(addr string) bool { // column accommodates email, phone E.164, and synthetic identifiers like // chat handles where case can be significant). // -// Confirming a brand new (source_id, address) pair changes which -// participants are owners for the source, so it bumps both the identity -// revision (the owner_participants cache dataset depends on it) and the -// account-identity revision (the message-baked is_from_me flag depends on -// it and can only be repaired by a full cache rebuild). Merging a new -// signal into an already-confirmed address does not change that mapping, -// so it leaves both revisions untouched. +// Confirming a brand new (source_id, address) pair changes which participants +// are owners for the source. The same transaction repairs is_from_me in the +// primary store and bumps both the identity revision (the owner_participants +// cache dataset depends on it) and the account-identity revision (Parquet +// shards bake the flag and need a full cache rebuild). Merging a new signal +// into an already-confirmed address does not change that mapping, so it leaves +// attribution and both revisions untouched. // // Concurrency: the read-modify-write runs inside a transaction that first // takes lockIdentityMutationTx's write lock, mirroring LinkParticipants so @@ -141,6 +141,48 @@ func (s *Store) AddAccountIdentityContext( ctx context.Context, sourceID int64, address, signal string, +) error { + return s.addAccountIdentityContext( + ctx, + sourceID, + address, + signal, + func(ctx context.Context, tx *loggedTx) error { + return refreshSourceMessageAttributionContext(ctx, tx, sourceID, "") + }, + ) +} + +// AddAccountIdentityAndRefreshMessageAttributionContext confirms an identity +// and, only when the identity is brand new, marks matching earlier messages in +// the source as sent by the account. Identity creation and attribution repair +// run in the same transaction. excludeSourceMessageID keeps the meeting +// currently being retried on its normal persistence path. +func (s *Store) AddAccountIdentityAndRefreshMessageAttributionContext( + ctx context.Context, + sourceID int64, + address, signal, excludeSourceMessageID string, +) error { + return s.addAccountIdentityContext( + ctx, + sourceID, + address, + signal, + func(ctx context.Context, tx *loggedTx) error { + return refreshSourceMessageAttributionContext( + ctx, tx, sourceID, excludeSourceMessageID, + ) + }, + ) +} + +type accountIdentityInsertHook func(context.Context, *loggedTx) error + +func (s *Store) addAccountIdentityContext( + ctx context.Context, + sourceID int64, + address, signal string, + onInsert accountIdentityInsertHook, ) error { addr := strings.TrimSpace(address) if addr == "" { @@ -156,7 +198,7 @@ func (s *Store) AddAccountIdentityContext( if err := ctx.Err(); err != nil { return err } - err := s.addAccountIdentityOnce(ctx, sourceID, addr, signal, match) + err := s.addAccountIdentityOnce(ctx, sourceID, addr, signal, match, onInsert) if err == nil { return nil } @@ -171,7 +213,11 @@ func (s *Store) AddAccountIdentityContext( // transaction. The caller's retry loop handles unique-violation // (concurrent INSERT race) and busy/snapshot errors (SQLite). func (s *Store) addAccountIdentityOnce( - ctx context.Context, sourceID int64, addr, signal string, match identifierMatch, + ctx context.Context, + sourceID int64, + addr, signal string, + match identifierMatch, + onInsert accountIdentityInsertHook, ) error { return s.withTxContext(ctx, func(tx *loggedTx) error { if err := s.lockIdentityMutationTxContext(ctx, tx); err != nil { @@ -201,6 +247,11 @@ func (s *Store) addAccountIdentityOnce( if err := s.bumpAccountIdentityRevisionContext(ctx, tx); err != nil { return err } + if onInsert != nil { + if err := onInsert(ctx, tx); err != nil { + return err + } + } case err != nil: return fmt.Errorf("read existing source_signal: %w", err) default: @@ -334,7 +385,10 @@ func (s *Store) RemoveAccountIdentityContext( if _, err := s.bumpIdentityRevisionContext(ctx, tx); err != nil { return err } - return s.bumpAccountIdentityRevisionContext(ctx, tx) + if err := s.bumpAccountIdentityRevisionContext(ctx, tx); err != nil { + return err + } + return refreshSourceMessageAttributionContext(ctx, tx, sourceID, "") }) if err != nil { return 0, err diff --git a/internal/store/account_identities_test.go b/internal/store/account_identities_test.go index 212061956..4ea1f6d2f 100644 --- a/internal/store/account_identities_test.go +++ b/internal/store/account_identities_test.go @@ -1,14 +1,176 @@ package store_test import ( + "database/sql" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/testutil" "go.kenn.io/msgvault/internal/testutil/storetest" ) +func TestRemoveAccountIdentityRecomputesOnlyIdentityDerivedAttribution(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := storetest.New(t) + st := f.Store + + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + identityDerived := f.NewMessage().WithSourceMessageID("identity-derived").Build() + identityDerived.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + identityDerivedID, err := st.UpsertMessage(identityDerived) + require.NoError(err, "persist identity-derived candidate") + + sourceNative := f.NewMessage(). + WithSourceMessageID("source-native"). + WithIsFromMe(true). + Build() + sourceNative.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + sourceNativeID, err := st.UpsertMessage(sourceNative) + require.NoError(err, "persist source-native message") + _, err = st.DB().Exec( + st.Rebind(`UPDATE messages SET source_is_from_me = NULL WHERE id = ?`), + sourceNativeID, + ) + require.NoError(err, "simulate legacy message without attribution provenance") + _, err = st.DB().Exec( + st.Rebind(`DELETE FROM applied_migrations WHERE name = ?`), + "message_attribution_provenance_v2", + ) + require.NoError(err, "reset attribution migration sentinel") + require.NoError(st.InitSchema(), "initialize legacy attribution provenance") + + require.NoError(st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual")) + afterAdd, err := st.GetMessageIsFromMe(identityDerivedID) + require.NoError(err, "read identity-derived attribution after add") + assert.True(afterAdd) + + removed, err := st.RemoveAccountIdentity(f.Source.ID, "owner@example.com") + require.NoError(err, "RemoveAccountIdentity") + require.Equal(int64(1), removed) + + afterRemove, err := st.GetMessageIsFromMe(identityDerivedID) + require.NoError(err, "read identity-derived attribution after remove") + assert.False(afterRemove, "removing the last matching identity must clear derived attribution") + nativeAfterRemove, err := st.GetMessageIsFromMe(sourceNativeID) + require.NoError(err, "read source-native attribution after remove") + assert.True(nativeAfterRemove, "identity removal must preserve source-native attribution") +} + +func TestAddAccountIdentityUpdatesOnlyChangedMessageAttribution(t *testing.T) { + testutil.SkipIfPostgres(t, "SQLite audit trigger measures updated message rows") + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + ownerID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + otherID := f.EnsureParticipant("other@example.com", "Other", "example.com") + + ownerMessage := f.NewMessage().WithSourceMessageID("owner-message").Build() + ownerMessage.SenderID = sql.NullInt64{Int64: ownerID, Valid: true} + ownerMessageID, err := st.UpsertMessage(ownerMessage) + require.NoError(err, "persist matching message") + + otherMessage := f.NewMessage().WithSourceMessageID("other-message").Build() + otherMessage.SenderID = sql.NullInt64{Int64: otherID, Valid: true} + otherMessageID, err := st.UpsertMessage(otherMessage) + require.NoError(err, "persist unrelated message") + + _, err = st.DB().Exec(` + DROP TRIGGER IF EXISTS trg_messages_last_modified; + CREATE TABLE message_update_audit (message_id INTEGER NOT NULL); + CREATE TRIGGER audit_message_update + AFTER UPDATE ON messages + BEGIN + INSERT INTO message_update_audit (message_id) VALUES (NEW.id); + END; + `) + require.NoError(err, "install message update audit") + + require.NoError( + st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual"), + "add matching identity", + ) + + var ownerUpdates, otherUpdates int + require.NoError(st.DB().QueryRow( + `SELECT COUNT(*) FROM message_update_audit WHERE message_id = ?`, + ownerMessageID, + ).Scan(&ownerUpdates)) + require.NoError(st.DB().QueryRow( + `SELECT COUNT(*) FROM message_update_audit WHERE message_id = ?`, + otherMessageID, + ).Scan(&otherUpdates)) + assert.Equal(1, ownerUpdates, "matching attribution must be updated") + assert.Zero(otherUpdates, "unchanged attribution must not rewrite the message") +} + +func TestUpsertMessageDerivesAttributionFromConfirmedIdentity(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + require.NoError( + st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual"), + "confirm identity before ingestion", + ) + + message := f.NewMessage().WithSourceMessageID("import-after-confirmation").Build() + message.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist message") + + var isFromMe, sourceIsFromMe, identityIsFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me, source_is_from_me, identity_is_from_me + FROM messages + WHERE id = ? + `), messageID).Scan(&isFromMe, &sourceIsFromMe, &identityIsFromMe)) + assert.True(isFromMe, "confirmed sender must be attributed during initial persistence") + assert.False(sourceIsFromMe, "incoming message did not carry source-native attribution") + assert.True(identityIsFromMe, "confirmed sender attribution must retain identity provenance") +} + +func TestUpsertMessagePreservesRepairedIdentityAttribution(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + message := f.NewMessage().WithSourceMessageID("resynced-message").Build() + message.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist unattributed message") + + require.NoError( + st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual"), + "confirm identity and repair existing message", + ) + isFromMe, err := st.GetMessageIsFromMe(messageID) + require.NoError(err, "read repaired attribution") + require.True(isFromMe, "identity confirmation must repair the existing message") + + message.Subject = sql.NullString{String: "resynced subject", Valid: true} + _, err = st.UpsertMessage(message) + require.NoError(err, "re-upsert message without caller-derived attribution") + + var sourceIsFromMe, identityIsFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT is_from_me, source_is_from_me, identity_is_from_me + FROM messages + WHERE id = ? + `), messageID).Scan(&isFromMe, &sourceIsFromMe, &identityIsFromMe)) + assert.True(isFromMe, "re-sync must preserve attribution from the confirmed sender") + assert.False(sourceIsFromMe, "re-sync did not introduce source-native attribution") + assert.True(identityIsFromMe, "re-sync must preserve identity-derived provenance") +} + func TestAddAndListAccountIdentities(t *testing.T) { require := require.New(t) assert := assert.New(t) diff --git a/internal/store/attribution_provenance_migration_test.go b/internal/store/attribution_provenance_migration_test.go new file mode 100644 index 000000000..094a6e9bc --- /dev/null +++ b/internal/store/attribution_provenance_migration_test.go @@ -0,0 +1,325 @@ +package store_test + +import ( + "database/sql" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/gcal" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil/storetest" +) + +func TestInitSchemaBackfillsLegacyIdentityDerivedAttribution(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + + type migratedMessage struct { + sourceID int64 + messageID int64 + } + migrated := make([]migratedMessage, 0, 2) + for _, sourceType := range []string{"granola", "circleback"} { + src, err := st.GetOrCreateSource(sourceType, sourceType+"-account") + require.NoError(err, "create %s source", sourceType) + convID, err := st.EnsureConversationWithType( + src.ID, + sourceType+"-conversation", + "meeting", + "Legacy meeting", + ) + require.NoError(err, "create %s conversation", sourceType) + messageID, err := st.UpsertMessage(&store.Message{ + SourceID: src.ID, + ConversationID: convID, + SourceMessageID: sourceType + "-message", + MessageType: "meeting", + SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + IsFromMe: true, + }) + require.NoError(err, "persist legacy %s message", sourceType) + require.NoError( + st.AddAccountIdentity(src.ID, "owner@example.com", "account-identifier"), + "add %s identity", + sourceType, + ) + migrated = append(migrated, migratedMessage{sourceID: src.ID, messageID: messageID}) + } + + nativeMessage := f.NewMessage(). + WithSourceMessageID("source-native-control"). + WithIsFromMe(true). + Build() + nativeMessage.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + nativeMessageID, err := st.UpsertMessage(nativeMessage) + require.NoError(err, "persist source-native control") + + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages + SET source_is_from_me = NULL, identity_is_from_me = FALSE + WHERE id IN (?, ?, ?) + `), migrated[0].messageID, migrated[1].messageID, nativeMessageID) + require.NoError(err, "simulate rows written before attribution provenance") + _, err = st.DB().Exec(st.Rebind(` + DELETE FROM applied_migrations + WHERE name = ? + `), "message_attribution_provenance_v2") + require.NoError(err, "reset attribution migration sentinel") + _, err = st.DB().Exec(` + INSERT INTO applied_migrations (name) + VALUES ('message_attribution_provenance_v1') + ON CONFLICT (name) DO NOTHING + `) + require.NoError(err, "record prior attribution migration") + + require.NoError(st.InitSchema(), "run production schema migration") + + for _, message := range migrated { + var sourceDerived sql.NullBool + var identityDerived bool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me, identity_is_from_me + FROM messages + WHERE id = ? + `), message.messageID).Scan(&sourceDerived, &identityDerived), + "read migrated provenance for message %d", + message.messageID, + ) + assert.True(sourceDerived.Valid, "migration must assign explicit source provenance") + assert.False(sourceDerived.Bool, "legacy meeting attribution is not source-native") + assert.True(identityDerived, "legacy meeting attribution must remain identity-derived") + + removed, removeErr := st.RemoveAccountIdentity(message.sourceID, "owner@example.com") + require.NoError(removeErr, "remove migrated identity") + require.Equal(int64(1), removed) + isFromMe, attributionErr := st.GetMessageIsFromMe(message.messageID) + require.NoError(attributionErr, "read attribution after identity removal") + assert.False(isFromMe, "identity removal must clear migrated attribution") + } + + var nativeSourceDerived sql.NullBool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me + FROM messages + WHERE id = ? + `), nativeMessageID).Scan(&nativeSourceDerived), + "read source-native control provenance", + ) + assert.True(nativeSourceDerived.Valid, "migration must initialize source-native provenance") + assert.True(nativeSourceDerived.Bool, "migration must preserve source-native attribution") +} + +func TestInitSchemaClearsLegacyAttributionWithoutRemainingIdentity(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + senderID := f.EnsureParticipant("former-owner@example.com", "Former Owner", "example.com") + + src, err := st.GetOrCreateSource("granola", "removed-identity-account") + require.NoError(err, "create granola source") + convID, err := st.EnsureConversationWithType( + src.ID, + "removed-identity-conversation", + "meeting", + "Legacy meeting", + ) + require.NoError(err, "create granola conversation") + messageID, err := st.UpsertMessage(&store.Message{ + SourceID: src.ID, + ConversationID: convID, + SourceMessageID: "removed-identity-message", + MessageType: "meeting", + SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + IsFromMe: true, + }) + require.NoError(err, "persist stale legacy attribution") + + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages + SET source_is_from_me = NULL, identity_is_from_me = FALSE + WHERE id = ? + `), messageID) + require.NoError(err, "simulate message written before attribution provenance") + _, err = st.DB().Exec(st.Rebind(` + DELETE FROM applied_migrations + WHERE name = ? + `), "message_attribution_provenance_v2") + require.NoError(err, "reset attribution migration sentinel") + + require.NoError(st.InitSchema(), "run production schema migration") + + var sourceDerived, identityDerived, isFromMe bool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me, identity_is_from_me, is_from_me + FROM messages + WHERE id = ? + `), messageID).Scan(&sourceDerived, &identityDerived, &isFromMe), + "read reconciled attribution", + ) + assert.False(sourceDerived, "legacy Granola attribution is not source-native") + assert.False(identityDerived, "removed identity must not remain as provenance") + assert.False(isFromMe, "migration must clear attribution whose identity was already removed") +} + +func TestInitSchemaReconcilesConfirmedIdentityForEverySource(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + require.NoError( + st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual"), + "confirm Gmail identity", + ) + message := f.NewMessage().WithSourceMessageID("legacy-gmail-identity").Build() + message.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist matching Gmail message") + + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages + SET is_from_me = FALSE, + source_is_from_me = NULL, + identity_is_from_me = FALSE + WHERE id = ? + `), messageID) + require.NoError(err, "simulate stale legacy Gmail attribution") + _, err = st.DB().Exec(st.Rebind(` + DELETE FROM applied_migrations + WHERE name = ? + `), "message_attribution_provenance_v2") + require.NoError(err, "reset attribution migration sentinel") + + revisionBefore, err := st.AccountIdentityRevision() + require.NoError(err, "read account identity revision before migration") + require.NoError(st.InitSchema(), "run production schema migration") + + var sourceDerived, identityDerived, isFromMe bool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me, identity_is_from_me, is_from_me + FROM messages + WHERE id = ? + `), messageID).Scan(&sourceDerived, &identityDerived, &isFromMe), + "read reconciled Gmail attribution", + ) + assert.False(sourceDerived, "incoming Gmail message was not source-native") + assert.True(identityDerived, "confirmed identity must be reconciled for Gmail") + assert.True(isFromMe, "effective attribution must include the confirmed identity") + + revisionAfter, err := st.AccountIdentityRevision() + require.NoError(err, "read account identity revision after migration") + assert.Equal( + revisionBefore+1, + revisionAfter, + "provenance migration must invalidate caches atomically", + ) +} + +func TestInitSchemaBackfillsLegacyCalendarAttributionProvenance(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + src, err := st.GetOrCreateSource(gcal.SourceType, "owner@example.com/primary") + require.NoError(err, "create calendar source") + require.NoError( + st.AddAccountIdentity(src.ID, "owner@example.com", "account-email"), + "confirm configured calendar account", + ) + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + + persistLegacyEvent := func(sourceMessageID, raw string) int64 { + convID, ensureErr := st.EnsureConversationWithType( + src.ID, + sourceMessageID+"-conversation", + gcal.ConversationType, + "Legacy calendar event", + ) + require.NoError(ensureErr, "create %s conversation", sourceMessageID) + messageID, upsertErr := st.UpsertMessage(&store.Message{ + SourceID: src.ID, + ConversationID: convID, + SourceMessageID: sourceMessageID, + MessageType: gcal.MessageTypeCalendarEvent, + SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + IsFromMe: true, + }) + require.NoError(upsertErr, "persist legacy %s message", sourceMessageID) + require.NoError( + st.UpsertMessageRawWithFormat(messageID, []byte(raw), gcal.RawFormat), + "persist legacy %s raw event", + sourceMessageID, + ) + return messageID + } + + identityMessageID := persistLegacyEvent( + "configured-account-organizer", + `{"organizer":{"email":"owner@example.com"}}`, + ) + nativeMessageID := persistLegacyEvent( + "google-self-organizer", + `{"organizer":{"email":"owner@example.com","self":true}}`, + ) + + _, err = st.DB().Exec(st.Rebind(` + UPDATE messages + SET source_is_from_me = NULL, + identity_is_from_me = FALSE, + is_from_me = TRUE + WHERE id IN (?, ?) + `), identityMessageID, nativeMessageID) + require.NoError(err, "simulate calendar rows written before attribution provenance") + _, err = st.DB().Exec(st.Rebind(` + DELETE FROM applied_migrations + WHERE name = ? + `), "message_attribution_provenance_v2") + require.NoError(err, "reset attribution migration sentinel") + + require.NoError(st.InitSchema(), "run production schema migration") + + var identitySourceDerived, identityDerived bool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me, identity_is_from_me + FROM messages + WHERE id = ? + `), identityMessageID).Scan(&identitySourceDerived, &identityDerived), + "read configured-account provenance", + ) + assert.False(identitySourceDerived, "configured-account match is not source-native") + assert.True(identityDerived, "configured-account match must remain identity-derived") + + var nativeSourceDerived bool + require.NoError( + st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me + FROM messages + WHERE id = ? + `), nativeMessageID).Scan(&nativeSourceDerived), + "read Organizer.Self provenance", + ) + assert.True(nativeSourceDerived, "Organizer.Self must remain source-native") + + removed, err := st.RemoveAccountIdentity(src.ID, "owner@example.com") + require.NoError(err, "remove configured calendar identity") + require.Equal(int64(1), removed) + + identityIsFromMe, err := st.GetMessageIsFromMe(identityMessageID) + require.NoError(err, "read configured-account attribution after identity removal") + assert.False(identityIsFromMe, "identity removal must clear configured-account attribution") + nativeIsFromMe, err := st.GetMessageIsFromMe(nativeMessageID) + require.NoError(err, "read Organizer.Self attribution after identity removal") + assert.True(nativeIsFromMe, "identity removal must preserve Organizer.Self attribution") +} diff --git a/internal/store/dialect_pg.go b/internal/store/dialect_pg.go index b4050027b..67640ed45 100644 --- a/internal/store/dialect_pg.go +++ b/internal/store/dialect_pg.go @@ -338,6 +338,8 @@ func (d *PostgreSQLDialect) LegacyColumnMigrations() []ColumnMigration { {`ALTER TABLE participants ADD COLUMN IF NOT EXISTS phone_number TEXT`, "phone_number"}, {`ALTER TABLE participants ADD COLUMN IF NOT EXISTS canonical_id TEXT`, "canonical_id"}, {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS sender_id BIGINT REFERENCES participants(id)`, "sender_id"}, + {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS source_is_from_me BOOLEAN`, "source_is_from_me"}, + {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS identity_is_from_me BOOLEAN NOT NULL DEFAULT FALSE`, "identity_is_from_me"}, {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS message_type TEXT NOT NULL DEFAULT 'email'`, "message_type"}, {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_count INTEGER DEFAULT 0`, "attachment_count"}, {`ALTER TABLE messages ADD COLUMN IF NOT EXISTS deleted_from_source_at TIMESTAMPTZ`, "deleted_from_source_at"}, diff --git a/internal/store/dialect_sqlite.go b/internal/store/dialect_sqlite.go index b0809e818..04d4eb584 100644 --- a/internal/store/dialect_sqlite.go +++ b/internal/store/dialect_sqlite.go @@ -274,6 +274,8 @@ func (d *SQLiteDialect) LegacyColumnMigrations() []ColumnMigration { {`ALTER TABLE participants ADD COLUMN phone_number TEXT`, "phone_number"}, {`ALTER TABLE participants ADD COLUMN canonical_id TEXT`, "canonical_id"}, {`ALTER TABLE messages ADD COLUMN sender_id INTEGER REFERENCES participants(id)`, "sender_id"}, + {`ALTER TABLE messages ADD COLUMN source_is_from_me BOOLEAN`, "source_is_from_me"}, + {`ALTER TABLE messages ADD COLUMN identity_is_from_me BOOLEAN NOT NULL DEFAULT FALSE`, "identity_is_from_me"}, {`ALTER TABLE messages ADD COLUMN message_type TEXT NOT NULL DEFAULT 'email'`, "message_type"}, {`ALTER TABLE messages ADD COLUMN attachment_count INTEGER DEFAULT 0`, "attachment_count"}, {`ALTER TABLE messages ADD COLUMN deleted_from_source_at DATETIME`, "deleted_from_source_at"}, diff --git a/internal/store/messages.go b/internal/store/messages.go index 3dd94ecfa..a36028a5c 100644 --- a/internal/store/messages.go +++ b/internal/store/messages.go @@ -25,6 +25,24 @@ type querier interface { QueryRow(query string, args ...any) *sql.Row } +type contextStatementQuerier interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +type boundQuerier struct { + ctx context.Context + q contextStatementQuerier +} + +func (q boundQuerier) Exec(query string, args ...any) (sql.Result, error) { + return q.q.ExecContext(q.ctx, query, args...) +} + +func (q boundQuerier) QueryRow(query string, args ...any) *sql.Row { + return q.q.QueryRowContext(q.ctx, query, args...) +} + type contextQuerier interface { ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) } @@ -37,6 +55,14 @@ type RecipientSet struct { DisplayNames []string } +// ParticipantPersistData describes one email participant to resolve inside a +// message persistence transaction. +type ParticipantPersistData struct { + EmailAddress string + DisplayName string + Domain string +} + // MessagePersistData bundles everything needed to atomically // persist a message and its related rows in a single transaction. type MessagePersistData struct { @@ -76,13 +102,16 @@ type Message struct { InternalDate sql.NullTime SenderID sql.NullInt64 IsFromMe bool - Subject sql.NullString - Snippet sql.NullString - SizeEstimate int64 - HasAttachments bool - AttachmentCount int - DeletedAt sql.NullTime - ArchivedAt time.Time + // IdentityDerivedIsFromMe reports that IsFromMe came from a confirmed + // account identity rather than a source-native sent-by-me signal. + IdentityDerivedIsFromMe bool + Subject sql.NullString + Snippet sql.NullString + SizeEstimate int64 + HasAttachments bool + AttachmentCount int + DeletedAt sql.NullTime + ArchivedAt time.Time } // MessageMetadataRecord is the archive identity and optional provider metadata @@ -584,13 +613,47 @@ func (s *Store) EnsureConversation(sourceID int64, sourceConversationID, title s // upsertMessageSQL returns the message upsert SQL with dialect-specific timestamp. func upsertMessageSQL(now string) string { return fmt.Sprintf(` + WITH attribution AS ( + SELECT + CAST(? AS BOOLEAN) AS source_is_from_me, + ( + CAST(? AS BOOLEAN) + OR EXISTS ( + SELECT 1 + FROM account_identities ai + JOIN participants p ON p.id = ? + WHERE ai.source_id = ? + AND p.email_address IS NOT NULL + AND LOWER(p.email_address) = LOWER(ai.address) + ) + OR EXISTS ( + SELECT 1 + FROM account_identities ai + JOIN participant_identifiers pi ON pi.participant_id = ? + WHERE ai.source_id = ? + AND ( + (pi.identifier_type = 'email' + AND LOWER(pi.identifier_value) = LOWER(ai.address)) + OR (pi.identifier_type <> 'email' + AND pi.identifier_value = ai.address) + ) + ) + ) AS identity_is_from_me + ) INSERT INTO messages ( conversation_id, source_id, source_message_id, rfc822_message_id, message_type, - sent_at, received_at, internal_date, sender_id, is_from_me, + sent_at, received_at, internal_date, sender_id, + is_from_me, source_is_from_me, identity_is_from_me, subject, snippet, size_estimate, has_attachments, attachment_count, archived_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, %s) + ) + SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, + (source_is_from_me OR identity_is_from_me), + source_is_from_me, identity_is_from_me, + ?, ?, ?, ?, ?, %s + FROM attribution + WHERE TRUE ON CONFLICT(source_id, source_message_id) DO UPDATE SET embed_gen = CASE WHEN COALESCE(messages.subject, '') <> COALESCE(excluded.subject, '') THEN NULL @@ -603,6 +666,8 @@ func upsertMessageSQL(now string) string { internal_date = excluded.internal_date, sender_id = excluded.sender_id, is_from_me = excluded.is_from_me, + source_is_from_me = excluded.source_is_from_me, + identity_is_from_me = excluded.identity_is_from_me, subject = excluded.subject, snippet = excluded.snippet, size_estimate = excluded.size_estimate, @@ -617,10 +682,15 @@ func (s *Store) UpsertMessage(msg *Message) (int64, error) { func upsertMessageWith(q querier, d Dialect, msg *Message) (int64, error) { sql := upsertMessageSQL(d.Now()) + sourceIsFromMe := msg.IsFromMe && !msg.IdentityDerivedIsFromMe + identityIsFromMe := msg.IsFromMe && msg.IdentityDerivedIsFromMe args := []any{ + sourceIsFromMe, identityIsFromMe, + msg.SenderID, msg.SourceID, + msg.SenderID, msg.SourceID, msg.ConversationID, msg.SourceID, msg.SourceMessageID, msg.RFC822MessageID, msg.MessageType, - msg.SentAt, msg.ReceivedAt, msg.InternalDate, msg.SenderID, msg.IsFromMe, + msg.SentAt, msg.ReceivedAt, msg.InternalDate, msg.SenderID, msg.Subject, msg.Snippet, msg.SizeEstimate, msg.HasAttachments, msg.AttachmentCount, } @@ -779,85 +849,257 @@ func (s *Store) GetMessageRaw(messageID int64) ([]byte, error) { return compressed, nil } +// GetMessageIsFromMe returns the baked account-attribution flag for a message. +func (s *Store) GetMessageIsFromMe(messageID int64) (bool, error) { + var isFromMe bool + err := s.db.QueryRow(` + SELECT COALESCE(is_from_me, FALSE) + FROM messages + WHERE id = ? + `, messageID).Scan(&isFromMe) + return isFromMe, err +} + +const messageIdentityAttributionMatch = `( + EXISTS ( + SELECT 1 + FROM account_identities ai + JOIN participants p ON p.id = messages.sender_id + WHERE ai.source_id = messages.source_id + AND p.email_address IS NOT NULL + AND LOWER(p.email_address) = LOWER(ai.address) + ) + OR EXISTS ( + SELECT 1 + FROM account_identities ai + JOIN participant_identifiers pi ON pi.participant_id = messages.sender_id + WHERE ai.source_id = messages.source_id + AND ( + (pi.identifier_type = 'email' AND LOWER(pi.identifier_value) = LOWER(ai.address)) + OR (pi.identifier_type <> 'email' AND pi.identifier_value = ai.address) + ) + ) +)` + +const messageSourceAttribution = `COALESCE(source_is_from_me, FALSE)` + +func refreshSourceMessageAttributionContext( + ctx context.Context, + q contextQuerier, + sourceID int64, + excludeSourceMessageID string, +) error { + // InitSchema assigns source provenance to every legacy row once. Runtime + // identity changes therefore only need to update the derived and effective + // values, and the change predicate avoids firing last_modified triggers for + // messages whose attribution already agrees with the current identity set. + _, err := q.ExecContext(ctx, fmt.Sprintf(` + UPDATE messages + SET identity_is_from_me = %[2]s, + is_from_me = (%[1]s OR %[2]s) + WHERE source_id = ? + AND (? = '' OR source_message_id <> ?) + AND ( + identity_is_from_me <> %[2]s + OR is_from_me IS NULL + OR is_from_me <> (%[1]s OR %[2]s) + ) + `, messageSourceAttribution, messageIdentityAttributionMatch), + sourceID, excludeSourceMessageID, excludeSourceMessageID) + if err != nil { + return fmt.Errorf("refresh source message attribution: %w", err) + } + return nil +} + +func refreshParticipantMessageAttributionContext( + ctx context.Context, + q contextQuerier, + participantIDs ...int64, +) error { + ids := make([]int64, 0, len(participantIDs)) + for _, participantID := range participantIDs { + if participantID != 0 { + ids = append(ids, participantID) + } + } + if len(ids) == 0 { + return nil + } + + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",") + args := make([]any, len(ids)) + for i, participantID := range ids { + args[i] = participantID + } + _, err := q.ExecContext(ctx, fmt.Sprintf(` + UPDATE messages + SET identity_is_from_me = %[2]s, + is_from_me = (%[1]s OR %[2]s) + WHERE sender_id IN (`+placeholders+`) + AND ( + identity_is_from_me <> %[2]s + OR is_from_me IS NULL + OR is_from_me <> (%[1]s OR %[2]s) + ) + `, messageSourceAttribution, messageIdentityAttributionMatch), args...) + if err != nil { + return fmt.Errorf("refresh participant message attribution: %w", err) + } + return nil +} + // PersistMessage atomically stores a message and its requested related // snapshots in one transaction. Existing email callers persist body, raw MIME, // recipients, and labels; non-email callers can additionally include // conversation state, metadata, provider raw data, and FTS. func (s *Store) PersistMessage(data *MessagePersistData) (int64, error) { + return s.PersistMessageContext(context.Background(), data) +} + +// PersistMessageContext is the request-aware form of PersistMessage. Every +// statement in the transaction observes ctx, and cancellation rolls the full +// message snapshot back. +func (s *Store) PersistMessageContext(ctx context.Context, data *MessagePersistData) (int64, error) { if data == nil || data.Message == nil { return 0, errors.New("persist message requires a message") } + return s.persistMessageWithParticipantsContext(ctx, nil, func([]int64) *MessagePersistData { + return data + }) +} + +// PersistMessageWithParticipantsContext resolves participants and persists the +// message snapshot in one request-aware transaction. build receives IDs in the +// same order as participants and must return the snapshot that references them. +func (s *Store) PersistMessageWithParticipantsContext( + ctx context.Context, + participants []ParticipantPersistData, + build func(participantIDs []int64) *MessagePersistData, +) (int64, error) { + if build == nil { + return 0, errors.New("persist message requires a participant builder") + } + return s.persistMessageWithParticipantsContext(ctx, participants, build) +} + +func (s *Store) persistMessageWithParticipantsContext( + ctx context.Context, + participants []ParticipantPersistData, + build func(participantIDs []int64) *MessagePersistData, +) (int64, error) { var messageID int64 - err := s.withTx(func(tx *loggedTx) error { - message := data.Message - if data.Conversation != nil { - conversationID, err := ensureConversationWithType( - tx, s.dialect, data.Message.SourceID, - data.Conversation.SourceConversationID, - data.Conversation.ConversationType, - data.Conversation.Title, + err := s.withTxContext(ctx, func(tx *loggedTx) error { + q := boundQuerier{ctx: ctx, q: tx} + participantIDs := make([]int64, len(participants)) + for idx, participant := range participants { + if err := ctx.Err(); err != nil { + return err + } + participantID, err := ensureParticipantWith( + q, + s.dialect, + participant.EmailAddress, + participant.DisplayName, + participant.Domain, ) if err != nil { - return fmt.Errorf("ensure conversation: %w", err) - } - if err := replaceConversationParticipantsTx( - tx, s.dialect, conversationID, data.Conversation.Participants, - ); err != nil { - return fmt.Errorf("replace conversation participants: %w", err) + return fmt.Errorf("ensure participant %d: %w", idx, err) } - messageCopy := *data.Message - messageCopy.ConversationID = conversationID - message = &messageCopy + participantIDs[idx] = participantID } - id, err := upsertMessageWith(tx, s.dialect, message) + if err := ctx.Err(); err != nil { + return err + } + data := build(participantIDs) + id, err := s.persistMessageWith(ctx, q, data) if err != nil { - return fmt.Errorf("upsert message: %w", err) + return err } messageID = id - if data.Metadata != nil { - if err := setMessageMetadataWith(tx, s.dialect, messageID, *data.Metadata); err != nil { - return fmt.Errorf("set metadata: %w", err) - } - } + return nil + }) + return messageID, err +} - if err := upsertMessageBody( - tx, s.dialect, s.fts5Available, messageID, data.BodyText, data.BodyHTML, +func (s *Store) persistMessageWith( + ctx context.Context, + q querier, + data *MessagePersistData, +) (int64, error) { + if data == nil || data.Message == nil { + return 0, errors.New("persist message requires a message") + } + message := data.Message + if data.Conversation != nil { + conversationID, err := ensureConversationWithType( + q, s.dialect, data.Message.SourceID, + data.Conversation.SourceConversationID, + data.Conversation.ConversationType, + data.Conversation.Title, + ) + if err != nil { + return 0, fmt.Errorf("ensure conversation: %w", err) + } + if err := replaceConversationParticipantsTx( + q, s.dialect, conversationID, data.Conversation.Participants, ); err != nil { - return fmt.Errorf("upsert body: %w", err) + return 0, fmt.Errorf("replace conversation participants: %w", err) } + messageCopy := *data.Message + messageCopy.ConversationID = conversationID + message = &messageCopy + } - if len(data.RawMIME) > 0 { - rawFormat := data.RawFormat - if rawFormat == "" { - rawFormat = "mime" - } - if err := upsertMessageRawWithFormat(tx, messageID, data.RawMIME, rawFormat); err != nil { - return fmt.Errorf("upsert raw: %w", err) - } + messageID, err := upsertMessageWith(q, s.dialect, message) + if err != nil { + return 0, fmt.Errorf("upsert message: %w", err) + } + if data.Metadata != nil { + if err := setMessageMetadataWith(q, s.dialect, messageID, *data.Metadata); err != nil { + return 0, fmt.Errorf("set metadata: %w", err) } + } - for _, rs := range data.Recipients { - if err := replaceMessageRecipientsTx(tx, messageID, rs); err != nil { - return fmt.Errorf("store %s recipients: %w", rs.Type, err) - } + if err := upsertMessageBody( + q, s.dialect, s.fts5Available, messageID, data.BodyText, data.BodyHTML, + ); err != nil { + return 0, fmt.Errorf("upsert body: %w", err) + } + + if len(data.RawMIME) > 0 { + if err := ctx.Err(); err != nil { + return 0, err + } + rawFormat := data.RawFormat + if rawFormat == "" { + rawFormat = "mime" + } + if err := upsertMessageRawWithFormat(q, messageID, data.RawMIME, rawFormat); err != nil { + return 0, fmt.Errorf("upsert raw: %w", err) } + } - if !data.PreserveLabels { - if err := replaceMessageLabelsTx(tx, messageID, data.LabelIDs); err != nil { - return fmt.Errorf("store labels: %w", err) - } + for _, rs := range data.Recipients { + if err := replaceMessageRecipientsTx(q, messageID, rs); err != nil { + return 0, fmt.Errorf("store %s recipients: %w", rs.Type, err) } - if data.FTS != nil && s.fts5Available { - fts := *data.FTS - fts.MessageID = messageID - if err := s.dialect.FTSUpsert(tx, fts); err != nil { - return fmt.Errorf("upsert fts: %w", err) - } + } + + if !data.PreserveLabels { + if err := replaceMessageLabelsTx(q, messageID, data.LabelIDs); err != nil { + return 0, fmt.Errorf("store labels: %w", err) } - return nil - }) - return messageID, err + } + if data.FTS != nil && s.fts5Available { + fts := *data.FTS + fts.MessageID = messageID + if err := s.dialect.FTSUpsert(q, fts); err != nil { + return 0, fmt.Errorf("upsert fts: %w", err) + } + } + return messageID, nil } // Participant represents a person in the participants table. @@ -876,6 +1118,32 @@ type Participant struct { // name and domain are left untouched on conflict to preserve any // hand-edited values. func (s *Store) EnsureParticipant(email, displayName, domain string) (int64, error) { + return s.EnsureParticipantContext(context.Background(), email, displayName, domain) +} + +// EnsureParticipantContext is the request-aware form of EnsureParticipant. +func (s *Store) EnsureParticipantContext( + ctx context.Context, + email, + displayName, + domain string, +) (int64, error) { + return ensureParticipantWith( + boundQuerier{ctx: ctx, q: s.db}, + s.dialect, + email, + displayName, + domain, + ) +} + +func ensureParticipantWith( + q querier, + dialect Dialect, + email, + displayName, + domain string, +) (int64, error) { // ON CONFLICT must mirror the partial unique index on // participants(email_address) WHERE email_address IS NOT NULL — both // PG and SQLite require the WHERE clause on the conflict target to @@ -883,13 +1151,13 @@ func (s *Store) EnsureParticipant(email, displayName, domain string) (int64, err // the same column) makes RETURNING fire for both INSERT and the // existing-row case, giving us the id either way. var id int64 - err := s.db.QueryRow(fmt.Sprintf(` + err := q.QueryRow(fmt.Sprintf(` INSERT INTO participants (email_address, display_name, domain, created_at, updated_at) VALUES (?, ?, ?, %s, %s) ON CONFLICT (email_address) WHERE email_address IS NOT NULL DO UPDATE SET email_address = EXCLUDED.email_address RETURNING id - `, s.dialect.Now(), s.dialect.Now()), email, displayName, domain).Scan(&id) + `, dialect.Now(), dialect.Now()), email, displayName, domain).Scan(&id) if err != nil { return 0, err } @@ -957,7 +1225,7 @@ func (s *Store) ReplaceMessageRecipients(messageID int64, recipientType string, }) } -func replaceMessageRecipientsTx(tx *loggedTx, messageID int64, rs RecipientSet) error { +func replaceMessageRecipientsTx(tx querier, messageID int64, rs RecipientSet) error { _, err := tx.Exec(` DELETE FROM message_recipients WHERE message_id = ? AND recipient_type = ? `, messageID, rs.Type) @@ -1240,7 +1508,7 @@ func (s *Store) ReplaceMessageLabels(messageID int64, labelIDs []int64) error { }) } -func replaceMessageLabelsTx(tx *loggedTx, messageID int64, labelIDs []int64) error { +func replaceMessageLabelsTx(tx querier, messageID int64, labelIDs []int64) error { _, err := tx.Exec(` DELETE FROM message_labels WHERE message_id = ? `, messageID) @@ -1847,7 +2115,31 @@ func (s *Store) backfillFTSBatchContext( // last_message_at, and last_message_preview from the current table state. // Safe to call multiple times — always produces the same result (idempotent). func (s *Store) RecomputeConversationStats(sourceID int64) error { - _, err := s.db.Exec(` + return s.recomputeConversationStats("source_id = ?", sourceID) +} + +// RecomputeConversationStatsForMessage updates the denormalized stats only for +// the conversation containing messageID. +func (s *Store) RecomputeConversationStatsForMessage(messageID int64) error { + return s.RecomputeConversationStatsForMessageContext(context.Background(), messageID) +} + +// RecomputeConversationStatsForMessageContext is the request-aware form of +// RecomputeConversationStatsForMessage. +func (s *Store) RecomputeConversationStatsForMessageContext(ctx context.Context, messageID int64) error { + return s.recomputeConversationStatsContext( + ctx, + "id = (SELECT conversation_id FROM messages WHERE id = ?)", + messageID, + ) +} + +func (s *Store) recomputeConversationStats(whereClause string, arg any) error { + return s.recomputeConversationStatsContext(context.Background(), whereClause, arg) +} + +func (s *Store) recomputeConversationStatsContext(ctx context.Context, whereClause string, arg any) error { + _, err := s.db.ExecContext(ctx, fmt.Sprintf(` UPDATE conversations SET message_count = ( SELECT COUNT(*) FROM messages @@ -1868,8 +2160,8 @@ func (s *Store) RecomputeConversationStats(sourceID int64) error { ORDER BY COALESCE(sent_at, received_at, internal_date) DESC, id DESC LIMIT 1 ) - WHERE source_id = ? - `, sourceID) + WHERE %s + `, whereClause), arg) if err != nil { return fmt.Errorf("recompute conversation stats: %w", err) } @@ -2186,6 +2478,13 @@ func (s *Store) MergeParticipants(oldID, newID int64) error { if _, err := tx.Exec(`UPDATE participant_identifiers SET participant_id = ? WHERE participant_id = ?`, newID, oldID); err != nil { return err } + // Sender and identifier repoints can add or remove identity evidence. + // Repair the primary-store provenance before committing the merge. + if err := refreshParticipantMessageAttributionContext( + context.Background(), tx, newID, + ); err != nil { + return err + } // Repoint (and, if needed, restructure) any link edges referencing // oldID before the delete below drops them via ON DELETE CASCADE. if err := s.rewriteLinksForMerge(tx, oldID, newID); err != nil { @@ -2200,10 +2499,9 @@ func (s *Store) MergeParticipants(oldID, newID int64) error { if _, err := s.bumpIdentityRevision(tx); err != nil { return err } - // Also bump the account-identity revision: the merge repoints - // messages.sender_id, so a merge involving the sender of any - // message with a baked is_from_me leaves that flag stale in the - // message Parquet shards, which only a full rebuild re-derives. + // Also bump the account-identity revision: the primary rows were + // repaired above, but existing message Parquet shards still bake the + // pre-merge attribution and require a full rebuild. if err := s.bumpAccountIdentityRevision(tx); err != nil { return err } @@ -2237,12 +2535,13 @@ func (s *Store) ParticipantByIdentifier(identifierType, identifierValue string) // search values and the participant_identifiers Parquet export), and the // derived-dataset refresh repairs that drift cheaply. When the changed // identifier additionally matches a confirmed account-identity address, it is -// owner evidence: the analytics cache bakes it into owner_participants, the -// per-row is_owner flags of the relationship activity index, and the -// export-derived is_from_me flag in message shards — so the mutation also -// bumps the identity and account-identity revisions the way MergeParticipants -// does, forcing the full rebuild that re-derives committed shards. No-op -// calls (the common importer re-run) bump nothing. +// owner evidence: the mutation repairs affected messages in the primary store, +// while the analytics cache bakes it into owner_participants, the per-row +// is_owner flags of the relationship activity index, and the export-derived +// is_from_me flag in message shards. It therefore also bumps the identity and +// account-identity revisions the way MergeParticipants does, forcing the full +// rebuild that re-derives committed shards. No-op calls (the common importer +// re-run) bump nothing. func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, identifierValue string) error { identifierType = strings.TrimSpace(identifierType) identifierValue = strings.TrimSpace(identifierValue) @@ -2252,8 +2551,10 @@ func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, id return s.withTx(func(tx *loggedTx) error { // Fast path first, read-only: importer re-runs hit the no-op case // constantly, and it must not take any write lock. - noop, err := participantIdentifierIsNoopTx(tx, participantID, identifierType, identifierValue) - if err != nil || noop { + existingParticipantID, exists, err := participantIdentifierTargetTx( + tx, identifierType, identifierValue, + ) + if err != nil || (exists && existingParticipantID == participantID) { return err } // The write path may bump the identity revision below (owner @@ -2266,8 +2567,10 @@ func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, id if err := s.lockIdentityMutationTx(tx); err != nil { return err } - noop, err = participantIdentifierIsNoopTx(tx, participantID, identifierType, identifierValue) - if err != nil || noop { + existingParticipantID, exists, err = participantIdentifierTargetTx( + tx, identifierType, identifierValue, + ) + if err != nil || (exists && existingParticipantID == participantID) { return err } if _, err := tx.Exec(` @@ -2293,6 +2596,11 @@ func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, id if !ownerEvidence { return nil } + if err := refreshParticipantMessageAttributionContext( + context.Background(), tx, existingParticipantID, participantID, + ); err != nil { + return err + } if _, err := s.bumpIdentityRevision(tx); err != nil { return err } @@ -2300,20 +2608,22 @@ func (s *Store) SetParticipantIdentifier(participantID int64, identifierType, id }) } -// participantIdentifierIsNoopTx reports whether (identifierType, -// identifierValue) already points at participantID, without taking any lock. -func participantIdentifierIsNoopTx( - tx *loggedTx, participantID int64, identifierType, identifierValue string, -) (bool, error) { +// participantIdentifierTargetTx returns the participant currently owning an +// identifier, if any, without taking any lock. +func participantIdentifierTargetTx( + tx *loggedTx, + identifierType, + identifierValue string, +) (int64, bool, error) { var existingID int64 err := tx.QueryRow(` SELECT participant_id FROM participant_identifiers WHERE identifier_type = ? AND identifier_value = ? `, identifierType, identifierValue).Scan(&existingID) if err != nil && !errors.Is(err, sql.ErrNoRows) { - return false, fmt.Errorf("lookup participant identifier: %w", err) + return 0, false, fmt.Errorf("lookup participant identifier: %w", err) } - return err == nil && existingID == participantID, nil + return existingID, err == nil, nil } func (s *Store) EnsureParticipantByIdentifier(identifierType, identifierValue, displayName string) (int64, error) { @@ -2719,7 +3029,7 @@ func (s *Store) ReplaceConversationParticipants(conversationID int64, participan }) } -func replaceConversationParticipantsTx(tx *loggedTx, dialect Dialect, conversationID int64, participants []ConversationParticipantRef) error { +func replaceConversationParticipantsTx(tx querier, dialect Dialect, conversationID int64, participants []ConversationParticipantRef) error { if _, err := tx.Exec(`DELETE FROM conversation_participants WHERE conversation_id = ?`, conversationID); err != nil { return err } diff --git a/internal/store/migrate_init_schema_ledger_test.go b/internal/store/migrate_init_schema_ledger_test.go index dff65a903..8074d1b76 100644 --- a/internal/store/migrate_init_schema_ledger_test.go +++ b/internal/store/migrate_init_schema_ledger_test.go @@ -9,13 +9,13 @@ import ( "github.com/stretchr/testify/require" ) -// TestInitSchema_OneShotMigrationsGatedOnLedger verifies the two data +// TestInitSchema_OneShotMigrationsGatedOnLedger verifies the data // migrations InitSchema used to re-verify on every start (the attachments -// dedupe and the messages.last_modified backfill — a full messages-table -// scan, the dominant daemon-startup cost on a large archive) are gated on -// the applied_migrations ledger: +// dedupe, attribution provenance reconciliation, and the +// messages.last_modified backfill — full messages-table scans on a large +// archive) are gated on the applied_migrations ledger: // -// 1. a fresh InitSchema runs them once and records both sentinels, +// 1. a fresh InitSchema runs them once and records all sentinels, // 2. a later InitSchema with the sentinel present skips the work, // 3. clearing the sentinel makes the next InitSchema run it again. // @@ -32,6 +32,7 @@ func TestInitSchema_OneShotMigrationsGatedOnLedger(t *testing.T) { for _, name := range []string{ migrationAttachmentsContentHashUnique, + migrationMessageAttributionProvenance, migrationMessagesLastModifiedBackfill, } { applied, err := st.IsMigrationApplied(name) diff --git a/internal/store/migrate_legacy_identity.go b/internal/store/migrate_legacy_identity.go index 11d079a60..a8ae2972c 100644 --- a/internal/store/migrate_legacy_identity.go +++ b/internal/store/migrate_legacy_identity.go @@ -146,6 +146,7 @@ func (s *Store) MigrateLegacyIdentityConfigContext( if !SourceTypeUsesEmailIdentity(src.SourceType) { continue } + insertedForSource := false for _, addr := range normalized { // Comparison rule (email-shaped → case-insensitive; // everything else → case-sensitive) is shared with @@ -173,6 +174,7 @@ func (s *Store) MigrateLegacyIdentityConfigContext( // this source, exactly like AddAccountIdentity's insert // branch — see the matching comment there. insertedAny = true + insertedForSource = true case qerr != nil: return fmt.Errorf("read existing identity (source=%d, addr=%s): %w", src.ID, addr, qerr) default: @@ -190,6 +192,11 @@ func (s *Store) MigrateLegacyIdentityConfigContext( } } } + if insertedForSource { + if err := refreshSourceMessageAttributionContext(ctx, tx, src.ID, ""); err != nil { + return fmt.Errorf("refresh migrated identity attribution (source=%d): %w", src.ID, err) + } + } } // A daemon that started before this migration ran must not keep diff --git a/internal/store/migrate_legacy_identity_test.go b/internal/store/migrate_legacy_identity_test.go index 9b0b8d4f1..5eab39f15 100644 --- a/internal/store/migrate_legacy_identity_test.go +++ b/internal/store/migrate_legacy_identity_test.go @@ -1,6 +1,7 @@ package store_test import ( + "database/sql" "testing" "github.com/stretchr/testify/assert" @@ -155,6 +156,33 @@ func TestMigrateLegacyIdentityConfig_BumpsRevisionsOnlyWhenItInserts(t *testing. "a no-op re-run must not bump the account identity revision") } +func TestMigrateLegacyIdentityConfigRefreshesExistingMessages(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + senderID := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + message := f.NewMessage().WithSourceMessageID("legacy-config-owner").Build() + message.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist message before identity migration") + + applied, _, _, _, err := st.MigrateLegacyIdentityConfig([]string{"owner@example.com"}) //nolint:dogsled // 5-return migration; test needs only applied+err + require.NoError(err, "MigrateLegacyIdentityConfig") + require.True(applied, "migration should insert the configured identity") + + var sourceDerived, identityDerived, isFromMe bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT source_is_from_me, identity_is_from_me, is_from_me + FROM messages + WHERE id = ? + `), messageID).Scan(&sourceDerived, &identityDerived, &isFromMe)) + assert.False(sourceDerived, "incoming Gmail message was not source-native") + assert.True(identityDerived, "migration must immediately repair identity provenance") + assert.True(isFromMe, "migration must immediately repair effective attribution") +} + func TestMigrateLegacyIdentityConfig_MergesExistingSignal(t *testing.T) { require := require.New(t) f := storetest.New(t) diff --git a/internal/store/migrate_phone_unique.go b/internal/store/migrate_phone_unique.go index 165b91872..d85fdf83c 100644 --- a/internal/store/migrate_phone_unique.go +++ b/internal/store/migrate_phone_unique.go @@ -250,10 +250,9 @@ func (s *Store) mergeParticipant(ctx context.Context, tx *loggedTx, winner, lose if _, err := s.bumpIdentityRevision(tx); err != nil { return fmt.Errorf("bump identity revision (loser=%d, winner=%d): %w", loser, winner, err) } - // Also bump the account-identity revision: the merge repoints - // messages.sender_id, so a merge involving the sender of any message - // with a baked is_from_me leaves that flag stale in the message - // Parquet shards, which only a full rebuild re-derives. + // Also bump the account-identity revision: the primary rows are repaired + // after the survivor metadata is finalized below, but existing message + // Parquet shards still require a full rebuild. if err := s.bumpAccountIdentityRevision(tx); err != nil { return fmt.Errorf("bump account identity revision (loser=%d, winner=%d): %w", loser, winner, err) } @@ -291,6 +290,17 @@ func (s *Store) mergeParticipant(ctx context.Context, tx *loggedTx, winner, lose ); err != nil { return fmt.Errorf("coalesce metadata onto winner (winner=%d, loser=%d): %w", winner, loser, err) } + // The sender repoint plus the survivor's final email/identifiers can add or + // remove identity evidence. Repair primary-store provenance atomically with + // the legacy merge. + if err := refreshParticipantMessageAttributionContext(ctx, tx, winner); err != nil { + return fmt.Errorf( + "refresh message attribution (winner=%d, loser=%d): %w", + winner, + loser, + err, + ) + } // (8) Finally drop the loser. participant_identifiers cascades; the // other FKs are already cleared by the repoints above. diff --git a/internal/store/migrate_phone_unique_test.go b/internal/store/migrate_phone_unique_test.go index 8aba8daed..70ae44e9c 100644 --- a/internal/store/migrate_phone_unique_test.go +++ b/internal/store/migrate_phone_unique_test.go @@ -82,6 +82,17 @@ func TestEnsureParticipantsPhoneUniqueIndex_LegacyNonUnique(t *testing.T) { // Make sure the legacy schema actually permitted the duplicate. require.NotEqual(winner, loser, "seeded participants must have distinct ids") + require.NoError( + st.AddAccountIdentity(source.ID, "+15555551234", "manual"), + "confirm duplicated phone as source identity", + ) + _, err = st.db.Exec(` + INSERT INTO participant_identifiers ( + participant_id, identifier_type, identifier_value, is_primary + ) + VALUES (?, 'phone', '+15555551234', TRUE) + `, winner) + require.NoError(err, "seed winner owner identifier") // Attach FK references to BOTH participants so we can prove the // repoint+dedupe logic runs end-to-end: @@ -183,6 +194,9 @@ func TestEnsureParticipantsPhoneUniqueIndex_LegacyNonUnique(t *testing.T) { "read msg-C sender") assert.True(msgCSender.Valid, "msg-C sender = %+v, want winner %d", msgCSender, winner) assert.Equal(winner, msgCSender.Int64, "msg-C sender") + msgCIsFromMe, err := st.GetMessageIsFromMe(msgC) + require.NoError(err, "read msg-C attribution") + assert.True(msgCIsFromMe, "sender repoint must refresh identity-derived attribution") // 6) The index is now UNIQUE. Verify via sqlite_master. var sqlDef string diff --git a/internal/store/migrations.go b/internal/store/migrations.go index f551d049b..bf40a3548 100644 --- a/internal/store/migrations.go +++ b/internal/store/migrations.go @@ -1,8 +1,13 @@ package store import ( + "bytes" + "compress/zlib" "context" + "database/sql" + "encoding/json" "fmt" + "io" ) // One-time data migrations run by InitSchema and gated on the @@ -13,9 +18,181 @@ import ( const ( migrationAttachmentsContentHashUnique = "attachments_content_hash_unique_index" migrationMessagesLastModifiedBackfill = "messages_last_modified_backfill" + migrationMessageAttributionProvenance = "message_attribution_provenance_v2" migrationArchiveIdentity = "archive_identity_v1" ) +func backfillLegacyMessageAttributionProvenance( + ctx context.Context, + tx *loggedTx, +) error { + if err := backfillLegacyCalendarAttribution(ctx, tx); err != nil { + return err + } + + _, err := tx.ExecContext(ctx, ` + UPDATE messages + SET source_is_from_me = FALSE, + identity_is_from_me = COALESCE(is_from_me, FALSE) + WHERE source_is_from_me IS NULL + AND source_id IN ( + SELECT id + FROM sources + WHERE source_type IN ('granola', 'circleback') + ) + `) + if err != nil { + return fmt.Errorf("backfill message attribution provenance: %w", err) + } + + _, err = tx.ExecContext(ctx, ` + UPDATE messages + SET source_is_from_me = COALESCE(is_from_me, FALSE), + identity_is_from_me = FALSE + WHERE source_is_from_me IS NULL + `) + if err != nil { + return fmt.Errorf("initialize source-native message attribution provenance: %w", err) + } + + rows, err := tx.QueryContext(ctx, ` + SELECT id + FROM sources + WHERE source_type IN ('granola', 'circleback') + OR EXISTS ( + SELECT 1 + FROM account_identities ai + WHERE ai.source_id = sources.id + ) + `) + if err != nil { + return fmt.Errorf("list identity-derived attribution sources: %w", err) + } + var sourceIDs []int64 + for rows.Next() { + var sourceID int64 + if err := rows.Scan(&sourceID); err != nil { + _ = rows.Close() + return fmt.Errorf("scan identity-derived attribution source: %w", err) + } + sourceIDs = append(sourceIDs, sourceID) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return fmt.Errorf("iterate identity-derived attribution sources: %w", err) + } + if err := rows.Close(); err != nil { + return fmt.Errorf("close identity-derived attribution sources: %w", err) + } + + for _, sourceID := range sourceIDs { + if err := refreshSourceMessageAttributionContext(ctx, tx, sourceID, ""); err != nil { + return fmt.Errorf("reconcile source %d attribution: %w", sourceID, err) + } + } + return nil +} + +func backfillLegacyCalendarAttribution( + ctx context.Context, + tx *loggedTx, +) error { + var lastMessageID int64 + for { + if err := ctx.Err(); err != nil { + return err + } + + rows, err := tx.QueryContext(ctx, ` + SELECT m.id, mr.raw_data, mr.compression + FROM messages m + JOIN sources s ON s.id = m.source_id + JOIN message_raw mr ON mr.message_id = m.id + WHERE m.source_is_from_me IS NULL + AND COALESCE(m.is_from_me, FALSE) = TRUE + AND s.source_type = 'gcal' + AND mr.raw_format = 'gcal_json' + AND m.id > ? + ORDER BY m.id + LIMIT 500 + `, lastMessageID) + if err != nil { + return fmt.Errorf("list legacy calendar attribution: %w", err) + } + + var ( + batchSize int + messageIDs []int64 + ) + for rows.Next() { + var ( + messageID int64 + rawData []byte + compression sql.NullString + ) + if err := rows.Scan(&messageID, &rawData, &compression); err != nil { + _ = rows.Close() + return fmt.Errorf("scan legacy calendar attribution: %w", err) + } + lastMessageID = messageID + batchSize++ + organizerSelf, ok := legacyCalendarOrganizerSelf(rawData, compression) + if ok && !organizerSelf { + messageIDs = append(messageIDs, messageID) + } + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return fmt.Errorf("iterate legacy calendar attribution: %w", err) + } + if err := rows.Close(); err != nil { + return fmt.Errorf("close legacy calendar attribution: %w", err) + } + + if err := execInChunks( + tx, + messageIDs, + nil, + `UPDATE messages + SET source_is_from_me = FALSE, + identity_is_from_me = COALESCE(is_from_me, FALSE) + WHERE id IN (%s)`, + ); err != nil { + return fmt.Errorf("backfill calendar attribution provenance: %w", err) + } + if batchSize < 500 { + return nil + } + } +} + +func legacyCalendarOrganizerSelf( + rawData []byte, + compression sql.NullString, +) (bool, bool) { + if compression.Valid && compression.String == "zlib" { + reader, err := zlib.NewReader(bytes.NewReader(rawData)) + if err != nil { + return false, false + } + defer func() { _ = reader.Close() }() + rawData, err = io.ReadAll(reader) + if err != nil { + return false, false + } + } + + var event struct { + Organizer *struct { + Self bool `json:"self"` + } `json:"organizer"` + } + if err := json.Unmarshal(rawData, &event); err != nil || event.Organizer == nil { + return false, false + } + return event.Organizer.Self, true +} + // IsMigrationApplied reports whether the named one-time data migration // has already run. func (s *Store) IsMigrationApplied(name string) (bool, error) { diff --git a/internal/store/participant_identifiers_test.go b/internal/store/participant_identifiers_test.go index d45ad1ca5..1923c6074 100644 --- a/internal/store/participant_identifiers_test.go +++ b/internal/store/participant_identifiers_test.go @@ -1,6 +1,7 @@ package store_test import ( + "database/sql" "testing" "github.com/stretchr/testify/assert" @@ -78,6 +79,51 @@ func TestSetParticipantIdentifierRepointOwnerEvidenceBumpsRevisions(t *testing.T assert.Equal(second, id, "identifier must point at the new participant") } +func TestSetParticipantIdentifierRepointRefreshesSenderAttribution(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + require.NoError(st.AddAccountIdentity(f.Source.ID, "+15550100", "manual"), "confirm owner identity") + first := f.EnsureParticipant("first@example.com", "First", "example.com") + second := f.EnsureParticipant("second@example.com", "Second", "example.com") + require.NoError(st.SetParticipantIdentifier(first, "phone", "+15550100"), "seed owner evidence") + + persistMessage := func(sourceMessageID string, senderID int64) int64 { + message := f.NewMessage().WithSourceMessageID(sourceMessageID).Build() + message.SenderID = sql.NullInt64{Int64: senderID, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist %s", sourceMessageID) + return messageID + } + firstMessageID := persistMessage("identifier-first", first) + secondMessageID := persistMessage("identifier-second", second) + + require.NoError(st.SetParticipantIdentifier(second, "phone", "+15550100"), "repoint owner evidence") + + firstIsFromMe, err := st.GetMessageIsFromMe(firstMessageID) + require.NoError(err, "read former owner attribution") + assert.False(firstIsFromMe, "reassignment must clear attribution from the former identifier owner") + secondIsFromMe, err := st.GetMessageIsFromMe(secondMessageID) + require.NoError(err, "read new owner attribution") + assert.True(secondIsFromMe, "reassignment must attribute messages from the new identifier owner") + + var firstIdentityDerived, secondIdentityDerived bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT identity_is_from_me + FROM messages + WHERE id = ? + `), firstMessageID).Scan(&firstIdentityDerived)) + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT identity_is_from_me + FROM messages + WHERE id = ? + `), secondMessageID).Scan(&secondIdentityDerived)) + assert.False(firstIdentityDerived, "former identifier owner provenance") + assert.True(secondIdentityDerived, "new identifier owner provenance") +} + // TestSetParticipantIdentifierNonOwnerEvidenceBumpsOnlyIdentifierRevision // verifies plain alternate identifiers (the common importer case) advance // only the participant-identifier revision — the identity directory bakes diff --git a/internal/store/participant_links_test.go b/internal/store/participant_links_test.go index edf7973e6..e1605aa6f 100644 --- a/internal/store/participant_links_test.go +++ b/internal/store/participant_links_test.go @@ -1,6 +1,7 @@ package store_test import ( + "database/sql" "errors" "fmt" "sync" @@ -400,6 +401,34 @@ func TestMergeParticipantsWithoutLinksStillBumpsRevisionButRewritesNoLinks(t *te assert.Equal(0, edgeCount, "merge without pre-existing links must not create any link edge") } +func TestMergeParticipantsRefreshesSenderAttribution(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + st := f.Store + + require.NoError( + st.AddAccountIdentity(f.Source.ID, "owner@example.com", "manual"), + "confirm owner identity", + ) + owner := f.EnsureParticipant("owner@example.com", "Owner", "example.com") + survivor := f.EnsureParticipant("other@example.com", "Other", "example.com") + message := f.NewMessage().WithSourceMessageID("merge-attribution").Build() + message.SenderID = sql.NullInt64{Int64: owner, Valid: true} + messageID, err := st.UpsertMessage(message) + require.NoError(err, "persist owner-attributed message") + + isFromMe, err := st.GetMessageIsFromMe(messageID) + require.NoError(err, "read attribution before merge") + require.True(isFromMe, "confirmed owner sender must begin attributed") + + require.NoError(st.MergeParticipants(owner, survivor), "merge owner into unmatched survivor") + + isFromMe, err = st.GetMessageIsFromMe(messageID) + require.NoError(err, "read attribution after merge") + assert.False(isFromMe, "merge must clear attribution after sender no longer matches") +} + // TestLinkParticipants_ConcurrentDisjointClusters covers the race that // lockIdentityMutationTx serializes against: two LinkParticipants calls // that each try to connect two already-linked, previously-disjoint diff --git a/internal/store/schema.sql b/internal/store/schema.sql index 527e56554..1c0f0b20c 100644 --- a/internal/store/schema.sql +++ b/internal/store/schema.sql @@ -162,6 +162,8 @@ CREATE TABLE IF NOT EXISTS messages ( -- Sender sender_id INTEGER REFERENCES participants(id), is_from_me BOOLEAN DEFAULT FALSE, + source_is_from_me BOOLEAN, + identity_is_from_me BOOLEAN NOT NULL DEFAULT FALSE, -- Content subject TEXT, -- email subject, NULL for chat diff --git a/internal/store/schema_pg.sql b/internal/store/schema_pg.sql index 9872d5fd4..d44d748b6 100644 --- a/internal/store/schema_pg.sql +++ b/internal/store/schema_pg.sql @@ -143,6 +143,8 @@ CREATE TABLE IF NOT EXISTS messages ( sender_id BIGINT REFERENCES participants(id), is_from_me BOOLEAN DEFAULT FALSE, + source_is_from_me BOOLEAN, + identity_is_from_me BOOLEAN NOT NULL DEFAULT FALSE, subject TEXT, snippet TEXT, diff --git a/internal/store/store.go b/internal/store/store.go index 67e3f28ca..f663f7748 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -515,10 +515,19 @@ func (s *Store) withTxContext(ctx context.Context, fn func(tx *loggedTx) error) } return err } + if err := ctx.Err(); err != nil { + _ = tx.Rollback() + return err + } if err := tx.Commit(); err != nil { slog.Warn("sql tx commit failed", "error", err.Error(), "duration_ms", time.Since(start).Milliseconds()) + if errors.Is(err, sql.ErrTxDone) { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + } return err } // A tx crossing the slow threshold is a diagnostic, not a problem — @@ -664,7 +673,9 @@ type chunkInsert struct { // parameter limit (999). valueBuilder generates the VALUES placeholders and // args for each chunk of row indices. Rebinding to the dialect's placeholder // form happens inside tx.Exec (loggedTx wraps the dialect's Rebind). -func insertInChunks(tx *loggedTx, c chunkInsert, valueBuilder func(start, end int) ([]string, []any)) error { +func insertInChunks(tx interface { + Exec(query string, args ...any) (sql.Result, error) +}, c chunkInsert, valueBuilder func(start, end int) ([]string, []any)) error { // SQLite default SQLITE_MAX_VARIABLE_NUMBER is 999 // Leave some margin for safety const maxParams = 900 @@ -832,6 +843,53 @@ func (s *Store) InitSchema() error { } } + // Initialize explicit attribution provenance for every legacy message once + // under the maintenance timeout escape hatch. Granola and Circleback + // historically derived is_from_me from confirmed organizer identities. + // Google Calendar combined that signal with Organizer.Self, so its archived + // event payload separates source-native ownership from identity-derived + // ownership. Other providers' existing values are source-native. Runtime + // identity mutations can then update only rows whose derived attribution + // actually changes instead of rewriting an entire source to initialize NULL + // provenance. + attributionMigrated, err := s.IsMigrationApplied(migrationMessageAttributionProvenance) + if err != nil { + return err + } + if !attributionMigrated { + if err := s.runMaintenance( + context.Background(), + func(ctx context.Context, tx *loggedTx) error { + if err := backfillLegacyMessageAttributionProvenance(ctx, tx); err != nil { + return err + } + + // Published message shards used to trust the effective + // is_from_me value. The provenance migration changes cache + // inputs even when no identity is added or removed, so advance + // the account-identity revision in the same transaction as the + // repaired rows. Empty archives have no stale shards to + // invalidate. + var hasMessages bool + if err := tx.QueryRowContext( + ctx, + `SELECT EXISTS (SELECT 1 FROM messages)`, + ).Scan(&hasMessages); err != nil { + return fmt.Errorf("check attribution migration cache impact: %w", err) + } + if !hasMessages { + return nil + } + return s.bumpAccountIdentityRevisionContext(ctx, tx) + }, + ); err != nil { + return err + } + if err := s.MarkMigrationApplied(migrationMessageAttributionProvenance); err != nil { + return err + } + } + // Partial expression indexes for live-message listing and date filtering. // The first is a covering index for the ListMessages page // (GET /api/v1/messages). diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 4d59a8950..32ef38769 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -1,11 +1,13 @@ package store_test import ( + "context" "database/sql" "fmt" "testing" "time" + "github.com/mattn/go-sqlite3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/msgvault/internal/mime" @@ -1590,6 +1592,73 @@ func TestStore_PersistMessage_Atomicity(t *testing.T) { assert.Empty(t, existing, "message should not exist after failed PersistMessage") } +func TestStore_PersistMessageContext_CancellationRollsBack(t *testing.T) { + testutil.SkipIfPostgres(t, "uses a SQLite trigger and registered function to pause persistence") + require := require.New(t) + assert := assert.New(t) + f := storetest.New(t) + f.Store.DB().SetMaxOpenConns(1) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + persistStarted := make(chan struct{}) + conn, err := f.Store.DB().Conn(context.Background()) + require.NoError(err, "get SQLite connection") + err = conn.Raw(func(driverConn any) error { + sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn) + require.True(ok, "driver connection is SQLite") + return sqliteConn.RegisterFunc("wait_for_persist_cancel", func() int { + close(persistStarted) + <-ctx.Done() + return 0 + }, true) + }) + require.NoError(err, "register cancellation function") + require.NoError(conn.Close(), "return SQLite connection to pool") + _, err = f.Store.DB().Exec(` + CREATE TRIGGER wait_before_meeting_raw_insert + BEFORE INSERT ON message_raw + WHEN NEW.raw_format = 'meeting_json' + BEGIN + SELECT wait_for_persist_cancel(); + END + `) + require.NoError(err, "create cancellation trigger") + + msg := storetest.NewMessage(f.Source.ID, f.ConvID). + WithSourceMessageID("persist-cancel"). + WithSubject("Canceled persistence"). + Build() + done := make(chan error, 1) + go func() { + _, persistErr := f.Store.PersistMessageContext(ctx, &store.MessagePersistData{ + Message: msg, + BodyText: sql.NullString{String: "must roll back", Valid: true}, + RawMIME: []byte(`{"meeting":"cancel"}`), + RawFormat: "meeting_json", + }) + done <- persistErr + }() + + select { + case <-persistStarted: + case <-time.After(time.Second): + require.FailNow("message persistence did not reach cancellation trigger") + } + cancel() + + select { + case err = <-done: + case <-time.After(time.Second): + require.FailNow("message persistence did not stop after cancellation") + } + require.ErrorIs(err, context.Canceled) + + existing, err := f.Store.MessageExistsBatch(f.Source.ID, []string{"persist-cancel"}) + require.NoError(err, "lookup canceled message") + assert.Empty(existing, "canceled message transaction must roll back") +} + func TestStore_OAuthAppColumn(t *testing.T) { require := require.New(t) assert := assert.New(t) diff --git a/internal/store/subset.go b/internal/store/subset.go index 0f79f50ff..fc508f9d9 100644 --- a/internal/store/subset.go +++ b/internal/store/subset.go @@ -22,6 +22,18 @@ type CopyResult struct { Elapsed time.Duration } +const messageCopyColumns = ` + id, conversation_id, source_id, source_message_id, + rfc822_message_id, message_type, + sent_at, received_at, read_at, delivered_at, internal_date, + sender_id, is_from_me, source_is_from_me, identity_is_from_me, + subject, snippet, reply_to_message_id, thread_position, + is_read, is_delivered, is_sent, is_edited, is_forwarded, + size_estimate, has_attachments, attachment_count, + deleted_at, deleted_from_source_at, delete_batch_id, + archived_at, indexing_version, last_modified, metadata, embed_gen +` + // CopySubset copies rowCount most recent messages (and all referenced // data) from srcDBPath into a new database in dstDir. The destination // schema is initialized using the embedded store schema. @@ -417,7 +429,9 @@ func copyData(tx *sql.Tx, rowCount int, includeIdentity bool) (*CopyResult, erro } if _, err := tx.Exec(` - INSERT INTO messages SELECT * FROM src.messages + INSERT INTO messages (` + messageCopyColumns + `) + SELECT ` + messageCopyColumns + ` + FROM src.messages WHERE id IN (SELECT id FROM selected_messages)`); err != nil { return nil, fmt.Errorf("copy messages: %w", err) } diff --git a/internal/store/subset_test.go b/internal/store/subset_test.go index de354522c..d72308e11 100644 --- a/internal/store/subset_test.go +++ b/internal/store/subset_test.go @@ -185,6 +185,72 @@ func TestCopySubset_Basic(t *testing.T) { assert.False(hasViolation, "foreign key violations found in destination database") } +func TestCopySubset_UpgradedMessageColumnOrder(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + srcDir := t.TempDir() + dstDir := filepath.Join(t.TempDir(), "dst") + srcDB := createTestSourceDB(t, srcDir, 1) + + st, err := Open(srcDB) + require.NoError(err, "open source for upgrade") + _, err = st.DB().Exec(` + ALTER TABLE messages DROP COLUMN identity_is_from_me; + ALTER TABLE messages DROP COLUMN source_is_from_me; + DELETE FROM applied_migrations + WHERE name = 'message_attribution_provenance_v2'; + `) + require.NoError(err, "simulate pre-attribution schema") + require.NoError(st.InitSchema(), "upgrade source schema") + _, err = st.DB().Exec(` + UPDATE messages + SET is_from_me = TRUE, + source_is_from_me = FALSE, + identity_is_from_me = TRUE, + metadata = '{"schema":"upgraded"}', + embed_gen = 7 + WHERE id = 1 + `) + require.NoError(err, "seed upgraded message columns") + require.NoError(st.Close(), "close upgraded source") + + result, err := CopySubset(srcDB, dstDir, 1, false) + require.NoError(err, "CopySubset from upgraded schema") + assert.Equal(int64(1), result.Messages) + + db, err := sql.Open("sqlite3", filepath.Join(dstDir, "msgvault.db")) + require.NoError(err, "open copied database") + defer func() { _ = db.Close() }() + + var sourceMessageID, messageType, subject, metadata string + var isFromMe, sourceIsFromMe, identityIsFromMe bool + var embedGen int64 + require.NoError(db.QueryRow(` + SELECT source_message_id, message_type, subject, + is_from_me, source_is_from_me, identity_is_from_me, + metadata, embed_gen + FROM messages + WHERE id = 1 + `).Scan( + &sourceMessageID, + &messageType, + &subject, + &isFromMe, + &sourceIsFromMe, + &identityIsFromMe, + &metadata, + &embedGen, + )) + assert.Equal("msg_1", sourceMessageID) + assert.Equal("email", messageType) + assert.Equal("Subject B", subject) + assert.True(isFromMe) + assert.False(sourceIsFromMe) + assert.True(identityIsFromMe) + assert.JSONEq(`{"schema":"upgraded"}`, metadata) + assert.Equal(int64(7), embedGen) +} + func TestCopySubset_AllRows(t *testing.T) { srcDir := t.TempDir() dstDir := filepath.Join(t.TempDir(), "dst") diff --git a/internal/store/sync.go b/internal/store/sync.go index 7c2c1797e..5f9604dfb 100644 --- a/internal/store/sync.go +++ b/internal/store/sync.go @@ -16,6 +16,8 @@ const ( SyncRunItemStatusError = "error" SyncRunItemStatusSkipped = "skipped" + + manualTransactionCleanupTimeout = 5 * time.Second ) // ErrSyncRunNotFound is returned by the sync-run getters (GetActiveSync, @@ -186,7 +188,11 @@ type SourceImportItem struct { // source via SELECT ... FOR UPDATE before doing the read-modify-write // on sync_runs. func (s *Store) StartSync(sourceID int64, syncType string) (int64, error) { - ctx := context.Background() + return s.StartSyncContext(context.Background(), sourceID, syncType) +} + +// StartSyncContext is the request-aware form of StartSync. +func (s *Store) StartSyncContext(ctx context.Context, sourceID int64, syncType string) (int64, error) { const maxAttempts = 5 for range maxAttempts { id, err := s.startSyncOnce(ctx, sourceID) @@ -213,7 +219,12 @@ func (s *Store) startSyncOnce(ctx context.Context, sourceID int64) (retID int64, committed := false defer func() { if !committed { - _, _ = conn.ExecContext(ctx, "ROLLBACK") + rollbackCtx, rollbackCancel := context.WithTimeout( + context.WithoutCancel(ctx), + manualTransactionCleanupTimeout, + ) + defer rollbackCancel() + _, _ = conn.ExecContext(rollbackCtx, "ROLLBACK") } }() @@ -267,7 +278,13 @@ func (s *Store) startSyncOnce(ctx context.Context, sourceID int64) (retID int64, // UpdateSyncCheckpoint saves progress for resumption. func (s *Store) UpdateSyncCheckpoint(syncID int64, cp *Checkpoint) error { - _, err := s.db.Exec(` + return s.UpdateSyncCheckpointContext(context.Background(), syncID, cp) +} + +// UpdateSyncCheckpointContext is the request-aware form of +// UpdateSyncCheckpoint. +func (s *Store) UpdateSyncCheckpointContext(ctx context.Context, syncID int64, cp *Checkpoint) error { + _, err := s.db.ExecContext(ctx, ` UPDATE sync_runs SET cursor_before = ?, messages_processed = ?, @@ -364,7 +381,12 @@ func (s *Store) ListSyncRunItems(syncRunID int64, status string, limit int) ([]S // CompleteSync marks a sync as successfully completed. func (s *Store) CompleteSync(syncID int64, finalHistoryID string) error { - _, err := s.db.Exec(fmt.Sprintf(` + return s.CompleteSyncContext(context.Background(), syncID, finalHistoryID) +} + +// CompleteSyncContext is the request-aware form of CompleteSync. +func (s *Store) CompleteSyncContext(ctx context.Context, syncID int64, finalHistoryID string) error { + _, err := s.db.ExecContext(ctx, fmt.Sprintf(` UPDATE sync_runs SET status = 'completed', completed_at = %s, diff --git a/internal/store/sync_context_test.go b/internal/store/sync_context_test.go new file mode 100644 index 000000000..13bff3d21 --- /dev/null +++ b/internal/store/sync_context_test.go @@ -0,0 +1,115 @@ +package store + +import ( + "context" + "database/sql" + "database/sql/driver" + "io" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type canceledStartSyncConnector struct { + cancel context.CancelFunc + rollbackCtx chan error +} + +func (c *canceledStartSyncConnector) Connect(context.Context) (driver.Conn, error) { + return &canceledStartSyncConn{cancel: c.cancel, rollbackCtx: c.rollbackCtx}, nil +} + +func (c *canceledStartSyncConnector) Driver() driver.Driver { + return &canceledStartSyncDriver{connector: c} +} + +type canceledStartSyncDriver struct { + connector *canceledStartSyncConnector +} + +func (d *canceledStartSyncDriver) Open(string) (driver.Conn, error) { + return d.connector.Connect(context.Background()) +} + +type canceledStartSyncConn struct { + cancel context.CancelFunc + rollbackCtx chan error +} + +func (c *canceledStartSyncConn) Prepare(string) (driver.Stmt, error) { + return nil, driver.ErrSkip +} + +func (c *canceledStartSyncConn) Close() error { return nil } + +func (c *canceledStartSyncConn) Begin() (driver.Tx, error) { + return nil, driver.ErrSkip +} + +func (c *canceledStartSyncConn) ExecContext( + ctx context.Context, + query string, + _ []driver.NamedValue, +) (driver.Result, error) { + if strings.TrimSpace(query) == "ROLLBACK" { + c.rollbackCtx <- ctx.Err() + } + return driver.RowsAffected(1), nil +} + +func (c *canceledStartSyncConn) QueryContext( + _ context.Context, + query string, + _ []driver.NamedValue, +) (driver.Rows, error) { + if strings.Contains(query, "INSERT INTO sync_runs") { + c.cancel() + return nil, context.Canceled + } + return &singleInt64Rows{value: 1}, nil +} + +type singleInt64Rows struct { + value int64 + read bool +} + +func (r *singleInt64Rows) Columns() []string { return []string{"id"} } +func (r *singleInt64Rows) Close() error { return nil } +func (r *singleInt64Rows) Next(dest []driver.Value) error { + if r.read { + return io.EOF + } + r.read = true + dest[0] = r.value + return nil +} + +func TestStartSyncContextRollbackOutlivesRequestCancellation(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + rollbackCtx := make(chan error, 1) + db := sql.OpenDB(&canceledStartSyncConnector{ + cancel: cancel, + rollbackCtx: rollbackCtx, + }) + t.Cleanup(func() { + require.NoError(db.Close()) + }) + st := &Store{ + db: newLoggedDB(db, identityRebind), + dialect: &SQLiteDialect{}, + } + + _, err := st.StartSyncContext(ctx, 1, "gmail") + require.ErrorIs(err, context.Canceled) + select { + case rollbackErr := <-rollbackCtx: + require.NoError(rollbackErr, "rollback must use a context independent of request cancellation") + case <-time.After(time.Second): + require.FailNow("manual transaction rollback was not attempted") + } +} diff --git a/internal/tui/meeting_mode_test.go b/internal/tui/meeting_mode_test.go index 44ca0ff63..f734c92b6 100644 --- a/internal/tui/meeting_mode_test.go +++ b/internal/tui/meeting_mode_test.go @@ -84,17 +84,31 @@ func TestMeetingAccountsExcludeUnrelatedSources(t *testing.T) { query.AccountInfo{ID: 2, SourceType: meetingSourceGranola, Identifier: "work-notes"}, query.AccountInfo{ID: 3, SourceType: meetingSourceCircleback, Identifier: "team-meetings"}, query.AccountInfo{ID: 4, SourceType: "teams", Identifier: "team-chat"}, + query.AccountInfo{ID: 5, SourceType: meetingSourceImported, Identifier: "local-meetings"}, ).Build() accounts := model.meetingAccounts() - require.Len(t, accounts, 2) - assert.Equal(t, []string{"work-notes", "team-meetings"}, []string{ + require.Len(t, accounts, 3) + assert.Equal(t, []string{"work-notes", "team-meetings", "local-meetings"}, []string{ accounts[0].Identifier, accounts[1].Identifier, + accounts[2].Identifier, }) } +func TestMeetingImportedSourceLabelUsesDisplayNameAndFallbacks(t *testing.T) { + model := NewBuilder().WithAccounts( + query.AccountInfo{ID: 5, SourceType: meetingSourceImported, Identifier: "local-meetings", DisplayName: "Imported Interviews"}, + query.AccountInfo{ID: 6, SourceType: meetingSourceImported, Identifier: "second-stream"}, + query.AccountInfo{ID: 7, SourceType: meetingSourceImported}, + ).Build() + + assert.Equal(t, "Imported Interviews", model.meetingSourceLabel(5)) + assert.Equal(t, "second-stream", model.meetingSourceLabel(6)) + assert.Equal(t, "Imported", model.meetingSourceLabel(7)) +} + func TestMeetingAccountSelectorUsesMeetingSources(t *testing.T) { assert := assert.New(t) selectedID := int64(3) diff --git a/internal/tui/meeting_state.go b/internal/tui/meeting_state.go index 1d017c83f..f4a4be12f 100644 --- a/internal/tui/meeting_state.go +++ b/internal/tui/meeting_state.go @@ -12,6 +12,7 @@ const meetingMessageType = "meeting_transcript" const ( meetingSourceGranola = "granola" meetingSourceCircleback = "circleback" + meetingSourceImported = "meeting_import" ) type meetingViewLevel int @@ -76,7 +77,7 @@ func (m Model) meetingAccounts() []query.AccountInfo { accounts := make([]query.AccountInfo, 0, len(m.accounts)) for _, account := range m.accounts { switch strings.ToLower(strings.TrimSpace(account.SourceType)) { - case meetingSourceGranola, meetingSourceCircleback: + case meetingSourceGranola, meetingSourceCircleback, meetingSourceImported: accounts = append(accounts, account) } } diff --git a/internal/tui/meeting_view.go b/internal/tui/meeting_view.go index 739db9330..34722f1b5 100644 --- a/internal/tui/meeting_view.go +++ b/internal/tui/meeting_view.go @@ -60,6 +60,14 @@ func (m Model) meetingSourceLabel(sourceID int64) string { return "Granola" case meetingSourceCircleback: return "Circleback" + case meetingSourceImported: + if account.DisplayName != "" { + return textutil.SanitizeTerminal(account.DisplayName) + } + if account.Identifier != "" { + return textutil.SanitizeTerminal(account.Identifier) + } + return "Imported" } if account.DisplayName != "" { return textutil.SanitizeTerminal(account.DisplayName) @@ -84,9 +92,9 @@ func meetingColumnWidths(width int) (date, title, organizer, source int) { func (m Model) meetingListView() string { if len(m.meetingState.messages) == 0 && !m.loading && m.err == nil && !m.meetingState.searchActive && m.meetingState.searchQuery == "" { - message := "No meetings found. Sync Granola or Circleback to import meetings." + message := "No meetings found. Sync a provider or import meetings." if len(m.meetingAccounts()) == 0 { - message = "No meeting sources configured. Add Granola or Circleback to begin." + message = "No meeting sources configured. Add a provider or import meetings to begin." } content := m.fillScreen(m.styles.normalRow.Render(padRight(message, m.width))) if m.modal != modalNone { diff --git a/internal/tui/meeting_view_test.go b/internal/tui/meeting_view_test.go index 111531165..3654752c2 100644 --- a/internal/tui/meeting_view_test.go +++ b/internal/tui/meeting_view_test.go @@ -62,8 +62,32 @@ func TestMeetingEmptyStateGuidesSourceSetup(t *testing.T) { view := stripANSI(model.renderView()) assert.Contains(t, view, "No meeting sources configured") - assert.Contains(t, view, "Granola") - assert.Contains(t, view, "Circleback") + assert.Contains(t, view, "provider") + assert.Contains(t, view, "import") +} + +func TestMeetingListViewShowsImportedSourceDisplayName(t *testing.T) { + model := NewBuilder().WithAccounts( + query.AccountInfo{ + ID: 5, + SourceType: meetingSourceImported, + Identifier: "local-meetings", + DisplayName: "Imported Interviews", + }, + ).WithSize(120, 24).Build() + model.mode = modeMeetings + model.loading = false + model.meetingState.messages = []query.MessageSummary{{ + ID: 10, + SourceID: 5, + Subject: "Synthetic interview", + SentAt: time.Date(2026, 7, 23, 18, 0, 0, 0, time.UTC), + MessageType: meetingMessageType, + }} + + view := stripANSI(model.renderView()) + + assert.Contains(t, view, "Imported ...") } func TestMeetingListViewShowsSearchInput(t *testing.T) { diff --git a/pkg/client/client.go b/pkg/client/client.go index 28c8ca7dd..b01c94927 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -222,6 +222,30 @@ func (c *Client) AddAccount( return nil, fmt.Errorf("add account: %w", err) } +// ImportMeeting accepts both documented success statuses. The generated +// convenience method treats only 201 as success even though an idempotent +// update returns 200. +func (c *Client) ImportMeeting( + ctx context.Context, + options *generated.ImportMeetingRequestOptions, + reqEditors ...runtime.RequestEditorFn, +) (*generated.ImportMeetingResponseJSON, error) { + resp, err := c.ImportMeetingWithResponse(ctx, options, reqEditors...) + if err != nil { + return nil, err + } + if resp.JSON201 != nil { + return resp.JSON201, nil + } + if resp.JSON200 != nil { + return resp.JSON200, nil + } + err = runtime.NewClientAPIError( + fmt.Errorf("unexpected status code: %d", resp.StatusCode), + runtime.WithStatusCode(resp.StatusCode)) + return nil, fmt.Errorf("import meeting: %w", err) +} + // StageDeletion accepts both documented success statuses. The generated // convenience method treats only 201 as success even though the daemon // returns 200 for dry-run staging requests. diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 04de81a4e..005bee0ed 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -336,6 +336,41 @@ func TestCreatePersonAcceptsIdempotentOK(t *testing.T) { assert.Equal(int64(2), got.Revision, "revision") } +func TestImportMeetingAcceptsIdempotentOK(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(http.MethodPost, r.Method, "method") + assert.Equal("/api/v1/import/meeting", r.URL.Path, "path") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte( + `{"status":"updated","source_id":3,"message_id":901,"source_message_id":"meeting:synthetic-meeting-42"}`, + )) + })) + t.Cleanup(server.Close) + + c, err := New(server.URL) + require.NoError(err, "New") + + got, err := c.ImportMeeting(context.Background(), &generated.ImportMeetingRequestOptions{ + Body: &generated.ImportMeetingBody{ + Source: generated.Source{ + Identifier: "local-meetings", + AccountEmail: "user@example.com", + }, + Meeting: generated.Meeting{ + ExternalID: "synthetic-meeting-42", + StartedAt: "2026-07-23T18:00:00Z", + }, + }, + }) + require.NoError(err, "ImportMeeting update") + + assert.Equal(generated.MeetingImportResponseStatusUpdated, got.Status, "status") + assert.Equal(int64(901), got.MessageID, "message id") +} + func TestStageDeletionAcceptsDryRunOK(t *testing.T) { assert := assert.New(t) require := require.New(t) diff --git a/pkg/client/generated/client.go b/pkg/client/generated/client.go index 6cb984d4b..b7ad7035d 100644 --- a/pkg/client/generated/client.go +++ b/pkg/client/generated/client.go @@ -303,6 +303,10 @@ type ClientInterface interface { UnlinkIdentityParticipants(ctx context.Context, options *UnlinkIdentityParticipantsRequestOptions, reqEditors ...runtime.RequestEditorFn) (*UnlinkIdentityParticipantsResponse, error) UnlinkIdentityParticipantsWithResponse(ctx context.Context, options *UnlinkIdentityParticipantsRequestOptions, reqEditors ...runtime.RequestEditorFn) (*UnlinkIdentityParticipantsResp, error) + // ImportMeeting Import one meeting + ImportMeeting(ctx context.Context, options *ImportMeetingRequestOptions, reqEditors ...runtime.RequestEditorFn) (*ImportMeetingResponseJSON, error) + ImportMeetingWithResponse(ctx context.Context, options *ImportMeetingRequestOptions, reqEditors ...runtime.RequestEditorFn) (*ImportMeetingResp, error) + // SearchIntegrationTasks Search tasks in the configured project SearchIntegrationTasks(ctx context.Context, options *SearchIntegrationTasksRequestOptions, reqEditors ...runtime.RequestEditorFn) (*SearchIntegrationTasksResponse, error) SearchIntegrationTasksWithResponse(ctx context.Context, options *SearchIntegrationTasksRequestOptions, reqEditors ...runtime.RequestEditorFn) (*SearchIntegrationTasksResp, error) @@ -4643,6 +4647,70 @@ func (c *Client) UnlinkIdentityParticipants(ctx context.Context, options *Unlink return responseParser(ctx, resp) } +// ImportMeeting Import one meeting +func (c *Client) ImportMeeting(ctx context.Context, options *ImportMeetingRequestOptions, reqEditors ...runtime.RequestEditorFn) (*ImportMeetingResponseJSON, error) { + var err error + reqParams := runtime.RequestOptionsParameters{ + RequestURL: c.apiClient.GetBaseURL() + "/api/v1/import/meeting", + Method: "POST", + Options: options, + ContentType: "application/json", + } + + req, err := c.apiClient.CreateRequest(ctx, reqParams, reqEditors...) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + + responseParser := func(ctx context.Context, resp *runtime.Response) (*ImportMeetingResponseJSON, error) { + bodyBytes := resp.Content + if resp.StatusCode != 201 { + target := new(ImportMeetingErrorResponse) + // Handle empty error response body gracefully - skip unmarshal if no content + if len(bodyBytes) > 0 { + if err = json.Unmarshal(bodyBytes, target); err != nil { + return nil, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "ImportMeetingErrorResponse", + Body: bodyBytes, + Err: err, + } + } + } + // Return error with (possibly empty) target + if errTarget, ok := any(*target).(error); ok { + return nil, runtime.NewClientAPIError(errTarget, runtime.WithStatusCode(resp.StatusCode)) + } + return nil, runtime.NewClientAPIError(fmt.Errorf("API error (status %d): %v", resp.StatusCode, *target), + runtime.WithStatusCode(resp.StatusCode)) + } + target := new(ImportMeetingResponseJSON) + // Handle empty response body gracefully + if len(bodyBytes) == 0 { + return target, nil + } + if err = json.Unmarshal(bodyBytes, target); err != nil { + return nil, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "ImportMeetingResponseJSON", + Body: bodyBytes, + Err: err, + } + } + return target, nil + } + + resp, err := c.apiClient.ExecuteRequest(ctx, req, "/api/v1/import/meeting") + if err != nil { + return nil, fmt.Errorf("error executing request: %w", err) + } + return responseParser(ctx, resp) +} + // SearchIntegrationTasks Search tasks in the configured project func (c *Client) SearchIntegrationTasks(ctx context.Context, options *SearchIntegrationTasksRequestOptions, reqEditors ...runtime.RequestEditorFn) (*SearchIntegrationTasksResponse, error) { var err error diff --git a/pkg/client/generated/client_options.go b/pkg/client/generated/client_options.go index 9798c6dbe..5a714cacb 100644 --- a/pkg/client/generated/client_options.go +++ b/pkg/client/generated/client_options.go @@ -2533,6 +2533,50 @@ func (o *UnlinkIdentityParticipantsRequestOptions) GetHeader() (map[string]strin return nil, nil } +// ImportMeetingRequestOptions is the options needed to make a request to ImportMeeting. +type ImportMeetingRequestOptions struct { + Body *ImportMeetingBody +} + +// Validate validates all the fields in the options. +// Use it if fields validation was not run. +func (o *ImportMeetingRequestOptions) Validate() error { + var errors runtime.ValidationErrors + + if o.Body != nil { + if v, ok := any(o.Body).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Body", err) + } + } + } + if len(errors) == 0 { + return nil + } + + return errors +} + +// GetPathParams returns the path params as a map. +func (o *ImportMeetingRequestOptions) GetPathParams() (map[string]any, error) { + return nil, nil +} + +// GetQuery returns the query params as a map. +func (o *ImportMeetingRequestOptions) GetQuery() (map[string]any, error) { + return nil, nil +} + +// GetBody returns the payload in any type that can be marshalled to JSON by the client. +func (o *ImportMeetingRequestOptions) GetBody() any { + return o.Body +} + +// GetHeader returns the headers as a map. +func (o *ImportMeetingRequestOptions) GetHeader() (map[string]string, error) { + return nil, nil +} + // SearchIntegrationTasksRequestOptions is the options needed to make a request to SearchIntegrationTasks. type SearchIntegrationTasksRequestOptions struct { Query *SearchIntegrationTasksQuery diff --git a/pkg/client/generated/client_with_response.go b/pkg/client/generated/client_with_response.go index 885308ebc..21e9d4178 100644 --- a/pkg/client/generated/client_with_response.go +++ b/pkg/client/generated/client_with_response.go @@ -4885,6 +4885,72 @@ func (c *Client) UnlinkIdentityParticipantsWithResponse(ctx context.Context, opt } } +// ImportMeeting Import one meeting +func (c *Client) ImportMeetingWithResponse(ctx context.Context, options *ImportMeetingRequestOptions, reqEditors ...runtime.RequestEditorFn) (*ImportMeetingResp, error) { + var err error + reqParams := runtime.RequestOptionsParameters{ + RequestURL: c.apiClient.GetBaseURL() + "/api/v1/import/meeting", + Method: "POST", + Options: options, + ContentType: "application/json", + } + + req, err := c.apiClient.CreateRequest(ctx, reqParams, reqEditors...) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + + resp, err := c.apiClient.ExecuteRequest(ctx, req, "/api/v1/import/meeting") + if err != nil { + return nil, fmt.Errorf("error executing request: %w", err) + } + + out := &ImportMeetingResp{ + HTTPResponse: resp.Raw, + Body: resp.Content, + StatusCode: resp.StatusCode, + } + + switch resp.StatusCode { + case 200: + out.JSON200 = new(ImportMeetingResponse) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON200); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "ImportMeetingResponse", + Body: bodyBytes, + Err: err, + } + } + } + return out, nil + case 201: + out.JSON201 = new(ImportMeetingResponseJSON) + bodyBytes := resp.Content + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, out.JSON201); err != nil { + return out, &runtime.ResponseDecodeError{ + StatusCode: resp.StatusCode, + ContentType: resp.Headers.Get("Content-Type"), + ContentLength: len(bodyBytes), + TargetType: "ImportMeetingResponseJSON", + Body: bodyBytes, + Err: err, + } + } + } + return out, nil + case 500: + return out, runtime.NewClientAPIError(fmt.Errorf("API error (status %d)", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + default: + return out, runtime.NewClientAPIError(fmt.Errorf("unexpected status code: %d", resp.StatusCode), runtime.WithStatusCode(resp.StatusCode)) + } +} + // SearchIntegrationTasks Search tasks in the configured project func (c *Client) SearchIntegrationTasksWithResponse(ctx context.Context, options *SearchIntegrationTasksRequestOptions, reqEditors ...runtime.RequestEditorFn) (*SearchIntegrationTasksResp, error) { var err error diff --git a/pkg/client/generated/enums.go b/pkg/client/generated/enums.go index 314ef88bb..b2e98a8ed 100644 --- a/pkg/client/generated/enums.go +++ b/pkg/client/generated/enums.go @@ -333,6 +333,23 @@ func (i IdentitySearchSortField) Validate() error { } } +type MeetingImportResponseStatus string + +const ( + MeetingImportResponseStatusCreated MeetingImportResponseStatus = "created" + MeetingImportResponseStatusUpdated MeetingImportResponseStatus = "updated" +) + +// Validate checks if the MeetingImportResponseStatus value is valid +func (m MeetingImportResponseStatus) Validate() error { + switch m { + case MeetingImportResponseStatusCreated, MeetingImportResponseStatusUpdated: + return nil + default: + return runtime.NewValidationErrorsFromString("Enum", fmt.Sprintf("must be a valid MeetingImportResponseStatus value, got: %v", m)) + } +} + type RemoveResultCacheState string const ( diff --git a/pkg/client/generated/payloads.go b/pkg/client/generated/payloads.go index 8d72fd20f..a0fc72624 100644 --- a/pkg/client/generated/payloads.go +++ b/pkg/client/generated/payloads.go @@ -68,6 +68,8 @@ type LinkIdentityParticipantsBody = IdentityLinkRequest type UnlinkIdentityParticipantsBody = IdentityLinkRequest +type ImportMeetingBody = MeetingImportRequest + type CreateOrLinkMessageTaskBody = TaskLinkMutationRequest type SearchPeopleBody = IdentitySearchHTTPRequest diff --git a/pkg/client/generated/responses.go b/pkg/client/generated/responses.go index f9186692d..e936626cb 100644 --- a/pkg/client/generated/responses.go +++ b/pkg/client/generated/responses.go @@ -841,6 +841,12 @@ type UnlinkIdentityParticipantsResponse = IdentityLinkResponse type UnlinkIdentityParticipantsErrorResponse = ErrorResponse +type ImportMeetingResponse = MeetingImportResponse + +type ImportMeetingResponseJSON = MeetingImportResponse + +type ImportMeetingErrorResponse = ErrorResponse + type SearchIntegrationTasksResponse = TaskSearchResponse type SearchIntegrationTasksErrorResponse = ErrorResponse @@ -1950,6 +1956,14 @@ type UnlinkIdentityParticipantsResp struct { JSON200 *UnlinkIdentityParticipantsResponse } +type ImportMeetingResp struct { + HTTPResponse *http.Response + Body []byte + StatusCode int + JSON200 *ImportMeetingResponse + JSON201 *ImportMeetingResponseJSON +} + type SearchIntegrationTasksResp struct { HTTPResponse *http.Response Body []byte diff --git a/pkg/client/generated/types.go b/pkg/client/generated/types.go index 495d660af..e6617ec62 100644 --- a/pkg/client/generated/types.go +++ b/pkg/client/generated/types.go @@ -2441,6 +2441,115 @@ type MatchSummary struct { StrongestExcerpt *string `json:"strongest_excerpt,omitempty"` } +type Meeting struct { + Attendees []MeetingPerson `json:"attendees,omitempty"` + EndedAt *string `json:"ended_at,omitempty"` + ExternalID string `json:"external_id" validate:"required,max=256"` + Metadata map[string]any `json:"metadata,omitempty"` + Organizer *MeetingPerson `json:"organizer,omitempty"` + StartedAt string `json:"started_at" validate:"required"` + SummaryMarkdown *string `json:"summary_markdown,omitempty"` + SummaryText *string `json:"summary_text,omitempty"` + Title *string `json:"title,omitempty" validate:"omitempty,max=4096"` + Transcript *string `json:"transcript,omitempty"` + TranscriptSegments []TranscriptSegment `json:"transcript_segments,omitempty"` +} + +func (m Meeting) Validate() error { + var errors runtime.ValidationErrors + for i, item := range m.Attendees { + if v, ok := any(item).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append(fmt.Sprintf("Attendees[%d]", i), err) + } + } + } + if err := typesValidator.Var(m.ExternalID, "required,max=256"); err != nil { + errors = errors.Append("ExternalID", err) + } + if m.Organizer != nil { + if v, ok := any(m.Organizer).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Organizer", err) + } + } + } + if err := typesValidator.Var(m.StartedAt, "required"); err != nil { + errors = errors.Append("StartedAt", err) + } + if m.Title != nil { + if err := typesValidator.Var(m.Title, "omitempty,max=4096"); err != nil { + errors = errors.Append("Title", err) + } + } + for i, item := range m.TranscriptSegments { + if v, ok := any(item).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append(fmt.Sprintf("TranscriptSegments[%d]", i), err) + } + } + } + if len(errors) == 0 { + return nil + } + return errors +} + +type MeetingImportRequest struct { + Meeting Meeting `json:"meeting"` + Source Source `json:"source"` +} + +func (m MeetingImportRequest) Validate() error { + var errors runtime.ValidationErrors + if v, ok := any(m.Meeting).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Meeting", err) + } + } + if v, ok := any(m.Source).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Source", err) + } + } + if len(errors) == 0 { + return nil + } + return errors +} + +type MeetingImportResponse struct { + MessageID int64 `json:"message_id"` + SourceID int64 `json:"source_id"` + SourceMessageID string `json:"source_message_id" validate:"required"` + Status MeetingImportResponseStatus `json:"status" validate:"required"` +} + +func (m MeetingImportResponse) Validate() error { + var errors runtime.ValidationErrors + if err := typesValidator.Var(m.SourceMessageID, "required"); err != nil { + errors = errors.Append("SourceMessageID", err) + } + if v, ok := any(m.Status).(runtime.Validator); ok { + if err := v.Validate(); err != nil { + errors = errors.Append("Status", err) + } + } + if len(errors) == 0 { + return nil + } + return errors +} + +type MeetingPerson struct { + Email string `json:"email" validate:"required"` + Name *string `json:"name,omitempty"` +} + +func (m MeetingPerson) Validate() error { + return runtime.ConvertValidatorError(typesValidator.Struct(m)) +} + type MessageDetail struct { Attachments []AttachmentInfo `json:"attachments,omitempty" validate:"required"` Bcc []string `json:"bcc,omitempty"` @@ -3557,6 +3666,16 @@ func (s SimilarSearchResponse) Validate() error { return errors } +type Source struct { + AccountEmail string `json:"account_email" validate:"required"` + DisplayName *string `json:"display_name,omitempty" validate:"omitempty,max=256"` + Identifier string `json:"identifier" validate:"required,max=128"` +} + +func (s Source) Validate() error { + return runtime.ConvertValidatorError(typesValidator.Struct(s)) +} + type SourceCount struct { Count int64 `json:"count"` SourceType string `json:"source_type" validate:"required"` @@ -4139,6 +4258,16 @@ type TotalStatsResponse struct { TotalSize int64 `json:"total_size"` } +type TranscriptSegment struct { + OffsetSeconds *float64 `json:"offset_seconds,omitempty" validate:"omitempty,gte=0"` + Speaker string `json:"speaker" validate:"required"` + Text string `json:"text" validate:"required"` +} + +func (t TranscriptSegment) Validate() error { + return runtime.ConvertValidatorError(typesValidator.Struct(t)) +} + type UpdateRequest struct { DisplayName string `json:"display_name" validate:"required"` Email string `json:"email" validate:"required"` diff --git a/pkg/client/openapi.yaml b/pkg/client/openapi.yaml index c9f3c55b8..b4e856564 100644 --- a/pkg/client/openapi.yaml +++ b/pkg/client/openapi.yaml @@ -2543,6 +2543,96 @@ components: strongest_excerpt: type: string type: object + Meeting: + additionalProperties: false + properties: + attendees: + items: + $ref: "#/components/schemas/MeetingPerson" + nullable: true + type: array + ended_at: + format: date-time + type: string + x-go-type: string + external_id: + maxLength: 256 + type: string + metadata: + additionalProperties: + x-go-type: any + type: object + organizer: + $ref: "#/components/schemas/MeetingPerson" + started_at: + format: date-time + type: string + x-go-type: string + summary_markdown: + type: string + summary_text: + type: string + title: + maxLength: 4096 + type: string + transcript: + type: string + transcript_segments: + items: + $ref: "#/components/schemas/TranscriptSegment" + nullable: true + type: array + required: + - external_id + - started_at + type: object + MeetingImportRequest: + additionalProperties: false + properties: + meeting: + $ref: "#/components/schemas/Meeting" + source: + $ref: "#/components/schemas/Source" + required: + - source + - meeting + type: object + MeetingImportResponse: + properties: + message_id: + format: int64 + type: integer + source_id: + format: int64 + type: integer + source_message_id: + type: string + status: + enum: + - created + - updated + type: string + x-enum-names: + - MeetingImportResponseStatusCreated + - MeetingImportResponseStatusUpdated + required: + - status + - source_id + - message_id + - source_message_id + type: object + MeetingPerson: + additionalProperties: false + properties: + email: + format: email + type: string + x-go-type: string + name: + type: string + required: + - email + type: object MessageDetail: properties: attachments: @@ -3651,6 +3741,23 @@ components: - generation - messages type: object + Source: + additionalProperties: false + properties: + account_email: + format: email + type: string + x-go-type: string + display_name: + maxLength: 256 + type: string + identifier: + maxLength: 128 + type: string + required: + - identifier + - account_email + type: object SourceCount: properties: count: @@ -4314,6 +4421,21 @@ components: - label_count - account_count type: object + TranscriptSegment: + additionalProperties: false + properties: + offset_seconds: + format: double + minimum: 0 + type: number + speaker: + type: string + text: + type: string + required: + - speaker + - text + type: object UpdateRequest: additionalProperties: false properties: @@ -4351,7 +4473,7 @@ components: type: apiKey info: title: msgvault API - version: 1.32.0 + version: 1.33.0 openapi: 3.0.3 paths: /api/ping: @@ -7069,6 +7191,39 @@ paths: summary: Remove a link edge between two participants tags: - API + /api/v1/import/meeting: + post: + operationId: importMeeting + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportResponse" + description: OK + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportResponse" + description: Created + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Import one meeting + tags: + - API /api/v1/integrations/tasks/search: get: operationId: searchIntegrationTasks diff --git a/web/src/lib/api/generated/schema.d.ts b/web/src/lib/api/generated/schema.d.ts index 68a8123cf..deefeca80 100644 --- a/web/src/lib/api/generated/schema.d.ts +++ b/web/src/lib/api/generated/schema.d.ts @@ -1032,6 +1032,23 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/import/meeting": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Import one meeting */ + post: operations["importMeeting"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/integrations/tasks/search": { parameters: { query?: never; @@ -2875,6 +2892,51 @@ export interface components { } & { [key: string]: unknown; }; + Meeting: { + attendees?: components["schemas"]["MeetingPerson"][] | null; + /** Format: date-time */ + ended_at?: string; + external_id: string; + metadata?: { + [key: string]: unknown; + }; + organizer?: components["schemas"]["MeetingPerson"]; + /** Format: date-time */ + started_at: string; + summary_markdown?: string; + summary_text?: string; + title?: string; + transcript?: string; + transcript_segments?: components["schemas"]["TranscriptSegment"][] | null; + } & (({ + summary_markdown: string; + } | { + summary_text: string; + } | { + transcript: string; + } | { + transcript_segments: unknown[]; + }) & unknown); + MeetingImportRequest: { + meeting: components["schemas"]["Meeting"]; + source: components["schemas"]["Source"]; + }; + MeetingImportResponse: { + /** Format: int64 */ + message_id: number; + /** Format: int64 */ + source_id: number; + source_message_id: string; + /** @enum {string} */ + status: "created" | "updated"; + } & { + [key: string]: unknown; + }; + MeetingPerson: { + /** Format: email */ + email: string; + name?: string; + }; MessageDetail: { attachments: components["schemas"]["AttachmentInfo"][] | null; bcc?: string[] | null; @@ -3371,6 +3433,12 @@ export interface components { } & { [key: string]: unknown; }; + Source: { + /** Format: email */ + account_email: string; + display_name?: string; + identifier: string; + }; SourceCount: { /** Format: int64 */ count: number; @@ -3685,6 +3753,12 @@ export interface components { } & { [key: string]: unknown; }; + TranscriptSegment: { + /** Format: double */ + offset_seconds?: number; + speaker: string; + text: string; + }; UpdateRequest: { display_name: string; email: string; @@ -6895,6 +6969,48 @@ export interface operations { }; }; }; + importMeeting: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["MeetingImportRequest"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MeetingImportResponse"]; + }; + }; + /** @description Created */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["MeetingImportResponse"]; + }; + }; + /** @description Error */ + default: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + }; + }; searchIntegrationTasks: { parameters: { query: { diff --git a/web/src/lib/components/sources/SourcesWorkspace.svelte b/web/src/lib/components/sources/SourcesWorkspace.svelte index 0668e873e..2aff03f26 100644 --- a/web/src/lib/components/sources/SourcesWorkspace.svelte +++ b/web/src/lib/components/sources/SourcesWorkspace.svelte @@ -74,6 +74,10 @@ return source.sync_unavailable_reason === 'sync_already_running'; } + function isOnDemandSource(source: Source): boolean { + return source.source_type === 'meeting_import'; + } + // allowIdle bypasses the active-sync gate below for error-path retries: // a status-load failure must be able to reschedule itself even when no // source is actively syncing and no accepted run is being awaited, or an @@ -243,6 +247,8 @@ {#if source.scheduled} Scheduled · {source.schedule ?? 'schedule unavailable'} {#if source.next_sync_at}Next {source.next_sync_at}{/if} + {:else if isOnDemandSource(source)} + On demand · imported through the API {:else}Not scheduled{/if} {#if source.scheduler_last_error}Scheduler: {source.scheduler_last_error}{/if} @@ -275,6 +281,8 @@
{#if source.can_sync}