diff --git a/ts/receiver/configMessage.ts b/ts/receiver/configMessage.ts index df61dd6394..a39754dab4 100644 --- a/ts/receiver/configMessage.ts +++ b/ts/receiver/configMessage.ts @@ -108,14 +108,21 @@ async function printDumpForDebug(prefix: string, variant: ConfigWrapperObjectTyp window.log.info(prefix, StringUtils.toHex(metaGroupDumps)); } +/** + * Note the `mergedCount` in the result. libSession's `merge()` reports which hashes it actually + * took in, and a message it could not merge is simply absent from that list — no throw, no error. + * So 2-of-3 looks exactly like 3-of-3 to a caller that only checks for an exception, and anything + * relying on "we incorporated what we fetched" has to compare the counts instead. + */ async function mergeUserConfigsWithIncomingUpdates( incomingConfigs: Array -): Promise> { +): Promise<{ results: Map; mergedCount: number }> { // first, group by namespaces so we do a single merge call // Note: this call throws if given a non user kind as this function should only handle user variants/kinds const groupedByNamespaces = byUserNamespace(incomingConfigs); const groupedResults: Map = new Map(); + let mergedCount = 0; const us = UserUtils.getOurPubKeyStrFromCache(); @@ -187,6 +194,8 @@ async function mergeUserConfigsWithIncomingUpdates( assertUnreachable(variant, `mergeConfigsWithInboxUpdates unhandled case "${variant}"`); } + mergedCount += hashesMerged.length; + const needsDump = await UserGenericWrapperActions.needsDump(variant); const needsPush = await UserGenericWrapperActions.needsPush(variant); const mergedTimestamps = sameVariant @@ -211,7 +220,7 @@ async function mergeUserConfigsWithIncomingUpdates( groupedResults.set(variant, incomingConfResult); } - return groupedResults; + return { results: groupedResults, mergedCount }; } catch (e) { window.log.error('mergeConfigsWithIncomingUpdates failed with', e); throw e; @@ -981,11 +990,16 @@ async function processUserMergingResults(results: Map -) { +): Promise { if (isEmpty(configMessages)) { - return; + return true; } window?.log?.debug( @@ -997,8 +1011,16 @@ async function handleUserConfigMessagesViaLibSession( )}` ); - const incomingMergeResult = await mergeUserConfigsWithIncomingUpdates(configMessages); - await processUserMergingResults(incomingMergeResult); + const { results, mergedCount } = await mergeUserConfigsWithIncomingUpdates(configMessages); + await processUserMergingResults(results); + + if (mergedCount !== configMessages.length) { + window.log.warn( + `handleUserConfigMessagesViaLibSession: merged ${mergedCount}/${configMessages.length} config messages` + ); + } + + return mergedCount === configMessages.length; } async function updateOurProfileFromLibSession({ diff --git a/ts/session/apis/snode_api/SnodeRequestTypes.ts b/ts/session/apis/snode_api/SnodeRequestTypes.ts index e4e7a6a82f..5867da2c35 100644 --- a/ts/session/apis/snode_api/SnodeRequestTypes.ts +++ b/ts/session/apis/snode_api/SnodeRequestTypes.ts @@ -94,6 +94,19 @@ abstract class ExpireSubRequest extends SnodeAPISubRequest<'expire'> { } } +/** + * The flag the storage server expects on the wire, for a given ShortenOrExtend. + * Note: the same value has to be used to build the signature, otherwise the request is signed + * for one behaviour and asks the server for another (and the server won't complain). + */ +function shortenOrExtendToParams(shortenOrExtend: ShortenOrExtend) { + return shortenOrExtend === 'extend' + ? { extend: true } + : shortenOrExtend === 'shorten' + ? { shorten: true } + : {}; +} + abstract class StoreSubRequest extends SnodeAPISubRequest<'store'> { public readonly getNow: () => number; @@ -775,13 +788,6 @@ export class UpdateExpiryOnNodeUserSubRequest extends ExpireSubRequest { ); } - const shortenOrExtend = - this.shortenOrExtend === 'extend' - ? { extend: true } - : this.shortenOrExtend === 'shorten' - ? { shorten: true } - : {}; - return { method: this.method, params: { @@ -790,7 +796,7 @@ export class UpdateExpiryOnNodeUserSubRequest extends ExpireSubRequest { signature: signResult.signature, messages: this.messageHashes, expiry: this.expiryMs, - ...shortenOrExtend, + ...shortenOrExtendToParams(this.shortenOrExtend), }, }; } @@ -807,6 +813,12 @@ export class UpdateExpiryOnNodeUserSubRequest extends ExpireSubRequest { export class UpdateExpiryOnNodeGroupSubRequest extends ExpireSubRequest { public readonly messageHashes: Array; public readonly expiryMs: number; + /** + * Same shape as the user request deliberately: the storage server supports shorten on group + * expiries too, so narrowing this to 'extend' would state something false about the endpoint. + * Note the value is read TWICE — for the signature and for the wire flag — and the two must + * agree, which is why both derive from this one field rather than from a literal. + */ public readonly shortenOrExtend: ShortenOrExtend; public readonly groupDetailsNeededForSignature: GroupDetailsNeededForSignature; @@ -846,18 +858,11 @@ export class UpdateExpiryOnNodeGroupSubRequest extends ExpireSubRequest { ); } - const shortenOrExtend = - this.shortenOrExtend === 'extend' - ? { extends: true } - : this.shortenOrExtend === 'shorten' - ? { shorten: true } - : {}; - return { method: this.method, params: { messages: this.messageHashes, - ...shortenOrExtend, + ...shortenOrExtendToParams(this.shortenOrExtend), ...signResult, // pubkey_ed25519 is forbidden for the group one @@ -966,8 +971,15 @@ abstract class StoreGroupConfigSubRequest< public readonly destination: GroupPubkeyType; public readonly ttlMs: number; public readonly encryptedData: Uint8Array; - // this is mandatory for a group config store, if it is null, we throw public readonly secretKey: Uint8Array | null; + /** + * A member's subaccount auth, used when we have no admin `secretKey`. + * + * Only config RECOVERY takes this path: a member cannot change group config, so the normal push + * always has the admin key. But a member CAN put its own unmodified copy back after it expires + * from the swarm, and its subaccount token carries Read+Write for exactly that. + */ + public readonly authData: Uint8Array | null; constructor( args: WithGroupPubkey & @@ -975,6 +987,7 @@ abstract class StoreGroupConfigSubRequest< namespace: T; encryptedData: Uint8Array; secretKey: Uint8Array | null; + authData?: Uint8Array | null; ttlMs: number; } ) { @@ -984,6 +997,7 @@ abstract class StoreGroupConfigSubRequest< this.ttlMs = args.ttlMs; this.encryptedData = args.encryptedData; this.secretKey = args.secretKey; + this.authData = args.authData ?? null; if (isEmpty(this.encryptedData)) { throw new Error('this.encryptedData cannot be empty'); @@ -991,8 +1005,10 @@ abstract class StoreGroupConfigSubRequest< if (!PubKey.is03Pubkey(this.destination)) { throw new Error('StoreGroupConfigSubRequest: group config namespace required a 03 pubkey'); } - if (isEmpty(this.secretKey)) { - throw new Error('StoreGroupConfigSubRequest needs secretKey to be set'); + // Either credential will do, but not neither: `getSnodeGroupSignature` prefers the admin key + // and falls back to the subaccount, and with both empty it cannot sign at all. + if (isEmpty(this.secretKey) && isEmpty(this.authData)) { + throw new Error('StoreGroupConfigSubRequest needs secretKey or authData to be set'); } } @@ -1006,7 +1022,7 @@ abstract class StoreGroupConfigSubRequest< const signDetails = await SnodeGroupSignature.getSnodeGroupSignature({ method: this.method, namespace: this.namespace, - group: { authData: null, pubkeyHex: this.destination, secretKey: this.secretKey }, + group: { authData: this.authData, pubkeyHex: this.destination, secretKey: this.secretKey }, }); if (!signDetails) { diff --git a/ts/session/apis/snode_api/configExpiryDetection.ts b/ts/session/apis/snode_api/configExpiryDetection.ts new file mode 100644 index 0000000000..d315d5aa93 --- /dev/null +++ b/ts/session/apis/snode_api/configExpiryDetection.ts @@ -0,0 +1,99 @@ +import { isArray, isEmpty } from 'lodash'; +import { ExpireMessageResultItem, ExpireMessagesResultsContent } from './types'; + +/** + * Deciding whether a config message has expired from the swarm, from the response to the `expire` + * request we piggyback on every poll. + * + * This is a normative rule shared with iOS and Android. The three clients each implement it + * separately, so if you change the behaviour here it has to change there too. Every rule below has + * a test vector. + */ + +export type ConfigExpiryDetection = + /** + * The response cannot answer the question. Either we didn't ask for `extend` (so the server + * omits `unchanged` entirely and every hash we didn't update *looks* missing), or no + * sub-response was usable. Nothing may be marked missing from this. + */ + | { status: 'unavailable' } + /** + * Every snode either failed or timed out. Distinct from "nothing is missing": we simply have no + * evidence either way. + */ + | { status: 'inconclusive' } + | { status: 'conclusive'; missingHashes: Array }; + +/** + * A sub-response contributes to the decision only if the snode actually answered. + * + * `failed: true` may come with `timeout`, `code`, `reason`, `bad_peer_response` or + * `query_failure` — none of that matters, `failed` alone is enough to exclude it. Treating a + * timeout as "that snode doesn't have the message" would turn every network blip into a re-push + * storm, which is the single worst thing this code could do. + */ +function isEligible(subResponse: ExpireMessageResultItem | undefined): subResponse is Eligible { + if (!subResponse || subResponse.failed || !isArray(subResponse.updated)) { + return false; + } + // The server sets `unchanged` whenever the request set `extend` (or `shorten`), even when it is + // empty. So if the key is absent, this response cannot tell presence from absence and has to be + // excluded rather than read as "nothing was unchanged". + return !!subResponse.unchanged; +} + +type Eligible = ExpireMessageResultItem & { unchanged: Record }; + +function subResponseHolds(subResponse: Eligible, hash: string) { + return subResponse.updated.includes(hash) || hash in subResponse.unchanged; +} + +/** + * @param requestedHashes the hashes the `expire` sub-request asked about + * @param swarm the per-snode `swarm` dict from the recursive `expire` response + * @param requestSetExtend whether the request we are reading the response of set `extend: true`. + * Note: this must be what *we* sent. The server silently forces extend-only semantics for a group + * member's subaccount without telling us, and does *not* return `unchanged` in that case. + */ +export function detectMissingConfigHashes({ + requestedHashes, + swarm, + requestSetExtend, +}: { + requestedHashes: Array; + swarm: ExpireMessagesResultsContent | null | undefined; + requestSetExtend: boolean; +}): ConfigExpiryDetection { + if (isEmpty(requestedHashes)) { + // We asked about nothing, so we learned nothing. The tempting short-circuit here is + // "no hashes requested, therefore none are missing" — but reporting that as *conclusive* makes + // detection the authority for a swarm it has no information about, and a conclusive result + // outranks the empty-fetch check. That check is precisely the one that should decide when we + // hold no hashes, and it could then never be reached. + return { status: 'inconclusive' }; + } + + if (!requestSetExtend) { + return { status: 'unavailable' }; + } + + if (!swarm || isEmpty(swarm)) { + return { status: 'inconclusive' }; + } + + const eligible = Object.values(swarm).filter(isEligible); + + if (!eligible.length) { + return { status: 'inconclusive' }; + } + + // One eligible snode reporting a hash absent is enough (D1). Presence elsewhere does not + // override it: re-storing is idempotent, so a false positive costs one redundant request, + // whereas waiting for a consensus leans on the swarm replication that is itself the unreliable + // part here. + const missingHashes = requestedHashes.filter(hash => + eligible.some(subResponse => !subResponseHolds(subResponse, hash)) + ); + + return { status: 'conclusive', missingHashes }; +} diff --git a/ts/session/apis/snode_api/configRecovery.ts b/ts/session/apis/snode_api/configRecovery.ts new file mode 100644 index 0000000000..52d2feb742 --- /dev/null +++ b/ts/session/apis/snode_api/configRecovery.ts @@ -0,0 +1,1226 @@ +/* eslint-disable no-await-in-loop */ +import AbortController from 'abort-controller'; +import { GroupPubkeyType, PubkeyType } from 'libsession_util_nodejs'; +import { chunk, isEmpty } from 'lodash'; +import { UserUtils } from '../../utils'; +import type { ConfigWrapperUser } from '../../../webworker/workers/browser/libsession_worker_functions'; +import { + MetaGroupWrapperActions, + UserGenericWrapperActions, + UserGroupsWrapperActions, +} from '../../../webworker/workers/browser/libsession_worker_interface'; +import { LibSessionUtil } from '../../utils/libsession/libsession_utils'; +import { DURATION, TTL_DEFAULT } from '../../constants'; +import { NetworkTime } from '../../../util/NetworkTime'; +import { MessageSender } from '../../sending/MessageSender'; +import { timeoutWithAbort } from '../../utils/Promise'; +import { + DeleteHashesFromGroupNodeSubRequest, + DeleteHashesFromUserNodeSubRequest, + MAX_SUBREQUESTS_COUNT, + StoreGroupInfoSubRequest, + StoreGroupKeysSubRequest, + StoreGroupMembersSubRequest, + StoreUserConfigSubRequest, +} from './SnodeRequestTypes'; +import { ConfigExpiryDetection } from './configExpiryDetection'; +import { ed25519Str, fromBase64ToArray } from '../../utils/String'; +import { PubKey } from '../../types'; +import { ConvoHub } from '../../conversations'; +import { SnodePool } from './snodePool'; +import { SnodeAPIRetrieve } from './retrieveRequest'; +import { SnodeNamespaces } from './namespaces'; + +type SnodeSubRequestForRecovery = StoreUserConfigSubRequest | DeleteHashesFromUserNodeSubRequest; + +type StoreGroupConfigSubRequestForRecovery = + | StoreGroupInfoSubRequest + | StoreGroupMembersSubRequest + | StoreGroupKeysSubRequest; +type GroupSubRequestForRecovery = + | StoreGroupConfigSubRequestForRecovery + | DeleteHashesFromGroupNodeSubRequest; + +/** + * An ACCOUNT pubkey — our own (`05…`) or a group's (`03…`). Never a snode's ed25519 key. + * + * Spelled out because everything here is "per swarm", and a swarm is identified by the account + * whose swarm it is; a bare `string` left a reader working that out from the call sites. + */ +type AccountPubkey = PubkeyType | GroupPubkeyType; + +/** + * Putting a config message back on the swarm after it expired from it. + * + * The whole thing rests on config encryption being deterministic: re-storing an *unchanged* config + * produces the identical message hash it had before, so this is the same message going back where + * it was, not a new one competing with existing state. Which is why nothing here is allowed to + * dirty a config to force an upload — that would bump the seqno and trigger a merge, and turn a + * repair into the destructive thing this design exists to avoid. + * + * Both our own configs and a group's are recovered here. The two differ only in how a config is + * inspected and put back, so the guards and the bookkeeping are shared and the split happens as + * late as possible. + * + * Three things about the group path are easy to mistake for bugs: + * - `GroupKeys` is recoverable ONLY from retained bytes. A keys message is admin-signed and its + * padding derives from the group secret key, so nobody can regenerate one — but bytes already + * held push back verbatim and land on the same hash, which is what lets a MEMBER repair a + * group's keys. Where no bytes are held it is unrecoverable BY THIS DEVICE and settles; another + * peer holding them can still put them back. + * - a MEMBER gets an empty obsolete-hash list. `push()` hands the superseded hashes back only + * `if (!is_readonly())` while clearing them either way (`base.cpp:809-813`), so empty is the + * expected result rather than a sign anything failed. + * - a member could not act on a non-empty list anyway: its subaccount token carries Read+Write + * but not Delete. Member-driven recovery re-stores and never prunes; the superseded messages + * wait for an admin's next push. + */ + +/** + * None of this state is persisted, deliberately rather than merely unimplemented: the level-with-swarm rule asks + * what has happened since this process started, so a verdict reloaded from disk would be answering + * that question about a previous run. + * + * The scoping is NOT uniform across these declarations, though it reads as if it should be and it + * once was — the two Sets below are session-scoped, `hashSettledAt` is time-bounded. + * + * SESSION-SCOPED HERE MEANS PROCESS-LIFETIME, WHICH ON DESKTOP IS WEEKS. Nothing ages these Sets + * out: `swarmsLevelWithLocalState` is added to on the first good poll of the process and removed + * only by the sticky merge-incomplete withdrawal. So `localStateIsLevelWithSwarm` answers "was + * level at SOME POINT since startup", never "is level now", and the staleness it permits is + * unbounded. + * + * That is correct for what reads it today: recovery is a cheap, idempotent re-store, so acting on + * a stale verdict costs a redundant request. It is NOT correct for anything irreversible or + * externally visible — a force-rekey encrypts to THIS DEVICE'S view of the members, so a stale + * "level" verdict authorises a write from a members list we already know may be behind, and + * silently drops anyone added since. That fails OPEN. + * + * Before reading either Set, check its lifetime against what YOU are about to do with it rather + * than against what its existing caller does. `hashSettledAt` was made time-bounded for exactly + * this reason and the same reasoning was never applied one declaration up. + */ +/** + * pubkey -> the poll token that was current when we last marked this swarm level. + * + * A Map rather than a Set because two different consumers ask two different questions of it, and + * one field answers both: + * + * has(pubkey) "were we EVER level this session" — recovery's precondition + * stored === current token "are we level AS OF THIS POLL" — the force rekey's + * + * Recovery is a cheap idempotent re-store, so acting on a stale verdict costs a redundant request + * and the sticky question is right for it. A rekey encrypts to this device's view of the members + * and cannot be undone, so it needs the poll-scoped one. Same value, two readings — deliberately + * NOT two fields, or they would drift. + */ +const swarmsLevelWithLocalState = new Map(); +/** + * pubkey -> a token that changes every time a poll begins for that swarm. + * + * Per swarm, not global: a poll of some other pubkey must not invalidate this one's mark. + */ +const currentPollToken = new Map(); +const swarmsWithIncompleteMerge = new Set(); +/** + * hash -> when it was settled, for either of two reasons that must not be conflated with a FAILED + * store, which stays retryable: + * - it was stored successfully; or + * - a guard ruled it out. + * + * A Map rather than a Set, and "at" rather than "this session", because the bar is TIME-BOUNDED + * — see HASH_BAR_MS. This was first written as a permanent, session-scoped bar, justified by the + * claim that no guard's verdict can change within a session. That sentence is false on any session + * measured in hours, which on Desktop is all of them (there is no foreground gate): a kicked group can + * be rejoined, a destroyed one replaced, a dirty config settle. Re-examining a guard costs no + * network call, so a permanent bar buys nothing and silently withdraws the device. + */ +const hashSettledAt = new Map(); +const missingHashesByPubkey = new Map>(); +/** + * Swarms with a recovery round currently running — see the guard at the top of recoverIfNeeded. + * + * Holds the round's promise rather than just a marker, so a caller that wants to know when the + * round finishes can await it. Nothing in production does (the poller deliberately does not wait), + * but it is what lets a test assert on the outcome of an unawaited round without sleeping. + */ +const recoveryInFlight = new Map>(); +/** + * Groups where a keys backfill has run and the bytes are STILL absent. + * + * In memory on purpose. A persisted record would be a sticky negative — it would let the rekey fire + * on evidence gathered weeks ago, after the swarm has changed underneath it. Forgetting on restart + * delays the rekey by one poll cycle, which is the safe direction for the one irreversible, + * externally visible write in this feature. + */ +const keysBackfillFailedAt = new Map(); +const recoveryAttemptsBySwarm = new Map< + AccountPubkey, + { consecutiveFailures: number; lastAttemptAt: number } +>(); + +/** + * Recovery attempts for one swarm are RATE-LIMITED, deliberately not capped. + * + * Releasing a failed attempt for retry and bounding the retries are a pair; either alone is wrong. + * Without the release, a partial failure is banked as done and never repaired. Without the bound, a + * persistently-failing store is retried on every poll — every few seconds — which is the re-push + * storm this design exists to avoid. + * + * Why a backoff and NOT a "give up after N rounds" cap, which is what this was first written as: + * a cap re-creates the very exclusion the rate limit was corrected to remove, one layer up. Three transient + * network failures would withdraw the device for the rest of the session — and a Desktop session can + * be days — while intermittent connectivity correlates with having been offline long enough for the + * config to expire in the first place. So the cap would exclude exactly the population the feature + * exists for. A backoff bounds the RATE without ever excluding anyone. + * + * That test — WHICH POPULATION DOES THIS EXCLUDE, AND DOES IT CORRELATE WITH NEEDING THE REPAIR? — + * is worth applying to any bound added here. It has caught this same mistake twice. + * + * Note the real request count is NOT one per attempt: `sendEncryptedDataToSnode` wraps each send in + * pRetry with `retries: 2`, so the worst case is 3 attempts x (parts + 1 delete) per entry below. + * That wrapper lives in MessageSender, a long way from here, and is easy not to know about. + */ +/** + * Read the clock through an indirection so the backoff is testable without faking global timers — + * freezing Date breaks mocha's own timeout accounting. Mirrors the `getNow` argument the store + * sub-requests already take. Local scheduling only, so Date.now is correct here rather than + * NetworkTime: nothing is compared against a value that came from the network. + */ +let nowMs: () => number = () => Date.now(); + +/** + * How long a successfully re-stored hash is barred from being re-stored again. + * + * NOT "for the session". A session is unbounded in time and the config TTL is 30 days, so on + * Desktop — which has no foreground gate and runs for weeks by design — a session-scoped + * bar can outlive the TTL. The hash would then expire from the swarm a second time and the bar + * would block the very recovery that should put it back, on exactly the long-lived sessions where + * configs expire: the bound would exclude exactly the population it exists to serve. + * + * One hour, standardised across the three clients. The figure is NOT load-bearing — the property + * is "hours" — so don't tune it as though something depends on it. It errs short because the two + * failure modes are asymmetric: too long re-creates the defect this bound exists to fix, while too + * short costs a byte-identical, idempotent re-store that changes nothing. When one side costs + * correctness and the other costs a redundant request, err toward the request. + */ +const HASH_BAR_MS = 1 * DURATION.HOURS; + +const RECOVERY_BACKOFF_BASE_MS = 60 * DURATION.SECONDS; +const RECOVERY_BACKOFF_CEILING_MS = 30 * DURATION.MINUTES; + +/** + * How long to wait before the next recovery round for a swarm, given consecutive FAILED rounds. + * 60s doubling, ceilinged at 30 minutes, reset to zero by any successful store. + * + * The ceiling bounds the INTERVAL, never the NUMBER OF ATTEMPTS. This must not become a + * consecutive-failure cap: that is the exclusion shape this feature has already produced twice, and it + * would exclude exactly the swarms most in need of repair. A permanently failing swarm keeps being + * retried, just rarely — ~48 rounds a day rather than ~1,440. + * + * Growth matters more here than on mobile: Desktop has no foreground gate, so a "session" + * is however long the app stays open, which is days rather than minutes. + */ +function backoffMsFor(consecutiveFailures: number) { + if (consecutiveFailures <= 0) { + return 0; + } + return Math.min( + RECOVERY_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), + RECOVERY_BACKOFF_CEILING_MS + ); +} + +/** + * Our local state must be level with the swarm before anything may be re-stored. + * That stops a long-offline device putting back state that has since been deliberately changed: + * the dangerous ordering is re-storing while the swarm still holds config we haven't merged. + * + * A successful poll makes us level in one of two ways, and BOTH count: + * + * - it returned config messages and we merged them; or + * - it returned no config messages at all, so there is nothing on the swarm we haven't already + * incorporated. + * + * The second one is not a technicality — it is the case this whole feature exists for. A device + * whose config has expired gets *nothing* back, so a guard that waits for a merge would never fire + * for exactly the devices being repaired, and would do it silently: detection runs, the guard + * declines, no error, no failing test. Requiring a merge is why this function is not called + * `markSwarmMerged`. + * + * A failed or errored poll counts for neither. + */ +/** + * Called when a poll STARTS for this swarm. Everything marked level before now becomes stale for + * any consumer asking the poll-scoped question. + */ +function beginPollForSwarm(pubkey: AccountPubkey) { + currentPollToken.set(pubkey, (currentPollToken.get(pubkey) ?? 0) + 1); +} + +function markLocalStateLevelWithSwarm(pubkey: AccountPubkey) { + if (swarmsWithIncompleteMerge.has(pubkey)) { + // withdrawn for the session — see markMergeIncompleteForSwarm + return; + } + swarmsLevelWithLocalState.set(pubkey, currentPollToken.get(pubkey) ?? 0); +} + +/** + * Withdraw a swarm for the rest of the session, because we fetched config we could not take in. + * + * This has to be STICKY, and the reason is not obvious. The lastHash cursor advances when a message + * is *fetched*, inside pollNodeForKey, before the merge is even attempted. So a message we failed to + * merge sits behind the cursor and the swarm never sends it again — which means the very next poll + * returns nothing, looks perfectly clean, and would re-authorise recovery over state we know we + * never incorporated. The failure doesn't just go unreported, it becomes unreachable: after that + * second poll there is no error, no log and no state anywhere recording that anything was missed. + * + * A per-poll check alone is therefore cosmetic. Note the fix is NOT to advance the cursor only on a + * successful merge — that would re-fetch a permanently unmergeable message forever. Recovery is a + * best-effort repair, so deferring it to the next app start costs almost nothing, where acting on a + * view we know to be incomplete is the thing this precondition exists to prevent. + * + * Known correlated exclusion, found by asking which population this excludes. "Deferred to the + * next app start" is only true for a + * TRANSIENT merge failure. If a config message on the swarm is *permanently* unmergeable — corrupt, + * or written by a client newer than we can parse — then every session fetches it, fails, and + * withdraws this swarm again, so recovery never runs on that device for that swarm. Ever. And a + * device holding an unmergeable config is plausibly one whose state needs repairing. + * + * Kept anyway, because the alternative is re-storing over state we know we could not read, which is + * worse than not repairing. Named here so nobody later re-derives it as harmless. + */ +function markMergeIncompleteForSwarm(pubkey: AccountPubkey) { + swarmsWithIncompleteMerge.add(pubkey); + swarmsLevelWithLocalState.delete(pubkey); +} + +function localStateIsLevelWithSwarm(pubkey: AccountPubkey) { + return swarmsLevelWithLocalState.has(pubkey); +} + +/** + * Were we level as of the poll currently running for this swarm — not merely at some point since + * the process started? + * + * Fails CLOSED. A swarm we have never polled, never marked, or withdrawn answers false, because + * the only consumer is an irreversible write and "we do not know" must not read as "yes". + */ +function localStateIsLevelAsOfCurrentPoll(pubkey: AccountPubkey) { + const markedAt = swarmsLevelWithLocalState.get(pubkey); + const current = currentPollToken.get(pubkey); + + return markedAt !== undefined && current !== undefined && markedAt === current; +} + +/** + * Detection runs on every poll, including ones we won't act on. Recording it separately from + * acting on it is what lets the precondition hold without throwing the detection away. + */ +function recordDetection(pubkey: AccountPubkey, detection: ConfigExpiryDetection) { + if (detection.status !== 'conclusive') { + // 'unavailable' and 'inconclusive' are not evidence of anything. + return; + } + + if (!detection.missingHashes.length) { + // Deliberately NOT clearing what earlier polls recorded. A hash whose store FAILED is exactly + // the thing we want a later poll to retry, and forgetting it is not how that retry is bounded — + // the backoff is. So a clearing step here could only destroy findings, including on a + // wrongly-conclusive result, without ever preventing a redundant re-store. Hashes are dropped + // once SETTLED instead — see pruneSettledDetections. + return; + } + + const known = missingHashesByPubkey.get(pubkey) ?? new Set(); + detection.missingHashes.forEach(hash => known.add(hash)); + missingHashesByPubkey.set(pubkey, known); +} + +/** + * Drop bars that have expired, rather than merely reading past them. + * + * The read below already ignores an expired entry, so omitting this looks correct and leaks for the + * life of the process instead. And the population it leaks against is long-lived sessions — which + * is exactly the population the time-bound was added for, so the leak would target the same people + * as the defect it fixes. + */ +/** + * Drop detections we have finished with, so the accumulator cannot grow for the life of the process. + * + * Same leak `pruneExpiredBars` was written for, one map over and against the same population: hashes + * rotate on every re-push, and a Desktop session runs for days by design, so every superseded hash + * would otherwise be retained forever. + * + * Only settled hashes are dropped. A hash still awaiting a retry must stay, or the retry never + * happens — this prunes what is done with, never what is outstanding. + */ +function pruneSettledDetections(pubkey: AccountPubkey) { + const known = missingHashesByPubkey.get(pubkey); + if (!known) { + return; + } + known.forEach(hash => { + if (hashSettledAt.has(hash)) { + known.delete(hash); + } + }); + if (!known.size) { + missingHashesByPubkey.delete(pubkey); + } +} + +function pruneExpiredBars() { + const now = nowMs(); + hashSettledAt.forEach((settledAt, hash) => { + if (now - settledAt >= HASH_BAR_MS) { + hashSettledAt.delete(hash); + } + }); +} + +function getMissingHashes(pubkey: AccountPubkey) { + return [...(missingHashesByPubkey.get(pubkey) ?? [])]; +} + +/** + * Which of our user configs need putting back, given the hashes reported missing. + * + * Applies two guards: clean configs only, and current hashes only (`activeHashes()` *is* the set of + * hashes the device believes are current, so a hash that has since been superseded simply isn't in + * it any more). + */ +async function userVariantsNeedingRestore(missingHashes: Array) { + const needingRestore: Array = []; + // the missing hashes a restorable variant actually claims. Anything left over was ruled out by a + // guard rather than merely un-attempted, which is a different outcome — see recoverIfNeeded. + const coveredHashes = new Set(); + // an inspection that THREW is not a guard rejection; that variant stays retryable + let inspectedEverything = true; + + for (let index = 0; index < LibSessionUtil.requiredUserVariants.length; index++) { + const variant = LibSessionUtil.requiredUserVariants[index]; + + try { + // Clean only: recovery re-uploads existing state, it never creates new state. A config with + // pending changes will be pushed by the UserSyncJob anyway, which supersedes this. + if (await UserGenericWrapperActions.needsPush(variant)) { + continue; + } + + const activeHashes = await UserGenericWrapperActions.activeHashes(variant); + + const claimed = activeHashes.filter(hash => missingHashes.includes(hash)); + if (claimed.length) { + needingRestore.push(variant); + claimed.forEach(hash => coveredHashes.add(hash)); + } + } catch (e) { + inspectedEverything = false; + window.log.warn( + `ConfigRecovery: could not inspect user variant ${variant}: ${e.message}. Skipping it.` + ); + } + } + + return { needingRestore, coveredHashes, inspectedEverything }; +} + +/** + * @returns `stored` — every part of every config landed, which is what bars a hash from retry ( + * a multipart config counts as stored only when all its parts do). + * @returns `progressed` — at least one config landed IN FULL, so its hashes are now barred and the + * next round is strictly smaller. Deliberately separate from `stored`: a swarm where one of several + * configs succeeded is converging, and backing off would penalise it for that. + * + * `progressed` must mean "a hash became BARRED", never "some sub-request returned 200". Every part + * of a multipart config goes back on every attempt, so a config whose parts half-land sends the + * IDENTICAL request next round; if that counted as progress, one part that always succeeds beside + * one that always fails would reset the failure counter forever — nothing barred, `backoffMsFor(0)` + * is 0, a full re-send on every poll. Measured at 10 rounds / 10 sends / 0 barred. Only a hash + * becoming barred makes the next round smaller, so only that is progress. + */ +async function restoreUserConfigs( + variants: Array +): Promise<{ stored: boolean; progressed: boolean }> { + const us = UserUtils.getOurPubKeyStrFromCache() as PubkeyType; + + /** one entry per config we are putting back, so success can be attributed per config */ + const restores: Array<{ + variant: ConfigWrapperUser; + stores: Array; + obsoleteHashes: Array; + activeHashes: Array; + }> = []; + + for (let index = 0; index < variants.length; index++) { + const variant = variants[index]; + const { data, hashes, namespace } = await UserGenericWrapperActions.push(variant); + + restores.push({ + variant, + // Every part of a multipart config goes back, not just the parts reported missing. The + // present ones re-encrypt to the same bytes, so they cost a no-op TTL refresh — and + // `activeHashes()` is unordered, so a part hash can't be mapped to its index here anyway. + stores: data.map( + ciphertext => + new StoreUserConfigSubRequest({ + encryptedData: ciphertext, + namespace, + ttlMs: TTL_DEFAULT.CONFIG_MESSAGE, + getNow: NetworkTime.now, + }) + ), + // push() drains the config's obsolete-hash list and clears it unconditionally, so this + // is the only time we will ever see these. Held per-config rather than pooled, because the + // delete must only cover configs whose stores actually landed. + obsoleteHashes: hashes, + activeHashes: await UserGenericWrapperActions.activeHashes(variant), + }); + } + + const allStores = restores.flatMap(r => r.stores); + if (!allStores.length) { + return { stored: false, progressed: false }; + } + + // The batch endpoint takes at most MAX_SUBREQUESTS_COUNT sub-requests INCLUSIVE, and an oversized + // one is rejected outright — a parse_error against the whole batch. (Our own helper throws before + // sending, which is worse for being silent: the throw lands in recoverIfNeeded's catch and an + // affected account simply never recovers.) + // + // So split across batches rather than dropping anything. The all-parts rule governs when a config + // COUNTS AS STORED, not which transport its parts travel in. Skipping instead would make a config + // over ~1.5MB permanently unrecoverable. + const landed = new Map(); + + const sendBatch = async (batch: Array) => { + const controller = new AbortController(); + const result = await timeoutWithAbort( + MessageSender.sendEncryptedDataToSnode({ + sortedSubRequests: batch, + destination: us, + method: 'sequence', + abortSignal: controller.signal, + allow401s: false, + }), + 30 * DURATION.SECONDS, + controller + ); + + if (!result || result.length !== batch.length) { + window.log.warn( + `ConfigRecovery: unexpected result length for ${ed25519Str(us)}: expected ${batch.length} but got ${result?.length}` + ); + return false; + } + + // A batch reports PER SUB-REQUEST, so the right number of results says nothing about whether + // they succeeded — reading the length alone would take a partial store for a complete one. + batch.forEach((request, i) => { + if (request instanceof StoreUserConfigSubRequest) { + landed.set(request, result[i].code === 200); + } + }); + + return result.every(m => m.code === 200); + }; + + const storeBatches = chunk(allStores, MAX_SUBREQUESTS_COUNT); + window.log.info( + `ConfigRecovery: re-storing ${allStores.length} config message(s) for ${ed25519Str(us)} in ${storeBatches.length} batch(es) (variants: ${variants.join(', ')})` + ); + + for (let i = 0; i < storeBatches.length; i++) { + // eslint-disable-next-line no-await-in-loop + const ok = await sendBatch(storeBatches[i]); + if (!ok) { + break; // later batches are pointless, and the delete below is now narrower + } + } + + const fullyLanded = restores.filter(r => r.stores.every(store => landed.get(store) === true)); + const progressed = fullyLanded.length > 0; + + // The delete covers only the configs that FULLY landed. An obsolete hash whose + // replacement did not store is the swarm's only older copy of that config — deleting it would + // leave a seed restore in that window with nothing rather than something stale. And in the case + // the sweep was actually written for, the delete is a no-op anyway: an obsolete hash is never + // TTL-extended (active_hashes() covers _curr_hashes only), so if the CURRENT hash lived long + // enough to expire, its predecessor necessarily expired before it. + const deletableHashes = fullyLanded.flatMap(r => r.obsoleteHashes); + + if (deletableHashes.length) { + // eslint-disable-next-line no-await-in-loop + await sendBatch([ + new DeleteHashesFromUserNodeSubRequest({ messagesHashes: [...new Set(deletableHashes)] }), + ]); + } + + fullyLanded.forEach(r => r.activeHashes.forEach(hash => hashSettledAt.set(hash, nowMs()))); + + if (fullyLanded.length) { + // push() mutated the wrappers (it drained their obsolete hashes), so that has to reach disk. + await LibSessionUtil.saveDumpsToDb(us); + } + + return { stored: fullyLanded.length === restores.length, progressed }; +} + +/** the group sub-configs recovery can put back */ +type RestorableGroupConfig = 'groupInfo' | 'groupMember' | 'groupKeys'; + +/** + * Which of a group's sub-configs claim one of the missing hashes. + * + * GroupKeys is restorable only if we RETAINED ITS BYTES. A keys message is admin-signed and padded + * from the group secret key, so nobody can regenerate one — but bytes already held can be pushed + * back verbatim and land on the same hash, which is what lets a MEMBER repair a group's keys rather + * than only an admin. Where we hold no bytes (a message loaded before the wrapper retained them), + * it is unrecoverable by this device and settles, exactly as before. + */ +async function groupConfigsNeedingRestore(groupPk: GroupPubkeyType, missingHashes: Array) { + const needingRestore: Array = []; + const coveredHashes = new Set(); + /** every keys hash gone AND we hold no bytes — the group is expired as far as this device goes */ + let keysUnrecoverableHere = false; + + try { + const group = await UserGroupsWrapperActions.getGroup(groupPk); + + // We are no longer entitled to write to this swarm, and for a destroyed group there is + // nothing to put back. Note both flags: `kicked` is false when the group was `destroyed`, so + // checking one alone silently misses the other population. + if (!group || group.kicked || group.destroyed) { + return { needingRestore, coveredHashes, inspectedEverything: true, keysUnrecoverableHere }; + } + + // Clean configs only — but this gate does NOT apply to GroupKeys. + // + // The gate exists so local state cannot overwrite newer remote state. Keys recovery replays the + // exact bytes the swarm already had — byte-identical, same hash — so it cannot overwrite + // anything, and a pending rekey produces a NEW message at a NEW generation, which says nothing + // about whether the retained ones are stale. Gating keys on a dirty groupInfo would be a + // correlated exclusion: a group with pending changes is exactly a group in active use. + const dirty = await MetaGroupWrapperActions.needsPush(groupPk); + + // Only hashes the wrapper still considers active. Per-config, because the answer differs + // per config: `activeHashes()` merges all three and cannot tell a restorable groupInfo hash + // from an unrestorable groupKeys one. + const byConfig = await MetaGroupWrapperActions.activeHashesByConfig(groupPk); + + const missingKeysHashes = byConfig.groupKeys.filter(hash => missingHashes.includes(hash)); + + // EVERY keys hash we asked about is gone. That — and only that — is what decides an expired + // group: one surviving keys hash still lets a new device in, which is why a partial miss is not + // expired. + const allKeysMissing = + byConfig.groupKeys.length > 0 && missingKeysHashes.length === byConfig.groupKeys.length; + + if (missingKeysHashes.length) { + const retained = await MetaGroupWrapperActions.activeKeyMessages(groupPk); + + if (isEmpty(retained)) { + // Unrecoverable BY THIS DEVICE rather than unrecoverable: another peer holding the bytes can + // still put them back. Covered so it settles instead of being re-examined every poll. + missingKeysHashes.forEach(hash => coveredHashes.add(hash)); + keysUnrecoverableHere = allKeysMissing; + window.log.warn( + `ConfigRecovery: ${missingKeysHashes.length} GroupKeys hash(es) missing for ${ed25519Str(groupPk)} and no retained bytes — cannot repair from here` + ); + } else { + needingRestore.push('groupKeys'); + missingKeysHashes.forEach(hash => coveredHashes.add(hash)); + } + } + + if (!dirty) { + (['groupInfo', 'groupMember'] as const).forEach(config => { + const claimed = byConfig[config].filter(hash => missingHashes.includes(hash)); + if (claimed.length) { + needingRestore.push(config); + claimed.forEach(hash => coveredHashes.add(hash)); + } + }); + } + + return { needingRestore, coveredHashes, inspectedEverything: true, keysUnrecoverableHere }; + } catch (e) { + // as on the user path: a throw is not a guard verdict, so nothing settles on this pass + window.log.warn( + `ConfigRecovery: could not inspect group ${ed25519Str(groupPk)}: ${e.message}. Skipping it.` + ); + return { + needingRestore: [], + coveredHashes, + inspectedEverything: false, + keysUnrecoverableHere: false, + }; + } +} + +/** + * Put a group's clean `groupInfo`/`groupMember` configs back on its swarm. + * + * Returns the same pair as the user path — see `restoreUserConfigs` for what `stored` and + * `progressed` mean and why they are separate. + */ +async function restoreGroupConfigs( + groupPk: GroupPubkeyType, + configs: Array +): Promise<{ stored: boolean; progressed: boolean }> { + const group = await UserGroupsWrapperActions.getGroup(groupPk); + if (!group) { + return { stored: false, progressed: false }; + } + + const needsPushed = configs.some(c => c !== 'groupKeys'); + + // `pushForRecovery` ignores needs_push() and hands back groupInfo and groupMember only. Keys are + // NOT in it and cannot be: it re-serialises current state, and a keys message is admin-signed + // with padding derived from the group secret key, so a member could not produce a valid one. + // + // It DRAINS the obsolete-hash list despite reading like a query, because it calls push() + // underneath. So this is the only time we will see those hashes — and it is why it is only called + // when a config that needs it is actually being restored. + const pushed = needsPushed ? await MetaGroupWrapperActions.pushForRecovery(groupPk) : null; + + const byConfig = await MetaGroupWrapperActions.activeHashesByConfig(groupPk); + + // Keys come from retained BYTES rather than from a re-serialise — that is the whole mechanism. + // + // ALL retained messages go back, not only the ones reported missing. A generation is the full + // rekey plus every supplemental issued against it, and a member who receives only part of a + // generation does not get the key — so a partial re-store can leave the group unreadable for + // someone. Re-storing everything is a superset of "every message of the affected generation", + // which is what the rule requires; the extras are byte-identical no-op TTL refreshes. + // (The accessor is keyed by hash and carries no generation, so grouping by generation is not + // expressible here. Re-storing all of them is correct regardless of how they group.) + const keyMessages = configs.includes('groupKeys') + ? await MetaGroupWrapperActions.activeKeyMessages(groupPk) + : {}; + + const storeArgs = { + groupPk, + secretKey: group.secretKey, + authData: group.authData, + ttlMs: TTL_DEFAULT.CONFIG_MESSAGE, + getNow: NetworkTime.now, + }; + + type GroupRestore = { + config: RestorableGroupConfig; + stores: Array; + obsoleteHashes: Array; + activeHashes: Array; + }; + + const restores: Array = configs.map((config): GroupRestore => { + if (config === 'groupKeys') { + return { + config, + stores: Object.values(keyMessages).map( + encryptedData => new StoreGroupKeysSubRequest({ ...storeArgs, encryptedData }) + ), + // a keys message supersedes nothing, so there is never anything to prune here + obsoleteHashes: [] as Array, + activeHashes: Object.keys(keyMessages), + }; + } + + return { + config, + // Every part goes back again here, not just the parts reported missing. + stores: (pushed?.[config].data ?? []).map(encryptedData => + config === 'groupInfo' + ? new StoreGroupInfoSubRequest({ ...storeArgs, encryptedData }) + : new StoreGroupMembersSubRequest({ ...storeArgs, encryptedData }) + ), + obsoleteHashes: pushed?.[config].hashes ?? [], + activeHashes: byConfig[config], + }; + }); + + const allStores = restores.flatMap(r => r.stores); + if (!allStores.length) { + return { stored: false, progressed: false }; + } + + const landed = new Map(); + + const sendBatch = async (batch: Array) => { + const controller = new AbortController(); + const result = await timeoutWithAbort( + MessageSender.sendEncryptedDataToSnode({ + sortedSubRequests: batch, + destination: groupPk, + method: 'sequence', + abortSignal: controller.signal, + allow401s: false, + }), + 30 * DURATION.SECONDS, + controller + ); + + if (!result || result.length !== batch.length) { + window.log.warn( + `ConfigRecovery: unexpected result length for ${ed25519Str(groupPk)}: expected ${batch.length} but got ${result?.length}` + ); + return false; + } + + batch.forEach((request, i) => { + if ( + request instanceof StoreGroupInfoSubRequest || + request instanceof StoreGroupMembersSubRequest || + request instanceof StoreGroupKeysSubRequest + ) { + landed.set(request, result[i].code === 200); + } + }); + + return result.every(m => m.code === 200); + }; + + const storeBatches = chunk(allStores, MAX_SUBREQUESTS_COUNT); + window.log.info( + `ConfigRecovery: re-storing ${allStores.length} config message(s) for group ${ed25519Str(groupPk)} in ${storeBatches.length} batch(es) (configs: ${configs.join(', ')})` + ); + + for (let i = 0; i < storeBatches.length; i++) { + const ok = await sendBatch(storeBatches[i]); + if (!ok) { + break; + } + } + + const fullyLanded = restores.filter(r => r.stores.every(store => landed.get(store) === true)); + const progressed = fullyLanded.length > 0; + + // Same sweep rule as the user path, but note what it means for a MEMBER, because it looks like a + // bug from either side: + // - push() hands the superseded hashes back only `if (!is_readonly())` while clearing them + // either way, so a member gets an EMPTY list. That is the expected result, not a failure. + // - a member could not act on a non-empty one anyway: its subaccount token carries Read+Write + // but NOT Delete, so the delete would 401. + // Member-driven recovery therefore re-stores but never prunes; the superseded messages wait for + // an admin's next push. Only attempt the delete when we hold the admin key. + const adminSecretKey = group.secretKey?.length ? group.secretKey : null; + const deletableHashes = adminSecretKey ? fullyLanded.flatMap(r => r.obsoleteHashes) : []; + + if (adminSecretKey && deletableHashes.length) { + await sendBatch([ + new DeleteHashesFromGroupNodeSubRequest({ + messagesHashes: [...new Set(deletableHashes)], + groupPk, + secretKey: adminSecretKey, + }), + ]); + } + + fullyLanded.forEach(r => r.activeHashes.forEach(hash => hashSettledAt.set(hash, nowMs()))); + + // A landed keys re-store clears an existing expired flag EAGERLY rather than leaving it to the + // poller's reactive clear. That path fires when config messages are received — but we just + // re-stored messages we already hold, so we may never receive or re-handle them, and the flag + // would stay set forever over keys that are back on the swarm. + if (fullyLanded.some(r => r.config === 'groupKeys')) { + try { + const convo = ConvoHub.use().get(groupPk); + if (convo?.getIsExpired03Group()) { + window.log.info( + `ConfigRecovery: keys restored for ${ed25519Str(groupPk)} — clearing its expired flag` + ); + convo.setIsExpired03Group(false); + await convo.commit(); + } + } catch (e) { + // best-effort: the repair itself succeeded, and the reactive path may still clear it + window.log.warn( + `ConfigRecovery: could not clear expired flag for ${ed25519Str(groupPk)}: ${e.message}` + ); + } + } + + if (fullyLanded.length) { + // pushForRecovery mutated the wrapper (it drained the obsolete hashes), so that has to persist + await LibSessionUtil.saveDumpsToDb(groupPk); + } + + return { stored: fullyLanded.length === restores.length, progressed }; +} + +/** + * Act on whatever detection has recorded for this swarm. Safe to call on every poll — the guards + * below are what make it a no-op almost every time. + * + * Note on the "foreground only" rule the mobile clients follow: it exists because on mobile the largest recovery + * coincides with a constrained background execution window. Desktop has no such window — the + * process is either running and polling or not running at all — so there is nothing here to + * defer to. Gating on window focus would only stop a minimised client from repairing itself. + */ +async function recoverIfNeeded(pubkey: AccountPubkey) { + // The caller does not await us — see the note at the call site in swarmPolling — so a round can + // still be in flight when the next poll comes round. A round that has not finished has not + // settled its hashes yet, so without this the second poll re-reads the same missing hashes and + // issues the same stores. Deterministic encryption makes those idempotent, so nothing corrupts; + // what it costs is duplicate traffic and doubled batch pressure aimed at the one swarm we already + // know is struggling. + // Check-then-set with no await between them, so the two cannot interleave. + if (recoveryInFlight.has(pubkey)) { + return false; + } + const round = runRecoveryRound(pubkey); + recoveryInFlight.set(pubkey, round); + + try { + return await round; + } finally { + // `finally`, not the end of the try: runRecoveryRound catches its own errors, but an in-flight + // entry that leaked on any path would withdraw the swarm permanently — the exclusion shape this + // design has already produced twice. + recoveryInFlight.delete(pubkey); + } +} + +async function runRecoveryRound(pubkey: AccountPubkey): Promise { + try { + const isUs = UserUtils.isUsFromCache(pubkey); + if (!isUs && !PubKey.is03Pubkey(pubkey)) { + // neither our swarm nor a group's: nothing here knows how to recover it + return false; + } + + // Nothing may be re-stored until we know our local state is level with the swarm — otherwise + // we would be re-uploading a view we already know is behind. + if (!localStateIsLevelWithSwarm(pubkey)) { + return false; + } + + const missingHashes = getMissingHashes(pubkey).filter( + // barred for a bounded interval rather than for the session + hash => { + const settledAt = hashSettledAt.get(hash); + return settledAt === undefined || nowMs() - settledAt >= HASH_BAR_MS; + } + ); + + // ORDER MATTERS, and it is the only reason this works. `pruneSettledDetections` reads + // `hashSettledAt` to decide what is finished with; `pruneExpiredBars` removes entries from it. + // Run the other way round, a bar that has just expired takes its hash out of `hashSettledAt` + // first, the detection then looks unfinished, and it is retained forever — the leak survives + // with both pruners present and looking correct. + pruneSettledDetections(pubkey); + pruneExpiredBars(); + + if (isEmpty(missingHashes)) { + return false; + } + + // The guards above and the bookkeeping below are identical for both; only the inspection and + // the restore know the difference between a user config and a group sub-config. + const inspection = isUs + ? await userVariantsNeedingRestore(missingHashes) + : await groupConfigsNeedingRestore(pubkey as GroupPubkeyType, missingHashes); + const { needingRestore, coveredHashes, inspectedEverything } = inspection; + + // Every keys hash gone and no bytes held: nothing here can repair it, so raise the banner. This + // is the only place it is raised from detection — the poller's empty-fetch branch cannot see + // this case at all, because it requires holding NO config hashes and we hold plenty. + if (!isUs && 'keysUnrecoverableHere' in inspection && inspection.keysUnrecoverableHere) { + await setGroupExpired(pubkey as GroupPubkeyType, true); + } + + // "not stored" is three outcomes, not two. A hash no restorable config claims was ruled out by a + // guard — not active any more, or belonging to a dirty config that will be pushed under a new + // hash anyway — so it is SETTLED rather than retryable. Folding these into "failed" costs no + // requests, because the rejection happens before any network call, which is exactly why it does + // not look like a problem: what it actually does is re-examine and re-log the same detection on + // every poll, forever. + // Settled here means barred for HASH_BAR_MS, NOT for the session — a guard's verdict CAN change + // over hours, so the bar buys quiet without withdrawing the hash permanently. + // An inspection that THREW is not a guard verdict, so nothing is settled on that pass. + if (inspectedEverything) { + missingHashes + .filter(hash => !coveredHashes.has(hash)) + .forEach(hash => hashSettledAt.set(hash, nowMs())); + } + + if (!needingRestore.length) { + // nothing attemptable, so no network call and no backoff slot consumed + return false; + } + + // The other half of releasing a failed attempt. A store that keeps failing leaves its + // hashes unmarked so the next poll retries, which is correct; unbounded, that retry is every few + // seconds forever. Rate-limited rather than capped, so a device with flaky connectivity keeps + // getting chances instead of being written off for the session. + const previous = recoveryAttemptsBySwarm.get(pubkey); + const consecutiveFailures = previous?.consecutiveFailures ?? 0; + if (previous && nowMs() - previous.lastAttemptAt < backoffMsFor(consecutiveFailures)) { + return false; + } + + const keysWereRestorable = !isUs && needingRestore.includes('groupKeys' as never); + + const { stored, progressed } = isUs + ? await restoreUserConfigs(needingRestore as Array) + : await restoreGroupConfigs( + pubkey as GroupPubkeyType, + needingRestore as Array + ); + + // We held the bytes and the re-store did not land, so the keys are still gone from the swarm + // and still not back. Deferring the banner was right while we had a repair in hand; once that + // repair fails the user needs to know. A later successful round clears it eagerly. + if (keysWereRestorable && !stored) { + await setGroupExpired(pubkey as GroupPubkeyType, true); + } + + // Reset on PROGRESS, not on completion — but progress means "something got BARRED", not + // "something returned 200". Those differ exactly when a multipart config half-lands, and that + // is the case that matters: all parts go back on every attempt, so a half-landing config sends + // the identical request next round. Treating that as progress reset the counter forever and + // re-sent in full on every poll. + // Still gated on its own value rather than reusing `stored`: with several configs, one landing + // in full genuinely shrinks the next round even though the swarm is not finished. + recoveryAttemptsBySwarm.set(pubkey, { + consecutiveFailures: progressed ? 0 : consecutiveFailures + 1, + lastAttemptAt: nowMs(), + }); + + return stored; + } catch (e) { + window.log.warn(`ConfigRecovery: recoverIfNeeded for ${ed25519Str(pubkey)} failed:`, e.message); + return false; + } +} + +/** + * Exported for tests only — the sets above are process-lifetime state by design. + */ +function setNowForTesting(fn: () => number) { + nowMs = fn; +} + +function resetForTesting() { + nowMs = () => Date.now(); + swarmsLevelWithLocalState.clear(); + currentPollToken.clear(); + swarmsWithIncompleteMerge.clear(); + recoveryAttemptsBySwarm.clear(); + hashSettledAt.clear(); + missingHashesByPubkey.clear(); + recoveryInFlight.clear(); + keysBackfillFailedAt.clear(); +} + +/** + * Do we hold the bytes to put this group's keys messages back ourselves? + * + * The poller asks before flagging a group expired. "Expired" means not recoverable BY THIS DEVICE, + * so a device retaining the keys messages must not raise it — it is about to repair the group. + * + * Deliberately tolerant: any failure to answer returns false, which keeps the existing behaviour + * rather than suppressing a flag we cannot justify suppressing. + */ +/** + * Which of a group's active keys hashes we hold NO BYTES for. + * + * `activeHashesByConfig().groupKeys` names every keys message still active; `activeKeyMessages()` + * returns only the ones whose bytes libSession retained. A hash in the first and not the second is + * a message that is still on the swarm and that we could not put back if it ever expired. + */ +async function keysHashesWeLackBytesFor(groupPk: GroupPubkeyType) { + const byConfig = await MetaGroupWrapperActions.activeHashesByConfig(groupPk); + const retained = await MetaGroupWrapperActions.activeKeyMessages(groupPk); + + return byConfig.groupKeys.filter(hash => !(hash in retained)); +} + +/** + * Re-fetch and re-merge a group's keys messages so libSession retains their bytes. + * + * PROACTIVE, NOT ON DETECTION, and that distinction is the whole value. Detection fires when the + * swarm has already LOST a hash — by then there is nothing left to fetch and this can do nothing. + * This fires while the message is still there, which is the only window in which it works. + * + * Re-loading a keys message we already hold the key for is a no-op for KEY STATE (insert_key + * early-returns) but NOT for RETENTION: that early-return path still stores the bytes and flags a + * dump. So this is cheap and safe against a group whose keys are perfectly healthy. + * + * @returns whether we now hold bytes for every active keys hash. + */ +async function backfillGroupKeys(groupPk: GroupPubkeyType): Promise { + if (isEmpty(await keysHashesWeLackBytesFor(groupPk))) { + return true; + } + + const swarm = await SnodePool.getSwarmFor(groupPk); + const targetNode = swarm[0]; + if (!targetNode) { + // Not an attempt — we never asked anyone. Throwing keeps the caller from recording a failure + // for a group we learned nothing about. + throw new Error('backfillGroupKeys: no snode in swarm'); + } + + // The retrieve layer DIRECTLY, never the poll wrapper. `pollNodeForKey` writes the namespace + // cursor from whatever it fetched (swarmPolling.ts:902), and this asks with NO last_hash, so + // routing through it would advance the cursor past messages the poll never consumed. Nothing + // below the retrieve writes the cursor — the only writers are that call site and the Data helper + // it calls — so staying outside it is sufficient here, which is not true on every platform. + const results = await SnodeAPIRetrieve.retrieveNextMessagesNoRetries( + targetNode, + groupPk, + [{ lastHash: '', namespace: SnodeNamespaces.ClosedGroupKeys }], + UserUtils.getOurPubKeyStrFromCache(), + null, + true + ); + + const keysMessages = (results ?? []) + .filter(r => r.namespace === SnodeNamespaces.ClosedGroupKeys) + .flatMap(r => r.messages?.messages ?? []) + .filter(m => !!m?.data && !!m?.hash && !!m?.storedAt) + .map(m => ({ + data: fromBase64ToArray(m.data), + hash: m.hash, + // `storedAt` is when the snode stored it, which is what the merge wants — NOT the envelope + // timestamp. The normal poll path uses the same field for keys messages. + timestampMs: m.storedAt, + })); + + if (isEmpty(keysMessages)) { + return false; + } + + await MetaGroupWrapperActions.metaMerge(groupPk, { + groupInfo: [], + groupKeys: keysMessages, + groupMember: [], + }); + + // The merge alone is not enough, and the difference is invisible in-process. Retention lives in + // the config DUMP, so bytes captured by a merge that never persists die with the process: the + // backfill appears to work and silently does not, and any test asserting within one run passes + // either way. + await LibSessionUtil.saveDumpsToDb(groupPk); + + return isEmpty(await keysHashesWeLackBytesFor(groupPk)); +} + +/** + * The entry point the poller calls. Records the outcome so the rekey can tell "a backfill has run + * and nothing can restore these" from "a backfill has never run" — two states no other predicate + * distinguishes. + */ +async function backfillGroupKeysIfNeeded(groupPk: GroupPubkeyType) { + try { + const lastFailure = keysBackfillFailedAt.get(groupPk); + if (lastFailure !== undefined && nowMs() - lastFailure < HASH_BAR_MS) { + return; + } + + if (await backfillGroupKeys(groupPk)) { + // CLEARED on success rather than left alone. This record is read as "this device cannot + // repair this group", and a device that just retained the bytes plainly can. + keysBackfillFailedAt.delete(groupPk); + return; + } + + // Means ATTEMPTED AND THE BYTES ARE STILL ABSENT — not "the fetch came back empty". A fetch + // returning messages that still do not restore the bytes is equally a failed attempt, and + // recording only the empty case leaves the group looking un-attempted forever while re-fetching + // the same useless messages every eligible poll. + // The two are indistinguishable in any fixture where the swarm holds nothing, which is the + // first fixture anyone writes — so the test that separates them needs a swarm that returns + // something. + keysBackfillFailedAt.set(groupPk, nowMs()); + } catch (e) { + // A throw is not an attempt: we never learned whether the bytes are obtainable, so recording a + // failure would let the rekey act on evidence we do not have. + window.log.warn( + `ConfigRecovery: keys backfill for ${ed25519Str(groupPk)} failed: ${e.message}` + ); + } +} + +/** Has a backfill run for this group and still come up short? Read by the rekey's precondition. */ +function keysBackfillHasFailedFor(groupPk: GroupPubkeyType) { + return keysBackfillFailedAt.has(groupPk); +} + +/** + * Raise or clear a group's expired banner. + * + * "Expired" means its keys are gone from the swarm and **this device cannot put them back** — so it + * is a not-available-to-you-right-now signal, not a statement about the group. A peer that still + * holds the bytes clears it by re-storing them. + */ +async function setGroupExpired(groupPk: GroupPubkeyType, expired: boolean) { + try { + const convo = ConvoHub.use().get(groupPk); + if (!convo || convo.getIsExpired03Group() === expired) { + return; + } + window.log.info( + `ConfigRecovery: marking ${ed25519Str(groupPk)} ${expired ? 'EXPIRED' : 'not expired'}` + ); + convo.setIsExpired03Group(expired); + await convo.commit(); + } catch (e) { + window.log.warn( + `ConfigRecovery: could not set expired flag for ${ed25519Str(groupPk)}: ${e.message}` + ); + } +} + +async function canRepairGroupKeys(groupPk: GroupPubkeyType) { + try { + return !isEmpty(await MetaGroupWrapperActions.activeKeyMessages(groupPk)); + } catch (e) { + window.log.warn( + `ConfigRecovery: canRepairGroupKeys failed for ${ed25519Str(groupPk)}: ${e.message}` + ); + return false; + } +} + +/** + * Exported for tests only. The poller does not await recovery, so a test that drives a poll has to + * be able to wait for the round it started; without this it would have to sleep and hope. + * Resolves immediately when no round is running. + */ +async function waitForRecoveryForTesting(pubkey: AccountPubkey) { + await recoveryInFlight.get(pubkey); +} + +/** exported for tests only — as with the bars, this leak is invisible from outside */ +function trackedDetectionCountForTesting(pubkey: AccountPubkey) { + return missingHashesByPubkey.get(pubkey)?.size ?? 0; +} + +/** exported for tests only — the leak this guards is otherwise unobservable from outside */ +function barredHashCountForTesting() { + return hashSettledAt.size; +} + +export const ConfigRecovery = { + barredHashCountForTesting, + trackedDetectionCountForTesting, + markLocalStateLevelWithSwarm, + beginPollForSwarm, + localStateIsLevelAsOfCurrentPoll, + setNowForTesting, + markMergeIncompleteForSwarm, + localStateIsLevelWithSwarm, + recordDetection, + getMissingHashes, + recoverIfNeeded, + canRepairGroupKeys, + backfillGroupKeysIfNeeded, + keysBackfillHasFailedFor, + resetForTesting, + waitForRecoveryForTesting, +}; diff --git a/ts/session/apis/snode_api/configRecoveryForceRekey.ts b/ts/session/apis/snode_api/configRecoveryForceRekey.ts new file mode 100644 index 0000000000..3563f64c32 --- /dev/null +++ b/ts/session/apis/snode_api/configRecoveryForceRekey.ts @@ -0,0 +1,125 @@ +import { GroupPubkeyType } from 'libsession_util_nodejs'; +import { isEmpty } from 'lodash'; +import { + MetaGroupWrapperActions, + UserGroupsWrapperActions, +} from '../../../webworker/workers/browser/libsession_worker_interface'; +import { LibSessionUtil } from '../../utils/libsession/libsession_utils'; +import { GroupSync } from '../../utils/job_runners/jobs/GroupSyncJob'; +import { ed25519Str } from '../../utils/String'; +import { DURATION } from '../../constants'; +import { ConfigRecovery } from './configRecovery'; + +/** + * Last resort for a group whose keys are gone from the swarm and which no device here can repair: + * an admin mints a new generation so the group becomes usable again. + * + * This is the only irreversible, universally visible write in config recovery. Every other part + * replays bytes the swarm already had — byte-identical, same hash, invisible to every other client. + * A rekey creates NEW state that every member on every version must process, and it cannot be + * undone. Everything below is about not doing it when it was not needed. + * + * The rekey encrypts the new key to THIS DEVICE'S view of the members. That is why the freshness + * precondition is not optional: a member added while we were away and not yet merged is simply + * absent from the list we encrypt to. + */ + +/** + * Groups we have already rekeyed this session. + * + * In memory, and it does not need to be otherwise: a rekey mints a new generation, so after a + * successful one the preconditions below stop holding on their own. This exists to stop a second + * attempt inside the window before that becomes observable, not to remember across restarts. + */ +const rekeyedThisSession = new Set(); + +/** how long after an attempt we refuse another for the same group, successful or not */ +const REKEY_COOLDOWN_MS = 24 * DURATION.HOURS; +const lastAttemptAt = new Map(); + +let nowMs: () => number = () => Date.now(); + +async function forceRekeyIfPossible(groupPk: GroupPubkeyType): Promise { + try { + // Asked of the store rather than taken as an argument. The store stamps each level mark with + // the poll it came from, so this compares that stamp against the poll running now — a caller + // cannot supply a value it likes, only be wrong about when it called. + // + // Our members list may otherwise be behind, and a rekey from a stale one silently drops whoever + // was added since. Refusing costs a poll cycle; proceeding costs someone their access. + if (!ConfigRecovery.localStateIsLevelAsOfCurrentPoll(groupPk)) { + return false; + } + + if (rekeyedThisSession.has(groupPk)) { + return false; + } + + const previous = lastAttemptAt.get(groupPk); + if (previous !== undefined && nowMs() - previous < REKEY_COOLDOWN_MS) { + return false; + } + + // A back-fill must have RUN and still come up short. "We hold no bytes" alone cannot + // distinguish that from a back-fill that has never run — on a fresh install, a restored backup, + // or a device that has not completed a poll, those look identical and only one of them + // justifies this. + if (!ConfigRecovery.keysBackfillHasFailedFor(groupPk)) { + return false; + } + + const group = await UserGroupsWrapperActions.getGroup(groupPk); + if (!group || group.kicked || group.destroyed) { + return false; + } + + // Members cannot rekey — the key is signed with the group secret key. + if (!group.secretKey?.length) { + return false; + } + + const byConfig = await MetaGroupWrapperActions.activeHashesByConfig(groupPk); + const retained = await MetaGroupWrapperActions.activeKeyMessages(groupPk); + + // Only when EVERY keys hash is beyond us. One surviving keys message still lets a new device + // in, so the group is not stuck and this is not warranted. + if (isEmpty(byConfig.groupKeys)) { + return false; + } + if (byConfig.groupKeys.some(hash => hash in retained)) { + return false; + } + + window.log.warn( + `ConfigRecovery: no device here can restore the keys for ${ed25519Str(groupPk)} — rekeying` + ); + + lastAttemptAt.set(groupPk, nowMs()); + + await MetaGroupWrapperActions.keyRekey(groupPk); + await LibSessionUtil.saveDumpsToDb(groupPk); + await GroupSync.queueNewJobIfNeeded(groupPk); + + rekeyedThisSession.add(groupPk); + return true; + } catch (e) { + window.log.warn(`ConfigRecovery: force rekey for ${ed25519Str(groupPk)} failed: ${e.message}`); + return false; + } +} + +function setNowForTesting(fn: () => number) { + nowMs = fn; +} + +function resetForTesting() { + nowMs = () => Date.now(); + rekeyedThisSession.clear(); + lastAttemptAt.clear(); +} + +export const ConfigRecoveryForceRekey = { + forceRekeyIfPossible, + setNowForTesting, + resetForTesting, +}; diff --git a/ts/session/apis/snode_api/retrieveRequest.ts b/ts/session/apis/snode_api/retrieveRequest.ts index 7f27dafda4..a6457d1bb2 100644 --- a/ts/session/apis/snode_api/retrieveRequest.ts +++ b/ts/session/apis/snode_api/retrieveRequest.ts @@ -1,4 +1,4 @@ -import { GroupPubkeyType } from 'libsession_util_nodejs'; +import { GroupPubkeyType, PubkeyType } from 'libsession_util_nodejs'; import { isArray } from 'lodash'; import { Snode } from '../../../data/types'; import { SnodeNamespace, SnodeNamespaces, SnodeNamespacesGroup } from './namespaces'; @@ -14,9 +14,16 @@ import { UpdateExpiryOnNodeUserSubRequest, } from './SnodeRequestTypes'; import { BatchRequests } from './batchRequest'; -import { RetrieveMessagesResultsBatched, RetrieveMessagesResultsContent } from './types'; +import { + ExpireMessagesResultsContent, + RetrieveMessagesResultsBatched, + RetrieveMessagesResultsContent, +} from './types'; import { ed25519Str } from '../../utils/String'; import { NetworkTime } from '../../../util/NetworkTime'; +import { detectMissingConfigHashes } from './configExpiryDetection'; +import { ConfigRecovery } from './configRecovery'; +import { BatchResultEntry } from './BatchResultEntry'; type RetrieveParams = { pubkey: string; @@ -127,7 +134,9 @@ async function buildRetrieveRequest( const request = new UpdateExpiryOnNodeUserSubRequest({ expiryMs, messagesHashes: configHashesToBump, - shortenOrExtend: '', + // extend-only: bumping a config TTL must never be able to shorten it, and it is what makes + // the server return the `unchanged` array we need to detect configs expired from the swarm. + shortenOrExtend: 'extend', }); retrieveRequestsParams.push(request); return retrieveRequestsParams; @@ -149,7 +158,9 @@ async function buildRetrieveRequest( new UpdateExpiryOnNodeGroupSubRequest({ expiryMs, messagesHashes: configHashesToBump, - shortenOrExtend: '', + // extend-only, same as the user path above: bumping a config TTL must never shorten it, + // and it is what makes the server return the `unchanged` array detection needs. + shortenOrExtend: 'extend', groupDetailsNeededForSignature: group, }) ); @@ -157,6 +168,51 @@ async function buildRetrieveRequest( return retrieveRequestsParams; } +/** + * Read the `expire` sub-response we piggyback on every poll to work out whether any of our config + * messages have expired from the swarm, and record it. Acting on it happens after the merge, in + * `swarmPolling`, once we know our local state is level with the swarm. + * + * This only ever records; it must not throw into the polling path. + */ +function detectExpiredConfigs({ + associatedWith, + configHashesToBump, + expireSubRequest, + expireResult, +}: { + associatedWith: PubkeyType | GroupPubkeyType; + configHashesToBump: Array; + expireSubRequest: RetrieveSubRequestType | undefined; + expireResult: BatchResultEntry; +}) { + try { + if (expireSubRequest?.method !== 'expire') { + return; + } + + // Note: read from the request we actually built rather than assuming. A response to a request + // that didn't set `extend` omits `unchanged` entirely, which makes every hash we didn't update + // look missing — so if that flag ever changes, detection has to switch itself off rather than + // report the whole config gone on every poll. + const detection = detectMissingConfigHashes({ + requestedHashes: configHashesToBump, + swarm: (expireResult.body as { swarm?: ExpireMessagesResultsContent })?.swarm, + requestSetExtend: expireSubRequest.shortenOrExtend === 'extend', + }); + + if (detection.status === 'conclusive' && detection.missingHashes.length) { + window.log.warn( + `SwarmPolling: ${detection.missingHashes.length} config message(s) missing from the swarm of ${ed25519Str(associatedWith)}` + ); + } + + ConfigRecovery.recordDetection(associatedWith, detection); + } catch (e) { + window.log.warn('detectExpiredConfigs failed with:', e.message); + } +} + /** * * @param targetNode the node to make the request to @@ -208,13 +264,20 @@ async function retrieveNextMessagesNoRetries( ); } - // the +1 is to take care of the extra `expire` method added once user config is released - if ( - results.length !== namespacesAndLastHashes.length && - results.length !== namespacesAndLastHashes.length + 1 - ) { + // One result per sub-request, and it must STAY a throw rather than becoming a filter or a + // tolerance: everything below pairs `results[index]` with `namespacesAndLastHashes[index]` by + // position, so a missing result does not drop a namespace — it shifts every later one onto its + // neighbour's messages. + // + // Compare against `rawRequests`, never against `namespacesAndLastHashes.length` or that +1. + // The `expire` sub-request is appended only when there are config hashes to bump, so a bound + // written to allow for both admits two lengths — and two accepted lengths is the same hole as a + // filter: with `expire` appended, a response that dropped one retrieve result lands on the lower + // bound, passes, and then the LAST namespace is handed the expire result as its messages. + // `rawRequests` already accounts for the conditional sub-request, so it admits exactly one. + if (results.length !== rawRequests.length) { throw new Error( - `We asked for updates about ${namespacesAndLastHashes.length} messages but got results of length ${results.length}` + `We asked for ${rawRequests.length} sub-requests but got results of length ${results.length}` ); } @@ -227,6 +290,10 @@ async function retrieveNextMessagesNoRetries( `_retrieveNextMessages - retrieve result is not 200 with ${targetNode.ip}:${targetNode.port} but ${firstResult.code}` ); } + // Safe to read both `length - 1` slots as a pair only because the check above admits exactly + // one length: `configHashesToBump` being set means `buildRetrieveRequest` appended the expire + // sub-request last, and the result array is now known to be the same length. Loosen that check + // and this pairs an expire request with a retrieve response. if (configHashesToBump?.length) { const lastResult = results[results.length - 1]; if (lastResult?.code !== 200) { @@ -234,6 +301,17 @@ async function retrieveNextMessagesNoRetries( window.log.warn( `the update expiry of our tracked config hashes didn't work: ${JSON.stringify(lastResult)}` ); + } else if (PubKey.is03Pubkey(associatedWith) || PubKey.is05Pubkey(associatedWith)) { + // Narrowed with a real check rather than a cast. `associatedWith` is a `string` all the way + // down the poller, but recovery keys its per-swarm state on an ACCOUNT pubkey, and anything + // that is neither `03` nor `05` names no swarm we could recover — so skipping is right, not + // merely type-convenient. + detectExpiredConfigs({ + associatedWith, + configHashesToBump, + expireSubRequest: rawRequests[rawRequests.length - 1], + expireResult: lastResult, + }); } } diff --git a/ts/session/apis/snode_api/swarmPolling.ts b/ts/session/apis/snode_api/swarmPolling.ts index 9212b2f576..7605cccbd8 100644 --- a/ts/session/apis/snode_api/swarmPolling.ts +++ b/ts/session/apis/snode_api/swarmPolling.ts @@ -35,6 +35,8 @@ import { MultiEncryptUtils } from '../../utils/libsession/libsession_utils_multi import { SnodeNamespace, SnodeNamespaces, SnodeNamespacesUserConfig } from './namespaces'; import { PollForGroup, PollForLegacy, PollForUs } from './pollingTypes'; import { SnodeAPIRetrieve } from './retrieveRequest'; +import { ConfigRecovery } from './configRecovery'; +import { ConfigRecoveryForceRekey } from './configRecoveryForceRekey'; import { SnodePool } from './snodePool'; import { SwarmPollingGroupConfig } from './swarm_polling_config/SwarmPollingGroupConfig'; import { SwarmPollingUserConfig } from './swarm_polling_config/SwarmPollingUserConfig'; @@ -106,11 +108,55 @@ function mergeMultipleRetrieveResults( } // Convert the merged map back to an array - return Array.from(mapped.entries()).map(([namespace, messagesMap]) => ({ - code: results.find(m => m.namespace === namespace)?.code || 200, - namespace, - messages: { messages: Array.from(messagesMap.values()) }, - })); + return Array.from(mapped.entries()).map(([namespace, messagesMap]) => { + // A namespace ANSWERED if any snode we polled returned 200 for it: the messages above are the + // union across snodes, so one snode failing does not cost us that namespace's content. + // + // The verdict has to come from ALL codes for the namespace, and a missing code must never + // default to a pass. Taking the first entry (`find`) makes it depend on which snode happens to + // come back first — arbitrary in BOTH directions, so not conservative either — and a `|| 200` + // fallback turns a missing or zero code into an answer, defaulting the one direction that + // must not default. + // + // The only consumer is allConfigNamespacesAnswered, which decides whether we are level with the + // swarm — so an unanswered namespace reading as answered is the failure that matters. + const codes = results.filter(m => m.namespace === namespace).map(m => m.code); + + return { + code: codes.includes(200) ? 200 : (codes[0] ?? 0), + namespace, + messages: { messages: Array.from(messagesMap.values()) }, + }; + }); +} + +/** + * Whether every config namespace we polled actually answered. + * + * Being "level with the swarm" is about knowing the swarm state for the configs we are about to act + * on. A poll fetches several namespaces at once and they can fail independently, so one namespace + * erroring while the others answer leaves us ignorant about exactly its configs — a partial answer, + * not a full one. + * + * We gate the WHOLE SWARM rather than the individual namespace, and the reason is local: gating per + * namespace means mapping each hash we are about to act on back to the namespace it came from, and + * keeping that mapping correct as either side changes. Get it wrong and we act on a config we are + * ignorant about, silently. Gating the whole swarm is coarser — we skip a repair we could safely + * have made — but its failure mode is doing nothing, which the next poll fixes. + */ +function allConfigNamespacesAnswered( + results: RetrieveMessagesResultsMergedBatched, + type: ConversationTypeEnum +) { + const isConfigNamespace = + type === ConversationTypeEnum.GROUPV2 + ? SnodeNamespace.isGroupConfigNamespace + : SnodeNamespace.isUserConfigNamespace; + + const configResults = results.filter(m => isConfigNamespace(m.namespace)); + + // no config namespace polled at all means there is nothing we could soundly act on either + return configResults.length > 0 && configResults.every(m => m.code === 200); } function swarmLog(msg: string) { @@ -361,21 +407,23 @@ export class SwarmPolling { type: ConversationTypeEnum; pubkey: string; confMessages: Array | null; - }) { + }): Promise { if (!confMessages) { - return; + // nothing was fetched, so nothing failed to be taken in + return true; } // first make sure to handle the shared user config message first if (type === ConversationTypeEnum.PRIVATE && UserUtils.isUsFromCache(pubkey)) { - // this does not throw, no matter what happens - await SwarmPollingUserConfig.handleUserSharedConfigMessages(confMessages); - return; + // Note: this does not throw, no matter what happens — a merge failure is swallowed and only + // logged, so its outcome has to come back as a value or the caller cannot see it at all. + return SwarmPollingUserConfig.handleUserSharedConfigMessages(confMessages); } if (type === ConversationTypeEnum.GROUPV2 && PubKey.is03Pubkey(pubkey)) { await sleepFor(100); - await SwarmPollingGroupConfig.handleGroupSharedConfigMessages(confMessages, pubkey); + return SwarmPollingGroupConfig.handleGroupSharedConfigMessages(confMessages, pubkey); } + return true; } public async handleRevokedMessages({ @@ -424,10 +472,20 @@ export class SwarmPolling { * Only exposed as public for testing */ public async pollOnceForKey([pubkey, type]: PollForUs | PollForLegacy | PollForGroup) { + // A poll for this swarm starts here, which invalidates any earlier level mark for anything + // asking the poll-scoped question. Minted at the START deliberately: a token taken at the end + // would be the poll that just finished, and the rekey would compare a mark against its own poll + // and always agree. + ConfigRecovery.beginPollForSwarm(pubkey); const namespaces = this.getNamespacesToPollFrom(type); const swarmSnodes = await SnodePool.getSwarmFor(pubkey); let resultsFromAllNamespaces: RetrieveMessagesResultsMergedBatched | null; + // An empty result set is ambiguous: it can mean "the swarm has nothing for us" or "every snode + // we asked failed". Only the first tells us anything, so track whether a snode actually + // answered rather than inferring it from the emptiness. + let atLeastOneSnodeAnswered = false; + let toPollFrom: Array = []; try { @@ -459,6 +517,12 @@ export class SwarmPolling { `SwarmPolling: pollNodeForKey of ${ed25519Str(pubkey)} namespaces: ${namespaces} returned ${resultsFromAllSnodesSettled.filter(m => m.status === 'fulfilled').length}/${RETRIEVE_SNODES_COUNT} fulfilled promises` ); + // pollNodeForKey resolves to null when that snode's poll failed, so a fulfilled promise + // carrying a non-null value is the only thing that counts as an answer. + atLeastOneSnodeAnswered = resultsFromAllSnodesSettled.some( + m => m.status === 'fulfilled' && m.value !== null + ); + resultsFromAllNamespaces = mergeMultipleRetrieveResults( compact( resultsFromAllSnodesSettled.filter(m => m.status === 'fulfilled').flatMap(m => m.value) @@ -479,6 +543,22 @@ export class SwarmPolling { pubkey, type, }); + + // A snode answered and had nothing for us, so there is no config on the swarm we have yet to + // merge — which is exactly what being level asks for. This is the path a device with expired + // configs takes on every poll, so returning without considering recovery here would make the + // whole feature a no-op for the devices it exists to repair. + if (atLeastOneSnodeAnswered) { + ConfigRecovery.markLocalStateLevelWithSwarm(pubkey); + // NOT awaited. A recovery round is up to 20 sub-requests per batch and possibly several + // batches, and this poll loop is shared by every other pubkey — holding it here delays their + // polls for a repair that is by design best-effort and can just as well finish after we + // return. + // The usual objection to `void` does not apply: recoverIfNeeded wraps its whole body in + // try/catch and logs, so it cannot produce an unhandled rejection. It also guards against + // overlapping rounds internally, which voiding it here is what makes necessary. + void ConfigRecovery.recoverIfNeeded(pubkey); + } return; } const { confMessages, otherMessages, revokedMessages } = filterMessagesPerTypeOfConvo( @@ -489,7 +569,53 @@ export class SwarmPolling { `SwarmPolling: received for ${ed25519Str(pubkey)} confMessages:${confMessages?.length || 0}, revokedMessages:${revokedMessages?.length || 0}, , otherMessages:${otherMessages?.length || 0}, ` ); // We always handle the config messages first (for groups 03 or our own messages) - await this.handleUserOrGroupConfMessages({ confMessages, pubkey, type }); + const mergedEverythingFetched = await this.handleUserOrGroupConfMessages({ + confMessages, + pubkey, + type, + }); + + // The level-with-swarm decision, evaluated in one place rather than inside the merge handler, + // because it depends + // on how the *poll* went and not on what the merge did. + // Three ways to fail to be level, and they fail differently, which is why all three are + // checked here rather than inferred from one another: + // - no snode answered at all; + // - a config namespace errored while others answered — a partial answer is not a full one; + // - the fetch succeeded but the merge failed. That one is neither a value nor an exception, + // only a log line, so it has to be reported back deliberately. + if (!mergedEverythingFetched) { + // Withdraw this swarm for the session rather than just skipping this poll. The lastHash + // cursor already advanced past the message we failed to merge, so it will never be offered + // again — the next poll would come back empty, look clean, and re-authorise recovery over + // state we know we never took in. A per-poll refusal alone is cosmetic here. + ConfigRecovery.markMergeIncompleteForSwarm(pubkey); + } + + const levelWithSwarmThisPoll = + atLeastOneSnodeAnswered && + allConfigNamespacesAnswered(resultsFromAllNamespaces, type) && + mergedEverythingFetched; + + if (levelWithSwarmThisPoll) { + ConfigRecovery.markLocalStateLevelWithSwarm(pubkey); + // not awaited — see the note on the other call site above + void ConfigRecovery.recoverIfNeeded(pubkey); + + // The keys backfill runs PROACTIVELY, beside recovery rather than inside it. Recovery acts on + // a hash the swarm has LOST; the backfill acts on a hash the swarm still HAS but whose bytes + // we never retained. Hanging it off the detection path would be nearly useless — by the time + // detection fires, the message it needed to fetch is gone. + if (PubKey.is03Pubkey(pubkey)) { + void ConfigRecovery.backfillGroupKeysIfNeeded(pubkey); + + // The freshness fact is computed HERE and passed in, because it is a property of this poll + // and nothing downstream can reconstruct it: the level marker is set once per process and + // never says whether it is still true. A rekey encrypts to our current view of the members, + // so a stale view silently drops anyone added since. + void ConfigRecoveryForceRekey.forceRekeyIfPossible(pubkey); + } + } await this.handleRevokedMessages({ revokedMessages, groupPk: pubkey, type }); @@ -739,6 +865,17 @@ export class SwarmPolling { window.log.info( `no configs before and after fetch of group: ${ed25519Str(pubkey)} from snode ${ed25519Str(snodeEdkey)}, but another snode has config hash fetched already (${ed25519Str(swarmSnodes?.[swarmIndex]?.pubkey_ed25519)}). Group is not expired.` ); + } else if (await ConfigRecovery.canRepairGroupKeys(pubkey)) { + // We hold the keys messages verbatim, so this is recoverable BY US: recovery will put + // them back on the next pass. Flagging expired here would tell the user the group is + // gone at the exact moment we are able to fix it. + // + // The verdict is DEFERRED behind bytes-held rather than merely corrected afterwards — + // setting the flag and clearing it a moment later is a visible flicker on a group that + // was never unrecoverable from this device. + window.log.info( + `no configs before and after fetch of group: ${ed25519Str(pubkey)}, but we retain its keys messages. Not flagging expired — recovery can repair it.` + ); } else { // the group appears to be expired. window.log.warn( @@ -759,6 +896,14 @@ export class SwarmPolling { convo.setIsExpired03Group(false); await convo.commit(); } + + // Note: the check above answers "we hold nothing and nobody gave us anything". It cannot + // see the case where we *do* hold config hashes and the swarm has since dropped them, + // because nothing new arriving looks identical to nothing having changed. That case is + // what the expire response tells us, and it is handled once the wrapper can attribute a + // hash to GroupKeys — which is what decides an 03-group expired. Deliberately not + // approximated in the meantime: a heuristic that can disagree with that rule is the signal + // proliferation this was supposed to remove. } if (!results.length) { return []; diff --git a/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingGroupConfig.ts b/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingGroupConfig.ts index c3d9ca3892..f330053298 100644 --- a/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingGroupConfig.ts +++ b/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingGroupConfig.ts @@ -195,10 +195,21 @@ async function handleMetaMergeResults(groupPk: GroupPubkeyType) { } } +/** + * @returns whether everything fetched was actually taken in. See the note on the user equivalent: + * the catch below is deliberate, which is exactly why the outcome has to be returned rather than + * inferred from the call completing. + * + * Note: this currently reports only whether the merge THREW, not whether it was lossy. libSession's + * metaMerge does return how many messages it took in, but the pinned wrapper's typings declare it + * `void` and discard the value — fixed in the wrapper alongside the group-recovery work, and the + * count comparison lands here once that release is pinned. Inert until then: nothing acts on a + * group swarm being marked level, because group recovery is not implemented yet. + */ async function handleGroupSharedConfigMessages( groupConfigMessages: Array, groupPk: GroupPubkeyType -) { +): Promise { try { window.log.info( `received groupConfigMessages count: ${groupConfigMessages.length} for groupPk:${ed25519Str( @@ -255,11 +266,13 @@ async function handleGroupSharedConfigMessages( groupPk, }) as any ); + return true; } catch (e) { window.log.warn( `handleGroupSharedConfigMessages of ${groupConfigMessages.length} failed with ${e.message}` ); // not rethrowing + return false; } } diff --git a/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingUserConfig.ts b/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingUserConfig.ts index 36a2e1c3a4..babc75de7c 100644 --- a/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingUserConfig.ts +++ b/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingUserConfig.ts @@ -1,9 +1,15 @@ import { ConfigMessageHandler } from '../../../../receiver/configMessage'; import { RetrieveMessageItemWithNamespace } from '../types'; +/** + * @returns whether everything fetched was actually taken in. The swallowing below is deliberate — + * one bad config message must not fail a whole poll — but that makes a merge failure a log line + * rather than a value or an exception, so a caller cannot otherwise tell success from silence. + * Anything relying on "our local state is level with the swarm" has to read this, not the fetch. + */ async function handleUserSharedConfigMessages( userConfigMessagesMerged: Array -) { +): Promise { try { if (userConfigMessagesMerged.length) { window.log.info( @@ -14,19 +20,26 @@ async function handleUserSharedConfigMessages( window.log.info( `handleConfigMessagesViaLibSession of "${userConfigMessagesMerged.length}" messages with libsession` ); - await ConfigMessageHandler.handleUserConfigMessagesViaLibSession(userConfigMessagesMerged); + // not just "did it throw" — a merge that took in only some of what it was given reports + // false here, and that is not something an exception would have told us. + return await ConfigMessageHandler.handleUserConfigMessagesViaLibSession( + userConfigMessagesMerged + ); } catch (e) { const allMessageHashes = userConfigMessagesMerged.map(m => m.hash).join(','); window.log.warn( `failed to handle messages hashes "${allMessageHashes}" with libsession. Error: "${e.message}"` ); + return false; } } + return true; } catch (e) { window.log.warn( `handleSharedConfigMessages of ${userConfigMessagesMerged.length} failed with ${e.message}` ); // not rethrowing + return false; } } diff --git a/ts/test/session/unit/disappearing_messages/ExpireRequest_test.ts b/ts/test/session/unit/disappearing_messages/ExpireRequest_test.ts index 56ad2f2931..db78cd0bf6 100644 --- a/ts/test/session/unit/disappearing_messages/ExpireRequest_test.ts +++ b/ts/test/session/unit/disappearing_messages/ExpireRequest_test.ts @@ -1,8 +1,13 @@ import chai, { expect } from 'chai'; import chaiAsPromised from 'chai-as-promised'; -import { PubkeyType } from 'libsession_util_nodejs'; +import { GroupPubkeyType, PubkeyType } from 'libsession_util_nodejs'; import Sinon from 'sinon'; -import { UpdateExpiryOnNodeUserSubRequest } from '../../../../session/apis/snode_api/SnodeRequestTypes'; +import { from_hex, to_hex } from 'libsodium-wrappers-sumo'; +import { ShortenOrExtend } from '../../../../session/types/with'; +import { + UpdateExpiryOnNodeGroupSubRequest, + UpdateExpiryOnNodeUserSubRequest, +} from '../../../../session/apis/snode_api/SnodeRequestTypes'; import { ExpireMessageWithExpiryOnSnodeProps, ExpireRequestResponseResults, @@ -11,10 +16,12 @@ import { verifyExpireMsgsResponseSignature, verifyExpireMsgsResponseSignatureProps, } from '../../../../session/apis/snode_api/expireRequest'; -import { UserUtils } from '../../../../session/utils'; +import { StringUtils, UserUtils } from '../../../../session/utils'; import { isValidUnixTimestamp } from '../../../../session/utils/Timestamps'; import { generateFakeSnode } from '../../../test-utils/utils'; import { NetworkTime } from '../../../../util/NetworkTime'; +import { getSodiumRenderer } from '../../../../session/crypto'; +import { fromBase64ToArray } from '../../../../session/utils/String'; chai.use(chaiAsPromised as any); @@ -128,6 +135,76 @@ describe('ExpireRequest', () => { }); }); + /** + * The group builder had no coverage at all, which is how it managed to send `extends` instead of + * `extend` for as long as it did: the server ignores keys it doesn't know about and raises no + * error, so nothing anywhere would have failed. + */ + describe('UpdateExpiryOnNodeGroupSubRequest', () => { + const messagesHashes = ['groupHash1', 'groupHash2']; + const expiryMs = 12340000 + 30 * 24 * 3600 * 1000; + + async function buildForGroup(shortenOrExtend: ShortenOrExtend = 'extend') { + const sodium = await getSodiumRenderer(); + const groupKeypair = sodium.crypto_sign_keypair(); + const groupPk: GroupPubkeyType = `03${to_hex(groupKeypair.publicKey)}`; + + const request = new UpdateExpiryOnNodeGroupSubRequest({ + expiryMs, + messagesHashes, + shortenOrExtend, + groupDetailsNeededForSignature: { + pubkeyHex: groupPk, + secretKey: groupKeypair.privateKey, + authData: null, + }, + }); + return { request, signedReq: await request.build(), groupPk }; + } + + it('sends "extend", not "extends"', async () => { + const { signedReq } = await buildForGroup(); + + expect(signedReq.params.extend, 'extend should be true').to.be.true; + expect( + (signedReq.params as Record).extends, + 'extends is not a key the storage server reads, so it must not be sent' + ).to.be.undefined; + expect(signedReq.params.shorten, 'shorten should be undefined').to.be.undefined; + }); + + it('supports shorten too — the type does not narrow what the server accepts', async () => { + // The only production call site passes 'extend', but the endpoint supports shorten and this + // request models it, same as the user one. Narrowing it here would state something false + // about the endpoint — and would make the group and user paths disagree about what the same + // operation is. + const { signedReq } = await buildForGroup('shorten'); + + expect(signedReq.params.shorten, 'shorten should be true').to.be.true; + expect(signedReq.params.extend, 'extend should be undefined').to.be.undefined; + }); + + it('signs the same shortenOrExtend that it puts on the wire', async () => { + const { signedReq, groupPk } = await buildForGroup(); + const sodium = await getSodiumRenderer(); + + // "expire" || ShortenOrExtend || expiry || messages[0] || ... || messages[N] + const verificationData = new Uint8Array( + StringUtils.encode(`expireextend${expiryMs}${messagesHashes.join('')}`, 'utf8') + ); + // note this pins the SIGNED value against the SENT one: the pair below is what the + // `extends` typo broke, and unifying the parameter must not reintroduce a way to diverge. + + const isValid = sodium.crypto_sign_verify_detached( + fromBase64ToArray(signedReq.params.signature as string), + verificationData, + from_hex(groupPk.slice(2)) + ); + + expect(isValid, 'the signature must cover "extend", matching the flag sent').to.be.true; + }); + }); + describe('verifyExpireMsgsResponseSignature', () => { const props: verifyExpireMsgsResponseSignatureProps = { pubkey: '058dc8432a63f9dda4d642bfc3eb5e037838bbd779f73e0a6dfb92b8040a1e7848', diff --git a/ts/test/session/unit/snode_api/configExpiryDetection_test.ts b/ts/test/session/unit/snode_api/configExpiryDetection_test.ts new file mode 100644 index 0000000000..43cc6ae970 --- /dev/null +++ b/ts/test/session/unit/snode_api/configExpiryDetection_test.ts @@ -0,0 +1,205 @@ +import chai from 'chai'; +import { describe } from 'mocha'; + +import { detectMissingConfigHashes } from '../../../../session/apis/snode_api/configExpiryDetection'; +import { ExpireMessagesResultsContent } from '../../../../session/apis/snode_api/types'; + +const { expect } = chai; + +/** + * The shared detection vectors, one test each. iOS and Android implement the same rule separately, + * and these vectors are the only thing keeping the three in agreement. + * + * So a vector failing here is a disagreement between clients, not a test that needs adjusting. If + * one of these looks wrong, the other two implementations are the thing to check first — relaxing + * it to make this client pass removes the only evidence that they have diverged. + */ + +const H1 = 'hash1'; +const H2 = 'hash2'; + +/** the fields detection doesn't read, but which are always on a real sub-response */ +const filler = { expiry: 1696915132498, signature: 'sig' }; + +function swarmOf(...subResponses: Array>) { + const swarm: ExpireMessagesResultsContent = {}; + subResponses.forEach((subResponse, index) => { + swarm[`snode${index}`] = { ...filler, updated: [], ...subResponse } as any; + }); + return swarm; +} + +function detect( + swarm: ExpireMessagesResultsContent | null, + { requestedHashes = [H1, H2], requestSetExtend = true } = {} +) { + return detectMissingConfigHashes({ requestedHashes, swarm, requestSetExtend }); +} + +describe('configExpiryDetection', () => { + it('V1: everything updated -> nothing missing', () => { + const result = detect(swarmOf({ updated: [H1, H2], unchanged: {} })); + + expect(result).to.be.deep.eq({ status: 'conclusive', missingHashes: [] }); + }); + + it('V2: unchanged counts as present', () => { + const result = detect(swarmOf({ updated: [H1], unchanged: { [H2]: 12345 } })); + + expect(result).to.be.deep.eq({ status: 'conclusive', missingHashes: [] }); + }); + + it('V3: absent from both arrays -> missing', () => { + const result = detect(swarmOf({ updated: [H1], unchanged: {} })); + + expect(result).to.be.deep.eq({ status: 'conclusive', missingHashes: [H2] }); + }); + + it('V4: one eligible snode reporting absence is sufficient', () => { + const result = detect( + swarmOf({ updated: [H1], unchanged: {} }, { updated: [H1, H2], unchanged: {} }) + ); + + expect(result).to.be.deep.eq({ status: 'conclusive', missingHashes: [H2] }); + }); + + // The failed nodes below CARRY an `unchanged` array on purpose, and it must stay. + // + // Without it they are already unreadable, so the eligibility check excludes them on that ground + // and never consults `failed` at all — the `failed` term could then be deleted with the whole + // suite green. Verified: it was, and nothing died until these two fixtures gained the array. + // + // Carrying it also makes them the dangerous shape rather than a harmless one. A node that says it + // failed but still reports arrays is exactly the input the term exists for: read as usable, its + // empty arrays become authority and EVERY requested hash is reported missing — a false positive + // that re-stores configs the swarm still holds, on the word of a node that told us it failed. + it('V5: a failed sub-response is excluded, not read as absence', () => { + const result = detect( + swarmOf({ updated: [H1, H2], unchanged: {} }, { + updated: [], + unchanged: {}, + failed: true, + timeout: true, + } as any) + ); + + expect(result).to.be.deep.eq({ status: 'conclusive', missingHashes: [] }); + }); + + it('V6: every sub-response failed -> inconclusive, no recovery', () => { + const result = detect( + swarmOf( + { updated: [], unchanged: {}, failed: true } as any, + { updated: [], unchanged: {}, failed: true, code: 500 } as any + ) + ); + + expect(result).to.be.deep.eq({ status: 'inconclusive' }); + }); + + it('V7: snode holds nothing -> both hashes missing', () => { + const result = detect(swarmOf({ updated: [], unchanged: {} })); + + expect(result).to.be.deep.eq({ status: 'conclusive', missingHashes: [H1, H2] }); + }); + + it('V8: request did not set extend -> detection unavailable', () => { + const result = detect(swarmOf({ updated: [H1] }), { requestSetExtend: false }); + + expect(result).to.be.deep.eq({ status: 'unavailable' }); + }); + + it('V9: multipart config parts are evaluated independently', () => { + const [P1, P2, P3] = ['part1', 'part2', 'part3']; + + const result = detect(swarmOf({ updated: [P1, P3], unchanged: {} }), { + requestedHashes: [P1, P2, P3], + }); + + // only the missing part is re-stored; "recovered" is the caller's call, and it needs all three + expect(result).to.be.deep.eq({ status: 'conclusive', missingHashes: [P2] }); + }); + + /** + * V10-V13 are guard and action rules rather than properties of the response, so they are + * covered where those live: V10 (poll+merge this session), V11 (obsolete hash set), V12 + * (kicked/destroyed group) and V13 (re-store once per session) are in + * configRecovery_test.ts. Detection itself still reports MISSING in all four cases, which is + * what these assert. + */ + it('V10-V13: detection still reports missing; acting on it is guarded elsewhere', () => { + const result = detect(swarmOf({ updated: [H1], unchanged: {} })); + + expect(result).to.be.deep.eq({ status: 'conclusive', missingHashes: [H2] }); + }); + + it('V15: an EMPTY unchanged is a valid answer; an ABSENT one is not — same fixture, opposite verdicts', () => { + // Split out of V7/V8b at planning's request, because the vector is about the DISTINCTION rather + // than either endpoint. Conflating the two would silently disable recovery in the total-loss + // case — the one case the feature exists for — so it is worth pinning as one assertion. + const empty = detect(swarmOf({ updated: [], unchanged: {} })); + const absent = detect(swarmOf({ updated: [] })); + + expect(empty, 'present-and-empty: the snode answered, and it holds neither hash').to.be.deep.eq( + { + status: 'conclusive', + missingHashes: [H1, H2], + } + ); + expect(absent, 'absent: this response cannot tell presence from absence at all').to.be.deep.eq({ + status: 'inconclusive', + }); + }); + + describe('rules that are easy to get wrong', () => { + it('an empty swarm is inconclusive, not "nothing missing"', () => { + expect(detect({})).to.be.deep.eq({ status: 'inconclusive' }); + expect(detect(null)).to.be.deep.eq({ status: 'inconclusive' }); + }); + + it('V8b: an ABSENT unchanged KEY excludes that sub-response — distinct from V8s flag', () => { + // if this were read as "nothing was unchanged", H2 would look missing + const result = detect(swarmOf({ updated: [H1] })); + + expect(result).to.be.deep.eq({ status: 'inconclusive' }); + }); + + it('V8c: one unreadable sub-response alongside a readable one — the readable one is honoured', () => { + // The two unreadable nodes are unreadable for DIFFERENT reasons, and each is excludable only + // by its own guard — otherwise this vector says "unreadable" while testing one route twice. + // The failed node therefore carries full arrays (only `failed` can exclude it) and the other + // omits `unchanged` (only the readability check can). + const result = detect( + swarmOf( + { updated: [], unchanged: {}, failed: true } as any, + { updated: [H1], unchanged: { [H2]: 1 } }, + { updated: [] } // no unchanged key -> excluded on that ground alone + ) + ); + + expect(result).to.be.deep.eq({ status: 'conclusive', missingHashes: [] }); + }); + + it('V14: asking about no hashes is INCONCLUSIVE, not "nothing missing"', () => { + // 'conclusive' is the natural short-circuit here, and it is wrong: a conclusive result + // outranks the empty-fetch check, so reporting one for a swarm detection never asked about + // would make detection the authority for it, and the check that should decide the no-hashes + // case could never fire. + const result = detect(swarmOf({ updated: [], unchanged: {} }), { requestedHashes: [] }); + + expect(result).to.be.deep.eq({ status: 'inconclusive' }); + }); + + it('V14: an empty ask is inconclusive even when the request did set extend', () => { + const result = detect(swarmOf({ updated: [], unchanged: {} }), { + requestedHashes: [], + requestSetExtend: true, + }); + + expect( + result.status, + 'having asked correctly about nothing is still asking nothing' + ).to.be.eq('inconclusive'); + }); + }); +}); diff --git a/ts/test/session/unit/snode_api/configRecoveryForceRekey_test.ts b/ts/test/session/unit/snode_api/configRecoveryForceRekey_test.ts new file mode 100644 index 0000000000..3adcc7e81f --- /dev/null +++ b/ts/test/session/unit/snode_api/configRecoveryForceRekey_test.ts @@ -0,0 +1,235 @@ +import chai from 'chai'; +import { beforeEach, describe } from 'mocha'; +import Sinon from 'sinon'; +import { GroupPubkeyType } from 'libsession_util_nodejs'; + +import { ConfigRecoveryForceRekey } from '../../../../session/apis/snode_api/configRecoveryForceRekey'; +import { ConfigRecovery } from '../../../../session/apis/snode_api/configRecovery'; +import { + MetaGroupWrapperActions, + UserGroupsWrapperActions, +} from '../../../../webworker/workers/browser/libsession_worker_interface'; +import { LibSessionUtil } from '../../../../session/utils/libsession/libsession_utils'; +import { GroupSync } from '../../../../session/utils/job_runners/jobs/GroupSyncJob'; +import { TestUtils } from '../../../test-utils'; + +const { expect } = chai; + +/** + * The force rekey — the only irreversible, universally visible write in config recovery. + * + * Every assertion below is about NOT doing it, which makes them all vulnerable to passing because + * the path died early rather than because a rule held. Each therefore starts from a fixture that + * WOULD rekey, and changes exactly one thing; the first test proves that fixture actually rekeys, + * so every later refusal is measured against a known-live baseline. + */ +describe('ConfigRecovery force rekey', () => { + let groupPk: GroupPubkeyType; + let rekeyStub: Sinon.SinonStub; + let backfillFailedStub: Sinon.SinonStub; + + /** the state in which a rekey IS warranted: admin, keys all gone, no bytes, backfill tried */ + function stubWarranted({ + secretKey = new Uint8Array(64).fill(7) as any, + kicked = false, + destroyed = false, + keysHashes = ['keyshash1'], + retained = {} as Record, + backfillFailed = true, + } = {}) { + Sinon.stub(UserGroupsWrapperActions, 'getGroup').resolves({ + pubkeyHex: groupPk, + secretKey, + authData: null, + kicked, + destroyed, + name: 'g', + invitePending: false, + } as any); + Sinon.stub(MetaGroupWrapperActions, 'activeHashesByConfig').resolves({ + groupInfo: [], + groupMember: [], + groupKeys: keysHashes, + }); + Sinon.stub(MetaGroupWrapperActions, 'activeKeyMessages').resolves(retained); + backfillFailedStub = Sinon.stub(ConfigRecovery, 'keysBackfillHasFailedFor').returns( + backfillFailed + ); + } + + beforeEach(() => { + TestUtils.stubWindowLog(); + ConfigRecoveryForceRekey.resetForTesting(); + ConfigRecovery.resetForTesting(); + groupPk = TestUtils.generateFakeClosedGroupV2PkStr(); + rekeyStub = Sinon.stub(MetaGroupWrapperActions, 'keyRekey').resolves(undefined as any); + Sinon.stub(LibSessionUtil, 'saveDumpsToDb').resolves(); + Sinon.stub(GroupSync, 'queueNewJobIfNeeded').resolves(); + }); + + afterEach(() => { + Sinon.restore(); + }); + + /** put the store into "level as of the poll running now" */ + function levelNow() { + ConfigRecovery.beginPollForSwarm(groupPk); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + } + + it('rekeys when nothing here can restore the keys — the baseline every refusal is measured against', async () => { + stubWarranted(); + levelNow(); + + const did = await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk); + + expect(did, 'the fixture genuinely rekeys').to.be.true; + expect(rekeyStub.calledOnceWith(groupPk)).to.be.true; + expect( + (LibSessionUtil.saveDumpsToDb as unknown as Sinon.SinonStub).calledWith(groupPk), + 'and the new generation is persisted, or it dies with the process' + ).to.be.true; + expect( + (GroupSync.queueNewJobIfNeeded as unknown as Sinon.SinonStub).called, + 'and queued for push, or nobody else ever sees it' + ).to.be.true; + }); + + it('REFUSES a stale members view, even though everything else warrants it', async () => { + // The rekey encrypts to this device's view of the members. If that view is behind, whoever was + // added since is silently left out — and this fires precisely on devices whose config state is + // known to be degraded, so "behind" is the expected condition rather than the unlucky one. + stubWarranted(); + + // no mark for the current poll — the store answers false + const did = await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk); + + expect(did).to.be.false; + expect(rekeyStub.called, 'nothing minted from a members list we know may be behind').to.be + .false; + }); + + it('REFUSES when a backfill has never run', async () => { + // "We hold no bytes" cannot distinguish "a backfill ran and found nothing" from "no backfill has + // ever run" — identical on a fresh install, a restored backup, or before the first poll + // completes. Only the first justifies this. + stubWarranted({ backfillFailed: false }); + levelNow(); + + const did = await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk); + + expect(did).to.be.false; + expect(backfillFailedStub.called, 'PREMISE: it actually consulted the record').to.be.true; + expect(rekeyStub.called).to.be.false; + }); + + it('REFUSES when one keys message is still recoverable', async () => { + // A single surviving keys hash still lets a new device in, so the group is not stuck. + stubWarranted({ + keysHashes: ['keyshash1', 'keyshash2'], + retained: { keyshash2: new Uint8Array([1]) }, + }); + levelNow(); + + expect(await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk)).to.be.false; + expect(rekeyStub.called).to.be.false; + }); + + it('REFUSES for a member — only an admin can mint a key', async () => { + stubWarranted({ secretKey: null }); + levelNow(); + + expect(await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk)).to.be.false; + expect(rekeyStub.called).to.be.false; + }); + + it('REFUSES for a kicked or destroyed group', async () => { + stubWarranted({ destroyed: true }); + levelNow(); + + expect(await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk)).to.be.false; + expect(rekeyStub.called).to.be.false; + }); + + it('V25e: refuses when the level mark is from a PREVIOUS poll', async () => { + // The whole point of the poll token. A device that was level yesterday and has not completed a + // poll since still answers true to the sticky question, and its members list may be behind by + // exactly the member a rekey would drop. Three states, one fixture, asserting on the rekey + // count rather than the return value so a refusal cannot be confused with a throw. + stubWarranted(); + + // 1. a poll has begun, nothing marked -> nothing to be level from + ConfigRecovery.beginPollForSwarm(groupPk); + await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk); + expect(rekeyStub.callCount, 'unmarked poll: refuse').to.be.eq(0); + + // 2. marked during THIS poll -> proceed + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk); + expect(rekeyStub.callCount, 'marked this poll: proceed').to.be.eq(1); + + // 3. a NEW poll begins, so the mark is now from the previous one -> refuse + ConfigRecoveryForceRekey.resetForTesting(); // clear the once-per-session guard, isolating this rule + ConfigRecovery.beginPollForSwarm(groupPk); + await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk); + expect( + rekeyStub.callCount, + 'a new poll makes the earlier mark stale, even though the sticky question still says level' + ).to.be.eq(1); + + // and the sticky question DOES still say level — otherwise this test passes for the wrong reason + expect( + ConfigRecovery.localStateIsLevelWithSwarm(groupPk), + 'PREMISE: the sticky reading is still true, so only the poll-scoped one refused' + ).to.be.true; + }); + + it('rekeys a group ONCE — a second call in the same session is refused', async () => { + // Without this, every poll that still sees the old preconditions mints another generation, and + // each one is a write every member on every version has to process. + stubWarranted(); + levelNow(); + + expect(await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk)).to.be.true; + expect(await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk)).to.be.false; + expect(rekeyStub.callCount, 'exactly one generation minted').to.be.eq(1); + }); + + it('the 24h cooldown lapses — a failed rekey is retried, but not before 24h', async () => { + // The cooldown is a SEPARATE guard from the once-per-session set above, and it is the only one + // of the two that can lapse: the session set is only written on success, so the cooldown's + // reachable job is throttling RETRIES after a rekey that threw. Untested, a guard that never + // lapses would leave such a group unable to ever try again, and — because the interval is a + // full day — would be indistinguishable from a working guard for that whole day. + stubWarranted(); + levelNow(); + + let now = 1_000_000; + ConfigRecoveryForceRekey.setNowForTesting(() => now); + + // an attempt that FAILS: the cooldown is stamped before the call, the session set is not + rekeyStub.rejects(new Error('swarm unreachable')); + expect(await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk)).to.be.false; + expect(rekeyStub.callCount, 'PREMISE: it genuinely attempted').to.be.eq(1); + + rekeyStub.resolves(undefined as any); + + // immediately after, and repeatedly: refused + await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk); + await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk); + expect(rekeyStub.callCount, 'inside the window, however many polls run').to.be.eq(1); + + // 2 hours later: still refused. This step is what pins the interval at 24h — without it a + // silent regression to a one-hour cooldown passes every other assertion here. + now += 2 * 60 * 60 * 1000; + await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk); + expect(rekeyStub.callCount, 'two hours is not a lapse').to.be.eq(1); + + // a further 23 hours: the guard lapses and the failed rekey is retried. + // This step also discriminates WHICH guard refused above — the once-per-session set never + // lapses, so if it had been the blocker this would still read 1. + now += 23 * 60 * 60 * 1000; + await ConfigRecoveryForceRekey.forceRekeyIfPossible(groupPk); + expect(rekeyStub.callCount, 'past 24h the group gets another chance').to.be.eq(2); + }); +}); diff --git a/ts/test/session/unit/snode_api/configRecoveryGroup_test.ts b/ts/test/session/unit/snode_api/configRecoveryGroup_test.ts new file mode 100644 index 0000000000..b751537864 --- /dev/null +++ b/ts/test/session/unit/snode_api/configRecoveryGroup_test.ts @@ -0,0 +1,781 @@ +import chai from 'chai'; +import { beforeEach, describe } from 'mocha'; +import Sinon from 'sinon'; +import { GroupPubkeyType, PubkeyType } from 'libsession_util_nodejs'; + +import { ConfigRecovery } from '../../../../session/apis/snode_api/configRecovery'; +import { + MetaGroupWrapperActions, + UserGroupsWrapperActions, +} from '../../../../webworker/workers/browser/libsession_worker_interface'; +import { LibSessionUtil } from '../../../../session/utils/libsession/libsession_utils'; +import { MessageSender } from '../../../../session/sending/MessageSender'; +import { UserUtils } from '../../../../session/utils'; +import { + DeleteHashesFromGroupNodeSubRequest, + StoreGroupInfoSubRequest, + StoreGroupKeysSubRequest, + StoreGroupMembersSubRequest, +} from '../../../../session/apis/snode_api/SnodeRequestTypes'; +import { TestUtils } from '../../../test-utils'; +import { ConvoHub } from '../../../../session/conversations'; +import { SnodePool } from '../../../../session/apis/snode_api/snodePool'; +import { SnodeAPIRetrieve } from '../../../../session/apis/snode_api/retrieveRequest'; +import { SnodeNamespaces } from '../../../../session/apis/snode_api/namespaces'; + +const { expect } = chai; + +/** + * Group recovery — the vectors that were blocked on the wrapper until v0.6.20 exposed + * `pushForRecovery()` and `activeHashesByConfig()`: V16, V16a, V16b, V19, V20, V21. + * + * The user-path vectors are in configRecovery_test.ts and are NOT repeated here. What is specific + * to groups is: which sub-config a hash belongs to (GroupKeys goes back only from retained bytes, + * verbatim), and whether we are an admin or a member (a member cannot delete). + * + * ON "ASSERTS THAT X DOES NOT HAPPEN" TESTS — same rule as the user file. Every absence assertion + * below is also satisfied by the path dying early, so each carries something proving it reached the + * decision. Where a vector's own premise is "it stops at a guard", the reachability anchor is the + * paired positive test using the SAME fixture, named in the test. + */ + +const INFO_HASH = 'infohash1'; +const MEMBER_HASH = 'memberhash1'; +const KEYS_HASH = 'keyshash1'; + +describe('ConfigRecovery (groups)', () => { + let groupPk: GroupPubkeyType; + let us: PubkeyType; + let sendStub: Sinon.SinonStub; + + /** a clean group we are an ADMIN of, holding one hash in each of the three sub-configs */ + function stubGroup({ + secretKey = new Uint8Array(64).fill(7) as any, + authData = null as any, + kicked = false, + destroyed = false, + needsPush = false, + infoHashes = [INFO_HASH], + memberHashes = [MEMBER_HASH], + keysHashes = [KEYS_HASH], + infoParts = [new Uint8Array([1])], + memberParts = [new Uint8Array([2])], + infoObsolete = [] as Array, + memberObsolete = [] as Array, + retainedKeyMessages = {} as Record, + } = {}) { + Sinon.stub(UserGroupsWrapperActions, 'getGroup').resolves({ + pubkeyHex: groupPk, + secretKey, + authData, + kicked, + destroyed, + name: 'g', + invitePending: false, + } as any); + Sinon.stub(MetaGroupWrapperActions, 'needsPush').resolves(needsPush); + Sinon.stub(MetaGroupWrapperActions, 'activeHashesByConfig').resolves({ + groupInfo: infoHashes, + groupMember: memberHashes, + groupKeys: keysHashes, + }); + // Must be stubbed even for tests that are not about keys. Without it the call throws, the + // inspection reports "could not inspect" and every keys assertion below passes through the + // error path instead of the rule it names. + Sinon.stub(MetaGroupWrapperActions, 'activeKeyMessages').resolves(retainedKeyMessages); + Sinon.stub(MetaGroupWrapperActions, 'pushForRecovery').resolves({ + groupInfo: { data: infoParts, seqno: 5, hashes: infoObsolete, namespace: 12 }, + groupMember: { data: memberParts, seqno: 5, hashes: memberObsolete, namespace: 13 }, + } as any); + } + + function allSubRequestsSent() { + return sendStub.getCalls().flatMap(c => c.args[0].sortedSubRequests as Array); + } + + function infoStoresSent() { + return allSubRequestsSent().filter(r => r instanceof StoreGroupInfoSubRequest); + } + + function memberStoresSent() { + return allSubRequestsSent().filter(r => r instanceof StoreGroupMembersSubRequest); + } + + function keysStoresSent() { + return allSubRequestsSent().filter( + (r): r is StoreGroupKeysSubRequest => r instanceof StoreGroupKeysSubRequest + ); + } + + function deleteRequestSent() { + return allSubRequestsSent().find( + (r): r is DeleteHashesFromGroupNodeSubRequest => + r instanceof DeleteHashesFromGroupNodeSubRequest + ); + } + + beforeEach(() => { + TestUtils.stubWindowLog(); + ConfigRecovery.resetForTesting(); + us = TestUtils.generateFakePubKeyStr(); + groupPk = TestUtils.generateFakeClosedGroupV2PkStr(); + Sinon.stub(UserUtils, 'getOurPubKeyStrFromCache').returns(us); + Sinon.stub(UserUtils, 'isUsFromCache').callsFake(pk => pk === us); + Sinon.stub(LibSessionUtil, 'saveDumpsToDb').resolves(); + sendStub = Sinon.stub(MessageSender, 'sendEncryptedDataToSnode').callsFake( + async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 200, body: { hash: 'newhash' } })) as any + ); + }); + + afterEach(() => { + Sinon.restore(); + }); + + function detectMissing(hashes: Array) { + ConfigRecovery.recordDetection(groupPk, { status: 'conclusive', missingHashes: hashes }); + } + + it('V19: a missing GroupInfo hash is re-stored, and GroupKeys is not flagged expired', async () => { + // The vector's point is that a missing groupInfo hash says nothing about the keys. An + // implementation that treats "any group hash missing" as "the group is gone" passes nothing + // else in this file and fails here. + stubGroup(); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran, 'the group config was put back').to.be.true; + expect(infoStoresSent().length, 'groupInfo re-stored').to.be.eq(1); + expect( + memberStoresSent().length, + 'and groupMember was NOT, it claimed no missing hash' + ).to.be.eq(0); + }); + + it('V16a: one GroupKeys hash missing while another is PRESENT — no re-store, group not expired', async () => { + // The reason is "we retain no bytes for it", NOT "a keys message can never be put back" — + // this fixture holds none. A device that DOES hold them re-stores instead, which is V23. Kept + // as the no-bytes case because groups predating retention are real. + stubGroup({ keysHashes: [KEYS_HASH, 'keyshash2'] }); + detectMissing([KEYS_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran, 'nothing was restorable').to.be.false; + expect(sendStub.called, 'and nothing was SENT — this fixture retains no bytes to send').to.be + .false; + }); + + it('V16/V23a: EVERY GroupKeys hash missing and NO retained bytes — no re-store attempted', async () => { + // The group predates keys retention, so there is nothing to push back. Unchanged behaviour, but + // note the reason: not "impossible to recover" — "not recoverable BY THIS DEVICE". A peer that + // holds the bytes can still repair it. + stubGroup({ retainedKeyMessages: {} }); + detectMissing([KEYS_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran).to.be.false; + expect(sendStub.called, 'nothing held, so nothing to send').to.be.false; + }); + + it('V23: every GroupKeys hash missing but the bytes ARE held — re-store them', async () => { + // The vector pins BYTES-HELD as the term, not the missing-ness: V23a has the identical missing + // set and does nothing. The only difference between them is the retained map. + stubGroup({ retainedKeyMessages: { [KEYS_HASH]: new Uint8Array([9, 9]) } }); + detectMissing([KEYS_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran, 'a member CAN put keys back, because it pushes the bytes verbatim').to.be.true; + expect(keysStoresSent().length, 'the keys message went out').to.be.eq(1); + expect( + keysStoresSent()[0].encryptedData, + 'and VERBATIM — re-signing is impossible, so any transformation breaks it' + ).to.deep.eq(new Uint8Array([9, 9])); + }); + + it('V23 (member): a non-admin with retained bytes repairs the keys', async () => { + // The point of the whole change. A member cannot sign a keys message, so this only works + // because the bytes are pushed back unchanged. + stubGroup({ + secretKey: null, + authData: new Uint8Array(100).fill(3), + retainedKeyMessages: { [KEYS_HASH]: new Uint8Array([9]) }, + }); + detectMissing([KEYS_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + expect(await ConfigRecovery.recoverIfNeeded(groupPk)).to.be.true; + expect(keysStoresSent().length).to.be.eq(1); + expect(deleteRequestSent(), 'a keys message supersedes nothing, so no delete ever').to.be + .undefined; + }); + + it('V23b: a supplemental is retained and re-stored too, not dropped', async () => { + // Storage is hash-keyed, not generation-keyed, and a generation is the full rekey PLUS every + // supplemental issued against it — a member receiving only one of them does not get the key. + // We cannot group by generation (the accessor carries none), so EVERY retained message goes + // back. That is a superset of the affected generation, which is what the rule protects. + stubGroup({ + keysHashes: [KEYS_HASH, 'supplemental1'], + retainedKeyMessages: { + [KEYS_HASH]: new Uint8Array([1]), + supplemental1: new Uint8Array([2]), + }, + }); + detectMissing([KEYS_HASH]); // only ONE reported missing + + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + await ConfigRecovery.recoverIfNeeded(groupPk); + + expect( + keysStoresSent().length, + 'both go back though only one was missing — a partial generation is unusable' + ).to.be.eq(2); + }); + + it('V23c: a FAILED keys re-store is not banked as success', async () => { + stubGroup({ retainedKeyMessages: { [KEYS_HASH]: new Uint8Array([9]) } }); + detectMissing([KEYS_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 500, body: {} })) + ); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran, 'a 500 is not a repair').to.be.false; + expect(keysStoresSent().length, 'but it was attempted — this is not an early return').to.be.eq( + 1 + ); + }); + + it('V16b: the device holds NO GroupKeys hashes at all, so no keys question was asked', async () => { + // Distinct from V16: there, the keys hashes exist and are gone. Here we never had any, so a + // missing groupInfo hash must still be recovered normally rather than the absence of keys + // hashes being read as "the keys are missing". + stubGroup({ keysHashes: [] }); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran, 'holding no keys hashes must not block an unrelated recovery').to.be.true; + expect(infoStoresSent().length).to.be.eq(1); + }); + + it('V20: a non-admin MEMBER re-stores a clean GroupInfo whose hash is missing', async () => { + // The trap this vector exists for is asserting the store is skipped for a member. It is not: + // a member's subaccount token carries Read+Write, and this is the whole point of member-driven + // recovery. Assert it SUCCEEDS. + stubGroup({ secretKey: null, authData: new Uint8Array(100).fill(3) }); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran, 'a member CAN put its own copy back').to.be.true; + expect(infoStoresSent().length, 'the store went out').to.be.eq(1); + }); + + it('V21: a member with an EMPTY obsolete-hash list still succeeds, and issues no delete', async () => { + // Two traps in one vector. push() hands the superseded hashes back only if !is_readonly(), so + // an empty list is EXPECTED for a member — asserting a non-empty one would be asserting a bug. + // And a member could not delete anyway: its token has no Delete permission. + stubGroup({ secretKey: null, authData: new Uint8Array(100).fill(3), infoObsolete: [] }); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran, 'the re-store still succeeds').to.be.true; + expect( + infoStoresSent().length, + 'proving we got past the store, not that we never tried' + ).to.be.eq(1); + expect(deleteRequestSent(), 'and no delete is attempted').to.be.undefined; + }); + + it('V21 (the gate itself): a MEMBER never deletes, even given a non-empty obsolete list', async () => { + // The test above cannot see this rule. Its fixture has an EMPTY obsolete list, so "no delete" + // is true there whether the admin check exists or not — found by mutation: removing the check + // left that test green. push() should never hand a member a non-empty list, so this state is + // not reachable through the wrapper today; the check is what stops it becoming a 401 storm if + // that ever changes. Asserting it needs a fixture the real path cannot produce, which is the + // point: an unreachable state is exactly what a defence-in-depth check is for. + stubGroup({ + secretKey: null, + authData: new Uint8Array(100).fill(3), + infoObsolete: ['oldinfo1'], + }); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran, 'the re-store still succeeds').to.be.true; + expect( + infoStoresSent().length, + 'and we got past the store rather than stopping short' + ).to.be.eq(1); + expect(deleteRequestSent(), 'but a member has no Delete permission, so no delete goes out').to + .be.undefined; + }); + + it('V21 counterpart: an ADMIN with a non-empty obsolete list DOES delete', async () => { + // The reachability control for the assertion above: without this, "no delete" would also pass + // against a group delete path that was never wired at all. + stubGroup({ infoObsolete: ['oldinfo1'] }); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + await ConfigRecovery.recoverIfNeeded(groupPk); + + expect( + deleteRequestSent()?.messageHashes, + 'the admin prunes what it superseded' + ).to.have.members(['oldinfo1']); + }); + + it('V23d: a successful keys re-store CLEARS an existing expired flag, eagerly', async () => { + // Not left to the poller's reactive clear. That fires when config messages are RECEIVED — but + // we just re-stored messages we already hold, so we may never receive or re-handle them, and + // the flag would sit set forever over keys that are back on the swarm. + const setExpired = Sinon.stub(); + const commit = Sinon.stub().resolves(); + Sinon.stub(ConvoHub, 'use').returns({ + get: () => ({ getIsExpired03Group: () => true, setIsExpired03Group: setExpired, commit }), + } as any); + + stubGroup({ retainedKeyMessages: { [KEYS_HASH]: new Uint8Array([9]) } }); + detectMissing([KEYS_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(keysStoresSent().length, 'the re-store happened').to.be.eq(1); + expect(setExpired.calledOnceWith(false), 'and the flag was cleared by it').to.be.true; + expect(commit.called, 'and persisted').to.be.true; + }); + + it('V23d counterpart: a FAILED keys re-store leaves the expired flag alone', async () => { + // The reachability control for the assertion above: without it, "cleared" would also pass + // against an implementation that cleared the flag unconditionally on every attempt. + const setExpired = Sinon.stub(); + Sinon.stub(ConvoHub, 'use').returns({ + get: () => ({ + getIsExpired03Group: () => true, + setIsExpired03Group: setExpired, + commit: Sinon.stub().resolves(), + }), + } as any); + + stubGroup({ retainedKeyMessages: { [KEYS_HASH]: new Uint8Array([9]) } }); + detectMissing([KEYS_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 500, body: {} })) + ); + + await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(keysStoresSent().length, 'it was attempted').to.be.eq(1); + expect(setExpired.called, 'but nothing landed, so the group is still expired').to.be.false; + }); + + it('canRepairGroupKeys: true only when bytes are actually held', async () => { + // What the poller asks before flagging a group expired. The flag means "not recoverable by this + // device", so holding the bytes must defer it rather than raise-then-clear. + stubGroup({ retainedKeyMessages: { [KEYS_HASH]: new Uint8Array([9]) } }); + expect(await ConfigRecovery.canRepairGroupKeys(groupPk)).to.be.true; + Sinon.restore(); + + TestUtils.stubWindowLog(); + stubGroup({ retainedKeyMessages: {} }); + expect(await ConfigRecovery.canRepairGroupKeys(groupPk)).to.be.false; + }); + + it('Q4/V16: all keys hashes gone and NO retained bytes -> the group is flagged EXPIRED', async () => { + // Detection is the only thing that can raise the flag for this case. The poller's empty-fetch + // branch cannot reach it by construction: that branch requires holding NO config hashes, and a + // device in this state holds plenty — the hashes are exactly what told us they were missing. + const setExpired = Sinon.stub(); + Sinon.stub(ConvoHub, 'use').returns({ + get: () => ({ + getIsExpired03Group: () => false, + setIsExpired03Group: setExpired, + commit: Sinon.stub().resolves(), + }), + } as any); + + stubGroup({ retainedKeyMessages: {} }); + detectMissing([KEYS_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(setExpired.calledOnceWith(true), 'the banner is raised').to.be.true; + }); + + it('Q4/V16a: only SOME keys hashes gone -> NOT expired, even with no bytes', async () => { + // The reachability control for the assertion above, and the vector's own point: one surviving + // keys hash still lets a new device in, so a partial miss is not an expired group. + const setExpired = Sinon.stub(); + Sinon.stub(ConvoHub, 'use').returns({ + get: () => ({ + getIsExpired03Group: () => false, + setIsExpired03Group: setExpired, + commit: Sinon.stub().resolves(), + }), + } as any); + + stubGroup({ keysHashes: [KEYS_HASH, 'keyshash2'], retainedKeyMessages: {} }); + detectMissing([KEYS_HASH]); // one of two + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(setExpired.called, 'one surviving keys hash is not an expired group').to.be.false; + }); + + it('Q4/V23c: bytes held but the keys re-store FAILS -> expired after all', async () => { + // The banner is deferred while we hold a repair in hand. Once that repair fails the keys are + // still gone and still not back, so the user needs to know. + const setExpired = Sinon.stub(); + Sinon.stub(ConvoHub, 'use').returns({ + get: () => ({ + getIsExpired03Group: () => false, + setIsExpired03Group: setExpired, + commit: Sinon.stub().resolves(), + }), + } as any); + + stubGroup({ retainedKeyMessages: { [KEYS_HASH]: new Uint8Array([9]) } }); + detectMissing([KEYS_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 500, body: {} })) + ); + + await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(keysStoresSent().length, 'the repair was attempted').to.be.eq(1); + expect(setExpired.calledWith(true), 'and having failed, the banner goes up').to.be.true; + }); + + it('Q10: a dirty groupInfo does NOT block KEYS recovery', async () => { + // The clean-only gate exists so local state cannot overwrite newer remote state. + // Keys recovery replays the exact bytes the swarm already had, so it cannot overwrite anything, + // and a pending rekey produces a NEW message at a NEW generation — which says nothing about + // whether the retained ones are stale. Gating keys on a dirty groupInfo excluded groups in + // active use, which is the population most likely to need them. + stubGroup({ needsPush: true, retainedKeyMessages: { [KEYS_HASH]: new Uint8Array([9]) } }); + detectMissing([KEYS_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran, 'keys go back even though the group is dirty').to.be.true; + expect(keysStoresSent().length).to.be.eq(1); + }); + + it('Q10 counterpart: a dirty group still blocks groupInfo/groupMember', async () => { + // The exemption is keys-only. Info and members are re-serialised from local state, so the gate + // is doing real work for them — without this, "dirty blocks nothing" would pass the test above. + stubGroup({ needsPush: true }); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran).to.be.false; + expect(infoStoresSent().length, 'GroupSync will push it under a new hash anyway').to.be.eq(0); + }); + + describe('keys backfill', () => { + // The backfill exists to capture BYTES for keys messages the swarm still holds, so that this + // device can repair the group later. It runs proactively — by the time detection fires, the + // message it would have fetched is gone. + + function stubBackfill({ + keysHashes = [KEYS_HASH], + retained = {} as Record, + retainedAfterMerge = null as Record | null, + fetched = [] as Array<{ hash: string; data: string; storedAt: number }>, + } = {}) { + Sinon.stub(SnodePool, 'getSwarmFor').resolves([ + { pubkey_ed25519: 'ed', ip: '1', port: 1 }, + ] as any); + const hashesStub = Sinon.stub(MetaGroupWrapperActions, 'activeHashesByConfig').resolves({ + groupInfo: [], + groupMember: [], + groupKeys: keysHashes, + }); + // second call (after the merge) reports the post-merge state when one is given + const keysStub = Sinon.stub(MetaGroupWrapperActions, 'activeKeyMessages'); + keysStub.onFirstCall().resolves(retained); + keysStub.resolves(retainedAfterMerge ?? retained); + const mergeStub = Sinon.stub(MetaGroupWrapperActions, 'metaMerge').resolves(undefined as any); + const retrieveStub = Sinon.stub(SnodeAPIRetrieve, 'retrieveNextMessagesNoRetries').resolves([ + { code: 200, namespace: SnodeNamespaces.ClosedGroupKeys, messages: { messages: fetched } }, + ] as any); + return { hashesStub, keysStub, mergeStub, retrieveStub }; + } + + it('does nothing when we already hold bytes for every active keys hash', async () => { + const { retrieveStub } = stubBackfill({ retained: { [KEYS_HASH]: new Uint8Array([1]) } }); + + await ConfigRecovery.backfillGroupKeysIfNeeded(groupPk); + + expect(retrieveStub.called, 'no fetch when there is nothing to capture').to.be.false; + expect(ConfigRecovery.keysBackfillHasFailedFor(groupPk)).to.be.false; + }); + + it('fetches the keys namespace with NO last_hash and merges what comes back', async () => { + const { retrieveStub, mergeStub } = stubBackfill({ + fetched: [{ hash: KEYS_HASH, data: 'AQID', storedAt: 111 }], + retainedAfterMerge: { [KEYS_HASH]: new Uint8Array([1]) }, + }); + + await ConfigRecovery.backfillGroupKeysIfNeeded(groupPk); + + expect(retrieveStub.calledOnce, 'it fetched').to.be.true; + const namespaces = retrieveStub.firstCall.args[2]; + expect(namespaces, 'the keys namespace, and only that').to.be.deep.eq([ + { lastHash: '', namespace: SnodeNamespaces.ClosedGroupKeys }, + ]); + expect(mergeStub.calledOnce, 'and merged').to.be.true; + expect( + mergeStub.firstCall.args[1].groupKeys!.length, + 'the fetched keys message went into the merge' + ).to.be.eq(1); + }); + + it('PERSISTS the dump — a merge that only captures in memory dies with the process', async () => { + // iOS hit this: retention lives in the config dump, so bytes captured by a merge that never + // persists are gone on restart. It passes every in-process assertion either way. + // + // On Desktop the hazard is worse: saveDumpsToDb is stubbed in this file's beforeEach for an + // unrelated reason, so an implementation that never persists passes the whole suite silently. + // Hence the PREMISE assertion first — without it "saveDumpsToDb was called" is also satisfied + // by a path that exited before the merge. + const { mergeStub } = stubBackfill({ + fetched: [{ hash: KEYS_HASH, data: 'AQID', storedAt: 111 }], + retainedAfterMerge: { [KEYS_HASH]: new Uint8Array([1]) }, + }); + const saveStub = LibSessionUtil.saveDumpsToDb as unknown as Sinon.SinonStub; + + await ConfigRecovery.backfillGroupKeysIfNeeded(groupPk); + + expect(mergeStub.called, 'PREMISE: the merge ran at all').to.be.true; + expect(saveStub.calledWith(groupPk), 'and the dump was persisted for this group').to.be.true; + }); + + it('records a failure when the fetch comes back EMPTY', async () => { + stubBackfill({ fetched: [] }); + + await ConfigRecovery.backfillGroupKeysIfNeeded(groupPk); + + expect(ConfigRecovery.keysBackfillHasFailedFor(groupPk)).to.be.true; + }); + + it('records a failure when messages ARRIVE but the bytes are still absent', async () => { + // The one that separates "attempted and still absent" from "the fetch was empty". Both look + // identical in any fixture where the swarm has nothing — which is the fixture above, and the + // first one anyone writes. An implementation that only records the empty case passes that one + // and fails this, and without this test it would refetch the same useless messages forever. + const { mergeStub } = stubBackfill({ + fetched: [{ hash: 'someotherhash', data: 'AQID', storedAt: 111 }], + retained: {}, + retainedAfterMerge: {}, // merged something, still hold no bytes for KEYS_HASH + }); + + await ConfigRecovery.backfillGroupKeysIfNeeded(groupPk); + + expect(mergeStub.called, 'PREMISE: it got as far as merging').to.be.true; + expect( + ConfigRecovery.keysBackfillHasFailedFor(groupPk), + 'a merge that did not restore the bytes is still a failed attempt' + ).to.be.true; + }); + + it('CLEARS the failure once the bytes are obtained', async () => { + // The record is read as "this device cannot repair this group". A device that just retained + // the bytes plainly can, so leaving it set would permanently misreport it to the rekey. + stubBackfill({ fetched: [] }); + await ConfigRecovery.backfillGroupKeysIfNeeded(groupPk); + expect(ConfigRecovery.keysBackfillHasFailedFor(groupPk), 'failed first').to.be.true; + + Sinon.restore(); + TestUtils.stubWindowLog(); + Sinon.stub(LibSessionUtil, 'saveDumpsToDb').resolves(); + ConfigRecovery.setNowForTesting(() => Date.now() + 2 * 60 * 60 * 1000); + stubBackfill({ retained: { [KEYS_HASH]: new Uint8Array([1]) } }); + + await ConfigRecovery.backfillGroupKeysIfNeeded(groupPk); + + expect(ConfigRecovery.keysBackfillHasFailedFor(groupPk), 'cleared once we hold them').to.be + .false; + }); + + it('a THROW is not an attempt — no failure recorded', async () => { + // We never learned whether the bytes are obtainable. Recording a failure would let the rekey + // act on evidence we do not have. + Sinon.stub(SnodePool, 'getSwarmFor').resolves([] as any); + Sinon.stub(MetaGroupWrapperActions, 'activeHashesByConfig').resolves({ + groupInfo: [], + groupMember: [], + groupKeys: [KEYS_HASH], + }); + Sinon.stub(MetaGroupWrapperActions, 'activeKeyMessages').resolves({}); + + await ConfigRecovery.backfillGroupKeysIfNeeded(groupPk); + + expect( + ConfigRecovery.keysBackfillHasFailedFor(groupPk), + 'an empty swarm tells us nothing about the bytes' + ).to.be.false; + }); + + it('does not re-attempt within the bar', async () => { + stubBackfill({ fetched: [] }); + await ConfigRecovery.backfillGroupKeysIfNeeded(groupPk); + const { retrieveStub } = { + retrieveStub: SnodeAPIRetrieve.retrieveNextMessagesNoRetries as unknown as Sinon.SinonStub, + }; + expect(retrieveStub.callCount, 'first attempt fetched').to.be.eq(1); + + await ConfigRecovery.backfillGroupKeysIfNeeded(groupPk); + + expect(retrieveStub.callCount, 'the second is barred, not retried').to.be.eq(1); + }); + }); + + it('a KICKED group is not re-stored', async () => { + stubGroup({ kicked: true }); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran).to.be.false; + expect(sendStub.called, 'we are not entitled to write to this swarm any more').to.be.false; + }); + + it('a DESTROYED group is not re-stored — kicked is FALSE in that case', async () => { + // Deliberately separate from the kicked test. libsession sets kicked=false when a group was + // destroyed, so an implementation checking only `kicked` passes the test above and fails here. + stubGroup({ kicked: false, destroyed: true }); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran).to.be.false; + expect(sendStub.called).to.be.false; + }); + + it('a group with pending changes is not re-stored', async () => { + stubGroup({ needsPush: true }); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran).to.be.false; + expect(sendStub.called, 'GroupSync will push it under a new hash anyway').to.be.false; + }); + + it('a group swarm not level with local state is not recovered', async () => { + stubGroup(); + detectMissing([INFO_HASH]); + // deliberately NOT marking level + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran).to.be.false; + expect(sendStub.called).to.be.false; + }); + + it('both sub-configs are re-stored when both claim a missing hash', async () => { + stubGroup(); + detectMissing([INFO_HASH, MEMBER_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran).to.be.true; + expect(infoStoresSent().length).to.be.eq(1); + expect(memberStoresSent().length).to.be.eq(1); + }); + + it('the in-flight guard: a second round for the same swarm while one is running is refused', async () => { + // Without this, `void`-ing the call in the poller means the next poll starts a second round + // over hashes the first has not settled yet — duplicate stores aimed at the swarm already + // being repaired. + stubGroup(); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + let releaseSend: () => void = () => {}; + const blocked = new Promise(resolve => { + releaseSend = resolve; + }); + sendStub.callsFake(async ({ sortedSubRequests }: any) => { + await blocked; + return sortedSubRequests.map(() => ({ code: 200, body: { hash: 'newhash' } })); + }); + + const first = ConfigRecovery.recoverIfNeeded(groupPk); + + // Let the first round get as far as the send before starting the second. It awaits the wrapper + // several times on the way, so without this the second call races it to an earlier await and + // the assertion below would be measuring the wrong moment. + const flush = () => + new Promise(resolve => { + setTimeout(resolve, 0); + }); + while (!sendStub.called) { + // eslint-disable-next-line no-await-in-loop + await flush(); + } + + // the first round is now parked inside the send, so this one must be turned away + const second = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(second, 'the overlapping round is refused').to.be.false; + expect(sendStub.callCount, 'and it sent nothing — one round is in flight, not two').to.be.eq(1); + + releaseSend(); + expect(await first, 'the original round still completes normally').to.be.true; + }); + + it('the in-flight guard releases after a FAILING round, or the swarm is withdrawn forever', async () => { + // The guard must clear on the failure path too. A marker that leaks there would be a permanent + // exclusion of exactly the swarm that needs repairing. + stubGroup(); + detectMissing([INFO_HASH]); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + sendStub.rejects(new Error('network gone')); + + expect(await ConfigRecovery.recoverIfNeeded(groupPk), 'first round fails').to.be.false; + + // same swarm, a later round must still be admitted + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 200, body: { hash: 'newhash' } })) + ); + ConfigRecovery.setNowForTesting(() => Date.now() + 60 * 60 * 1000); + detectMissing([INFO_HASH]); + + expect( + await ConfigRecovery.recoverIfNeeded(groupPk), + 'the guard released, so the retry is admitted' + ).to.be.true; + }); +}); diff --git a/ts/test/session/unit/snode_api/configRecovery_test.ts b/ts/test/session/unit/snode_api/configRecovery_test.ts new file mode 100644 index 0000000000..94808c0ed1 --- /dev/null +++ b/ts/test/session/unit/snode_api/configRecovery_test.ts @@ -0,0 +1,640 @@ +import chai from 'chai'; +import { beforeEach, describe } from 'mocha'; +import Sinon from 'sinon'; +import { PubkeyType } from 'libsession_util_nodejs'; + +import { ConfigRecovery } from '../../../../session/apis/snode_api/configRecovery'; +import { + UserGenericWrapperActions, + UserGroupsWrapperActions, +} from '../../../../webworker/workers/browser/libsession_worker_interface'; +import { LibSessionUtil } from '../../../../session/utils/libsession/libsession_utils'; +import { MessageSender } from '../../../../session/sending/MessageSender'; +import { UserUtils } from '../../../../session/utils'; +import { SnodeNamespaces } from '../../../../session/apis/snode_api/namespaces'; +import { + DeleteHashesFromUserNodeSubRequest, + StoreUserConfigSubRequest, +} from '../../../../session/apis/snode_api/SnodeRequestTypes'; +import { TestUtils } from '../../../test-utils'; + +const { expect } = chai; + +/** + * The shared guard and action vectors — V10-V13, V17 and V18. + * The response-shaped vectors live in configExpiryDetection_test.ts. + * + * V22 is deliberately NOT here. It is about the polling path reaching this module at all, and + * supplying that precondition in setup — as every test below does — is exactly what hides the bug + * it guards against. It lives in SwarmPolling_configRecovery_test.ts. + * + * ON "ASSERTS THAT X DOES NOT HAPPEN" TESTS. Every such assertion here is also satisfied by the path + * dying early, so each needs something proving it got as far as the decision. Most do. Three cannot, + * and it is deliberate rather than an omission — V10, "nothing missing" and V12/V16 all have a + * premise that IS "recoverIfNeeded returns at its first guard", so there is no later stage to + * witness. Don't add a reachability assertion to those; there is nothing to assert. Their + * protection is that a harness death would break the tests around them that DO have one. + */ + +const H1 = 'hash1'; +const H2 = 'hash2'; + +describe('ConfigRecovery', () => { + let us: PubkeyType; + let sendStub: Sinon.SinonStub; + + /** what a clean, healthy ContactsConfig holding H1 and H2 looks like to the wrapper */ + function stubWrappers({ + activeHashes = [H1, H2], + needsPush = false, + parts = [new Uint8Array([1])], + obsoleteHashes = [] as Array, + } = {}) { + const needsPushStub = Sinon.stub(UserGenericWrapperActions, 'needsPush').callsFake( + async variant => (variant === 'ContactsConfig' ? needsPush : false) + ); + const activeHashesStub = Sinon.stub(UserGenericWrapperActions, 'activeHashes').callsFake( + async variant => (variant === 'ContactsConfig' ? activeHashes : []) + ); + Sinon.stub(UserGenericWrapperActions, 'push').resolves({ + data: parts, + seqno: 5, + hashes: obsoleteHashes, + namespace: SnodeNamespaces.UserContacts, + }); + + return { needsPushStub, activeHashesStub }; + } + + /** across every batch: the delete goes in its own, after the stores have run */ + function allSubRequestsSent() { + return sendStub.getCalls().flatMap(c => c.args[0].sortedSubRequests as Array); + } + + function storeRequestsSent() { + return allSubRequestsSent().filter(r => r instanceof StoreUserConfigSubRequest); + } + + function deleteRequestSent() { + return allSubRequestsSent().find( + (r): r is DeleteHashesFromUserNodeSubRequest => + r instanceof DeleteHashesFromUserNodeSubRequest + ); + } + + beforeEach(() => { + TestUtils.stubWindowLog(); + ConfigRecovery.resetForTesting(); + us = TestUtils.generateFakePubKeyStr(); + Sinon.stub(UserUtils, 'getOurPubKeyStrFromCache').returns(us); + Sinon.stub(UserUtils, 'isUsFromCache').callsFake(pk => pk === us); + Sinon.stub(LibSessionUtil, 'saveDumpsToDb').resolves(); + // one result per sub-request sent, which is what the caller checks for + sendStub = Sinon.stub(MessageSender, 'sendEncryptedDataToSnode').callsFake( + async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 200, body: { hash: 'newhash' } })) as any + ); + }); + + afterEach(() => { + Sinon.restore(); + }); + + function detectMissing(hashes: Array) { + ConfigRecovery.recordDetection(us, { status: 'conclusive', missingHashes: hashes }); + } + + it('V10: no successful poll AT ALL this session -> recovery MUST NOT run', async () => { + stubWrappers(); + detectMissing([H2]); + + // deliberately not marking the swarm level. Note this is "no poll succeeded", NOT "no merge + // happened" — a successful poll that returned nothing also makes us level, and V22 covers it. + const ran = await ConfigRecovery.recoverIfNeeded(us); + + expect(ran, 'recovery must not run before any successful poll').to.be.false; + expect(sendStub.called, 'nothing may be sent').to.be.false; + }); + + it('the same detection recovers once we are level with the swarm', async () => { + stubWrappers(); + detectMissing([H2]); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + const ran = await ConfigRecovery.recoverIfNeeded(us); + + expect(ran, 'recovery should run after a merge').to.be.true; + expect(storeRequestsSent().length).to.be.eq(1); + }); + + it('V10b: another swarm being level does NOT satisfy THIS swarm', async () => { + // Per-swarm keying. A flat or global "we polled successfully" flag passes every other vector + // in the table and fails only here. + const otherSwarm = TestUtils.generateFakePubKeyStr(); + stubWrappers(); + detectMissing([H2]); + ConfigRecovery.markLocalStateLevelWithSwarm(otherSwarm); + + expect( + ConfigRecovery.localStateIsLevelWithSwarm(otherSwarm), + 'the other swarm really is level — otherwise this passes for the wrong reason' + ).to.be.true; + expect( + await ConfigRecovery.recoverIfNeeded(us), + 'but ours is not, and being level is per-swarm' + ).to.be.false; + expect(sendStub.called).to.be.false; + }); + + it('expired bars are PRUNED, not just read past — the leak is otherwise invisible', async () => { + // Reading past an expired entry gives the right answer, so every behavioural test passes with + // or without pruning. What is left is a map that grows for the life of the process — against + // long-lived sessions, which is the exact population the time-bound was added to protect. + // Note the hashes must ROTATE between rounds. Re-storing the same hashes overwrites the same + // keys, so the map stays the same size whether or not it prunes — a version of this test that + // reuses one hash set passes with pruning removed, which is how the first draft of it did. + let fakeNow = 1_700_000_000_000; + ConfigRecovery.setNowForTesting(() => fakeNow); + let currentHashes = ['R1', 'R2']; + Sinon.stub(UserGenericWrapperActions, 'needsPush').resolves(false); + Sinon.stub(UserGenericWrapperActions, 'activeHashes').callsFake(async variant => + variant === 'ContactsConfig' ? currentHashes : [] + ); + Sinon.stub(UserGenericWrapperActions, 'push').resolves({ + data: [new Uint8Array([1])], + seqno: 5, + hashes: [], + namespace: SnodeNamespaces.UserContacts, + }); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + detectMissing(['R1']); + await ConfigRecovery.recoverIfNeeded(us); + expect(ConfigRecovery.barredHashCountForTesting(), 'two hashes barred').to.be.eq(2); + + // the config is re-pushed in the meantime, so it now occupies different hashes + fakeNow += 61 * 60 * 1000; + currentHashes = ['R3', 'R4']; + detectMissing(['R3']); + await ConfigRecovery.recoverIfNeeded(us); + + // Three, not two: R3 and R4 were stored, and R1 comes back as GUARD-settled because it is no + // longer in activeHashes. R2 is the one that must be gone — it expired and nothing re-settled + // it. Without pruning this is four. + expect( + ConfigRecovery.barredHashCountForTesting(), + 'the cleanly-expired entry is dropped rather than accumulating' + ).to.be.eq(3); + }); + + it('V11: a hash no longer in the active set MUST NOT be re-stored', async () => { + // H2 was reported missing, but this config has since moved on and no longer claims it + const { activeHashesStub } = stubWrappers({ activeHashes: [H1] }); + detectMissing([H2]); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + const ran = await ConfigRecovery.recoverIfNeeded(us); + + expect(activeHashesStub.called, 'the config must actually have been inspected').to.be.true; + expect(ran, 'an obsolete hash is not ours to put back').to.be.false; + expect(sendStub.called).to.be.false; + }); + + it('guard 4.2: a dirty config MUST NOT be re-stored', async () => { + const { needsPushStub } = stubWrappers({ needsPush: true }); + detectMissing([H2]); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + const ran = await ConfigRecovery.recoverIfNeeded(us); + + expect(needsPushStub.called, 'cleanliness must actually have been checked').to.be.true; + expect(ran, 'recovery re-uploads existing state, it never creates new state').to.be.false; + expect(sendStub.called).to.be.false; + }); + + it('V13: a hash is re-stored ONCE however many polls report it, within the bar interval', async () => { + stubWrappers(); + detectMissing([H2]); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + expect(await ConfigRecovery.recoverIfNeeded(us), 'first poll recovers').to.be.true; + + // the next poll still sees it missing, because the swarm hasn't caught up yet + detectMissing([H2]); + + expect(await ConfigRecovery.recoverIfNeeded(us), 'second poll must not re-store').to.be.false; + // "one send" holds for the BAR INTERVAL, not the session — the clock does not move in this + // test, so the two are indistinguishable here and only V13g can tell them apart. + expect(sendStub.callCount, 'exactly one send while the bar holds').to.be.eq(1); + }); + + it('V13g: the bar EXPIRES — a barred hash is re-stored after the interval, same session', async () => { + // Driven by advancing the clock, NOT by restarting: a restart clears in-memory state and would + // pass on the session-scoped version too. That is the trap this vector exists for — a + // session-scoped bar passes V13, V13a and V13b and fails only here. + // + // Why it matters on Desktop specifically: there is no foreground gate, so a session + // runs for weeks. "Never again this session" can outlive the 30-day TTL, and the hash the bar + // is protecting can expire from the swarm a second time inside it. + let fakeNow = 1_700_000_000_000; + ConfigRecovery.setNowForTesting(() => fakeNow); + stubWrappers(); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + detectMissing([H2]); + expect(await ConfigRecovery.recoverIfNeeded(us), 'stored, and now barred').to.be.true; + + fakeNow += 5 * 60 * 1000; // five minutes later, still barred + detectMissing([H2]); + expect(await ConfigRecovery.recoverIfNeeded(us), 'the bar holds against a burst').to.be.false; + + fakeNow += 61 * 60 * 1000; // past HASH_BAR_MS (1 hour) + detectMissing([H2]); + + expect( + await ConfigRecovery.recoverIfNeeded(us), + 'the config expired again in a long-lived session, so it is ours to put back' + ).to.be.true; + expect(sendStub.callCount, 'two re-stores, an interval apart').to.be.eq(2); + }); + + it('V13h: a config too big for one batch is SPLIT across batches, not skipped', async () => { + // 25 parts against a limit of 20. Skipping would make a config over ~1.5MB permanently + // unrecoverable — the largest accounts, excluded by the fix written for them. The all-parts rule governs + // when it counts as stored, not which transport the parts travel in. + const many = Array.from({ length: 25 }, (_, i) => new Uint8Array([i])); + stubWrappers({ activeHashes: ['P1'], parts: many, obsoleteHashes: ['old1'] }); + detectMissing(['P1']); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + const ran = await ConfigRecovery.recoverIfNeeded(us); + + expect(ran, 'every part landed, across however many batches it took').to.be.true; + // 20 + 5 stores, then the delete in its own batch once they have all landed + expect(sendStub.callCount, 'two store batches plus the delete').to.be.eq(3); + + const sizes = sendStub.getCalls().map(c => c.args[0].sortedSubRequests.length); + expect(Math.max(...sizes), 'no batch may exceed the limit').to.be.at.most(20); + expect(storeRequestsSent().length, 'and nothing is dropped').to.be.eq(25); + expect(deleteRequestSent(), 'the delete still goes out').to.not.be.undefined; + }); + + it('V13h (failure half): a batch failing part-way does NOT send the delete for the rest', async () => { + const many = Array.from({ length: 25 }, (_, i) => new Uint8Array([i])); + stubWrappers({ activeHashes: ['P1'], parts: many, obsoleteHashes: ['old1'] }); + detectMissing(['P1']); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 500, body: {} })) + ); + + const ran = await ConfigRecovery.recoverIfNeeded(us); + + expect(ran).to.be.false; + expect(sendStub.callCount, 'it stops at the first failing batch').to.be.eq(1); + }); + + it('V17: the obsolete hashes push() returns are deleted, not dropped', async () => { + // push() drains _old_hashes unconditionally, even for a clean config. If we discard them here + // nothing ever reports them again and those messages leak on the swarm permanently. + // Note: V17 is the user/admin path specifically. A read-only group member is never handed the + // hashes at all (the hand-back is gated on !is_readonly(), the clear is not), so an empty list + // there is correct rather than a failure — that's V21, and it needs group recovery first. + stubWrappers({ obsoleteHashes: ['oldhash1', 'oldhash2'] }); + detectMissing([H2]); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + await ConfigRecovery.recoverIfNeeded(us); + + const deleteRequest = deleteRequestSent(); + expect(deleteRequest, 'a delete request must be sent for the drained hashes').to.not.be + .undefined; + expect(deleteRequest?.messageHashes).to.have.members(['oldhash1', 'oldhash2']); + }); + + it('no delete for a config whose store did NOT land', async () => { + // Deleting an obsolete hash whose replacement failed to store removes the swarm's only older + // copy — a seed restore in that window then gets nothing rather than something stale. Note the + // pair below: the absence assertion alone would also pass against a delete path that was never + // wired at all. + stubWrappers({ obsoleteHashes: ['oldhash1'] }); + detectMissing([H2]); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 500, body: {} })) + ); + + await ConfigRecovery.recoverIfNeeded(us); + + expect(storeRequestsSent().length, 'the store was attempted').to.be.eq(1); + expect(deleteRequestSent(), 'but its obsolete hash is left alone').to.be.undefined; + }); + + it('counterpart: the SAME fixture DOES delete once the store lands', async () => { + stubWrappers({ obsoleteHashes: ['oldhash1'] }); + detectMissing([H2]); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + await ConfigRecovery.recoverIfNeeded(us); + + expect(storeRequestsSent().length, 'same store').to.be.eq(1); + expect( + deleteRequestSent()?.messageHashes, + 'and now the obsolete hash is safe to drop' + ).to.have.members(['oldhash1']); + }); + + it('V17b: no delete request when push() returns no obsolete hashes', async () => { + stubWrappers({ obsoleteHashes: [] }); + detectMissing([H2]); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + await ConfigRecovery.recoverIfNeeded(us); + + // "no delete request" is also true when nothing was sent at all, so prove the re-store ran + // first. Found by breaking the stubWrappers helper: this was the one survivor with no excuse. + expect(storeRequestsSent().length, 'the re-store must actually have happened').to.be.eq(1); + expect(deleteRequestSent()).to.be.undefined; + }); + + it('V18: every part of a multipart config is re-stored, not just the missing one', async () => { + const parts = [new Uint8Array([1]), new Uint8Array([2]), new Uint8Array([3])]; + stubWrappers({ activeHashes: ['P1', 'P2', 'P3'], parts }); + detectMissing(['P2']); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + await ConfigRecovery.recoverIfNeeded(us); + + expect(storeRequestsSent().length, 'all three parts go back').to.be.eq(3); + }); + + it('a persistently half-landing config is RATE-LIMITED, not retried on every poll', async () => { + // The storm this replaces: the reset was keyed on "any sub-request returned 200", so a config + // with one part that always succeeds beside one that always fails reset the counter every + // round. Nothing barred, backoffMsFor(0) is 0, identical full re-send on every poll — measured + // at 10 rounds / 10 sends / 0 barred. + // + // This test pins the SIZE of the retry, not its existence. The version it replaces asserted + // only that a second attempt happened, which is true of the storm too — that is precisely why + // the defect survived: every part goes back on every attempt, so a half-landing config + // never shrinks its next round and "it retried" cannot distinguish progress from a loop. + const fakeNow = 1_700_000_000_000; // deliberately not advanced + ConfigRecovery.setNowForTesting(() => fakeNow); + stubWrappers({ activeHashes: ['P1', 'P2'], parts: [new Uint8Array([1]), new Uint8Array([2])] }); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + // one part lands, one does not — repeatedly + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map((_r: unknown, i: number) => ({ code: i === 0 ? 200 : 500, body: {} })) + ); + + for (let poll = 0; poll < 10; poll++) { + detectMissing(['P1']); + // eslint-disable-next-line no-await-in-loop + await ConfigRecovery.recoverIfNeeded(us); + } + + expect( + sendStub.callCount, + 'ten polls collapse to one attempt — nothing was barred, so this is not progress' + ).to.be.eq(1); + expect(ConfigRecovery.barredHashCountForTesting(), 'and nothing is barred').to.be.eq(0); + }); + + it('but one config landing IN FULL beside a failing one DOES reset the backoff', async () => { + // The counterpart, and the reason the reset is not simply keyed on total success: when several + // configs are in flight and one lands completely, its hashes ARE barred and the next round is + // genuinely smaller. That is convergence and must not be penalised. + // + // Without this pair, "reset on full success only" would look equally correct and would back off + // against a swarm that is demonstrably making ground. + const fakeNow = 1_700_000_000_000; + ConfigRecovery.setNowForTesting(() => fakeNow); + Sinon.stub(UserGenericWrapperActions, 'needsPush').resolves(false); + Sinon.stub(UserGenericWrapperActions, 'activeHashes').callsFake(async variant => + variant === 'ContactsConfig' ? ['C1'] : variant === 'UserConfig' ? ['U1'] : [] + ); + Sinon.stub(UserGenericWrapperActions, 'push').resolves({ + data: [new Uint8Array([1])], + seqno: 5, + hashes: [], + namespace: SnodeNamespaces.UserContacts, + }); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + // the first config's store lands, the second's does not + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map((_r: unknown, i: number) => ({ code: i === 0 ? 200 : 500, body: {} })) + ); + + detectMissing(['C1', 'U1']); + await ConfigRecovery.recoverIfNeeded(us); + expect(sendStub.callCount).to.be.eq(1); + expect( + ConfigRecovery.barredHashCountForTesting(), + 'the config that landed in full is barred — this is what makes the next round smaller' + ).to.be.greaterThan(0); + + // and because something was barred, the very next poll is allowed to try again + detectMissing(['U1']); + await ConfigRecovery.recoverIfNeeded(us); + expect(sendStub.callCount, 'a converging swarm is not made to wait').to.be.eq(2); + }); + + it('settled detections are PRUNED — the accumulator does not grow for the life of the process', async () => { + // Same leak pruneExpiredBars was written for, one map over and against the same population. + // Hashes rotate on every re-push and a Desktop session runs for days, so every superseded hash + // would otherwise be retained forever. + // + // The hashes must ROTATE between rounds, exactly as in the bars test. Re-detecting the same + // hashes writes the same Set entries, so the size is unchanged whether or not it prunes — a + // version of this reusing one hash set passes with the pruning removed. + let fakeNow = 1_700_000_000_000; + ConfigRecovery.setNowForTesting(() => fakeNow); + let currentHashes = ['R1', 'R2']; + Sinon.stub(UserGenericWrapperActions, 'needsPush').resolves(false); + Sinon.stub(UserGenericWrapperActions, 'activeHashes').callsFake(async variant => + variant === 'ContactsConfig' ? currentHashes : [] + ); + Sinon.stub(UserGenericWrapperActions, 'push').resolves({ + data: [new Uint8Array([1])], + seqno: 5, + hashes: [], + namespace: SnodeNamespaces.UserContacts, + }); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + for (let round = 0; round < 5; round++) { + currentHashes = [`R${round}a`, `R${round}b`]; + detectMissing([`R${round}a`]); + // eslint-disable-next-line no-await-in-loop + await ConfigRecovery.recoverIfNeeded(us); + fakeNow += 61 * 60 * 1000; // past the bar, so the next round is admitted + } + + // ONE, not five, and not zero. The property is that it stays CONSTANT as rounds accumulate — + // the single entry is the round just completed, whose hashes were settled by a restore that ran + // after this round's prune and will be dropped by the next one. Without pruning this is 5 and + // climbs with every poll for the life of the process. + expect( + ConfigRecovery.trackedDetectionCountForTesting(us), + 'the accumulator is bounded by the round in flight, not by how many rounds have run' + ).to.be.eq(1); + }); + + it('V13e: a hash ruled out by a GUARD is settled, not re-examined every poll', async () => { + // "not stored" is three outcomes, not two. Stored -> barred; store FAILED -> + // retryable; ruled out by a guard -> barred, because no guard's verdict changes within a + // session. Folding guard-rejections into "failure" costs no requests — the rejection happens + // before any network call — which is exactly why it does not look like a problem: it silently + // re-examines and re-logs the same detection on every poll for the life of the session. + const { activeHashesStub } = stubWrappers({ activeHashes: [H1] }); + detectMissing([H2]); // H2 is no longer active, so a guard will rule it out + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + await ConfigRecovery.recoverIfNeeded(us); + const inspectionsAfterFirstPoll = activeHashesStub.callCount; + expect(inspectionsAfterFirstPoll, 'the first poll must actually have inspected').to.be.above(0); + + // a later poll re-reports the same hash, as a real swarm would + detectMissing([H2]); + await ConfigRecovery.recoverIfNeeded(us); + + expect( + activeHashesStub.callCount, + 'H2 was settled by the guard, so the second poll must not inspect again' + ).to.be.eq(inspectionsAfterFirstPoll); + }); + + it('V13a + V13b: a store whose SUB-RESPONSE failed is retried, though the batch returned 200', async () => { + // Two vectors, one fixture, because they are two claims about the same situation: + // V13a — the bar keys on SUCCESS, not on attempt. Read as a pair with V13, which alone + // passes on a bars-on-attempt implementation — which is why that reading went + // unnoticed for a long time. V13 cannot catch it; only this pairing can. + // V13b — "success" means every SUB-RESPONSE's own code, not that the outer batch returned. + // A sequence returns 200 while its sub-requests carry their own codes, so barring on + // "did not throw" would settle a hash having written nothing. + // Either misreading excludes, for the whole session, exactly the device this feature is for. + let fakeNow = 1_700_000_000_000; + ConfigRecovery.setNowForTesting(() => fakeNow); + stubWrappers({ activeHashes: ['P1', 'P2'], parts: [new Uint8Array([1]), new Uint8Array([2])] }); + detectMissing(['P1']); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map((_r: unknown, i: number) => ({ + code: i === 0 ? 200 : 500, + body: {}, + })) + ); + + expect(await ConfigRecovery.recoverIfNeeded(us), 'a partial store is not a success').to.be + .false; + + // and it must not be banked as done: a later poll gets another go, once the backoff on the + // failed attempt has elapsed + fakeNow += 61 * 1000; + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 200, body: {} })) + ); + detectMissing(['P1']); + + expect( + await ConfigRecovery.recoverIfNeeded(us), + 'the hash was never successfully stored, so it is still ours to retry' + ).to.be.true; + }); + + it('V13c + V13d: a failing store is rate-limited, but NEVER permanently excluded', async () => { + // Release-without-bound is the re-push storm; a hard cap re-creates the exclusion the rule was + // corrected to remove (three transient failures would write the device off for a session that + // can last days). So: rate-limited, never excluded. Asserts BOTH halves with exact counts. + // a swappable clock rather than Sinon fake timers: faking global time deadlocks mocha, and + // faking only Date breaks its timeout accounting. + let fakeNow = 1_700_000_000_000; + ConfigRecovery.setNowForTesting(() => fakeNow); + stubWrappers(); + sendStub.callsFake(async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 500, body: {} })) + ); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + for (let poll = 0; poll < 10; poll++) { + detectMissing([H2]); + // eslint-disable-next-line no-await-in-loop + expect(await ConfigRecovery.recoverIfNeeded(us), 'a failing store never reports success').to + .be.false; + } + + expect(sendStub.callCount, 'ten rapid polls collapse to one attempt').to.be.eq(1); + + fakeNow += 61 * 1000; // past the first backoff step (60s) + detectMissing([H2]); + await ConfigRecovery.recoverIfNeeded(us); + expect(sendStub.callCount, 'and it tries again once the backoff elapses').to.be.eq(2); + + // V13c: the SECOND wait DOUBLES. A flat-rate implementation passes everything above and fails + // only here, which is why it needs its own assertion rather than a longer tick. + fakeNow += 61 * 1000; + detectMissing([H2]); + await ConfigRecovery.recoverIfNeeded(us); + expect(sendStub.callCount, '60s is not enough after a second failure — it is 120s').to.be.eq(2); + + fakeNow += 61 * 1000; // 122s total since the second failure + detectMissing([H2]); + await ConfigRecovery.recoverIfNeeded(us); + expect(sendStub.callCount, 'but 120s is').to.be.eq(3); + + // V13d: the retry never stops. A ceiling on the INTERVAL, never on the entitlement — walk a + // long failing session and confirm it is still trying at the end of it. + let expected = 3; + for (let hour = 0; hour < 24; hour++) { + fakeNow += 60 * 60 * 1000; + detectMissing([H2]); + // eslint-disable-next-line no-await-in-loop + await ConfigRecovery.recoverIfNeeded(us); + expected += 1; + } + expect( + sendStub.callCount, + 'a day of failures later it is STILL retrying — capped interval, uncapped attempts' + ).to.be.eq(expected); + }); + + it('does nothing when detection reported nothing missing', async () => { + stubWrappers(); + ConfigRecovery.recordDetection(us, { status: 'conclusive', missingHashes: [] }); + ConfigRecovery.markLocalStateLevelWithSwarm(us); + + expect(await ConfigRecovery.recoverIfNeeded(us)).to.be.false; + expect(sendStub.called).to.be.false; + }); + + it('an inconclusive or unavailable detection records nothing', () => { + detectMissing([H2]); + ConfigRecovery.recordDetection(us, { status: 'inconclusive' }); + ConfigRecovery.recordDetection(us, { status: 'unavailable' }); + + // and crucially, neither clears what a conclusive one already established + expect(ConfigRecovery.getMissingHashes(us)).to.be.deep.eq([H2]); + }); + + describe('groups', () => { + it('a group whose details cannot be read is not recovered, and the poll survives it', async () => { + // This replaces a test asserting "group swarms are not re-stored on Desktop", which stopped + // being true when group recovery landed. It had kept passing — but only because this file + // never stubbed the group wrappers, so the lookup threw and the early return did the work. + // The stub below makes the throw deliberate instead of incidental; the real group vectors + // live in configRecoveryGroup_test.ts. + Sinon.stub(UserGroupsWrapperActions, 'getGroup').rejects(new Error('no such group')); + const groupPk = TestUtils.generateFakeClosedGroupV2PkStr(); + ConfigRecovery.recordDetection(groupPk, { status: 'conclusive', missingHashes: [H2] }); + ConfigRecovery.markLocalStateLevelWithSwarm(groupPk); + + const ran = await ConfigRecovery.recoverIfNeeded(groupPk); + + expect(ran, 'nothing inspectable, so nothing to put back').to.be.false; + expect(sendStub.called, 'and nothing sent').to.be.false; + }); + }); +}); diff --git a/ts/test/session/unit/snode_api/retrieveNextMessages_test.ts b/ts/test/session/unit/snode_api/retrieveNextMessages_test.ts index a71a903f9f..c0c2b60c71 100644 --- a/ts/test/session/unit/snode_api/retrieveNextMessages_test.ts +++ b/ts/test/session/unit/snode_api/retrieveNextMessages_test.ts @@ -15,6 +15,7 @@ import { WithShortenOrExtend } from '../../../../session/types/with'; import { TestUtils } from '../../../test-utils'; import { expectAsyncToThrow, stubLibSessionWorker } from '../../../test-utils/utils'; import { NetworkTime } from '../../../../util/NetworkTime'; +import { TTL_DEFAULT } from '../../../../session/constants'; const { expect } = chai; @@ -44,8 +45,10 @@ function expectExpireWith({ } & WithShortenOrExtend) { expect(request.messageHashes).to.be.deep.eq(hashes); expect(request.shortenOrExtend).to.be.eq(shortenOrExtend); - expect(request.expiryMs).to.be.above(NetworkTime.now() + 14 * 24 * 3600 * 1000 - 100); - expect(request.expiryMs).to.be.above(NetworkTime.now() + 14 * 24 * 3600 * 1000 + 100); + // Both bounds are required, and the second must be `below`: two `above` assertions typecheck, + // read as a range, and leave the expiry unbounded upwards. + expect(request.expiryMs).to.be.above(NetworkTime.now() + TTL_DEFAULT.CONFIG_MESSAGE - 1000); + expect(request.expiryMs).to.be.below(NetworkTime.now() + TTL_DEFAULT.CONFIG_MESSAGE + 1000); } describe('SnodeAPI:buildRetrieveRequest', () => { @@ -153,7 +156,7 @@ describe('SnodeAPI:buildRetrieveRequest', () => { expectExpireWith({ request: req3, hashes: ['hashbump1', 'hashbump2'], - shortenOrExtend: '', + shortenOrExtend: 'extend', }); }); @@ -172,7 +175,7 @@ describe('SnodeAPI:buildRetrieveRequest', () => { expectExpireWith({ request: req1, hashes: ['hashbump1', 'hashbump2'], - shortenOrExtend: '', + shortenOrExtend: 'extend', }); }); @@ -308,7 +311,7 @@ describe('SnodeAPI:buildRetrieveRequest', () => { expectExpireWith({ request: req3, hashes: ['hashbump1', 'hashbump2'], - shortenOrExtend: '', + shortenOrExtend: 'extend', }); }); @@ -327,7 +330,7 @@ describe('SnodeAPI:buildRetrieveRequest', () => { expectExpireWith({ request: req1, hashes: ['hashbump1', 'hashbump2'], - shortenOrExtend: '', + shortenOrExtend: 'extend', }); }); diff --git a/ts/test/session/unit/swarm_polling/SwarmPolling_configRecovery_test.ts b/ts/test/session/unit/swarm_polling/SwarmPolling_configRecovery_test.ts new file mode 100644 index 0000000000..2a120644b0 --- /dev/null +++ b/ts/test/session/unit/swarm_polling/SwarmPolling_configRecovery_test.ts @@ -0,0 +1,495 @@ +import chai from 'chai'; +import { describe } from 'mocha'; +import Sinon from 'sinon'; +import { PubkeyType } from 'libsession_util_nodejs'; + +import { ConversationTypeEnum } from '../../../../models/types'; +import { Convo } from '../../../../models/conversation'; +import { getSwarmPollingInstance } from '../../../../session/apis/snode_api'; +import { SnodeAPIRetrieve } from '../../../../session/apis/snode_api/retrieveRequest'; +import { SnodePool } from '../../../../session/apis/snode_api/snodePool'; +import { SwarmPollingUserConfig } from '../../../../session/apis/snode_api/swarm_polling_config/SwarmPollingUserConfig'; +import { SwarmPolling } from '../../../../session/apis/snode_api/swarmPolling'; +import { ConfigRecovery } from '../../../../session/apis/snode_api/configRecovery'; +import { ConvoHub } from '../../../../session/conversations'; +import { UserUtils } from '../../../../session/utils'; +import { UserSync } from '../../../../session/utils/job_runners/jobs/UserSyncJob'; +import { LibSessionUtil } from '../../../../session/utils/libsession/libsession_utils'; +import { MessageSender } from '../../../../session/sending/MessageSender'; +import { + ContactsWrapperActions, + UserGenericWrapperActions, +} from '../../../../webworker/workers/browser/libsession_worker_interface'; +import { SnodeNamespaces } from '../../../../session/apis/snode_api/namespaces'; +import { TestUtils } from '../../../test-utils'; +import { generateFakeSnodes, stubData } from '../../../test-utils/utils'; +import { ReduxOnionSelectors } from '../../../../state/selectors/onions'; + +const { expect } = chai; + +/** + * V22 — the vector that has to go through the POLLING PATH rather than through + * ConfigRecovery directly. + * + * A device whose config has expired gets nothing back when it polls, because there is nothing left + * on the swarm to return. So the level-with-swarm rule must not demand a *merge*: read as "a + * successful poll AND merge" it can never be satisfied by exactly those devices, and it fails + * silently — detection runs, the guard declines, no error and no failing test. + * + * A unit test against ConfigRecovery cannot catch that, because supplying the precondition in + * setup is what hides it. So this one drives `pollOnceForKey` with an empty swarm and asserts the + * re-store actually goes out. Asserting that recovery is skipped here is a FAIL, not a pass. + */ +describe('SwarmPolling: config recovery on an empty poll (V22)', () => { + const ourPubkey = TestUtils.generateFakePubKey(); + const ourNumber = ourPubkey.key as PubkeyType; + const missingHash = 'expiredhash1'; + + let swarmPolling: SwarmPolling; + let sendStub: Sinon.SinonStub; + let retrieveStub: Sinon.SinonStub; + + beforeEach(async () => { + // Note: these come first because they create `global.window` if it is missing, and + // ConvoHub.reset() reads it. Other swarm-polling suites get away with the opposite order only + // because an earlier test file happened to create it. + TestUtils.stubWindowFeatureFlags(); + TestUtils.stubWindowLog(); + ConvoHub.use().reset(); + ConfigRecovery.resetForTesting(); + Sinon.stub(UserSync, 'queueNewJobIfNeeded').resolves(); + Sinon.stub(UserUtils, 'getOurPubKeyStrFromCache').returns(ourNumber); + Sinon.stub(UserUtils, 'isUsFromCache').callsFake(pk => pk === ourNumber); + // a poll that gets past the early return goes on to decrypt, which needs our keypair + Sinon.stub(UserUtils, 'getUserED25519KeyPairBytes').resolves({ + pubKeyBytes: new Uint8Array(32), + privKeyBytes: new Uint8Array(64), + }); + + stubData('getAllConversations').resolves([]); + TestUtils.stubData('getItemById'); + stubData('saveConversation').resolves(); + stubData('getSwarmNodesForPubkey').resolves(); + stubData('getLastHashBySnode').resolves(); + // needed as soon as a poll actually returns a message: pollNodeForKey records its last hash, + // and an unstubbed failure there makes pollNodeForKey return null — which would make the + // negative vectors below pass for entirely the wrong reason. + stubData('updateLastHash').resolves(); + stubData('createOrUpdateItem').resolves(); + Sinon.stub(Convo, 'commitConversationAndRefreshWrapper').resolves(); + + TestUtils.stubLibSessionWorker(undefined); + // needed once a poll gets past the early-return branch: pollOnceForKey then reaches + // shouldLeaveNotPolledGroup, which reads the user-groups wrapper. + TestUtils.stubUserGroupWrapper('getAllGroups', []); + TestUtils.stubUserGroupWrapper('getAllLegacyGroups', []); + Sinon.stub(SnodePool, 'getSwarmFor').resolves(generateFakeSnodes(5)); + Sinon.stub(ReduxOnionSelectors, 'isOnlineOutsideRedux').returns(true); + TestUtils.stubWindow('inboxStore', undefined); + TestUtils.stubWindow('isOnline', true); + + // the swarm has nothing for us — this is what an expired-config device sees on every poll + retrieveStub = Sinon.stub(SnodeAPIRetrieve, 'retrieveNextMessagesNoRetries').resolves([]); + + // a clean local config that still believes it owns the hash the swarm has lost + Sinon.stub(UserGenericWrapperActions, 'needsPush').resolves(false); + Sinon.stub(UserGenericWrapperActions, 'activeHashes').callsFake(async variant => + variant === 'ContactsConfig' ? [missingHash] : [] + ); + Sinon.stub(UserGenericWrapperActions, 'push').resolves({ + data: [new Uint8Array([1, 2, 3])], + seqno: 7, + hashes: [], + namespace: SnodeNamespaces.UserContacts, + }); + Sinon.stub(LibSessionUtil, 'saveDumpsToDb').resolves(); + + sendStub = Sinon.stub(MessageSender, 'sendEncryptedDataToSnode').callsFake( + async ({ sortedSubRequests }: any) => + sortedSubRequests.map(() => ({ code: 200, body: { hash: 'newhash' } })) as any + ); + + const convoController = ConvoHub.use(); + await convoController.load(); + ConvoHub.use().getOrCreate(ourPubkey.key, ConversationTypeEnum.PRIVATE); + + swarmPolling = getSwarmPollingInstance(); + swarmPolling.resetSwarmPolling(); + }); + + afterEach(() => { + ConvoHub.use().reset(); + ConfigRecovery.resetForTesting(); + Sinon.restore(); + }); + + it('V22: re-stores after a successful poll that returned NO config messages', async () => { + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect( + ConfigRecovery.localStateIsLevelWithSwarm(ourNumber), + 'an empty poll leaves us level with the swarm — there is nothing unmerged out there' + ).to.be.true; + expect( + sendStub.called, + 'recovery MUST run here; asserting it is skipped would encode the bug this vector exists for' + ).to.be.true; + }); + + it('V22a: an all-snodes-failed poll does NOT satisfy the guard, despite looking identical', async () => { + // The trap: a failed poll and an empty swarm both arrive as an empty result set, so keying off + // emptiness would treat "we couldn't reach anyone" as "there is nothing out there" and re-store + // against a swarm we never actually read. + retrieveStub.rejects(new Error('every snode timed out')); + + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect(retrieveStub.called, 'the poll must actually have been attempted').to.be.true; + expect( + ConfigRecovery.localStateIsLevelWithSwarm(ourNumber), + 'no snode answered, so we know nothing about the swarm' + ).to.be.false; + expect(sendStub.called, 'a failed poll must not license a re-store').to.be.false; + }); + + it('V22b: one config namespace failing does NOT satisfy the guard, even though others answered', async () => { + // A half-fix that checks "some namespace answered" passes V22 and V22a and fails only here. + // With one config namespace erroring we do not know the swarm state for the configs in it, so + // there is nothing level to act on. + retrieveStub.resolves([ + { code: 200, messages: { messages: [] }, namespace: SnodeNamespaces.UserProfile }, + { code: 500, messages: { messages: [] }, namespace: SnodeNamespaces.UserContacts }, + { code: 200, messages: { messages: [] }, namespace: SnodeNamespaces.Default }, + ] as any); + + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect(retrieveStub.called, 'the poll must actually have been attempted').to.be.true; + expect( + ConfigRecovery.localStateIsLevelWithSwarm(ourNumber), + 'a partial answer is not a full one' + ).to.be.false; + expect(sendStub.called, 'must not re-store on incomplete knowledge of the swarm').to.be.false; + }); + + it('V22 (realistic shape): an empty swarm answers per namespace, and still recovers', async () => { + // The production shape of "the swarm has nothing": every namespace answers 200 with an empty + // message list, rather than the whole result being absent. Worth asserting separately because + // it takes a different branch of pollOnceForKey from the bare-empty case above. + retrieveStub.resolves([ + { code: 200, messages: { messages: [] }, namespace: SnodeNamespaces.UserProfile }, + { code: 200, messages: { messages: [] }, namespace: SnodeNamespaces.UserContacts }, + { code: 200, messages: { messages: [] }, namespace: SnodeNamespaces.Default }, + ] as any); + + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect(ConfigRecovery.localStateIsLevelWithSwarm(ourNumber)).to.be.true; + expect(sendStub.called, 'this is the real expired-device path — it MUST recover').to.be.true; + }); + + it('V22c: a swallowed MERGE failure does NOT satisfy the guard, though the fetch succeeded', async () => { + // The merge deliberately swallows and only logs, so its failure is neither a value nor an + // exception. If the marker is reached anyway we assert "level with the swarm" over a config we + // just failed to take in — and on Desktop that pairs with recovery to re-store stale state. + retrieveStub.resolves([ + { + code: 200, + messages: { messages: [{ hash: 'h1', expiration: 1, data: 'x', timestamp: 1 }] }, + namespace: SnodeNamespaces.UserContacts, + }, + ] as any); + const mergeHandler = Sinon.stub( + SwarmPollingUserConfig, + 'handleUserSharedConfigMessages' + ).resolves(false); + + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + // Proves the path reached the decision rather than dying earlier. Without this the assertion + // below passes over a dead harness, because a poll that never ran also leaves us "not level" — + // confirmed by deleting a setup stub and watching this test stay green. + expect(mergeHandler.callCount, 'the merge must actually have been reached').to.be.eq(1); + expect( + ConfigRecovery.localStateIsLevelWithSwarm(ourNumber), + 'a fetch we could not merge leaves us behind the swarm, not level with it' + ).to.be.false; + expect(sendStub.called, 'must not re-store over a config we failed to merge').to.be.false; + }); + + it('V22c positive counterpart: the same path DOES recover when the merge succeeds', async () => { + // Without this, V22c cannot tell "the guard worked" from "nothing ran" — a negative test + // cannot validate its own harness. + retrieveStub.resolves([ + { + code: 200, + messages: { messages: [{ hash: 'h1', expiration: 1, data: 'x', timestamp: 1 }] }, + namespace: SnodeNamespaces.UserContacts, + }, + ] as any); + Sinon.stub(SwarmPollingUserConfig, 'handleUserSharedConfigMessages').resolves(true); + + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect(ConfigRecovery.localStateIsLevelWithSwarm(ourNumber)).to.be.true; + expect(sendStub.called, 'proves the path in V22c actually runs').to.be.true; + }); + + it('V22c (lossy): a merge that took in 1 of 2 does NOT satisfy the guard, despite reporting success', async () => { + // libSession skips what it cannot merge and carries on — correct, and it means a partial merge + // raises no error. The only way to see it is to compare what came back against what went in. + const twoMessages = [ + { hash: 'h1', expiration: 1, data: 'a', timestamp: 1, storedAt: 1 }, + { hash: 'h2', expiration: 1, data: 'b', timestamp: 2, storedAt: 2 }, + ]; + retrieveStub.resolves([ + { + code: 200, + messages: { messages: twoMessages }, + namespace: SnodeNamespaces.UserContacts, + }, + ] as any); + + // exactly one of the two merges. Asserting "fewer than 2" would also pass if BOTH failed, + // which is a different case and not the one this vector is about. + const mergeStub = Sinon.stub(ContactsWrapperActions, 'merge').resolves(['h1']); + + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect(mergeStub.callCount, 'the merge must actually have been reached').to.be.eq(1); + expect( + mergeStub.firstCall.args[0].length, + 'the fixture must really be the partial case: 2 handed in' + ).to.be.eq(2); + expect( + ConfigRecovery.localStateIsLevelWithSwarm(ourNumber), + '1 of 2 merged is not "we took in what we fetched"' + ).to.be.false; + expect(sendStub.called, 'must not re-store having only partly caught up').to.be.false; + }); + + it('V22c (lossy) positive counterpart: merging BOTH does satisfy the guard', async () => { + const twoMessages = [ + { hash: 'h1', expiration: 1, data: 'a', timestamp: 1, storedAt: 1 }, + { hash: 'h2', expiration: 1, data: 'b', timestamp: 2, storedAt: 2 }, + ]; + retrieveStub.resolves([ + { + code: 200, + messages: { messages: twoMessages }, + namespace: SnodeNamespaces.UserContacts, + }, + ] as any); + + Sinon.stub(ContactsWrapperActions, 'merge').resolves(['h1', 'h2']); + + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect(ConfigRecovery.localStateIsLevelWithSwarm(ourNumber)).to.be.true; + expect(sendStub.called, 'proves the partial case above is a real distinction').to.be.true; + }); + + it('V22d: a failed merge withdraws the swarm for the SESSION, not just for that poll', async () => { + // The defeat this guards: lastHash advances when a message is FETCHED, before the merge is + // attempted. So the message we failed to merge is never offered again, and poll N+1 comes back + // empty — indistinguishable from a healthy empty swarm. Without a sticky verdict, the correct + // refusal on poll N is undone by a poll that looks perfectly clean. + retrieveStub.resolves([ + { + code: 200, + messages: { messages: [{ hash: 'h1', expiration: 1, data: 'x', timestamp: 1 }] }, + namespace: SnodeNamespaces.UserContacts, + }, + ] as any); + const mergeHandler = Sinon.stub( + SwarmPollingUserConfig, + 'handleUserSharedConfigMessages' + ).resolves(false); + + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + // Every other assertion in this test is satisfied by "nothing happened", so without this the + // whole vector is a false green under any death that stops the poll early — found by reading + // the SURVIVORS of a harness-death run rather than its failures. + expect(mergeHandler.callCount, 'poll N must actually have reached the merge').to.be.eq(1); + expect(ConfigRecovery.localStateIsLevelWithSwarm(ourNumber), 'poll N refuses').to.be.false; + + // poll N+1: the unmergeable message is behind the cursor, so the swarm has nothing for us and + // there is now no error, no log and no state anywhere recording that anything was missed. + retrieveStub.resolves([]); + mergeHandler.resolves(true); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect( + ConfigRecovery.localStateIsLevelWithSwarm(ourNumber), + 'a clean-looking later poll MUST NOT undo the earlier refusal' + ).to.be.false; + expect(sendStub.called, 'recovery stays withdrawn until the process restarts').to.be.false; + }); + + it('V22e: a failed FETCH does NOT stick — a later clean poll restores level', async () => { + // Read as a pair with V22d, which pulls the opposite way. An incomplete MERGE is permanently + // disqualifying because the cursor moved past what we could not take in; a failed FETCH loses + // nothing, so withdrawing the swarm for the session over a network blip would be wrong. + // Without this vector, "simplifying" the narrow scoping into an unconditional sticky passes + // everything else in the suite. + retrieveStub.rejects(new Error('every snode timed out')); + + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + expect(ConfigRecovery.localStateIsLevelWithSwarm(ourNumber), 'poll N failed').to.be.false; + + // the swarm comes back; nothing was lost, so this must recover + retrieveStub.resolves([]); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect( + ConfigRecovery.localStateIsLevelWithSwarm(ourNumber), + 'a transient fetch failure must not withdraw the swarm for the session' + ).to.be.true; + expect(sendStub.called, 'and recovery proceeds once we can see the swarm again').to.be.true; + }); + + it('V13f: a THROWING inspection must not fail the poll it rides on', async () => { + // Recovery is a best-effort repair riding on the poll. If a throw escapes into pollOnceForKey it + // fails the whole poll — no messages processed, no configs merged — and it recurs every poll, + // because the condition that threw does not clear. That would make the repair destroy the + // mechanism it depends on. libSession's push() throws for a config with no encryption keys, + // and the inspection is the part that looks like a pure read, which is why it goes unwrapped. + retrieveStub.resolves([ + { + code: 200, + messages: { messages: [{ hash: 'h1', expiration: 1, data: 'x', timestamp: 1 }] }, + namespace: SnodeNamespaces.UserContacts, + }, + ] as any); + const mergeHandler = Sinon.stub( + SwarmPollingUserConfig, + 'handleUserSharedConfigMessages' + ).resolves(true); + + (UserGenericWrapperActions.push as Sinon.SinonStub).rejects( + new Error('Cannot push data without an encryption key!') + ); + + ConfigRecovery.recordDetection(ourNumber, { + status: 'conclusive', + missingHashes: [missingHash], + }); + + // the load-bearing half: the poll itself must complete + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect( + mergeHandler.callCount, + 'the poll did its real work — configs were still merged' + ).to.be.eq(1); + expect( + ConfigRecovery.localStateIsLevelWithSwarm(ourNumber), + 'and the poll completed far enough to reach the level-with-swarm marker' + ).to.be.true; + + // and the recovery half: nothing barred, nothing consumed, still retryable + (UserGenericWrapperActions.push as Sinon.SinonStub).resolves({ + data: [new Uint8Array([1, 2, 3])], + seqno: 7, + hashes: [], + namespace: SnodeNamespaces.UserContacts, + }); + retrieveStub.resolves([]); + + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect( + sendStub.called, + 'a thrown inspection consumed no backoff and barred nothing — the hash is still ours to fix' + ).to.be.true; + }); + + it('being level is a precondition, not a trigger: nothing detected -> no re-store', async () => { + await swarmPolling.pollOnceForKey([ourNumber, ConversationTypeEnum.PRIVATE]); + // the poller deliberately does not await recovery, so wait for the round it started + await ConfigRecovery.waitForRecoveryForTesting(ourNumber); + + expect(ConfigRecovery.localStateIsLevelWithSwarm(ourNumber)).to.be.true; + expect(sendStub.called, 'being level is a precondition, not a trigger').to.be.false; + }); +}); diff --git a/ts/webworker/workers/browser/libsession_worker_interface.ts b/ts/webworker/workers/browser/libsession_worker_interface.ts index 492c59a921..1d5cf3db74 100644 --- a/ts/webworker/workers/browser/libsession_worker_interface.ts +++ b/ts/webworker/workers/browser/libsession_worker_interface.ts @@ -650,6 +650,10 @@ export const MetaGroupWrapperActions: MetaGroupWrapperActionsCalls = { callLibSessionWorker([`MetaGroupConfig-${groupPk}`, 'activeHashesByConfig']) as Promise< ReturnType >, + activeKeyMessages: async (groupPk: GroupPubkeyType) => + callLibSessionWorker([`MetaGroupConfig-${groupPk}`, 'activeKeyMessages']) as Promise< + ReturnType + >, loadKeyMessage: async ( groupPk: GroupPubkeyType, hash: string,