From f06d94bed8c01cf82ae1ec8f9489c8fb1a384b59 Mon Sep 17 00:00:00 2001 From: Kevin Bowling Date: Sun, 14 Jun 2026 02:15:56 -0700 Subject: [PATCH 1/2] history: add hissqlite, a SQLite-based history method Add hissqlite, a SQLite history method selectable instead of hisv6 (hismethod = hissqlite) when INN is built with SQLite. It keeps the whole history in a single transactional database and, unlike ovsqlite, needs no server daemon: innd is the primary writer and updates the database directly, while nnrpd readers and the offline tools open the WAL database read-only, and expiration runs in place rather than rebuilding and swapping files. - The backend uses a single WITHOUT ROWID table clustered on the 16-byte MD5 of the Message-ID (a lookup is one clustered-leaf access); a row is a real article, a remembered Message-ID, or absent. Writes autocommit under WAL with synchronous=NORMAL, giving dbz's durability contract (recent writes may be lost on power loss, never corrupted, peers resend) without batching, which would only starve the second writer. The WAL is checkpointed in the background and truncated on a clean shutdown. - Expiration is two-horizon and in place (real articles past retention become remembered, remembered entries past /remember/ are deleted), streamed in hash-keyset pages. A new HISCTLG_INPLACEEXPIRE capability lets expire(8) reopen the backend read/write with no ICCpause; -d/-f (side-file rebuild) and a bare -x are refused, with -t as the in-place dry run; hisv6's path is unchanged. - The SQL prepared-statement codegen (sqlite-helper) moves from storage/ovsqlite to lib/ so the ovsqlite and hissqlite backends share one copy; libinn now carries the libsqlite3 dependency (LTVERSION bump). - Page size, writer and per-nnrpd reader cache sizes, and mmap size are tunable through inn.conf; mmap defaults off, as in ovsqlite. - hissqlite-convert migrates an existing history by walking the source backend and bulk-loading by hash, faithfully preserving the remembered entries and timestamps a from-spool makehistory rebuild cannot. Comes with the hissqlite-util inspection tool, man pages, a NEWS entry, and two runtime tests wired into tests/TESTS: a full-vtable backend test and an end-to-end hissqlite-convert test (seed hisv6 -> convert -> verify tokens, timestamps and remembered entries survive). --- .gitignore | 11 + MANIFEST | 31 +- doc/pod/Makefile | 8 +- doc/pod/expire.pod | 13 +- doc/pod/hissqlite-convert.pod | 74 ++ doc/pod/hissqlite-util.pod | 92 ++ doc/pod/hissqlite.pod | 162 ++++ doc/pod/history.pod | 8 +- doc/pod/inn.conf.pod | 42 +- doc/pod/makehistory.pod | 12 +- expire/expire.c | 51 + history/Makefile | 5 +- history/hissqlite/hismethod.config | 5 + history/hissqlite/hismethod.mk | 32 + history/hissqlite/hissqlite-convert.c | 223 +++++ history/hissqlite/hissqlite-init.sql | 53 ++ history/hissqlite/hissqlite-main.sql | 107 +++ history/hissqlite/hissqlite-private.h | 65 ++ history/hissqlite/hissqlite-read.sql | 24 + history/hissqlite/hissqlite-util.in | 230 +++++ history/hissqlite/hissqlite.c | 885 ++++++++++++++++++ history/hissqlite/hissqlite.h | 48 + include/inn/history.h | 10 +- include/inn/innconf.h | 7 +- .../ovsqlite => include/inn}/sqlite-helper.h | 0 lib/Makefile | 26 +- lib/innconf.c | 15 + .../ovsqlite => lib}/sqlite-helper-gen.in | 2 +- {storage/ovsqlite => lib}/sqlite-helper.c | 2 +- storage/Makefile | 14 +- storage/ovsqlite/ovmethod.config | 3 +- storage/ovsqlite/ovmethod.mk | 17 +- storage/ovsqlite/ovsqlite.c | 2 +- support/mkmanifest | 6 +- tests/Makefile | 7 + tests/TESTS | 2 + tests/lib/hissqlite-convert-t.c | 202 ++++ tests/lib/hissqlite-t.c | 390 ++++++++ 38 files changed, 2841 insertions(+), 45 deletions(-) create mode 100644 doc/pod/hissqlite-convert.pod create mode 100644 doc/pod/hissqlite-util.pod create mode 100644 doc/pod/hissqlite.pod create mode 100644 history/hissqlite/hismethod.config create mode 100644 history/hissqlite/hismethod.mk create mode 100644 history/hissqlite/hissqlite-convert.c create mode 100644 history/hissqlite/hissqlite-init.sql create mode 100644 history/hissqlite/hissqlite-main.sql create mode 100644 history/hissqlite/hissqlite-private.h create mode 100644 history/hissqlite/hissqlite-read.sql create mode 100644 history/hissqlite/hissqlite-util.in create mode 100644 history/hissqlite/hissqlite.c create mode 100644 history/hissqlite/hissqlite.h rename {storage/ovsqlite => include/inn}/sqlite-helper.h (100%) rename {storage/ovsqlite => lib}/sqlite-helper-gen.in (99%) rename {storage/ovsqlite => lib}/sqlite-helper.c (98%) create mode 100644 tests/lib/hissqlite-convert-t.c create mode 100644 tests/lib/hissqlite-t.c diff --git a/.gitignore b/.gitignore index 30cc70949..870ca7aed 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,14 @@ /history/buildconfig /history/hismethods.c /history/hismethods.h +/history/hissqlite/hissqlite-convert +/history/hissqlite/hissqlite-init.c +/history/hissqlite/hissqlite-init.h +/history/hissqlite/hissqlite-main.c +/history/hissqlite/hissqlite-main.h +/history/hissqlite/hissqlite-read.c +/history/hissqlite/hissqlite-read.h +/history/hissqlite/hissqlite-util /include/config.h /include/config.h.in /include/inn/paths.h @@ -143,6 +151,7 @@ /innfeed/imapfeed /innfeed/innfeed /innfeed/procbatch +/lib/sqlite-helper-gen /nnrpd/nnrpd /perl/INN/Config.pm /perl/INN/Utils/Shlock.pm @@ -216,6 +225,8 @@ /tests/lib/hashtab.t /tests/lib/headers.t /tests/lib/hex.t +/tests/lib/hissqlite-convert.t +/tests/lib/hissqlite.t /tests/lib/inet_aton.t /tests/lib/inet_ntoa.t /tests/lib/inet_ntop.t diff --git a/MANIFEST b/MANIFEST index 3ed2818fb..ca75fc263 100644 --- a/MANIFEST +++ b/MANIFEST @@ -157,6 +157,9 @@ doc/man/fastrm.1 Manpage for fastrm utility doc/man/gencancel.1 Manpage for gencancel program doc/man/getlist.1 Manpage for getlist frontend doc/man/grephistory.1 Manpage for grephistory +doc/man/hissqlite-convert.8 Manpage for hissqlite-convert +doc/man/hissqlite-util.8 Manpage for hissqlite-util +doc/man/hissqlite.5 Manpage for the hissqlite history method doc/man/history.5 Manpage for history database doc/man/ident.8 Manpage for ident resolver doc/man/incoming.conf.5 Manpage for incoming.conf config file @@ -276,6 +279,9 @@ doc/pod/gencancel.pod Master file for gencancel.1 doc/pod/getlist.pod Master file for getlist.1 doc/pod/grephistory.pod Master file for grephistory.1 doc/pod/hacking.pod Master file for HACKING +doc/pod/hissqlite-convert.pod Master file for hissqlite-convert.8 +doc/pod/hissqlite-util.pod Master file for hissqlite-util.8 +doc/pod/hissqlite.pod Master file for hissqlite.5 doc/pod/history.pod Master file for history.5 doc/pod/hook-perl.pod Master file for doc/hook-perl doc/pod/hook-python.pod Master file for doc/hook-python @@ -402,6 +408,23 @@ history/his.c History API glue implementation history/hisinterface.h History API interface history/hismethods.c Generated table of history methods history/hismethods.h Generated interface to history methods +history/hissqlite hissqlite history method (Directory) +history/hissqlite/hismethod.config buildconfig definition for hissqlite +history/hissqlite/hismethod.mk Make rules for hissqlite +history/hissqlite/hissqlite-convert.c Convert a history database to hissqlite +history/hissqlite/hissqlite-init.c Generated database setup implementation +history/hissqlite/hissqlite-init.h Generated database setup interface +history/hissqlite/hissqlite-init.sql SQLite code for database setup +history/hissqlite/hissqlite-main.c Generated daily operation implementation +history/hissqlite/hissqlite-main.h Generated daily operation interface +history/hissqlite/hissqlite-main.sql SQLite code for daily operation +history/hissqlite/hissqlite-private.h Private header for hissqlite +history/hissqlite/hissqlite-read.c Generated read-only query implementation +history/hissqlite/hissqlite-read.h Generated read-only query interface +history/hissqlite/hissqlite-read.sql SQLite code for direct reader queries +history/hissqlite/hissqlite-util.in Utility program for hissqlite +history/hissqlite/hissqlite.c hissqlite history method +history/hissqlite/hissqlite.h Header for hissqlite history history/hisv6 History v6 method (Directory) history/hisv6/hismethod.config hisbuildconfig definition history/hisv6/hisv6-private.h Private header file for hisv6 @@ -440,6 +463,7 @@ include/inn/paths.h.in Header file for paths include/inn/qio.h Header file for quick I/O package include/inn/secrets.h Header file for the secrets struct include/inn/sequence.h Header file for sequence space arithmetic +include/inn/sqlite-helper.h SQLite code package interface include/inn/storage.h Header file for storage API include/inn/timer.h Header file for generic timers include/inn/tombstone.h Header file for cancel tombstone log @@ -585,6 +609,8 @@ lib/setenv.c setenv replacement lib/seteuid.c seteuid replacement lib/setproctitle.c setproctitle replacement lib/snprintf.c snprintf and vsnprintf replacement +lib/sqlite-helper-gen.in Package SQLite code for convenient use +lib/sqlite-helper.c SQLite code package implementation lib/strcasecmp.c Case-insensitive string comparison (BSD) lib/strlcat.c strlcat replacement lib/strlcpy.c strlcpy replacement @@ -798,9 +824,6 @@ storage/ovsqlite/sql-main.sql SQLite code for daily operation storage/ovsqlite/sql-read.c Generated read-only query implementation storage/ovsqlite/sql-read.h Generated read-only query interface storage/ovsqlite/sql-read.sql SQLite code for direct reader queries -storage/ovsqlite/sqlite-helper-gen.in Package SQLite code for convenient use -storage/ovsqlite/sqlite-helper.c SQLite code package implementation -storage/ovsqlite/sqlite-helper.h SQLite code package interface storage/timecaf timecaf storage method (Directory) storage/timecaf/README.CAF README the CAF file format storage/timecaf/caf.c CAF file implementation @@ -969,6 +992,8 @@ tests/lib/hash-t.c Tests for lib/hash.c tests/lib/hashtab-t.c Tests for lib/hashtab.c tests/lib/headers-t.c Tests for lib/headers.c tests/lib/hex-t.c Tests for lib/hex.c +tests/lib/hissqlite-convert-t.c Tests for the hissqlite-convert tool +tests/lib/hissqlite-t.c Tests for the hissqlite history method tests/lib/inet_aton-t.c Tests for lib/inet_aton.c tests/lib/inet_ntoa-t.c Tests for lib/inet_ntoa.c tests/lib/inet_ntop-t.c Tests for lib/inet_ntop.c diff --git a/doc/pod/Makefile b/doc/pod/Makefile index 29a8b2604..403ffdd02 100644 --- a/doc/pod/Makefile +++ b/doc/pod/Makefile @@ -23,6 +23,7 @@ MAN3 = ../man/libinn.3 ../man/libinn_clientlib.3 ../man/libinn_dbz.3 \ MAN5 = ../man/active.5 ../man/active.times.5 ../man/buffindexed.conf.5 \ ../man/control.ctl.5 ../man/cycbuff.conf.5 ../man/distrib.pats.5 \ ../man/distributions.5 ../man/expire.ctl.5 ../man/history.5 \ + ../man/hissqlite.5 \ ../man/incoming.conf.5 ../man/inn.conf.5 ../man/innfeed.conf.5 \ ../man/innreport.conf.5 ../man/inn-secrets.conf.5 \ ../man/innwatch.ctl.5 ../man/moderators.5 \ @@ -38,7 +39,9 @@ MAN8 = ../man/actsync.8 ../man/archive.8 ../man/auth_krb5.8 \ ../man/controlchan.8 ../man/ctlinnd.8 ../man/cvtbatch.8 \ ../man/docheckgroups.8 \ ../man/domain.8 ../man/expire.8 ../man/expireover.8 \ - ../man/expirerm.8 ../man/ident.8 \ + ../man/expirerm.8 \ + ../man/hissqlite-convert.8 ../man/hissqlite-util.8 \ + ../man/ident.8 \ ../man/innbind.8 ../man/inncheck.8 ../man/innd.8 ../man/inndf.8 \ ../man/innfeed.8 ../man/innreport.8 ../man/innstat.8 \ ../man/innupgrade.8 \ @@ -108,6 +111,7 @@ maintclean: distclean ../man/distributions.5: distributions.pod ; $(POD2MAN) -s 5 $? > $@ ../man/expire.ctl.5: expire.ctl.pod ; $(POD2MAN) -s 5 $? > $@ ../man/history.5: history.pod ; $(POD2MAN) -s 5 $? > $@ +../man/hissqlite.5: hissqlite.pod ; $(POD2MAN) -s 5 $? > $@ ../man/incoming.conf.5: incoming.conf.pod ; $(POD2MAN) -s 5 $? > $@ ../man/inn.conf.5: inn.conf.pod ; $(POD2MAN) -s 5 $? > $@ ../man/innfeed.conf.5: innfeed.conf.pod ; $(POD2MAN) -s 5 $? > $@ @@ -145,6 +149,8 @@ maintclean: distclean ../man/expire.8: expire.pod ; $(POD2MAN) -s 8 $? > $@ ../man/expireover.8: expireover.pod ; $(POD2MAN) -s 8 $? > $@ ../man/expirerm.8: expirerm.pod ; $(POD2MAN) -s 8 $? > $@ +../man/hissqlite-convert.8: hissqlite-convert.pod ; $(POD2MAN) -s 8 $? > $@ +../man/hissqlite-util.8: hissqlite-util.pod ; $(POD2MAN) -s 8 $? > $@ ../man/ident.8: ident.pod ; $(POD2MAN) -s 8 $? > $@ ../man/innbind.8: innbind.pod ; $(POD2MAN) -s 8 $? > $@ ../man/inncheck.8: inncheck.pod ; $(POD2MAN) -s 8 $? > $@ diff --git a/doc/pod/expire.pod b/doc/pod/expire.pod index 29456e146..bc8bd0909 100644 --- a/doc/pod/expire.pod +++ b/doc/pod/expire.pod @@ -49,6 +49,15 @@ be better to only use self-expiring backends like CNFS, as the history entries will still get cleaned up by B when it detects that an article no longer exists in that backend. +When the C history method is in use, B updates the +history database in place instead of building a new file and swapping it. +It does not pause B (the two coordinate through the database). The +B<-d> and B<-f> options, which write a rebuilt history to a side file, are +rejected; use B<-t> for a dry run (it reports what would be removed without +changing the history or the spool). B<-x> on its own is also rejected, +because suppressing the history rewrite while still removing articles would +leave history entries for deleted articles. See hissqlite(5). + =head1 OPTIONS =over 4 @@ -193,7 +202,7 @@ POD by Julien Elie. =head1 SEE ALSO -ctlinnd(8), expire.ctl(5), expireover(8), history(5), inn.conf(5), innd(8), -libinn_dbz(3), libinn_inndcomm(3), news.daily(8). +ctlinnd(8), expire.ctl(5), expireover(8), hissqlite(5), history(5), +inn.conf(5), innd(8), libinn_dbz(3), libinn_inndcomm(3), news.daily(8). =cut diff --git a/doc/pod/hissqlite-convert.pod b/doc/pod/hissqlite-convert.pod new file mode 100644 index 000000000..c6b518b26 --- /dev/null +++ b/doc/pod/hissqlite-convert.pod @@ -0,0 +1,74 @@ +=head1 NAME + +hissqlite-convert - Convert an INN history database to the hissqlite method + +=head1 SYNOPSIS + +B [B<-m> I] I I + +=head1 DESCRIPTION + +B performs a faithful, one-time migration of an existing +INN history database into a new hissqlite (SQLite) database. It reads every +entry from the source history through the standard history interface and +bulk-loads it into a freshly created hissqlite database. + +The conversion is keyed by hash: the history records only the MD5 of each +Message-ID, not the Message-ID itself, so the original Message-IDs cannot be +recovered. Unlike a from-spool rebuild with B, this preserves +both the exact arrival, posting, and expiration timestamps and the +I (token-less) entries, which a rebuild cannot reconstruct. + +The new database is built in a temporary file and atomically renamed into +place only on full success, so an interrupted or failed run never leaves a +usable-but-incomplete destination. B refuses to run if +the destination already exists. + +B must not be writing the source history while it is being converted. +After a successful conversion, set I to C in +F and restart INN; see hissqlite(5). + +=head1 OPTIONS + +=over 4 + +=item B<-m> I + +The history method of the source database. The default is C. + +=back + +=head1 ARGUMENTS + +=over 4 + +=item I + +The path to the source history (for C, the F file without +any extension), as given by the I parameter in F. + +=item I + +The base path of the hissqlite database to create. A F<.sqlite> suffix is +appended, so I should match I (the resulting file is +F in I). + +=back + +=head1 EXAMPLES + +Convert the default C history into a hissqlite database alongside it: + + hissqlite-convert /news/db/history /news/db/history + +This creates F. + +=head1 HISTORY + +Written by Kevin Bowling for InterNetNews. + +=head1 SEE ALSO + +hissqlite(5), hissqlite-util(8), history(5), inn.conf(5), makehistory(8). + +=cut diff --git a/doc/pod/hissqlite-util.pod b/doc/pod/hissqlite-util.pod new file mode 100644 index 000000000..34db0f1bf --- /dev/null +++ b/doc/pod/hissqlite-util.pod @@ -0,0 +1,92 @@ +=head1 NAME + +hissqlite-util - History manipulation utility for hissqlite + +=head1 SYNOPSIS + +B [B<-AcdhV>] [B<-p> I] + +=head1 DESCRIPTION + +B is an administrative interface to the hissqlite history +method for INN. It only works on hissqlite history databases, not on any +other type of INN history. It allows the administrator to audit the +database for problems, dump its contents, and report entry counts and the +schema version. + +All of its actions are read-only, so it is safe to run while INN is running. +B opens the database directly; because B holds the +write lock for only a single statement at a time, it waits at most briefly +for any in-progress write. + +=head1 REQUIREMENTS + +The C Perl module with the C driver must be installed for +this program to work. They are present in most distributions as a package +named like C, C or C. +They can also be directly installed from CPAN (C and C +distributions available at L and +L). + +Naturally, a hissqlite history database also needs being in use, as detailed +in the hissqlite(5) manual page. + +=head1 OPTIONS + +Only one action may be given at a time. With no action, B<-A> is assumed. + +=over 4 + +=item B<-A> + +Audit the history database for problems and report them to standard error, +without trying to fix them. This reports the schema version and entry +counts, checks for rows with a missing arrival time, and runs SQLite's +C. This is the default action. + +=item B<-c> + +Print the total, real (token-bearing), and remembered (token-less) entry +counts. + +=item B<-d> + +Dump all history records as text, one per line: the hash in hexadecimal, +the arrival, posting, and expiration timestamps (in seconds since epoch), +and the storage API token in hexadecimal (or C for a token-less +entry). + +=item B<-h> + +Print a usage message and exit. + +=item B<-p> I + +Act on the F database in I instead of the default +I directory from F. + +=item B<-V> + +Print the version of this tool and the database schema version. + +=back + +=head1 EXAMPLES + +Audit the history database for problems: + + hissqlite-util -A + +Print entry counts for a database in F: + + hissqlite-util -c -p /news/db + +=head1 HISTORY + +Written by Kevin Bowling for InterNetNews. + +=head1 SEE ALSO + +hissqlite(5), hissqlite-convert(8), history(5), inn.conf(5). + +=cut diff --git a/doc/pod/hissqlite.pod b/doc/pod/hissqlite.pod new file mode 100644 index 000000000..b357e0cdd --- /dev/null +++ b/doc/pod/hissqlite.pod @@ -0,0 +1,162 @@ +=head1 NAME + +hissqlite - SQLite-based history storage method for INN + +=head1 DESCRIPTION + +This method uses SQLite to store the history database, as an alternative to +the default C method. It requires version 3.8.2 or later of the +SQLite library (3.20.0+ recommended). + +SQLite source, documentation, etc. are available at +L. One of the stated goals of the SQLite file +format is long-term stability and compatibility, which makes it a good +choice for a history database. + +Unlike C, hissqlite keeps everything in a single SQLite database +file: there is no separate F text file and no I index files. +Consequently the B and B text-file tools do not +apply, and the I path simply names the database file (see +L). Each entry is keyed on the 16-byte MD5 hash of its +Message-ID and is in one of three states: a real article (with a storage +API token), a I Message-ID (no token, recorded so a cancelled +or expired article is not re-offered), or absent. + +Only one schema version of the hissqlite method currently exists, +implemented since S. + +This method comes with a migration tool (B) and a +support utility (B). See their manual pages for details. + +=head1 INSTALLATION + +The configure script will automatically enable SQLite support if it finds +the SQLite library. If the library isn't installed in a standard location, +you may have to specify the B<--with-sqlite3> option to help configure find +it. For complicated cases, you can use separate B<--with-sqlite3-include> +and B<--with-sqlite3-lib> options. Finally, if you I want SQLite +support even when your system has the SQLite library, you can use the +B<--without-sqlite3> option. + +=head1 CONFIGURATION + +To select hissqlite as your history method, set the I parameter +in F to C. The database is stored at the location +named by the I parameter with a F<.sqlite> suffix appended +(by default, F in I). After changing I, +migrate or rebuild your history (see L) and restart INN. + +The following F parameters tune the database; their defaults suit +most sites. + +=over 4 + +=item I + +The SQLite page size in bytes, applied when the database is created (it +cannot be changed afterwards without rebuilding). Must be a power of two +between 512 and 65536. The default is C<4096>, matching the common +filesystem block size so that a random single-row insert or point lookup +on the random-MD5 keys touches exactly one block (no read-modify-write). +For best results the page size should match the underlying storage block +size: on ZFS, create the history dataset with C. A small +page on a coarse-recordsize ZFS dataset (the 128k default) forces +read-modify-write on every random write and will perform worse, not +better. + +=item I + +The page cache size in kilobytes for the B writer connection. The +default is C<65536> (64 MB). + +=item I + +The page cache size in kilobytes for each B reader process, which +opens the database directly for read-only access. Each reader has its own +cache, so this is kept small; history lookups are not B's hot path +(overview is), and readers share the operating system page cache. The +default is C<2000> (about 2 MB). + +=item I + +The maximum number of bytes of the database that SQLite memory-maps, for both +the writer and the direct readers. When enabled, memory-mapped pages are +served from the shared operating system page cache, which can keep the B-tree +hot and lets reader processes share physical memory. This is of little +benefit on ZFS and other systems without a unified buffer cache, where the +pages would simply be cached twice, and memory-mapped I/O turns a read error +into a fatal signal (B) rather than a recoverable error. The default +value of C<0> therefore disables it, matching ovsqlite(5). A good starting +value, if you enable it after benchmarking, is the size of the database file +or the amount of available memory, whichever is smaller. + +=back + +The database is opened in WAL journal mode with C set to +C (see L). + +=head1 DURABILITY + +hissqlite uses WAL (Write-Ahead Logging) journal mode with C +set to C. A committed write is a buffered append to the WAL: it is +durable across an application crash (for instance, if B is killed), +but a sudden power loss or kernel panic can lose the transactions written +since the last checkpoint. The database itself is never corrupted. This +matches the durability contract of the C/I backend, where +history updates are likewise periodically flushed rather than synchronously +committed; in both cases the affected articles are simply re-offered by +peers. The WAL is checkpointed in the background and truncated on a clean +shutdown. + +B WAL mode requires shared-memory support and does not work on +network filesystems such as NFS if duties are split across multiple machines. +Keep I on local storage or only use a single instance to run +B and other programs like B. + +=head1 MIGRATION + +To convert an existing C history into a hissqlite database without +losing remembered entries or exact timestamps, use B; +see hissqlite-convert(8). Alternatively, you can rebuild the history from +the article spool with B, though a from-spool rebuild cannot +reconstruct remembered Message-IDs. + +=head1 RUNNING + +hissqlite has no separate server daemon. B writes the database +directly and is the steady-state writer; the only other writer is B, +which runs in place (see L) rather than rebuilding and swapping +files, so B is not paused for it. SQLite's WAL locking and busy +timeout serialize the two writers, and because writes autocommit there is no +broker process to funnel them through (unlike ovsqlite). Reader processes +and the offline tools open the same WAL database read-only. + +While B is running, B can safely read the database: +B holds the write lock for only a single statement at a time, so it +waits at most briefly. (B is a one-time migration tool +that reads a I source history, which must not be written while it +runs; see hissqlite-convert(8).) + +=head1 EXPIRATION + +B runs against the live database in place: real articles past their +retention are transitioned to remembered entries (their token is cleared), +and remembered entries older than the F threshold in +F are deleted. Because hissqlite expires in place, B +cannot write a rebuilt history to a side file, so its B<-d> and B<-f> options +are rejected. Use B<-t> for a dry run: it lists the tokens that would be +removed and changes neither the history nor the spool (no F file +is produced, unlike with C). B<-x> on its own is also rejected, since +suppressing the history rewrite while still removing articles would leave the +history pointing at deleted articles. See expire(8). + +=head1 HISTORY + +Written by Kevin Bowling for InterNetNews. + +=head1 SEE ALSO + +expire(8), expire.ctl(5), hissqlite-convert(8), hissqlite-util(8), +history(5), inn.conf(5), makehistory(8), ovsqlite(5). + +=cut diff --git a/doc/pod/history.pod b/doc/pod/history.pod index 1e796222a..cd6a19705 100644 --- a/doc/pod/history.pod +++ b/doc/pod/history.pod @@ -9,6 +9,11 @@ stored in the news system, as well as those that have been received but since expired. In a typical production environment, this file will be many megabytes. +The text format described here applies to the default C history +method. The C method stores the same information in a single +SQLite database instead of this text file and its I indexes; see +hissqlite(5). + The file consists of text lines. Each line corresponds to one article. The file is normally kept sorted in the order in which articles are received, although this is not a requirement. B appends a new @@ -59,6 +64,7 @@ into POD by Julien Elie. =head1 SEE ALSO -expire(8), inn.conf(5), innd(8), libinn_dbz(3), makehistory(8). +expire(8), hissqlite(5), inn.conf(5), innd(8), libinn_dbz(3), +makehistory(8). =cut diff --git a/doc/pod/inn.conf.pod b/doc/pod/inn.conf.pod index dd5182c41..ab5ad05b6 100644 --- a/doc/pod/inn.conf.pod +++ b/doc/pod/inn.conf.pod @@ -484,15 +484,15 @@ is C<10> and probably doesn't need to be changed. =head2 History Settings -The following parameter affect the history database. +The following parameters affect the history database. =over 4 =item I -Which history storage method to use. The only currently supported -value is C. There is no default value; this parameter must -be set. +Which history storage method to use. The supported values are C +and, when INN is built with SQLite, C. There is no default +value; this parameter must be set. =over 4 @@ -504,8 +504,42 @@ v6 format, or tagged hash format, depending on the build options. Separation of these two is a project which has not yet been undertaken. +=item C + +Stores history data in a single SQLite database file, as an alternative +to C. This requires INN to be built with SQLite support. See +hissqlite(5). + =back +=item I + +The SQLite page cache size in kilobytes for the B writer connection +when the C history method is used. The default value is C<65536> +(64 MB). See hissqlite(5). + +=item I + +The maximum number of bytes of the C history database that SQLite +will memory-map, for both the writer and the direct readers. The default +value of C<0> disables memory-mapped I/O (as with ovsqlite); see hissqlite(5) +for the trade-offs before enabling it. + +=item I + +The SQLite page size in bytes for a newly created C history +database; it has no effect on an existing one. Must be a power of two +between 512 and 65536. The default value is C<4096>, which matches the +common filesystem block size; for best results pair it with the storage +block size (on ZFS, a C dataset). See hissqlite(5). + +=item I + +The SQLite page cache size in kilobytes for each B reader process +using the C history method. Each reader has its own cache, so +this is typically kept small. The default value is C<2000> (about 2 MB). +See hissqlite(5). + =back =head2 Article Storage diff --git a/doc/pod/makehistory.pod b/doc/pod/makehistory.pod index fbd786f3e..3c535558a 100644 --- a/doc/pod/makehistory.pod +++ b/doc/pod/makehistory.pod @@ -28,6 +28,13 @@ If a malformed article is found in the news spool, in a way which prevents its integration into the history or overview data, a log line will be output and the malformed article will just be skipped. +When the C history method is configured, B writes the +new history into a hissqlite database just as it does for C, and the +makedbz(8) step above does not apply. A from-spool rebuild cannot recover +I Message-IDs, so to migrate an existing C history into +hissqlite without losing them, use hissqlite-convert(8) instead. See +hissqlite(5). + =head1 OVERVIEW REBUILD I: If you're trying to rebuild the overview database, be sure to @@ -283,7 +290,8 @@ updated by various other people since. =head1 SEE ALSO -active(5), ctlinnd(8), history(5), inn.conf(5), innd(8), libinn_dbz(3), -makedbz(8), ovdb_init(8), overchan(8), ovsqlite-server(8). +active(5), ctlinnd(8), hissqlite(5), hissqlite-convert(8), history(5), +inn.conf(5), innd(8), libinn_dbz(3), makedbz(8), ovdb_init(8), overchan(8), +ovsqlite-server(8). =cut diff --git a/expire/expire.c b/expire/expire.c index 42082d425..c6a5946ba 100644 --- a/expire/expire.c +++ b/expire/expire.c @@ -710,6 +710,7 @@ main(int ac, char *av[]) bool Writing; bool UnlinkFile; bool val; + bool inplace; time_t TimeWarp; size_t Size = 0; @@ -880,6 +881,56 @@ main(int ac, char *av[]) HISctl(History, HISCTLS_NPAIRS, &Size); } + /* In-place backends (e.g. hissqlite) mutate the live database via a writer + * rather than rebuilding a new file and swapping it, so they must be opened + * read/write (even for a dry run: the expire scan runs on the writer + * connection). Backends that do not implement HISCTLG_INPLACEEXPIRE leave + * the flag untouched, so the hisv6 rebuild-and-swap path is unchanged. + * + * The modes that decouple the history rewrite from article removal need + * care here, because a run that does one but not the other corrupts the + * spool<->history correspondence (e.g. delete articles but keep their + * history rows). All exits go through CleanupAndExit, not die(), so the + * server reservation taken above is released. + * + * -t (tracing) is the dry run: EXPremove() already suppresses the + * SMcancel(), and we additionally force Writing off so the live history + * is not mutated -- it lists what would be removed and changes nothing. + * (Unlike hisv6 it produces no history.n side file.) + * + * -d/-f ask for a rebuilt history written to a side file, which an + * in-place backend cannot produce, so they are refused. + * + * -x on its own (suppress the history rewrite, but still remove articles) + * is exactly the inconsistent run the dry run replaces, so it is refused + * too; -t is the supported way to preview an expire. */ + inplace = false; + HISctl(History, HISCTLG_INPLACEEXPIRE, &inplace); + if (inplace) { + if (EXPtracing) { + Writing = false; + } else if (NHistory != NULL) { + warn("%s expires in place and cannot write a separate history file" + " (-d/-f); use -t for a dry run", + innconf->hismethod); + CleanupAndExit(Server, false, 1); + } else if (!Writing) { + warn("%s expires in place: -x without -t would remove articles but" + " leave their history entries; use -t for a dry run", + innconf->hismethod); + CleanupAndExit(Server, false, 1); + } + HISclose(History); + History = HISopen(HistoryText, innconf->hismethod, HIS_RDWR); + if (!History) { + warn("cannot reopen history read/write for in-place expire"); + CleanupAndExit(Server, false, 1); + } + HISctl(History, HISCTLS_IGNOREOLD, &IgnoreOld); + if (Size != 0) + HISctl(History, HISCTLS_NPAIRS, &Size); + } + val = true; if (!SMsetup(SM_RDWR, (void *) &val) || !SMsetup(SM_PREOPEN, (void *) &val)) { diff --git a/history/Makefile b/history/Makefile index 0cc3b6c04..a60e873a3 100644 --- a/history/Makefile +++ b/history/Makefile @@ -9,7 +9,7 @@ include ../Makefile.global LTVERSION = 3:9:0 top = .. -CFLAGS = $(GCFLAGS) -I. +CFLAGS = $(GCFLAGS) -I. $(SQLITE3_CPPFLAGS) SOURCES = his.c hismethods.c $(METHOD_SOURCES) OBJECTS = $(SOURCES:.c=.o) @@ -28,6 +28,9 @@ warnings: install: all $(LI_LPUB) libinnhist.$(EXTLIB) $D$(PATHLIB)/libinnhist.$(EXTLIB) + for F in $(PROGRAMS) ; do \ + $(LI_XPRI) $$F $D$(PATHBIN)/`basename $$F` ; \ + done bootstrap: Make.methods diff --git a/history/hissqlite/hismethod.config b/history/hissqlite/hismethod.config new file mode 100644 index 000000000..8b64b2f6d --- /dev/null +++ b/history/hissqlite/hismethod.config @@ -0,0 +1,5 @@ +name = hissqlite +number = 1 +sources = hissqlite.c hissqlite-init.c hissqlite-main.c hissqlite-read.c +extra-sources = hissqlite-convert.c +programs = hissqlite-convert hissqlite-util diff --git a/history/hissqlite/hismethod.mk b/history/hissqlite/hismethod.mk new file mode 100644 index 000000000..30b0aaa74 --- /dev/null +++ b/history/hissqlite/hismethod.mk @@ -0,0 +1,32 @@ +## Build rules for the hissqlite history method. Included into +## history/Make.methods by buildconfig. The .c/.h are generated from the +## .sql files by the shared sqlite-helper-gen, which lives in lib/ (built +## before history, so it is available here). + +hissqlite/hissqlite-init.c: hissqlite/hissqlite-init.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen hissqlite/hissqlite-init.sql +hissqlite/hissqlite-init.h: hissqlite/hissqlite-init.c ; + +hissqlite/hissqlite-main.c: hissqlite/hissqlite-main.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen hissqlite/hissqlite-main.sql +hissqlite/hissqlite-main.h: hissqlite/hissqlite-main.c ; + +hissqlite/hissqlite-read.c: hissqlite/hissqlite-read.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen hissqlite/hissqlite-read.sql +hissqlite/hissqlite-read.h: hissqlite/hissqlite-read.c ; + +## hissqlite.c and the generated translation units include the generated +## headers; make that explicit so a parallel build orders correctly. +hissqlite/hissqlite.o hissqlite/hissqlite.lo: \ + hissqlite/hissqlite-init.h hissqlite/hissqlite-main.h \ + hissqlite/hissqlite-read.h + +## Programs: the migration converter (C) and the inspect/dump utility (Perl). +hissqlite/hissqlite-convert: hissqlite/hissqlite-convert.o libinnhist.la \ + $(LIBSTORAGE) $(LIBINN) + $(LIBLD) $(LDFLAGS) $(SQLITE3_LDFLAGS) -o $@ \ + hissqlite/hissqlite-convert.o libinnhist.la \ + $(LIBSTORAGE) $(LIBINN) $(STORAGE_LIBS) $(LIBS) + +hissqlite/hissqlite-util: hissqlite/hissqlite-util.in $(FIXSCRIPT) + $(FIX) hissqlite/hissqlite-util.in diff --git a/history/hissqlite/hissqlite-convert.c b/history/hissqlite/hissqlite-convert.c new file mode 100644 index 000000000..f0fac2c19 --- /dev/null +++ b/history/hissqlite/hissqlite-convert.c @@ -0,0 +1,223 @@ +/* +** Convert an existing history database to the hissqlite (SQLite) backend. +** +** Usage: hissqlite-convert [-m srcmethod] +** +** Reads the source history (default method "hisv6") via HISwalk and bulk-loads +** every entry into a freshly created hissqlite database keyed by hash. This +** is a faithful migration (the makedbz analog): it preserves exact +** timestamps AND remembered (token-less) entries, which a from-spool rebuild +** (makehistory) cannot reconstruct. +** +** Insertion is by HASH, not Message-ID: the history text stores only the MD5, +** and the original Message-ID is unrecoverable, so HISwrite (which re-hashes a +** Message-ID) cannot be used. We therefore create the canonical schema via the +** hissqlite backend, then bulk-insert by hash through a raw SQLite connection. +** +** Written by Kevin Bowling in 2026. +*/ + +#include "portable/system.h" + +#include "inn/history.h" +#include "inn/innconf.h" +#include "inn/libinn.h" +#include "inn/messages.h" +#include "inn/storage.h" + +#ifndef HAVE_SQLITE3 + +int +main(void) +{ + die("INN was built without SQLite support"); +} + +#else + +# include +# include + +/* Commit the load transaction every this many rows. */ +# define COMMIT_EVERY 50000 + +struct convstate { + sqlite3 *db; + sqlite3_stmt *insert; + unsigned long count; /* rows successfully inserted */ + unsigned long dups; /* duplicate-hash rows skipped (anomalous) */ +}; + +static void +checked(sqlite3 *db, int status, const char *what) +{ + if (status != SQLITE_OK) + die("%s: %s", what, sqlite3_errmsg(db)); +} + +static void +exec(sqlite3 *db, const char *sql) +{ + char *errmsg = NULL; + + if (sqlite3_exec(db, sql, NULL, NULL, &errmsg) != SQLITE_OK) + die("%s: %s", sql, errmsg ? errmsg : "(unknown)"); +} + +/* +** HISwalk callback: insert one source entry into the hissqlite DB by hash. +** token == NULL marks a remembered entry (stored with a NULL token column). +*/ +static bool +convert_cb(void *cookie, const HASH *hash, time_t arrived, time_t posted, + time_t expires, const TOKEN *token) +{ + struct convstate *st = cookie; + sqlite3_stmt *ins = st->insert; + int rc; + + sqlite3_bind_blob(ins, 1, hash, sizeof(HASH), SQLITE_STATIC); + sqlite3_bind_int64(ins, 2, (sqlite3_int64) arrived); + sqlite3_bind_int64(ins, 3, (sqlite3_int64) posted); + sqlite3_bind_int64(ins, 4, (sqlite3_int64) expires); + if (token == NULL) + sqlite3_bind_null(ins, 5); + else + sqlite3_bind_blob(ins, 5, token, sizeof(TOKEN), SQLITE_STATIC); + + rc = sqlite3_step(ins); + sqlite3_reset(ins); + if (rc == SQLITE_DONE) { + st->count++; + } else if ((rc & 0xFF) == SQLITE_CONSTRAINT) { + /* Duplicate hash in the source: anomalous for a valid history, but not + fatal -- count it rather than silently dropping it. */ + st->dups++; + return true; + } else { + /* A real error (I/O, disk full, corruption): abort. The half-built + database is a temp file that is never renamed into place. */ + die("insert failed after %lu entries: %s", st->count, + sqlite3_errmsg(st->db)); + } + + if (st->count % COMMIT_EVERY == 0) { + exec(st->db, "commit"); + exec(st->db, "begin"); + if (st->count % (COMMIT_EVERY * 20) == 0) + notice("%lu entries...", st->count); + } + return true; +} + +int +main(int argc, char *argv[]) +{ + const char *srcmethod = "hisv6"; + const char *srcpath, *dstpath; + char *finaldb, *tempbase, *tempdb; + struct history *src, *tmp; + struct convstate st; + struct stat sb; + int opt; + + message_program_name = "hissqlite-convert"; + + while ((opt = getopt(argc, argv, "m:")) != -1) { + switch (opt) { + case 'm': + srcmethod = optarg; + break; + default: + die("usage: hissqlite-convert [-m srcmethod] "); + } + } + if (argc - optind != 2) + die("usage: hissqlite-convert [-m srcmethod] "); + srcpath = argv[optind]; + dstpath = argv[optind + 1]; + + /* The created database honors the hissqlite* tunables (page size in + particular) from inn.conf, matching what innd will later use. */ + if (!innconf_read(NULL)) + exit(1); + + /* Refuse to overwrite an existing destination: a re-run or a typo must not + clobber or merge into a live history. */ + finaldb = concat(dstpath, ".sqlite", (char *) NULL); + if (stat(finaldb, &sb) == 0) + die("destination %s already exists; refusing to overwrite", finaldb); + + /* Build into a temp file and rename it into place only on full success, so + a crash or error never leaves a usable-but-incomplete destination (this + is also what makes journal_mode=off below safe). Clear any stale temp + from a previous failed run. */ + tempbase = concat(dstpath, ".new", (char *) NULL); + tempdb = concat(tempbase, ".sqlite", (char *) NULL); + unlink(tempdb); + + /* Create the temp DB with the canonical hissqlite schema, then reopen the + raw SQLite file for a fast by-hash bulk load. */ + tmp = HISopen(tempbase, "hissqlite", HIS_CREAT | HIS_RDWR); + if (tmp == NULL) + die("cannot create destination %s", tempdb); + HISclose(tmp); + + memset(&st, 0, sizeof(st)); + if (sqlite3_open(tempdb, &st.db) != SQLITE_OK) + die("cannot open %s: %s", tempdb, sqlite3_errmsg(st.db)); + + /* Bulk-load tuning: no per-row durability (safe -- a failure leaves the + temp file, never the final), and drop the secondary index so it is + built once at the end. (The CREATE INDEX statement mirrors + hissqlite-init.sql.) */ + exec(st.db, "pragma synchronous = off"); + exec(st.db, "pragma journal_mode = off"); + /* A large page cache keeps the random-MD5 B-tree hot during the load, so + inserts don't constantly evict and re-fetch leaf pages. (The first innd + open flips the finished DB to WAL.) */ + exec(st.db, "pragma cache_size = -262144"); /* 256 MiB (negative = KiB) */ + exec(st.db, "drop index if exists hist_remember"); + + /* Plain INSERT (not OR IGNORE): a duplicate hash returns SQLITE_CONSTRAINT + so convert_cb can count it instead of silently dropping the row. */ + checked(st.db, + sqlite3_prepare_v2(st.db, + "insert into hist" + "(hash, arrived, posted, expires, token)" + " values(?1, ?2, ?3, ?4, ?5)", + -1, &st.insert, NULL), + "prepare insert"); + + src = HISopen(srcpath, srcmethod, HIS_RDONLY); + if (src == NULL) + die("cannot open source %s (method %s)", srcpath, srcmethod); + + exec(st.db, "begin"); + if (!HISwalk(src, NULL, &st, convert_cb)) + die("HISwalk failed: %s", HISerror(src)); + exec(st.db, "commit"); + HISclose(src); + + sqlite3_finalize(st.insert); + notice("rebuilding index..."); + exec(st.db, + "create index hist_remember on hist(posted, arrived)" + " where token is null"); + if (sqlite3_close(st.db) != SQLITE_OK) + die("cannot close %s", tempdb); + + /* Atomically publish the completed database. */ + if (rename(tempdb, finaldb) != 0) + sysdie("cannot rename %s to %s", tempdb, finaldb); + + if (st.dups > 0) + warn("%lu duplicate hashes in the source were skipped", st.dups); + notice("converted %lu entries into %s", st.count, dstpath); + free(finaldb); + free(tempbase); + free(tempdb); + return 0; +} + +#endif /* HAVE_SQLITE3 */ diff --git a/history/hissqlite/hissqlite-init.sql b/history/hissqlite/hissqlite-init.sql new file mode 100644 index 000000000..fdc052c19 --- /dev/null +++ b/history/hissqlite/hissqlite-init.sql @@ -0,0 +1,53 @@ +-- hissqlite schema version 1 +-- +-- Run once at database creation. Schema 1: a single WITHOUT ROWID table +-- clustered on the 16-byte MD5 of the Message-ID, so a Message-ID -> token +-- lookup is one clustered-leaf access (the token lives in the leaf). +-- +-- Three logical states of a row: +-- token IS NOT NULL -> a real, stored article +-- token IS NULL -> a "remembered" entry (article seen/rejected or +-- expired; kept so it is not re-accepted) +-- no row -> never seen +-- +-- The token-NULL state is a first-class lifecycle stage, NOT a tombstone for +-- deletion: when a real article reaches its retention it is UPDATEd to +-- remembered (token=NULL), and only DELETEd once the remembered entry passes +-- the /remember/ posting-time threshold. See hissqlite.c hissqlite_expire(). + +-- Page size (pragma page_size) is applied by hissqlite.c from the +-- hissqlitepagesize inn.conf parameter before this DDL runs: it must be set +-- before any table exists (changing it later needs a VACUUM). The default, +-- 4096, matches the common filesystem block (and a ZFS recordsize=4k dataset), +-- so a random single-row insert or a point lookup touches exactly one block -- +-- no read-modify-write, no super-block read amplification. Benchmarking the +-- random-MD5-key workload measured 4k beating 8k on both writes and reads at +-- equal footprint. + +create table hist ( + hash blob not null primary key, -- HashMessageID(): 16-byte MD5 + arrived integer not null, -- time_t + posted integer, -- time_t; 0/NULL if unknown + expires integer, -- time_t article expiry; 0/NULL = none + token blob -- raw TOKEN (sizeof TOKEN); NULL = remembered +) without rowid; + +-- Find remember-threshold delete candidates (remembered -> gone): a partial +-- index over only the token-NULL rows, so expire pass 2 scans the remembered +-- entries without touching the real articles. (Real-article expiry needs no +-- index: pass 1 evaluates every token-bearing row via the policy callback, in +-- clustered hash order -- see hissqlite_expire().) +create index hist_remember on hist(posted, arrived) where token is null; + +-- Schema/version marker, mirroring ovsqlite's misc table. +create table misc ( + key text not null primary key, + value not null +) without rowid; + +-- .get_version +select value from misc where key = 'version'; + +-- .set_version +insert into misc(key, value) values('version', ?1) + on conflict(key) do update set value = excluded.value; diff --git a/history/hissqlite/hissqlite-main.sql b/history/hissqlite/hissqlite-main.sql new file mode 100644 index 000000000..911a48992 --- /dev/null +++ b/history/hissqlite/hissqlite-main.sql @@ -0,0 +1,107 @@ +-- Writer prepared statements for hissqlite (used by innd and the offline +-- tools: makehistory, prunehistory, the migration converter). +-- +-- Implementation follows ovsqlite's limits/patterns; the operator-facing +-- durability contract matches dbz (lose recent on power loss, never corrupt, +-- peer resends). WAL + synchronous=NORMAL: a commit is durable across an +-- application crash; a power loss can lose transactions since the last +-- checkpoint, but the DB is never corrupted -- parity with dbz's periodic +-- msync, not a regression. + +pragma journal_mode = wal; +pragma synchronous = normal; +pragma busy_timeout = 10000; +pragma foreign_keys = off; + +-- Writer cache + mmap (pragma cache_size / mmap_size) are applied by +-- hissqlite.c from the hissqlitecachesize and hissqlitemmapsize inn.conf +-- parameters after this section runs. innd is a single process, so a larger +-- per-connection page cache is affordable; mmap lets reads come from the shared +-- OS page cache and keeps the random-MD5 B-tree hot. + +-- Writes are NOT batched: each statement autocommits (one implicit transaction +-- per write). Under synchronous=NORMAL a commit is a buffered WAL append, not +-- an fsync, so there is no fsync cost to amortise; batching would only hold the +-- write lock across the batch and starve the other writer (expire) in the +-- two-writer model. Batching is reserved for the offline bulk load +-- (hissqlite-convert.c). See hissqlite.c. + +-- HISlookup: Message-ID -> token. A row with token IS NULL (remembered) +-- returns found=true with an empty token; ARTopenbyid treats TOKEN_EMPTY as +-- "no article". +-- .lookup +select arrived, posted, expires, token + from hist + where hash = ?1; + +-- HIScheck: existence only (real OR remembered both count -> refuse re-offers). +-- his.c keeps an in-memory existence cache in front of this; this is the +-- authoritative fallback. +-- .check +select 1 from hist where hash = ?1; + +-- HISwrite: a real, stored article. The accept path runs HIScheck first, so +-- this is normally a fresh insert. DO NOTHING on a hash conflict keeps the +-- existing row rather than failing: a duplicate Message-ID during a makehistory +-- rebuild must warn and continue, not abort the whole rebuild, as in hisv6 +-- (hissqlite.c detects the no-op via sqlite3_changes()). +-- .write +insert into hist(hash, arrived, posted, expires, token) + values(?1, ?2, ?3, ?4, ?5) + on conflict(hash) do nothing; + +-- HISremember: record a seen/rejected Message-ID with no token. Must NEVER +-- downgrade an existing real article to remembered, hence DO NOTHING. +-- .remember +insert into hist(hash, arrived, posted, expires, token) + values(?1, ?2, ?3, 0, null) + on conflict(hash) do nothing; + +-- HISreplace: the only token-changing op (prunehistory real->remembered by +-- passing a null token; remembered->real upgrade). Full upsert. +-- .replace +insert into hist(hash, arrived, posted, expires, token) + values(?1, ?2, ?3, ?4, ?5) + on conflict(hash) do update set + arrived = excluded.arrived, + posted = excluded.posted, + expires = excluded.expires, + token = excluded.token; + +-- HISwalk: full iteration (feeds the bloom build, dump/migration). Full +-- clustered B-tree scan: each leaf page read once, semi-sequential. +-- .walk +select hash, arrived, posted, expires, token from hist; + +-- HISexpire pass 1: iterate token-bearing entries so the policy callback can +-- decide keep / transition-to-remembered. (expire.ctl needs every entry +-- evaluated, as hisv6 does.) Keyset-paginated by the hash PK: ?1 is the last +-- hash of the previous page (empty blob for the first), ?2 the page size. The +-- caller streams pages so the read snapshot is released between them; the PK +-- range scan is in hash order, and rows transitioned to token=NULL drop out. +-- .expire_scan +select hash, arrived, posted, expires, token + from hist + where token is not null + and hash > ?1 + order by hash + limit ?2; + +-- HISexpire pass 1 action: transition a real article to remembered. +-- .transition_remember +update hist set token = null, expires = 0 where hash = ?1; + +-- HISexpire pass 1 action: callback kept the article but rewrote the token. +-- .update_token +update hist set token = ?2, expires = ?3 where hash = ?1; + +-- HISexpire pass 2: delete remembered entries older than the /remember/ +-- posting-time threshold (indexed by hist_remember). Bounded to a chunk per +-- step (LIMIT) so each autocommit holds the write lock only briefly; the caller +-- loops until a step deletes nothing. +-- .expire_remembered +delete from hist where hash in ( + select hash from hist + where token is null + and (case when posted > 0 then posted else arrived end) < ?1 + limit 10000); diff --git a/history/hissqlite/hissqlite-private.h b/history/hissqlite/hissqlite-private.h new file mode 100644 index 000000000..892f3d2c4 --- /dev/null +++ b/history/hissqlite/hissqlite-private.h @@ -0,0 +1,65 @@ +/* +** Private declarations for the hissqlite history backend. +*/ + +#ifndef HISSQLITE_PRIVATE_H +#define HISSQLITE_PRIVATE_H + +#include "config.h" +#include "inn/history.h" +#include "inn/libinn.h" +#include "inn/storage.h" + +#include + +/* +** Generated by sqlite-helper-gen from the .sql files at build time: +** hissqlite-init.{c,h} -> hissqlite_init_t / hissqlite_init_helper +** hissqlite-main.{c,h} -> hissqlite_main_t / hissqlite_main_helper +** hissqlite-read.{c,h} -> hissqlite_read_t / hissqlite_read_helper +** sqlite-helper.{c,h} and sqlite-helper-gen are shared with the ovsqlite +** overview backend (lib/ and include/inn/). +*/ +#include "hissqlite-init.h" +#include "hissqlite-main.h" +#include "hissqlite-read.h" + +#define HISSQLITE_SCHEMA_VERSION 1 + +/* Checkpoint the WAL once it has grown past this many frames (mirrors + ovsqlite's threshold). Done as a non-blocking PASSIVE checkpoint from + HISsync; TRUNCATE (disk reclaim) is left to close. */ +#define HISSQLITE_WAL_CKPT_PAGES 1000 + +/* Expire streams pass 1 in hash-keyset pages of this many rows: bounds the + in-memory action buffer and releases the read snapshot between pages so a + long expire does not pin the WAL. */ +#define HISSQLITE_EXPIRE_BATCH 10000 + +/* Fallback values for the inn.conf tunables (hissqlitepagesize, + hissqlitecachesize, hissqlitemmapsize), used when inn.conf has not been read + -- offline tools and unit tests that open history without innconf_read(). + When inn.conf is loaded these are superseded by the configured values (whose + own defaults, set in lib/innconf.c, match these). */ +#define HISSQLITE_DEF_PAGE_SIZE 4096 /* bytes; before first table write */ +#define HISSQLITE_DEF_CACHE_SIZE 65536 /* writer page cache, kB */ +#define HISSQLITE_DEF_MMAP_SIZE 0 /* bytes; 0 disables mmap (as ovsqlite) */ +#define HISSQLITE_DEF_RCACHE_SIZE 2000 /* per-nnrpd reader page cache, kB */ + +struct hissqlite { + char *path; /* SQLite database file path. */ + struct history *history; /* For his_seterror(). */ + sqlite3 *db; + int flags; /* HIS_RDONLY / HIS_RDWR / HIS_CREAT / ... */ + + /* Writer side (NULL for a pure read-only direct reader). Writes + autocommit -- there is no batching state (see hissqlite.c). */ + hissqlite_main_t main; + int wal_pages; /* WAL frame count, updated by the wal_hook. */ + + /* Reader side (used when opened HIS_RDONLY in WAL mode). */ + hissqlite_read_t read; + bool direct_reader; +}; + +#endif /* HISSQLITE_PRIVATE_H */ diff --git a/history/hissqlite/hissqlite-read.sql b/history/hissqlite/hissqlite-read.sql new file mode 100644 index 000000000..9d3a071b3 --- /dev/null +++ b/history/hissqlite/hissqlite-read.sql @@ -0,0 +1,24 @@ +-- Read-only prepared statements for hissqlite (nnrpd and other readers). +-- +-- In WAL mode readers open the SQLite file read-only and query it directly, +-- bypassing any writer process. + +pragma query_only = 1; +pragma busy_timeout = 10000; + +-- Reader cache + mmap (pragma cache_size / mmap_size) are applied by +-- hissqlite.c from the hissqlitereadercachesize and hissqlitemmapsize inn.conf +-- parameters after this section runs. nnrpd runs one process PER connection, +-- so the per-connection cache is kept small by default; readers lean on the +-- shared OS page cache instead. + +-- .lookup +select arrived, posted, expires, token + from hist + where hash = ?1; + +-- .check +select 1 from hist where hash = ?1; + +-- .walk +select hash, arrived, posted, expires, token from hist; diff --git a/history/hissqlite/hissqlite-util.in b/history/hissqlite/hissqlite-util.in new file mode 100644 index 000000000..efb440419 --- /dev/null +++ b/history/hissqlite/hissqlite-util.in @@ -0,0 +1,230 @@ +#! /usr/bin/perl +# fixscript will replace this line with code to load INN::Config + +## History manipulation utility for hissqlite. +## +## Initial version written in 2026, modeled on ovsqlite-util. + +use strict; +use warnings; + +use Getopt::Std; +use POSIX qw(strftime); ## no critic (ProhibitUnusedImport) + +$0 =~ s!.*/!!; + +# Version of this tool. +my $VERSION = "1.0"; + +# Bail out if the needed DBI Perl module is not installed. +eval { + require DBI; + require DBD::SQLite; + 1; +} or die "DBI Perl module with SQLite driver needed" + . " (usually packaged as libdbd-sqlite3-perl, perl-DBD-SQLite," + . " or p5-DBD-SQLite)"; + +# Name of the database file. +my $dbfile = "history.sqlite"; + +my $usage = "Usage: + $0 [-AcdhV] [-p path] + +Options: + -A Audit the history database for problems, and report them to + standard error, without trying to fix them. This is the + default action. + -c Print total, real, and remembered entry counts. + -d Dump all history records as text, one per line. + -h Print this help message. + -p path Read $dbfile database file in path directory instead of + default \$INN::Config::pathdb directory. + -V Print the version of this tool and the database schema + version. +"; + +sub HELP_MESSAGE { + print $usage; + exit(0); +} + +my %opt; +getopts("AcdhVp:", \%opt) || die $usage; + +HELP_MESSAGE() if defined($opt{'h'}); + +my $modes = 0; +$modes++ if defined($opt{'A'}); +$modes++ if defined($opt{'c'}); +$modes++ if defined($opt{'d'}); +$modes++ if defined($opt{'V'}); + +die "Only one action allowed at the same time\n\n$usage" + if $modes > 1; + +my $dbdir = $opt{'p'} || $INN::Config::pathdb; +my $datasource = "dbi:SQLite:dbname=$dbdir/$dbfile"; + +# All implemented modes are read-only, so open the database read-only. +# SQLITE_OPEN_READONLY is 0x00000001. +my $SQLITE_OPEN_READONLY = 0x00000001; + +# Open the connection. The username and password fields are left empty. +# Enabling RaiseError permits not checking every return error codes. +my $dbh = DBI->connect( + $datasource, '', '', + { + PrintError => 0, + RaiseError => 1, + AutoCommit => 1, + sqlite_open_flags => $SQLITE_OPEN_READONLY, + }, +); + +# Extra safety net against any accidental write in the read-only modes. +$dbh->do("pragma query_only = 1;"); + +# Raise the busy timeout value to a huge number of milliseconds to prevent +# timeouts because of database locks while innd holds the writer. +$dbh->sqlite_busy_timeout(999999999); + +if (defined($opt{'V'})) { + print_version(); +} elsif (defined($opt{'c'})) { + count_entries(); +} elsif (defined($opt{'d'})) { + dump_history(); +} else { + # Default action: audit. + audit_history(); +} + +# Close the connection properly. +$dbh->disconnect; + +exit(0); + +# Return the schema version stored in the misc table, or undef if absent. +sub get_version { + my $statement = $dbh->prepare("select value from misc where key = ?;"); + my ($version) = $dbh->selectrow_array($statement, undef, "version"); + return $version; +} + +# Return a list of (total, real, remembered) entry counts. +sub get_counts { + my ($total) = $dbh->selectrow_array("select count(*) from hist;"); + my ($real) + = $dbh->selectrow_array("select count(*) from hist" + . " where token is not null;"); + my ($remembered) + = $dbh->selectrow_array("select count(*) from hist" + . " where token is null;"); + return ($total, $real, $remembered); +} + +# Print the tool version and the database schema version (-V option). +sub print_version { + my $version = get_version(); + $version = defined($version) ? $version : "unknown"; + print "$0 version $VERSION\n"; + print "hissqlite database schema version $version\n"; +} + +# Print total, real, and remembered entry counts (-c option). +sub count_entries { + my ($total, $real, $remembered) = get_counts(); + print "total $total\n"; + print "real $real\n"; + print "remembered $remembered\n"; +} + +# Dump all history records (-d option). +sub dump_history { + my $statement; + + # quote(token) renders a BLOB as X'...' uppercase hex, or NULL for a + # remembered entry; hex(hash) renders the hash as uppercase hex. + $statement = $dbh->prepare( + q{ + select hex(hash), arrived, posted, expires, quote(token) + from hist; + }, + ); + $statement->execute(); + + while (my @row = $statement->fetchrow_array()) { + my ($hash, $arrived, $posted, $expires, $token) = @row; + $posted = defined($posted) ? $posted : '-'; + $expires = defined($expires) ? $expires : '-'; + if (!defined($token) or uc($token) eq 'NULL') { + $token = 'remembered'; + } else { + # quote() returns the token in the form X'...'; keep the hex part. + $token = uc(substr($token, 2, -1)); + } + print "$hash $arrived $posted $expires $token\n"; + } +} + +# Audit the history database for problems and report them (-A option, default). +sub audit_history { + my $problems = 0; + + # Schema version. + my $version = get_version(); + if (defined($version)) { + print "Schema version: $version\n"; + } else { + print STDERR "No schema version found in misc table\n"; + $problems++; + } + + # Entry counts. + my ($total, $real, $remembered) = get_counts(); + print "Total entries: $total\n"; + print "Real entries (token not null): $real\n"; + print "Remembered entries (token null): $remembered\n"; + + # Arrived time range. + my ($minarrived, $maxarrived) + = $dbh->selectrow_array( + "select min(arrived), max(arrived) from hist;"); + if (defined($minarrived) and defined($maxarrived)) { + printf "Arrived range: %s .. %s\n", + strftime('%Y-%m-%d %H:%M:%S', localtime($minarrived)), + strftime('%Y-%m-%d %H:%M:%S', localtime($maxarrived)); + } else { + print "Arrived range: (empty database)\n"; + } + + # Rows with a NULL arrived value (arrived is declared not null, so any + # such row indicates corruption). + my ($nullarrived) + = $dbh->selectrow_array( + "select count(*) from hist where arrived is null;"); + if (defined($nullarrived) and $nullarrived > 0) { + print STDERR "$nullarrived row(s) with NULL arrived value\n"; + $problems++; + } + + # SQLite low-level integrity check. + my $statement = $dbh->prepare("pragma integrity_check;"); + $statement->execute(); + while (my @row = $statement->fetchrow_array()) { + next if $row[0] eq 'ok'; + print STDERR "integrity_check: $row[0]\n"; + $problems++; + } + + if ($problems > 0) { + printf STDERR ( + "%d problem%s found\n", + $problems, ($problems > 1) ? "s" : "", + ); + exit(1); + } else { + print "No problems found\n"; + } +} diff --git a/history/hissqlite/hissqlite.c b/history/hissqlite/hissqlite.c new file mode 100644 index 000000000..e0f331a2a --- /dev/null +++ b/history/hissqlite/hissqlite.c @@ -0,0 +1,885 @@ +/* +** The SQLite history method (hissqlite). +** +** A transactional replacement for hisv6/dbz. Schema 1: a single WITHOUT +** ROWID table clustered on the 16-byte MD5 of the Message-ID, so a +** Message-ID -> token lookup is one clustered-leaf access. WAL + +** synchronous=NORMAL, autocommit writes; readers use the WAL direct path. +** +** innd owns its write connection directly and is the steady-state writer; the +** only other writer is expire, which runs in place (it reopens the live DB +** read/write instead of rebuilding and swapping a file, so innd is not +** paused). SQLite's WAL write lock and busy_timeout serialize the two without +** a broker server process (unlike ovsqlite): this works because writes +** autocommit, so the lock is held only per statement and the periodic, light +** expire does not contend with innd for long. See hissqlite(5). +** +** Written by Kevin Bowling in 2026. +*/ + +#include "portable/system.h" + +#include "inn/messages.h" +#include "../hisinterface.h" +#include "hissqlite.h" + +#ifdef HAVE_SQLITE3 + +#include "inn/innconf.h" +#include "hissqlite-private.h" + +/* +** Record a SQLite error against the history handle. +*/ +static void +hissqlite_seterror(struct hissqlite *h, const char *context) +{ + if (h->history != NULL) + his_seterror(h->history, + concat("hissqlite: ", context, ": ", + h->db ? sqlite3_errmsg(h->db) : "(no db)", NULL)); +} + +/* +** Bind the MD5 of a Message-ID as the hash key (parameter 1). +*/ +static void +bind_key(sqlite3_stmt *stmt, const char *key) +{ + HASH hash = HashMessageID(key); + + sqlite3_bind_blob(stmt, 1, &hash, sizeof(HASH), SQLITE_TRANSIENT); +} + +/* +** Copy a column blob into a fixed-size destination, verifying its length +** first. A wrong-length blob means a corrupt row; report it (so corruption +** is visible) and return false rather than over-reading or silently producing +** a bogus value. +*/ +static bool +copy_blob(struct hissqlite *h, sqlite3_stmt *stmt, int col, void *dst, + size_t size, const char *what) +{ + if (sqlite3_column_bytes(stmt, col) != (int) size) { + /* A length mismatch is the error itself; the generic seterror would + append an unrelated sqlite3_errmsg, so report "what" on its own. */ + if (h->history != NULL) + his_seterror(h->history, concat("hissqlite: ", what, (char *) NULL)); + return false; + } + memcpy(dst, sqlite3_column_blob(stmt, col), size); + return true; +} + +/* +** Writes are NOT batched: each HISwrite/HISremember/HISreplace autocommits +** (one implicit transaction per statement). With synchronous=NORMAL a commit +** is a buffered WAL append, not an fsync, so there is no fsync cost to +** amortise; batching would buy only a little WAL overhead, at the price of +** holding the write lock across the batch. Holding the lock is exactly wrong +** for the two-writer model (innd + a separate expire process): a batch +** would starve the other writer. Autocommit holds the lock for a single +** statement, so innd and expire interleave cleanly, and there is no open +** transaction to leave dangling on error or to wrap the expire scan. +** Batching only makes sense for the single-writer, offline bulk load -- see +** hissqlite-convert.c. +*/ + +/* +** Track the WAL frame count so HISsync can decide when to checkpoint. +*/ +static int +hissqlite_wal_hook(void *arg, sqlite3 *db UNUSED, const char *name UNUSED, + int pages) +{ + ((struct hissqlite *) arg)->wal_pages = pages; + return SQLITE_OK; +} + +/* +** A PASSIVE checkpoint never blocks: it writes back as many WAL frames as the +** oldest reader allows and lets the WAL be reused (not grown) on the next +** write. Safe to call from innd's HISsync timer without stalling accepts. +** Disk reclaim (TRUNCATE) is left to close, where readers are expected gone. +*/ +static void +hissqlite_wal_passive(struct hissqlite *h) +{ + int log = 0, ckpt = 0, rc; + + rc = sqlite3_wal_checkpoint_v2(h->db, NULL, SQLITE_CHECKPOINT_PASSIVE, &log, + &ckpt); + if (rc != SQLITE_OK) { + /* A real failure (e.g. I/O error): surface it so a WAL that stops + shrinking -- and could fill the disk -- is visible, not silent. */ + warn("hissqlite: WAL checkpoint failed: %s", sqlite3_errmsg(h->db)); + return; + } + if (log == ckpt) + h->wal_pages = 0; /* Fully flushed; the WAL resets on the next write. */ + /* else a reader still pins older frames; wal_pages stays high and we retry + on the next HISsync (not logged per-call -- that is normal, transient). */ +} + +/* +** Has the schema already been created? HIS_CREAT must initialise a brand-new +** database but must NOT re-run the DDL (or re-stamp the version) on an existing +** one -- otherwise opening an existing DB with HIS_CREAT fails on "table hist +** already exists" and would mask a version mismatch. +*/ +static bool +hissqlite_initialized(struct hissqlite *h) +{ + sqlite3_stmt *stmt; + bool exists = false; + + if (sqlite3_prepare_v2( + h->db, + "select 1 from sqlite_master where type = 'table' and name = 'hist'", + -1, &stmt, NULL) + == SQLITE_OK) { + exists = (sqlite3_step(stmt) == SQLITE_ROW); + sqlite3_finalize(stmt); + } + return exists; +} + +/* +** Refuse to open a database whose schema version we do not understand, so a +** mismatch fails cleanly rather than as obscure later query errors. +*/ +static bool +hissqlite_check_version(struct hissqlite *h) +{ + sqlite3_stmt *stmt; + long version = -1; + char buf[160]; + + if (sqlite3_prepare_v2(h->db, + "select value from misc where key = 'version'", -1, + &stmt, NULL) + != SQLITE_OK) { + hissqlite_seterror(h, "prepare version check"); + return false; + } + if (sqlite3_step(stmt) == SQLITE_ROW) + version = (long) sqlite3_column_int64(stmt, 0); + sqlite3_finalize(stmt); + if (version == HISSQLITE_SCHEMA_VERSION) + return true; + snprintf(buf, sizeof(buf), + "hissqlite: %s has schema version %ld, expected %d", h->path, + version, HISSQLITE_SCHEMA_VERSION); + his_seterror(h->history, xstrdup(buf)); + return false; +} + +/* +** Is the database in WAL journal mode? A direct reader requires it; a +** non-WAL reader would take SHARED locks that conflict with the writer's +** commits. +*/ +static bool +hissqlite_is_wal(struct hissqlite *h) +{ + sqlite3_stmt *stmt; + bool wal = false; + + if (sqlite3_prepare_v2(h->db, "pragma journal_mode", -1, &stmt, NULL) + == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + const char *mode = (const char *) sqlite3_column_text(stmt, 0); + wal = (mode != NULL && strcmp(mode, "wal") == 0); + } + sqlite3_finalize(stmt); + } + return wal; +} + +/* +** Open (or create) the database named by h->path and prepare the statement +** sets, using h->flags. Factored out of hissqlite_open so the deferred-path +** pattern -- HISopen(NULL, ...) followed by HISCTLS_PATH, as makehistory uses +** -- can drive the real open once the path is known. On failure it reports +** the error, tears down any half-open database, and returns false; the handle +** itself is left for the caller to free or (for HISCTLS_PATH) retry. +*/ +static bool +hissqlite_doopen(struct hissqlite *h) +{ + char *errmsg = NULL; + char pragma[64]; + unsigned long pagesize = HISSQLITE_DEF_PAGE_SIZE; + unsigned long cachesize = HISSQLITE_DEF_CACHE_SIZE; + unsigned long mmapsize = HISSQLITE_DEF_MMAP_SIZE; + unsigned long rcachesize = HISSQLITE_DEF_RCACHE_SIZE; + int oflags; + + /* Performance tunables come from inn.conf when it has been read (innd, + expire, makehistory, the converter); offline callers and unit tests that + open history without innconf_read() fall back to the built-in defaults. */ + if (innconf != NULL) { + pagesize = innconf->hissqlitepagesize; + cachesize = innconf->hissqlitecachesize; + mmapsize = innconf->hissqlitemmapsize; + rcachesize = innconf->hissqlitereadercachesize; + } + + if (!(h->flags & HIS_RDWR)) { + /* Read-only open (HIS_RDONLY is 0, but callers may OR in hint flags + such as HIS_ONDISK, so test for the absence of HIS_RDWR rather than + exact equality). WAL direct reader: open read-only, verify WAL, + prepare read stmts. */ + if (sqlite3_open_v2(h->path, &h->db, SQLITE_OPEN_READONLY, NULL) + != SQLITE_OK) { + hissqlite_seterror(h, "open read-only"); + goto fail; + } + if (!hissqlite_is_wal(h)) { + hissqlite_seterror(h, "history is not in WAL mode; direct reader" + " unavailable (NFS history needs a server)"); + goto fail; + } + if (sqlite_helper_init(&hissqlite_read_helper, + (sqlite3_stmt **) &h->read, h->db, + SQLITE_PREPARE_PERSISTENT, &errmsg) + != SQLITE_OK) { + hissqlite_seterror(h, "prepare read statements"); + goto fail; + } + if (!hissqlite_check_version(h)) { + sqlite_helper_term(&hissqlite_read_helper, + (sqlite3_stmt **) &h->read); + goto fail; + } + /* Reader cache + mmap from inn.conf. Each nnrpd has its own cache, so + it is kept small by default (history is not nnrpd's hot path -- + overview is); readers lean on the shared OS page cache instead. */ + snprintf(pragma, sizeof(pragma), "pragma cache_size = -%lu;", + rcachesize); + sqlite3_exec(h->db, pragma, NULL, NULL, NULL); + snprintf(pragma, sizeof(pragma), "pragma mmap_size = %lu;", mmapsize); + sqlite3_exec(h->db, pragma, NULL, NULL, NULL); + h->direct_reader = true; + return true; + } + + oflags = SQLITE_OPEN_READWRITE; + if (h->flags & HIS_CREAT) + oflags |= SQLITE_OPEN_CREATE; + if (sqlite3_open_v2(h->path, &h->db, oflags, NULL) != SQLITE_OK) { + hissqlite_seterror(h, "open read/write"); + goto fail; + } + sqlite3_extended_result_codes(h->db, 1); + + if ((h->flags & HIS_CREAT) && !hissqlite_initialized(h)) { + /* Brand-new database: run the DDL (unnamed init section of + hissqlite-init.sql) and stamp the schema version. Skipped for an + existing DB so HIS_CREAT is idempotent. Mirrors ovsqlite-server.c. */ + hissqlite_init_t init; + + /* page_size only takes effect before the first table is written, so + apply it here, ahead of the DDL. (innconf validates it.) */ + snprintf(pragma, sizeof(pragma), "pragma page_size = %lu;", pagesize); + sqlite3_exec(h->db, pragma, NULL, NULL, NULL); + + if (sqlite_helper_init(&hissqlite_init_helper, + (sqlite3_stmt **) &init, h->db, 0, &errmsg) + != SQLITE_OK) { + hissqlite_seterror(h, "create schema"); + goto fail; + } + sqlite3_bind_int64(init.set_version, 1, HISSQLITE_SCHEMA_VERSION); + if (sqlite3_step(init.set_version) != SQLITE_DONE) + hissqlite_seterror(h, "set version"); + sqlite3_reset(init.set_version); + sqlite_helper_term(&hissqlite_init_helper, (sqlite3_stmt **) &init); + } + + /* Prepare writer statements; the unnamed init section applies the WAL / + synchronous / busy_timeout pragmas. PERSISTENT: these are long-lived. */ + if (sqlite_helper_init(&hissqlite_main_helper, (sqlite3_stmt **) &h->main, + h->db, SQLITE_PREPARE_PERSISTENT, &errmsg) + != SQLITE_OK) { + hissqlite_seterror(h, "prepare main statements"); + goto fail; + } + if (!hissqlite_check_version(h)) { + sqlite_helper_term(&hissqlite_main_helper, (sqlite3_stmt **) &h->main); + goto fail; + } + /* Track WAL growth so HISsync can keep it bounded (the .sql init section + has already put us in WAL mode). */ + sqlite3_wal_hook(h->db, hissqlite_wal_hook, h); + + /* Writer cache + mmap from inn.conf, applied after the .sql WAL pragmas. */ + snprintf(pragma, sizeof(pragma), "pragma cache_size = -%lu;", cachesize); + sqlite3_exec(h->db, pragma, NULL, NULL, NULL); + snprintf(pragma, sizeof(pragma), "pragma mmap_size = %lu;", mmapsize); + sqlite3_exec(h->db, pragma, NULL, NULL, NULL); + return true; + +fail: + if (errmsg != NULL) + sqlite3_free(errmsg); + if (h->db != NULL) { + sqlite3_close_v2(h->db); /* _v2 zombie-closes even if a stmt lingers */ + h->db = NULL; + } + return false; +} + +void * +hissqlite_open(const char *path, int flags, struct history *history) +{ + struct hissqlite *h; + + h = xcalloc(1, sizeof(*h)); + h->history = history; + h->flags = flags; + + /* makehistory opens with a NULL path and supplies the real one afterwards + via HISCTLS_PATH; defer the database open until then. Until the path is + set the handle is unusable, exactly as with hisv6. */ + if (path == NULL) + return h; + + h->path = concat(path, ".sqlite", (char *) NULL); + if (!hissqlite_doopen(h)) { + free(h->path); + free(h); + return NULL; + } + return h; +} + +bool +hissqlite_sync(void *history) +{ + struct hissqlite *h = history; + + /* HISsync is deliberately NOT a durability barrier, matching dbz: hisv6_sync + does fflush(3) + msync(MS_ASYNC), neither of which waits for the disk, so + dbz history is only kernel-buffered after a sync (survives an innd crash, + lost on power loss). Under synchronous=NORMAL each autocommit already + write(2)s the WAL into the kernel page cache, so hissqlite sits + continuously in that same state: there is nothing application-buffered + for HISsync to flush. (Per-commit durability would be synchronous=FULL, a + possible future knob, not an fsync here.) innd calls HISsync on a timer, + so it is simply the convenient place to keep the WAL bounded: once it has + grown past the threshold, do a non-blocking PASSIVE checkpoint. */ + if (!h->direct_reader && h->wal_pages >= HISSQLITE_WAL_CKPT_PAGES) + hissqlite_wal_passive(h); + return true; +} + +bool +hissqlite_close(void *history) +{ + struct hissqlite *h = history; + bool ok = true; + + /* A deferred handle (HISopen(NULL) whose HISCTLS_PATH was never set or + failed) never opened a database, so there is nothing to checkpoint or + close; just free it. Skipping this would call sqlite3_wal_checkpoint_v2() + on a NULL connection, which is not a no-op like sqlite3_close_v2(NULL). */ + if (h->db == NULL) { + free(h->path); + free(h); + return true; + } + + /* Flush and reclaim the WAL on a clean shutdown: PASSIVE first (never + blocks, even if direct-reader processes still hold the DB open), then + TRUNCATE only if every frame was checkpointed (no readers pinning it) so + we don't hang waiting for readers to disconnect. Mirrors ovsqlite. */ + if (!h->direct_reader) { + int log = 0, ckpt = 0; + + if (sqlite3_wal_checkpoint_v2(h->db, NULL, SQLITE_CHECKPOINT_PASSIVE, + &log, &ckpt) + == SQLITE_OK + && log == ckpt) + sqlite3_wal_checkpoint_v2(h->db, NULL, SQLITE_CHECKPOINT_TRUNCATE, + NULL, NULL); + sqlite_helper_term(&hissqlite_main_helper, (sqlite3_stmt **) &h->main); + } else { + sqlite_helper_term(&hissqlite_read_helper, (sqlite3_stmt **) &h->read); + } + + if (sqlite3_close_v2(h->db) != SQLITE_OK) + ok = false; + free(h->path); + free(h); + return ok; +} + +bool +hissqlite_lookup(void *history, const char *key, time_t *arrived, + time_t *posted, time_t *expires, struct token *token) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->direct_reader ? h->read.lookup : h->main.lookup; + bool found = false; + int status; + + bind_key(stmt, key); + status = sqlite3_step(stmt); + if (status == SQLITE_ROW) { + found = true; + if (arrived != NULL) + *arrived = (time_t) sqlite3_column_int64(stmt, 0); + if (posted != NULL) + *posted = (time_t) sqlite3_column_int64(stmt, 1); + if (expires != NULL) + *expires = (time_t) sqlite3_column_int64(stmt, 2); + if (token != NULL) { + if (sqlite3_column_type(stmt, 3) == SQLITE_NULL) { + /* Remembered entry: found, but no article. */ + memset(token, 0, sizeof(TOKEN)); + token->type = TOKEN_EMPTY; + } else if (!copy_blob(h, stmt, 3, token, sizeof(TOKEN), + "corrupt token blob in lookup")) { + /* Wrong-length token = corruption. Treat as a lookup failure + (reported above) rather than masking it as a no-article. */ + found = false; + } + } + } else if (status != SQLITE_DONE) { + hissqlite_seterror(h, "lookup"); + } + sqlite3_reset(stmt); + return found; +} + +bool +hissqlite_check(void *history, const char *key) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->direct_reader ? h->read.check : h->main.check; + bool found; + int status; + + /* Existence only: real OR remembered both count (refuse re-offers). + his.c keeps an in-memory existence cache in front of this. */ + bind_key(stmt, key); + status = sqlite3_step(stmt); + found = (status == SQLITE_ROW); + if (status != SQLITE_ROW && status != SQLITE_DONE) + hissqlite_seterror(h, "check"); + sqlite3_reset(stmt); + return found; +} + +bool +hissqlite_write(void *history, const char *key, time_t arrived, time_t posted, + time_t expires, const struct token *token) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->main.write; + bool ok; + + bind_key(stmt, key); + sqlite3_bind_int64(stmt, 2, (sqlite3_int64) arrived); + sqlite3_bind_int64(stmt, 3, (sqlite3_int64) posted); + sqlite3_bind_int64(stmt, 4, (sqlite3_int64) expires); + sqlite3_bind_blob(stmt, 5, token, sizeof(TOKEN), SQLITE_TRANSIENT); + ok = (sqlite3_step(stmt) == SQLITE_DONE); + if (!ok) + hissqlite_seterror(h, "write"); + else if (sqlite3_changes(h->db) == 0 && h->history != NULL) + /* The .write INSERT used ON CONFLICT DO NOTHING, so a duplicate hash + changed nothing. Report it (his_seterror logs a warning) but still + return success, so a makehistory rebuild over a spool with duplicate + Message-IDs warns and continues instead of aborting -- as hisv6 does + (hisv6.c hisv6_writedbz, DBZSTORE_EXISTS). */ + his_seterror(h->history, + concat("hissqlite: duplicate message-id, write ignored: ", + key, (char *) NULL)); + sqlite3_reset(stmt); + return ok; +} + +bool +hissqlite_remember(void *history, const char *key, time_t arrived, + time_t posted) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->main.remember; + bool ok; + + /* INSERT ... ON CONFLICT DO NOTHING: never downgrade a real article. */ + bind_key(stmt, key); + sqlite3_bind_int64(stmt, 2, (sqlite3_int64) arrived); + sqlite3_bind_int64(stmt, 3, (sqlite3_int64) posted); + ok = (sqlite3_step(stmt) == SQLITE_DONE); + if (!ok) + hissqlite_seterror(h, "remember"); + sqlite3_reset(stmt); + return ok; +} + +bool +hissqlite_replace(void *history, const char *key, time_t arrived, + time_t posted, time_t expires, const struct token *token) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->main.replace; + bool ok; + + /* The only token-changing op. A NULL token downgrades real->remembered + (prunehistory); a non-NULL token upgrades remembered->real. */ + bind_key(stmt, key); + sqlite3_bind_int64(stmt, 2, (sqlite3_int64) arrived); + sqlite3_bind_int64(stmt, 3, (sqlite3_int64) posted); + sqlite3_bind_int64(stmt, 4, (sqlite3_int64) expires); + if (token == NULL) + sqlite3_bind_null(stmt, 5); + else + sqlite3_bind_blob(stmt, 5, token, sizeof(TOKEN), SQLITE_TRANSIENT); + ok = (sqlite3_step(stmt) == SQLITE_DONE); + if (!ok) + hissqlite_seterror(h, "replace"); + sqlite3_reset(stmt); + return ok; +} + +bool +hissqlite_walk(void *history, const char *reason UNUSED, void *cookie, + bool (*callback)(void *, const HASH *, time_t, time_t, time_t, + const TOKEN *)) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->direct_reader ? h->read.walk : h->main.walk; + bool ok = true; + int status; + + /* WAL gives a consistent read snapshot, unlike hisv6, no + ICCpause/straggler re-scan is needed even while innd writes. */ + while ((status = sqlite3_step(stmt)) == SQLITE_ROW) { + HASH hash; + TOKEN token, *tp; + time_t arrived, posted, expires; + + if (!copy_blob(h, stmt, 0, &hash, sizeof(HASH), + "corrupt hash blob in walk")) { + ok = false; + break; + } + arrived = (time_t) sqlite3_column_int64(stmt, 1); + posted = (time_t) sqlite3_column_int64(stmt, 2); + expires = (time_t) sqlite3_column_int64(stmt, 3); + if (sqlite3_column_type(stmt, 4) == SQLITE_NULL) { + tp = NULL; /* Remembered entry: callback sees no token. */ + } else { + if (!copy_blob(h, stmt, 4, &token, sizeof(TOKEN), + "corrupt token blob in walk")) { + ok = false; + break; + } + tp = &token; + } + if (!(*callback)(cookie, &hash, arrived, posted, expires, tp)) { + ok = false; + break; + } + } + if (status != SQLITE_DONE && ok) { + hissqlite_seterror(h, "walk"); + ok = false; + } + sqlite3_reset(stmt); + return ok; +} + +/* A pending in-place action collected during the pass-1 scan, so we do not + mutate the table through the same cursor we are scanning. */ +struct expire_action { + HASH hash; + TOKEN token; + time_t expires; + bool transition; /* true: real -> remembered; false: keep, rewrite token */ +}; + +bool +hissqlite_expire(void *history, const char *path UNUSED, + const char *reason UNUSED, bool writing, void *cookie, + time_t threshold, + bool (*decide)(void *, time_t, time_t, time_t, struct token *)) +{ + struct hissqlite *h = history; + struct expire_action *acts; + HASH last; + bool ok = true, more = true, first = true; + + /* + * hisv6 rewrites the whole file; we mutate in place. Pass 1: evaluate + * every token-bearing entry via the policy callback. A real article whose + * retention has passed is UPDATEd to remembered (token=NULL) NOT deleted + * so the Message-ID is not re-accepted. Pass 2: delete remembered + * entries older than the /remember/ posting-time threshold. + * + * The keep/remove decision is made entirely by the caller's callback + * (expire.c's EXPdoline), including the groupbaseexpiry case where article + * retention is driven by expireover plus storage existence and the tombstone + * log rather than expire.ctl: this backend is decision-agnostic and only + * applies the verdict in place. + * + * Concurrency: expire is a separate writer process with its own r/w + * connection, coordinating with innd via the WAL write-lock + busy_timeout + * (the two-writer model described in the file header). + * Nothing is wrapped in a transaction: each apply autocommits a single + * UPDATE/DELETE so the lock is held one statement at a time. Pass 1 streams + * the scan in hash-keyset pages: each page reads under its own short read + * snapshot which is reset before the page is applied, so a long expire never + * pins the WAL (which would block innd's checkpoints) and RAM is bounded to + * one page rather than the whole change set. + */ + + memset(&last, 0, sizeof(last)); + acts = xmalloc(HISSQLITE_EXPIRE_BATCH * sizeof(*acts)); + + while (ok && more) { + sqlite3_stmt *scan = h->main.expire_scan; + HASH maxhash; + size_t n = 0; /* actions collected this page */ + int rows = 0; /* rows read this page */ + int status; + + memset(&maxhash, 0, sizeof(maxhash)); + /* Resume after the previous page's last hash (an empty blob sorts + before every hash, so the first page starts at the beginning). */ + if (first) + sqlite3_bind_blob(scan, 1, "", 0, SQLITE_STATIC); + else + sqlite3_bind_blob(scan, 1, &last, sizeof(HASH), SQLITE_TRANSIENT); + sqlite3_bind_int(scan, 2, HISSQLITE_EXPIRE_BATCH); + + while ((status = sqlite3_step(scan)) == SQLITE_ROW) { + TOKEN token, ltoken; + time_t arrived, posted, expires; + bool keep; + + /* maxhash tracks every row read (rows come in hash order), so + after the page it is the resume point. */ + if (!copy_blob(h, scan, 0, &maxhash, sizeof(HASH), + "corrupt hash blob in expire scan")) { + ok = false; + break; + } + arrived = (time_t) sqlite3_column_int64(scan, 1); + posted = (time_t) sqlite3_column_int64(scan, 2); + expires = (time_t) sqlite3_column_int64(scan, 3); + if (!copy_blob(h, scan, 4, &token, sizeof(TOKEN), + "corrupt token blob in expire scan")) { + ok = false; + break; + } + rows++; + ltoken = token; + + keep = (*decide)(cookie, arrived, posted, expires, <oken); + if (keep && memcmp(<oken, &token, sizeof(TOKEN)) == 0) + continue; /* Unchanged: nothing to do. */ + + acts[n].hash = maxhash; + acts[n].transition = !keep; + acts[n].token = ltoken; + acts[n].expires = keep ? expires : 0; + n++; + } + if (status != SQLITE_DONE && ok) { + hissqlite_seterror(h, "expire scan"); + ok = false; + } + sqlite3_reset(scan); /* release the read snapshot before applying */ + + /* Apply this page (only when writing; the expire(8) -t dry run passes + writing=false, so it just scans and reports via the callback). Each + statement autocommits, holding the write lock one row at a time so + innd's accepts interleave. */ + for (size_t i = 0; writing && ok && i < n; i++) { + sqlite3_stmt *stmt; + + if (acts[i].transition) { + stmt = h->main.transition_remember; + sqlite3_bind_blob(stmt, 1, &acts[i].hash, sizeof(HASH), + SQLITE_TRANSIENT); + } else { + stmt = h->main.update_token; + sqlite3_bind_blob(stmt, 1, &acts[i].hash, sizeof(HASH), + SQLITE_TRANSIENT); + sqlite3_bind_blob(stmt, 2, &acts[i].token, sizeof(TOKEN), + SQLITE_TRANSIENT); + sqlite3_bind_int64(stmt, 3, (sqlite3_int64) acts[i].expires); + } + if (sqlite3_step(stmt) != SQLITE_DONE) { + hissqlite_seterror(h, "expire apply"); + ok = false; + } + sqlite3_reset(stmt); + } + + if (rows < HISSQLITE_EXPIRE_BATCH) + more = false; /* short page -> end of table */ + else + last = maxhash; /* resume after the last row read */ + first = false; + } + free(acts); + + /* Pass 2: delete remembered entries past the /remember/ threshold, in + bounded chunks (the statement deletes up to a LIMIT per step). Each step + autocommits, so the write lock is released between chunks and innd's + accepts interleave; loop until a chunk deletes nothing. */ + if (writing && ok) { + sqlite3_stmt *stmt = h->main.expire_remembered; + + do { + sqlite3_bind_int64(stmt, 1, (sqlite3_int64) threshold); + if (sqlite3_step(stmt) != SQLITE_DONE) { + hissqlite_seterror(h, "expire remembered"); + ok = false; + } + sqlite3_reset(stmt); + } while (ok && sqlite3_changes(h->db) > 0); + } + return ok; +} + +bool +hissqlite_ctl(void *history, int selector, void *val) +{ + struct hissqlite *h = history; + + switch (selector) { + case HISCTLG_PATH: + *(char **) val = h->path; + return true; + case HISCTLS_SYNCCOUNT: + /* No-op: writes autocommit, so there is no batch size to set. */ + return true; + case HISCTLS_STATINTERVAL: + /* No-op: WAL readers always see a consistent snapshot; there is no + swapped-inode poll as in hisv6. */ + return true; + case HISCTLS_NPAIRS: + case HISCTLS_IGNOREOLD: + /* No-op: the B-tree grows itself; there is no fixed-size table to + presize and no rebuild-to-grow. */ + return true; + case HISCTLG_INPLACEEXPIRE: + /* hissqlite expires in place (UPDATE/DELETE on the live DB), so + expire(8) must open it read/write, not the hisv6 rebuild-and-swap + way. See hissqlite_expire(). */ + *(bool *) val = true; + return true; + case HISCTLS_PATH: + /* Deferred-open path (makehistory): HISopen(NULL) left the database + unopened; set the real path now and perform the open/create. Refuse + a second path set, as hisv6 does. */ + if (h->path != NULL) { + his_seterror(h->history, + concat("hissqlite: path already set in handle", + (char *) NULL)); + return false; + } + h->path = concat((char *) val, ".sqlite", (char *) NULL); + if (!hissqlite_doopen(h)) { + free(h->path); + h->path = NULL; + return false; + } + return true; + default: + return false; + } +} + +#else /* ! HAVE_SQLITE3 */ + +void * +hissqlite_open(const char *path UNUSED, int flags UNUSED, + struct history *history UNUSED) +{ + warn("hissqlite: SQLite support not enabled"); + return NULL; +} + +bool +hissqlite_close(void *history UNUSED) +{ + return false; +} + +bool +hissqlite_sync(void *history UNUSED) +{ + return false; +} + +bool +hissqlite_lookup(void *history UNUSED, const char *key UNUSED, + time_t *arrived UNUSED, time_t *posted UNUSED, + time_t *expires UNUSED, struct token *token UNUSED) +{ + return false; +} + +bool +hissqlite_check(void *history UNUSED, const char *key UNUSED) +{ + return false; +} + +bool +hissqlite_write(void *history UNUSED, const char *key UNUSED, + time_t arrived UNUSED, time_t posted UNUSED, + time_t expires UNUSED, const struct token *token UNUSED) +{ + return false; +} + +bool +hissqlite_replace(void *history UNUSED, const char *key UNUSED, + time_t arrived UNUSED, time_t posted UNUSED, + time_t expires UNUSED, const struct token *token UNUSED) +{ + return false; +} + +bool +hissqlite_expire(void *history UNUSED, const char *path UNUSED, + const char *reason UNUSED, bool writing UNUSED, + void *cookie UNUSED, time_t threshold UNUSED, + bool (*decide)(void *, time_t, time_t, time_t, + struct token *) UNUSED) +{ + return false; +} + +bool +hissqlite_walk(void *history UNUSED, const char *reason UNUSED, + void *cookie UNUSED, + bool (*callback)(void *, const HASH *, time_t, time_t, time_t, + const struct token *) UNUSED) +{ + return false; +} + +bool +hissqlite_remember(void *history UNUSED, const char *key UNUSED, + time_t arrived UNUSED, time_t posted UNUSED) +{ + return false; +} + +bool +hissqlite_ctl(void *history UNUSED, int selector UNUSED, void *val UNUSED) +{ + return false; +} + +#endif /* ! HAVE_SQLITE3 */ diff --git a/history/hissqlite/hissqlite.h b/history/hissqlite/hissqlite.h new file mode 100644 index 000000000..4c9e7da56 --- /dev/null +++ b/history/hissqlite/hissqlite.h @@ -0,0 +1,48 @@ +/* +** Internal history API interface for the SQLite (hissqlite) backend. +** +** Exposes the HIS_METHOD vtable functions to history/hismethods.c (which is +** auto-generated by buildconfig from the per-method hismethod.config files). +** See hissqlite(5) for the design rationale. +*/ + +#ifndef HISSQLITE_H +#define HISSQLITE_H + +#include "inn/libinn.h" + +struct token; +struct history; + +void *hissqlite_open(const char *path, int flags, struct history *); + +bool hissqlite_close(void *); + +bool hissqlite_sync(void *); + +bool hissqlite_lookup(void *, const char *key, time_t *arrived, time_t *posted, + time_t *expires, struct token *token); + +bool hissqlite_check(void *, const char *key); + +bool hissqlite_write(void *, const char *key, time_t arrived, time_t posted, + time_t expires, const struct token *token); + +bool hissqlite_replace(void *, const char *key, time_t arrived, time_t posted, + time_t expires, const struct token *token); + +bool hissqlite_expire(void *, const char *, const char *, bool, void *, + time_t threshold, + bool (*exists)(void *, time_t, time_t, time_t, + struct token *)); + +bool hissqlite_walk(void *, const char *, void *, + bool (*)(void *, const HASH *, time_t, time_t, time_t, + const struct token *)); + +bool hissqlite_remember(void *, const char *key, time_t arrived, + time_t posted); + +bool hissqlite_ctl(void *, int, void *); + +#endif /* HISSQLITE_H */ diff --git a/include/inn/history.h b/include/inn/history.h index 895242aa5..290722b77 100644 --- a/include/inn/history.h +++ b/include/inn/history.h @@ -83,7 +83,15 @@ enum { /* (time_t) interval, in s, between stats of the history database * for * detecting a replacement, or 0 to disable (no checks); * defaults {hisv6, taggedhash} */ - HISCTLS_STATINTERVAL + HISCTLS_STATINTERVAL, + + /* (bool) does this backend expire IN PLACE (mutating the live database + * via a writer) rather than rebuilding a new file and swapping it? + * Backends that do not implement this selector leave the value untouched, + * so the caller must initialise it to false. expire(8) uses this to + * decide whether to open the history read/write (in place) or read-only + * (rebuild-and-swap, the hisv6 model). */ + HISCTLG_INPLACEEXPIRE }; struct history *HISopen(const char *, const char *, int); diff --git a/include/inn/innconf.h b/include/inn/innconf.h index 91a40376a..a1703efff 100644 --- a/include/inn/innconf.h +++ b/include/inn/innconf.h @@ -72,7 +72,12 @@ struct innconf { unsigned long wipexpire; /* How long to keep pending article record */ /* History settings */ - char *hismethod; /* Which history method to use */ + char *hismethod; /* Which history method to use */ + unsigned long hissqlitecachesize; /* hissqlite writer page cache, in kB */ + unsigned long hissqlitemmapsize; /* hissqlite mmap size in bytes; 0 = off */ + unsigned long hissqlitepagesize; /* hissqlite database page size, in bytes */ + unsigned long + hissqlitereadercachesize; /* hissqlite per-nnrpd reader cache, in kB */ /* Article Storage */ unsigned long cnfscheckfudgesize; /* Additional CNFS integrity checking */ diff --git a/storage/ovsqlite/sqlite-helper.h b/include/inn/sqlite-helper.h similarity index 100% rename from storage/ovsqlite/sqlite-helper.h rename to include/inn/sqlite-helper.h diff --git a/lib/Makefile b/lib/Makefile index 7299cd71b..f8890e882 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -6,7 +6,7 @@ include ../Makefile.global # recover from make update. We can't use .OLD extensions for the shared # library since ldconfig will think .OLD sorts after the regular library and # will point the binaries at the old library. -LTVERSION = 9:2:0 +LTVERSION = 9:3:0 top = .. CFLAGS = $(GCFLAGS) @@ -24,7 +24,7 @@ SOURCES = argparse.c artnumber.c bloom.c buffer.c cleanfrom.c \ remopen.c reservedfd.c resource.c secrets.c sendarticle.c \ sendpass.c \ sequence.c timer.c tst.c uwildmat.c vector.c wire.c \ - xfopena.c xmalloc.c xsignal.c xwrite.c + xfopena.c xmalloc.c xsignal.c xwrite.c sqlite-helper.c # Sources for additional functions only built to replace missing system ones. EXTRA_SOURCES = asprintf.c daemon.c fseeko.c ftello.c getaddrinfo.c \ @@ -39,7 +39,7 @@ LOBJECTS = $(OBJECTS:.o=.lo) .SUFFIXES: .lo -all: libinn.$(EXTLIB) canlock.o perl.o +all: libinn.$(EXTLIB) canlock.o perl.o sqlite-helper-gen warnings: $(MAKE) COPT="$(COPT) $(WARNINGS)" all @@ -52,11 +52,15 @@ bootstrap: clean clobber distclean maintclean: rm -f *.o *.lo libinn.la libinn.a rm -f profiled perl$(PROFSUFFIX).o libinn$(PROFSUFFIX).a - rm -f libinn_pure_*.a .pure + rm -f libinn_pure_*.a .pure sqlite-helper-gen rm -rf .libs +# libinn includes sqlite-helper.lo, so it must record the SQLite dependency +# (a no-op, empty libs, when configured --without-sqlite3) -- otherwise every +# executable linking libinn would need to name -lsqlite3 itself. libinn.la: $(OBJECTS) $(LOBJECTS) - $(LIBLD) $(LDFLAGS) -o $@ $(LOBJECTS) $(LIBS) \ + $(LIBLD) $(LDFLAGS) -o $@ $(LOBJECTS) \ + $(SQLITE3_LDFLAGS) $(SQLITE3_LIBS) $(LIBS) \ -rpath $(PATHLIB) -version-info $(LTVERSION) libinn.a: $(OBJECTS) @@ -72,6 +76,18 @@ canlock.o: canlock.c perl.o: perl.c $(CC) $(CFLAGS) $(PERL_CPPFLAGS) -c perl.c +# sqlite-helper.c needs the SQLite headers; scope the flags to just this object +# (the file is a no-op when built without HAVE_SQLITE3). +sqlite-helper.o sqlite-helper.lo: sqlite-helper.c ../include/inn/sqlite-helper.h + $(LIBCC) $(CFLAGS) $(SQLITE3_CPPFLAGS) -c sqlite-helper.c + +# Shared SQL prepared-statement codegen, used by the ovsqlite overview method +# and the hissqlite history method. +FIX = $(SHELL) $(FIXSCRIPT) + +sqlite-helper-gen: sqlite-helper-gen.in $(FIXSCRIPT) + $(FIX) -i sqlite-helper-gen.in + ../include/inn/system.h: (cd ../include && $(MAKE)) diff --git a/lib/innconf.c b/lib/innconf.c index 52d18c2d2..8335cca7c 100644 --- a/lib/innconf.c +++ b/lib/innconf.c @@ -226,6 +226,10 @@ static const struct config config_table[] = { /* The following settings are specific to the history subsystem. */ {K(hismethod), STRING(NULL) }, + {K(hissqlitecachesize), UNUMBER(65536) }, + {K(hissqlitemmapsize), UNUMBER(0) }, + {K(hissqlitepagesize), UNUMBER(4096) }, + {K(hissqlitereadercachesize), UNUMBER(2000) }, /* The following settings are specific to rc.news. */ {K(docnfsstat), BOOL(false) }, @@ -491,6 +495,17 @@ innconf_validate(struct config_group *group) innconf->datamovethreshold = 1024 * 1024; } + /* hissqlitepagesize must be a power of two between 512 and 65536, as + required by SQLite's pragma page_size. */ + if (innconf->hissqlitepagesize < 512 || innconf->hissqlitepagesize > 65536 + || (innconf->hissqlitepagesize & (innconf->hissqlitepagesize - 1)) + != 0) { + config_error_param(group, "hissqlitepagesize", + "hissqlitepagesize must be a power of 2 between 512" + " and 65536"); + innconf->hissqlitepagesize = 4096; + } + if (innconf->docancels != NULL && strcasecmp(innconf->docancels, "require-auth") != 0 && strcasecmp(innconf->docancels, "auth") != 0 diff --git a/storage/ovsqlite/sqlite-helper-gen.in b/lib/sqlite-helper-gen.in similarity index 99% rename from storage/ovsqlite/sqlite-helper-gen.in rename to lib/sqlite-helper-gen.in index 87688aa64..550d92043 100644 --- a/storage/ovsqlite/sqlite-helper-gen.in +++ b/lib/sqlite-helper-gen.in @@ -108,7 +108,7 @@ $preamble #ifndef $guard #define $guard 1 -#include "sqlite-helper.h" +#include "inn/sqlite-helper.h" #ifdef HAVE_SQLITE3 diff --git a/storage/ovsqlite/sqlite-helper.c b/lib/sqlite-helper.c similarity index 98% rename from storage/ovsqlite/sqlite-helper.c rename to lib/sqlite-helper.c index 607b04117..300daea8c 100644 --- a/storage/ovsqlite/sqlite-helper.c +++ b/lib/sqlite-helper.c @@ -1,4 +1,4 @@ -#include "sqlite-helper.h" +#include "inn/sqlite-helper.h" #ifdef HAVE_SQLITE3 diff --git a/storage/Makefile b/storage/Makefile index 02f787bd3..e9b7544db 100644 --- a/storage/Makefile +++ b/storage/Makefile @@ -283,13 +283,9 @@ ovsqlite/ovsqlite.o: ovsqlite/ovsqlite.c ovsqlite/ovsqlite.h ../include/config.h ../include/inn/confparse.h ../include/inn/fdflag.h \ ../include/inn/portable-socket.h ../include/inn/innconf.h \ ../include/inn/newsuser.h ../include/inn/paths.h \ - ovsqlite/../ovinterface.h ovsqlite/sql-read.h ovsqlite/sqlite-helper.h + ovsqlite/../ovinterface.h ovsqlite/sql-read.h ../include/inn/sqlite-helper.h ovsqlite/sql-read.o: ovsqlite/sql-read.c ovsqlite/sql-read.h \ - ovsqlite/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ - ../include/inn/portable-macros.h ../include/inn/options.h \ - ../include/inn/system.h ../include/portable/stdbool.h -ovsqlite/sqlite-helper.o: ovsqlite/sqlite-helper.c ovsqlite/sqlite-helper.h \ - ../include/config.h ../include/inn/macros.h \ + ../include/inn/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ ../include/inn/portable-macros.h ../include/inn/options.h \ ../include/inn/system.h ../include/portable/stdbool.h timecaf/caf.o: timecaf/caf.c ../include/portable/system.h ../include/config.h \ @@ -418,13 +414,13 @@ ovsqlite/ovsqlite-server.o: ovsqlite/ovsqlite-server.c ../include/config.h \ ../include/inn/innconf.h ../include/inn/libinn.h \ ../include/inn/xmalloc.h ../include/inn/system.h ../include/inn/xwrite.h \ ../include/inn/storage.h ../include/inn/options.h ovsqlite/sql-init.h \ - ovsqlite/sqlite-helper.h ovsqlite/sql-main.h + ../include/inn/sqlite-helper.h ovsqlite/sql-main.h ovsqlite/sql-init.o: ovsqlite/sql-init.c ovsqlite/sql-init.h \ - ovsqlite/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ + ../include/inn/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ ../include/inn/portable-macros.h ../include/inn/options.h \ ../include/inn/system.h ../include/portable/stdbool.h ovsqlite/sql-main.o: ovsqlite/sql-main.c ovsqlite/sql-main.h \ - ovsqlite/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ + ../include/inn/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ ../include/inn/portable-macros.h ../include/inn/options.h \ ../include/inn/system.h ../include/portable/stdbool.h tradindexed/tdx-util.o: tradindexed/tdx-util.c ../include/portable/system.h \ diff --git a/storage/ovsqlite/ovmethod.config b/storage/ovsqlite/ovmethod.config index 5ba6f5612..31d7aa914 100644 --- a/storage/ovsqlite/ovmethod.config +++ b/storage/ovsqlite/ovmethod.config @@ -1,7 +1,6 @@ name = ovsqlite number = 5 -sources = ovsqlite.c ovsqlite-private.c sqlite-helper.c sql-read.c +sources = ovsqlite.c ovsqlite-private.c sql-read.c extra-sources = ovsqlite-server.c sql-main.c sql-init.c programs = ovsqlite-server ovsqlite-util -clean = sqlite-helper-gen maintclean = sql-init.c sql-init.h sql-main.c sql-main.h sql-read.c sql-read.h diff --git a/storage/ovsqlite/ovmethod.mk b/storage/ovsqlite/ovmethod.mk index 6fc0f0044..fa53b971b 100644 --- a/storage/ovsqlite/ovmethod.mk +++ b/storage/ovsqlite/ovmethod.mk @@ -11,21 +11,20 @@ ovsqlite/ovsqlite-server: $(OVSQLITEOBJECTS) libinnstorage.$(EXTLIB) ovsqlite/ovsqlite-util: ovsqlite/ovsqlite-util.in $(FIXSCRIPT) $(FIX) ovsqlite/ovsqlite-util.in -ovsqlite/sqlite-helper-gen: ovsqlite/sqlite-helper-gen.in $(FIXSCRIPT) - $(FIX) -i ovsqlite/sqlite-helper-gen.in +## sqlite-helper-gen now lives in lib/ (shared with the hissqlite history +## method); lib is built before storage so it is available here. -ovsqlite/sql-main.c: ovsqlite/sql-main.sql ovsqlite/sqlite-helper-gen - ovsqlite/sqlite-helper-gen ovsqlite/sql-main.sql +ovsqlite/sql-main.c: ovsqlite/sql-main.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen ovsqlite/sql-main.sql ovsqlite/sql-main.h: ovsqlite/sql-main.c ; -ovsqlite/sql-init.c: ovsqlite/sql-init.sql ovsqlite/sqlite-helper-gen - ovsqlite/sqlite-helper-gen ovsqlite/sql-init.sql +ovsqlite/sql-init.c: ovsqlite/sql-init.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen ovsqlite/sql-init.sql ovsqlite/sql-init.h: ovsqlite/sql-init.c ; -ovsqlite/sql-read.c: ovsqlite/sql-read.sql ovsqlite/sqlite-helper-gen - ovsqlite/sqlite-helper-gen ovsqlite/sql-read.sql +ovsqlite/sql-read.c: ovsqlite/sql-read.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen ovsqlite/sql-read.sql ovsqlite/sql-read.h: ovsqlite/sql-read.c ; - diff --git a/storage/ovsqlite/ovsqlite.c b/storage/ovsqlite/ovsqlite.c index eae8cd6b6..964ffb633 100644 --- a/storage/ovsqlite/ovsqlite.c +++ b/storage/ovsqlite/ovsqlite.c @@ -37,7 +37,7 @@ # include "../ovinterface.h" # include "sql-read.h" -# include "sqlite-helper.h" +# include "inn/sqlite-helper.h" # include # ifdef HAVE_ZLIB diff --git a/support/mkmanifest b/support/mkmanifest index a8f73377e..b0db9a285 100755 --- a/support/mkmanifest +++ b/support/mkmanifest @@ -275,7 +275,11 @@ storage/buffindexed/buffindexed_d storage/buildconfig storage/ovsqlite/ovsqlite-server storage/ovsqlite/ovsqlite-util -storage/ovsqlite/sqlite-helper-gen +lib/sqlite-helper-gen +history/hissqlite/hissqlite-convert +history/hissqlite/hissqlite-util +tests/lib/hissqlite.t +tests/lib/hissqlite-convert.t storage/tradindexed/tdx-util support/fixconfig support/fixscript diff --git a/tests/Makefile b/tests/Makefile index d2d3004c6..7351985e2 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -19,6 +19,7 @@ LIBM_LDFLAGS = '-lm' TESTS = authprogs/ident.t expire/tombstone.t expire/tombstone-hisexpire.t \ innd/artparse.t innd/chan.t lib/artnumber.t \ lib/asprintf.t lib/bloom.t lib/bloom-hiswalk.t lib/buffer.t \ + lib/hissqlite.t lib/hissqlite-convert.t \ lib/canlock.t lib/concat.t lib/conffile.t \ lib/confparse.t lib/daemon.t lib/date.t \ lib/dispatch.t lib/fdflag.t \ @@ -123,6 +124,12 @@ lib/bloom.t: lib/bloom-t.o tap/basic.o $(LIBINN) lib/bloom-hiswalk.t: lib/bloom-hiswalk-t.o tap/basic.o $(STORAGEDEPS) $(LINKDEPS) lib/bloom-hiswalk-t.o tap/basic.o $(STORAGELIBS) $(LIBS) +lib/hissqlite.t: lib/hissqlite-t.o tap/basic.o $(STORAGEDEPS) + $(LINKDEPS) lib/hissqlite-t.o tap/basic.o $(STORAGELIBS) $(LIBS) + +lib/hissqlite-convert.t: lib/hissqlite-convert-t.o tap/basic.o $(STORAGEDEPS) + $(LINKDEPS) lib/hissqlite-convert-t.o tap/basic.o $(STORAGELIBS) $(LIBS) + lib/buffer.t: lib/buffer-t.o tap/basic.o $(LIBINN) $(LINK) lib/buffer-t.o tap/basic.o $(LIBINN) diff --git a/tests/TESTS b/tests/TESTS index 1455d2c38..7cecb2506 100644 --- a/tests/TESTS +++ b/tests/TESTS @@ -27,6 +27,8 @@ lib/hash lib/hashtab lib/headers lib/hex +lib/hissqlite +lib/hissqlite-convert lib/inet_aton lib/inet_ntoa lib/inet_ntop diff --git a/tests/lib/hissqlite-convert-t.c b/tests/lib/hissqlite-convert-t.c new file mode 100644 index 000000000..a3ba527ee --- /dev/null +++ b/tests/lib/hissqlite-convert-t.c @@ -0,0 +1,202 @@ +/* +** End-to-end test for the hissqlite-convert migration tool. +** +** Seeds a small hisv6 history (real + remembered entries, distinct +** timestamps), runs the hissqlite-convert binary against it, and verifies the +** resulting hissqlite database through the public HIS API: the faithful +** migration must preserve tokens, timestamps and -- crucially -- the +** remembered (token-less) entries that a from-spool rebuild cannot +** reconstruct. Also checks the refuse-to-overwrite guard and that the +** temporary build file is cleaned up. +** +** This drives the real binary (via system()) rather than the library, so it +** needs inn.conf: a minimal one is written to the temp dir and pointed at with +** INNCONF, exactly as the shell integration tests do. +** +** Written by Kevin Bowling in 2026. +*/ + +#include "portable/system.h" + +#include "inn/history.h" +#include "inn/innconf.h" +#include "inn/libinn.h" +#include "inn/messages.h" +#include "inn/storage.h" +#include "tap/basic.h" + +#ifndef HAVE_SQLITE3 + +int +main(void) +{ + skip_all("not built with SQLite"); + return 0; +} + +#else + +# include +# include + +# define BASE ((time_t) 1600000000) + +static const char convert[] = "../../history/hissqlite/hissqlite-convert"; + +/* Run " " with output suppressed; return its exit code, + or -1 if it could not be run. */ +static int +run_convert(const char *src, const char *dst) +{ + char cmd[768]; /* Should provide a PATH_MAX fill in build somewhere */ + int status; + + snprintf(cmd, sizeof(cmd), "%s %s %s >/dev/null 2>&1", convert, src, dst); + status = system(cmd); + if (status == -1 || !WIFEXITED(status)) + return -1; + return WEXITSTATUS(status); +} + +int +main(void) +{ + char tmpdir[64], conf[128], src[128], dst[128], dstdb[160], tempdb[160]; + struct history *h; + FILE *f; + TOKEN t1, t2, got; + bool reals_ok, remembered_ok, absent_ok; + struct stat sb; + + /* runtests runs us from tests/; move into our own directory so the + relative path to the binary resolves */ + if (access("hissqlite-convert.t", F_OK) < 0) + if (access("lib/hissqlite-convert.t", F_OK) == 0) + if (chdir("lib") < 0) + sysbail("cannot chdir to lib"); + if (access(convert, X_OK) != 0) + skip_all("hissqlite-convert not built"); + + test_init(8); + + strlcpy(tmpdir, "hissqlite-conv-XXXXXX", sizeof(tmpdir)); + if (mkdtemp(tmpdir) == NULL) + sysbail("can't create temp directory"); + snprintf(conf, sizeof(conf), "%s/inn.conf", tmpdir); + snprintf(src, sizeof(src), "%s/src", tmpdir); + snprintf(dst, sizeof(dst), "%s/dst", tmpdir); + snprintf(dstdb, sizeof(dstdb), "%s/dst.sqlite", tmpdir); + snprintf(tempdb, sizeof(tempdb), "%s/dst.new.sqlite", tmpdir); + + /* Minimal inn.conf so both this process and the convert binary can read a + valid configuration (convert calls innconf_read). */ + f = fopen(conf, "w"); + if (f == NULL) + sysbail("can't write %s", conf); + fprintf(f, + "domain: news.example.com\n" + "mta: \"/bin/true %%s\"\n" + "hismethod: hisv6\n" + "enableoverview: false\n" + "wireformat: true\n" + "pathnews: %s\n" + "pathdb: %s\n" + "pathetc: %s\n", + tmpdir, tmpdir, tmpdir); + fclose(f); + if (setenv("INNCONF", conf, 1) != 0) + sysbail("can't set INNCONF"); + if (!innconf_read(NULL)) + bail("can't read test inn.conf"); + + /* Seed a hisv6 source: two real articles with distinct timestamps/tokens + and two remembered (token-less) entries. */ + h = HISopen(src, "hisv6", HIS_CREAT | HIS_RDWR); + if (h == NULL) + bail("can't create hisv6 source"); + memset(&t1, 0, sizeof(t1)); + t1.type = 1; + memcpy(t1.token, "0123456789abcdef", sizeof(t1.token)); + memset(&t2, 0, sizeof(t2)); + t2.type = 2; + memcpy(t2.token, "fedcba9876543210", sizeof(t2.token)); + if (!HISwrite(h, "", BASE + 1, BASE + 1, BASE + 5000, &t1) + || !HISwrite(h, "", BASE + 2, BASE + 7, (time_t) 0, &t2) + || !HISremember(h, "", BASE + 3, BASE + 3) + || !HISremember(h, "", BASE + 4, BASE + 4)) + bail("can't seed hisv6 source"); + HISsync(h); + HISclose(h); + + /* Convert hisv6 -> hissqlite. */ + ok(1, run_convert(src, dst) == 0); + + /* Verify the result. The converter leaves the database in a non-WAL + journal mode (the first innd open flips it to WAL), so open read/write + rather than via the WAL-only direct reader. */ + h = HISopen(dst, "hissqlite", HIS_RDWR); + if (h == NULL) + bail("can't open converted hissqlite database"); + + { + time_t a = 0, p = 0, e = -1; + bool r1, r2; + + memset(&got, 0xff, sizeof(got)); + r1 = HISlookup(h, "", &a, &p, &e, &got) && got.type == 1 + && memcmp(got.token, t1.token, sizeof(t1.token)) == 0 + && a == BASE + 1 && p == BASE + 1 && e == BASE + 5000; + memset(&got, 0xff, sizeof(got)); + r2 = HISlookup(h, "", &a, &p, &e, &got) && got.type == 2 + && memcmp(got.token, t2.token, sizeof(t2.token)) == 0 + && a == BASE + 2 && p == BASE + 7; + reals_ok = r1 && r2; + } + ok(2, reals_ok); /* real tokens and timestamps round-trip */ + + { + TOKEN g1, g2; + bool f1, f2; + + memset(&g1, 0xff, sizeof(g1)); + memset(&g2, 0xff, sizeof(g2)); + /* Remembered entries are found but carry no token (TOKEN_EMPTY). */ + f1 = HISlookup(h, "", NULL, NULL, NULL, &g1) + && g1.type == TOKEN_EMPTY; + f2 = HISlookup(h, "", NULL, NULL, NULL, &g2) + && g2.type == TOKEN_EMPTY; + remembered_ok = f1 && f2; + } + ok(3, remembered_ok); /* remembered entries survived the migration */ + + absent_ok = !HIScheck(h, ""); + ok(4, absent_ok); + + { + bool counts; + /* Existence of every seeded entry, nothing spurious: 2 real + 2 + remembered, all present and distinguishable. */ + counts = HIScheck(h, "") && HIScheck(h, "") + && HIScheck(h, "") && HIScheck(h, ""); + ok(5, counts); + } + HISclose(h); + + /* Refuse to overwrite an existing destination: a second run must fail and + leave the database in place. */ + ok(6, run_convert(src, dst) != 0); + ok(7, stat(dstdb, &sb) == 0); /* existing database left in place */ + + /* The temporary build file must not survive a successful conversion. */ + ok(8, access(tempdb, F_OK) != 0); + + { + char cmd[160]; + snprintf(cmd, sizeof(cmd), "/bin/rm -rf %s", tmpdir); + if (system(cmd) < 0) + sysdiag("can't clean up %s", tmpdir); + } + return 0; +} + +#endif /* HAVE_SQLITE3 */ diff --git a/tests/lib/hissqlite-t.c b/tests/lib/hissqlite-t.c new file mode 100644 index 000000000..21f1209fb --- /dev/null +++ b/tests/lib/hissqlite-t.c @@ -0,0 +1,390 @@ +/* +** Runtime test for the hissqlite history backend. +** +** Exercises the full HIS_METHOD vtable through the public HIS API against a +** real SQLite database: open/create, write, remember, check, lookup, replace, +** walk, the two-horizon expire (real->remembered transition + remember-delete), +** sync, close, and persistence across reopen. +** +** Written by Kevin Bowling in 2026. +*/ + +#include "portable/system.h" + +#include "inn/history.h" +#include "inn/libinn.h" +#include "inn/messages.h" +#include "inn/storage.h" +#include "tap/basic.h" + +#ifndef HAVE_SQLITE3 + +int +main(void) +{ + skip_all("not built with SQLite"); + return 0; +} + +#else + +# include +# include + +# define N_TOKEN 100 /* entries with a storage token */ +# define N_REMEMBER 20 /* remembered (token-less) entries */ +# define BASE ((time_t) 1600000000) + +/* msgid 0..N_TOKEN-1 are token entries; N_TOKEN..N_TOKEN+N_REMEMBER-1 are + remembered. arrived == posted == BASE + i for entry i. */ +static char * +make_msgid(unsigned long n) +{ + char buf[64]; + snprintf(buf, sizeof(buf), "", n); + return xstrdup(buf); +} + +struct walkcount { + unsigned long total; + unsigned long with_token; +}; + +static bool +walk_cb(void *cookie, const HASH *hash UNUSED, time_t arrived UNUSED, + time_t posted UNUSED, time_t expires UNUSED, const TOKEN *token) +{ + struct walkcount *w = cookie; + w->total++; + if (token != NULL) + w->with_token++; + return true; +} + +/* Expire policy: keep token entries with arrived >= cutoff, drop (->remember) + the older ones. */ +static time_t expire_cutoff; +static bool +decide_drop_old(void *cookie UNUSED, time_t arrived, time_t posted UNUSED, + time_t expires UNUSED, TOKEN *token UNUSED) +{ + return arrived >= expire_cutoff; +} + +static bool +decide_keep_all(void *cookie UNUSED, time_t arrived UNUSED, time_t posted UNUSED, + time_t expires UNUSED, TOKEN *token UNUSED) +{ + return true; +} + +/* Does HISlookup find this msgid, and (optionally) does it carry a token? */ +static bool +present_with_token(struct history *h, unsigned long n, bool *has_token, + TOKEN *out) +{ + char *msgid = make_msgid(n); + TOKEN token; + bool found; + + memset(&token, 0, sizeof(token)); + found = HISlookup(h, msgid, NULL, NULL, NULL, &token); + if (has_token != NULL) + *has_token = (found && token.type != TOKEN_EMPTY); + if (out != NULL) + *out = token; + free(msgid); + return found; +} + +int +main(void) +{ + struct history *h; + char tmpdir[64], histpath[128]; + TOKEN token, t2; + unsigned long i; + struct walkcount wc; + bool has_token; + + test_init(27); + + strlcpy(tmpdir, "hissqlite-XXXXXX", sizeof(tmpdir)); + if (mkdtemp(tmpdir) == NULL) + sysbail("can't create temp directory"); + snprintf(histpath, sizeof(histpath), "%s/history", tmpdir); + + /* create + populate */ + h = HISopen(histpath, "hissqlite", HIS_CREAT | HIS_RDWR); + ok(1, h != NULL); + if (h == NULL) + bail("can't create hissqlite history at %s", histpath); + + memset(&token, 0, sizeof(token)); + token.type = 1; + memcpy(token.token, "0123456789abcdef", sizeof(token.token)); + + for (i = 0; i < N_TOKEN; i++) { + char *msgid = make_msgid(i); + if (!HISwrite(h, msgid, BASE + i, BASE + i, (time_t) 0, &token)) + bail("HISwrite %lu failed: %s", i, HISerror(h)); + free(msgid); + } + for (i = N_TOKEN; i < N_TOKEN + N_REMEMBER; i++) { + char *msgid = make_msgid(i); + if (!HISremember(h, msgid, BASE + i, BASE + i)) + bail("HISremember %lu failed: %s", i, HISerror(h)); + free(msgid); + } + ok(2, HISsync(h)); + ok(3, HISclose(h)); + + /* reopen read/write and verify */ + h = HISopen(histpath, "hissqlite", HIS_RDWR); + ok(4, h != NULL); + if (h == NULL) + bail("can't reopen hissqlite history"); + + /* check: real present, remembered present, absent absent */ + { + char *m_real = make_msgid(5); + char *m_rem = make_msgid(N_TOKEN + 1); + char *m_abs = make_msgid(99999); + ok(5, HIScheck(h, m_real)); + ok(6, HIScheck(h, m_rem)); + ok(7, !HIScheck(h, m_abs)); + free(m_real); + free(m_rem); + free(m_abs); + } + + /* lookup: real -> token round-trips; remembered -> empty token */ + { + char *m = make_msgid(5); + time_t arrived = 0, posted = 0, expires = -1; + memset(&t2, 0xff, sizeof(t2)); + ok(8, HISlookup(h, m, &arrived, &posted, &expires, &t2) + && arrived == BASE + 5 && posted == BASE + 5 + && t2.type == 1 + && memcmp(t2.token, token.token, sizeof(token.token)) == 0); + free(m); + } + ok(9, present_with_token(h, N_TOKEN + 2, &has_token, NULL) && !has_token); + ok(10, !present_with_token(h, 99999, NULL, NULL)); + + /* walk (before mutations): total and token counts */ + wc.total = wc.with_token = 0; + HISwalk(h, NULL, &wc, walk_cb); + ok(11, wc.total == N_TOKEN + N_REMEMBER); + ok(12, wc.with_token == N_TOKEN); + + /* replace: change a token; downgrade real->remembered */ + memset(&t2, 0, sizeof(t2)); + t2.type = 2; + memcpy(t2.token, "fedcba9876543210", sizeof(t2.token)); + { + char *m = make_msgid(0); + TOKEN got; + HISreplace(h, m, BASE, BASE, 0, &t2); + present_with_token(h, 0, &has_token, &got); + ok(13, has_token && got.type == 2 + && memcmp(got.token, t2.token, sizeof(t2.token)) == 0); + free(m); + } + { + char *m = make_msgid(1); /* real -> remembered (NULL token) */ + HISreplace(h, m, BASE + 1, BASE + 1, 0, NULL); + ok(14, present_with_token(h, 1, &has_token, NULL) && !has_token); + free(m); + } + + /* expire pass 1: drop old token entries -> remembered */ + expire_cutoff = BASE + N_TOKEN / 2; /* keep i>=50, drop i<50 */ + ok(15, + HISexpire(h, NULL, NULL, true, NULL, (time_t) (BASE - 1), decide_drop_old)); + /* #10 (arrived BASE+10 < cutoff) -> now remembered; #90 still a token */ + { + bool r10 = present_with_token(h, 10, &has_token, NULL); + bool t10 = has_token; + bool r90 = present_with_token(h, 90, &has_token, NULL); + bool t90 = has_token; + ok(16, r10 && !t10 && r90 && t90); + } + + /* expire pass 2: remember-delete past the threshold */ + ok(17, HISexpire(h, NULL, NULL, true, NULL, BASE + 1000000, decide_keep_all)); + /* original remembered (#N_TOKEN+5) gone; surviving token (#90) stays */ + { + char *m = make_msgid(N_TOKEN + 5); + bool rem_gone = !HIScheck(h, m); + bool tok_stays = present_with_token(h, 90, &has_token, NULL) && has_token; + ok(18, rem_gone && tok_stays); + free(m); + } + + /* HISCTLG_INPLACEEXPIRE capability contract (expire.c relies on it) */ + { + bool inplace = false; + ok(19, HISctl(h, HISCTLG_INPLACEEXPIRE, &inplace) && inplace); + } + HISclose(h); + + /* A backend that does not implement the selector must leave the flag + untouched (hisv6 -> stays false, i.e. rebuild-and-swap). */ + { + struct history *hv; + bool inplace = false; + char hv6path[160]; + + snprintf(hv6path, sizeof(hv6path), "%s/hv6", tmpdir); + hv = HISopen(hv6path, "hisv6", HIS_CREAT | HIS_RDWR); + if (hv != NULL) + HISctl(hv, HISCTLG_INPLACEEXPIRE, &inplace); + ok(20, hv != NULL && !inplace); + if (hv != NULL) + HISclose(hv); + } + + /* corrupt token -> reported, and schema-version mismatch -> refused. + Both need DB-level tampering there is no HIS API for, so reach in with + SQLite directly (this whole test is already HAVE_SQLITE3-only). */ + { + char vpath[160], vdb[176]; + struct history *vh; + sqlite3 *raw; + TOKEN tk; + + snprintf(vpath, sizeof(vpath), "%s/v", tmpdir); + snprintf(vdb, sizeof(vdb), "%s.sqlite", vpath); + vh = HISopen(vpath, "hissqlite", HIS_CREAT | HIS_RDWR); + memset(&tk, 0, sizeof(tk)); + tk.type = 1; + HISwrite(vh, "", BASE, BASE, 0, &tk); + HISsync(vh); + HISclose(vh); + + /* Truncate the stored token to a wrong length: lookup must report the + entry as not-found rather than mask the corruption (MI2). */ + if (sqlite3_open(vdb, &raw) == SQLITE_OK) { + sqlite3_exec(raw, "update hist set token = x'0011'", NULL, NULL, + NULL); + sqlite3_close(raw); + } + vh = HISopen(vpath, "hissqlite", HIS_RDONLY); + if (vh != NULL) { + TOKEN got; + memset(&got, 0, sizeof(got)); + ok(21, !HISlookup(vh, "", NULL, NULL, NULL, &got)); + HISclose(vh); + } else { + ok(21, false); + } + + /* Bump the stored schema version: reopen must refuse cleanly (MA2). */ + if (sqlite3_open(vdb, &raw) == SQLITE_OK) { + sqlite3_exec(raw, + "update misc set value = 99 where key = 'version'", + NULL, NULL, NULL); + sqlite3_close(raw); + } + vh = HISopen(vpath, "hissqlite", HIS_RDONLY); + ok(22, vh == NULL); + if (vh != NULL) + HISclose(vh); + } + + /* deferred-open path: HISopen(NULL) then HISCTLS_PATH, as makehistory + drives it. The handle must defer the open until the path is set, then + create the real database at that path (not an anonymous temp DB), and a + write must land in that file. */ + { + char dpath[160], ddb[176]; + struct history *dh; + struct stat sb; + TOKEN tk; + + snprintf(dpath, sizeof(dpath), "%s/deferred", tmpdir); + snprintf(ddb, sizeof(ddb), "%s.sqlite", dpath); + dh = HISopen(NULL, "hissqlite", HIS_CREAT | HIS_RDWR); + ok(23, dh != NULL && HISctl(dh, HISCTLS_PATH, dpath) + && stat(ddb, &sb) == 0); + if (dh != NULL) { + memset(&tk, 0, sizeof(tk)); + tk.type = 1; + HISwrite(dh, "", BASE, BASE, 0, &tk); + HISsync(dh); + HISclose(dh); + } + /* Reopen the real file and confirm the entry persisted there, proving + the write did not go to a throwaway temporary database. */ + dh = HISopen(dpath, "hissqlite", HIS_RDWR); + ok(24, dh != NULL && HIScheck(dh, "")); + if (dh != NULL) + HISclose(dh); + + /* Closing a deferred handle whose path was never set (so no database + was ever opened) must be a clean no-op, not a NULL-connection crash. */ + dh = HISopen(NULL, "hissqlite", HIS_CREAT | HIS_RDWR); + ok(25, dh != NULL && HISclose(dh)); + } + + /* duplicate HISwrite is tolerated (makehistory over a spool with a + duplicate Message-ID must warn and continue, not abort): both calls + succeed and the entry exists. */ + { + char wpath[160]; + struct history *wh; + TOKEN w1, w2; + bool first, second; + + snprintf(wpath, sizeof(wpath), "%s/dup", tmpdir); + wh = HISopen(wpath, "hissqlite", HIS_CREAT | HIS_RDWR); + memset(&w1, 0, sizeof(w1)); + w1.type = 1; + memset(&w2, 0, sizeof(w2)); + w2.type = 2; + first = wh != NULL && HISwrite(wh, "", BASE, BASE, 0, &w1); + second = wh != NULL && HISwrite(wh, "", BASE, BASE, 0, &w2); + ok(26, first && second && wh != NULL && HIScheck(wh, "")); + if (wh != NULL) + HISclose(wh); + } + + /* a dry-run expire (writing == false, as expire(8) -t now passes for an + in-place backend) scans but must change nothing: an entry the policy + would drop stays a real token. */ + { + char epath[160]; + struct history *eh; + TOKEN et, got; + bool kept; + + snprintf(epath, sizeof(epath), "%s/dry", tmpdir); + eh = HISopen(epath, "hissqlite", HIS_CREAT | HIS_RDWR); + memset(&et, 0, sizeof(et)); + et.type = 1; + if (eh != NULL) + HISwrite(eh, "", BASE, BASE, 0, &et); + expire_cutoff = BASE + 100; /* decide_drop_old would drop arrived=BASE */ + if (eh != NULL) + HISexpire(eh, NULL, NULL, false, NULL, BASE + 1000000, + decide_drop_old); + memset(&got, 0, sizeof(got)); + kept = eh != NULL + && HISlookup(eh, "", NULL, NULL, NULL, &got) + && got.type == 1; /* still real, not transitioned to remembered */ + ok(27, kept); + if (eh != NULL) + HISclose(eh); + } + + { + char cmd[160]; + snprintf(cmd, sizeof(cmd), "/bin/rm -rf %s", tmpdir); + if (system(cmd) < 0) + sysdiag("can't clean up %s", tmpdir); + } + return 0; +} + +#endif /* HAVE_SQLITE3 */ From 33b3535596f7f527158e203850a470c9239a8e91 Mon Sep 17 00:00:00 2001 From: Kevin Bowling Date: Sat, 20 Jun 2026 21:06:14 -0700 Subject: [PATCH 2/2] tests: add history-bench, a hisv6/hissqlite history benchmark A standalone benchmark (deliberately not part of the TAP suite) for comparing the hisv6 and hissqlite history backends at scale. It opens each backend exactly as innd does including the in-core index hint and reads tuning from inn.conf, so results reflect production settings; with no inn.conf it falls back to each backend's built-in defaults, which match the inn.conf defaults, so the comparison stays fair either way. Per method it times create+write (one autocommit per article, like innd's steady state), sequential check, full walk (the expireover bloom-rebuild and migration scan), random lookup, and missing lookup, reports the on-disk footprint, and validates correctness. It fails if any inserted entry is missing or any absent entry is found. Lookups use a fixed-seed PRNG reset per phase so every method replays the identical order. Output is both a human readable table and a CSV row. Defaults to 100M entries (sized for large local testing); build with "make -C tests benchmarks" and run tests/lib/history-bench (see -h). --- .gitignore | 1 + MANIFEST | 1 + support/mkmanifest | 1 + tests/Makefile | 12 +- tests/lib/history-bench.c | 649 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 663 insertions(+), 1 deletion(-) create mode 100644 tests/lib/history-bench.c diff --git a/.gitignore b/.gitignore index 870ca7aed..1e07d1326 100644 --- a/.gitignore +++ b/.gitignore @@ -227,6 +227,7 @@ /tests/lib/hex.t /tests/lib/hissqlite-convert.t /tests/lib/hissqlite.t +/tests/lib/history-bench /tests/lib/inet_aton.t /tests/lib/inet_ntoa.t /tests/lib/inet_ntop.t diff --git a/MANIFEST b/MANIFEST index ca75fc263..12b0b6f53 100644 --- a/MANIFEST +++ b/MANIFEST @@ -994,6 +994,7 @@ tests/lib/headers-t.c Tests for lib/headers.c tests/lib/hex-t.c Tests for lib/hex.c tests/lib/hissqlite-convert-t.c Tests for the hissqlite-convert tool tests/lib/hissqlite-t.c Tests for the hissqlite history method +tests/lib/history-bench.c Benchmark for the history backends tests/lib/inet_aton-t.c Tests for lib/inet_aton.c tests/lib/inet_ntoa-t.c Tests for lib/inet_ntoa.c tests/lib/inet_ntop-t.c Tests for lib/inet_ntop.c diff --git a/support/mkmanifest b/support/mkmanifest index b0db9a285..da1a940d1 100755 --- a/support/mkmanifest +++ b/support/mkmanifest @@ -280,6 +280,7 @@ history/hissqlite/hissqlite-convert history/hissqlite/hissqlite-util tests/lib/hissqlite.t tests/lib/hissqlite-convert.t +tests/lib/history-bench storage/tradindexed/tdx-util support/fixconfig support/fixscript diff --git a/tests/Makefile b/tests/Makefile index 7351985e2..74084b491 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -42,17 +42,24 @@ EXTRA = runtests clients/server-list docs/pod.t lib/xmalloc \ overview/ovsqlite-read.t overview/ovsqlite-write.t \ perl/minimum-version.t +BENCHMARKS = lib/history-bench + all check test tests: $(TESTS) $(EXTRA) ./runtests -l TESTS build: $(TESTS) $(EXTRA) +benchmarks: $(BENCHMARKS) + +benchmark-history: $(BENCHMARKS) + ./lib/history-bench -n 100M + warnings: $(MAKE) COPT="$(COPT) $(WARNINGS)" build clean clobber distclean maintclean: rm -f *.o *.lo */*.o */*.lo */*/*.o */*/*.o \ - .pure */.pure */*/.pure $(TESTS) $(EXTRA) + .pure */.pure */*/.pure $(TESTS) $(EXTRA) $(BENCHMARKS) rm -rf .libs */.libs */*/.libs $(FIXSCRIPT): @@ -130,6 +137,9 @@ lib/hissqlite.t: lib/hissqlite-t.o tap/basic.o $(STORAGEDEPS) lib/hissqlite-convert.t: lib/hissqlite-convert-t.o tap/basic.o $(STORAGEDEPS) $(LINKDEPS) lib/hissqlite-convert-t.o tap/basic.o $(STORAGELIBS) $(LIBS) +lib/history-bench: lib/history-bench.o $(STORAGEDEPS) + $(LINKDEPS) lib/history-bench.o $(STORAGELIBS) $(LIBS) + lib/buffer.t: lib/buffer-t.o tap/basic.o $(LIBINN) $(LINK) lib/buffer-t.o tap/basic.o $(LIBINN) diff --git a/tests/lib/history-bench.c b/tests/lib/history-bench.c new file mode 100644 index 000000000..7770557c5 --- /dev/null +++ b/tests/lib/history-bench.c @@ -0,0 +1,649 @@ +/* +** Benchmark hisv6 and hissqlite history backends. +** +** This is intentionally not part of the TAP test suite. The default run is +** sized for large local performance testing (100M entries) and can consume a +** lot of time and disk. +** +** Fairness / representativeness: +** +** Both methods are opened exactly as innd opens them, including the in-core +** index hint (HIS_MMAP when INND_DBZINCORE, else HIS_ONDISK). Without it, +** hisv6 would run dbz fully on disk (INCORE_NO plus writethrough), slower +** than production, while hissqlite, which ignores the storage hint flags, +** would be unaffected. +** +** Tuning comes from inn.conf, so each backend uses the operator's real +** settings. Point at a config with the INNCONF environment variable: +** +** INNCONF=/path/to/inn.conf ./history-bench -n 100M +** +** If neither $INNCONF nor the compiled default inn.conf is present, both +** backends fall back to their built-in defaults (which equal the inn.conf +** defaults), so the comparison stays fair either way. The knobs that most +** affect these results: +** +** hissqlitecachesize writer page cache (KB). The default (64 MB) is +** far smaller than a 100M-entry B-tree, so +** random-key inserts thrash; raise it to isolate +** "SQLite is slower at random-key writes" from +** "under-cached". +** hissqlitemmapsize SQLite mmap window (bytes). Default 0 (no mmap) +** reads the DB via pread; set it to compare in-core +** against hisv6's mmap'd index on equal footing. +** hissqlitepagesize B-tree page size (bytes); affects tree depth and +** per-insert WAL write amplification. +** +** Note the write phase models innd's steady-state, one-autocommit-per-article +** path, not a makehistory rebuild: bulk loading uses the batched converter +** (hissqlite-convert.c) and is faster than the per-row figure here. +** +** Written by Kevin Bowling in 2026. +*/ + +#include "portable/system.h" + +#include "inn/history.h" +#include "inn/innconf.h" +#include "inn/libinn.h" +#include "inn/messages.h" +#include "inn/options.h" + +#if defined(__clang__) +# pragma clang diagnostic ignored "-Wc++-keyword" +#endif +#include "inn/storage.h" + +#ifdef HAVE_SYS_TIME_H +# include +#endif +#include +#include +#include + +#define DEFAULT_ENTRIES ((uint64_t) 100000000) +#define DEFAULT_RANDOM ((uint64_t) 1000000) +#define DEFAULT_SYNC ((uint64_t) 10000) +#define PROGRESS_STEPS 20 +#define BASE_TIME ((time_t) 1700000000) + +/* +** Open hisv6 the way innd does, not with a bare HIS_RDWR/HIS_RDONLY. Without +** an in-core hint, hisv6 hands dbz pag_incore = exists_incore = INCORE_NO +** with writethrough on, its slowest fully-on-disk index mode, so every check +** and lookup hits the disk and writes thrash the on-disk index. innd instead +** opens with HIS_MMAP (INND_DBZINCORE defaults to 1; HIS_ONDISK otherwise), +** keeping the index mmap()ed in core. hissqlite ignores these storage hint +** flags (its journal/cache behaviour is fixed), so passing them does not +** change hissqlite; it only stops the benchmark from unfairly crippling +** hisv6. +*/ +#define HIS_INCORE_HINT (INND_DBZINCORE ? HIS_MMAP : HIS_ONDISK) + +struct bench_config { + uint64_t entries; + uint64_t random_lookups; + uint64_t sync_every; + const char *root; + bool keep; + const char **methods; + size_t method_count; +}; + +struct bench_result { + const char *method; + uint64_t entries; + uint64_t random_lookups; + double write_seconds; + double sync_seconds; + double close_seconds; + double open_ro_seconds; + double check_seconds; + double walk_seconds; + double lookup_seconds; + double missing_seconds; + uint64_t bytes; + const char *path; + bool skipped; +}; + +static double +now_seconds(void) +{ + struct timeval tv; + + if (gettimeofday(&tv, NULL) < 0) + sysdie("gettimeofday failed"); + return (double) tv.tv_sec + ((double) tv.tv_usec / 1000000.0); +} + +static uint64_t +parse_count(const char *value) +{ + char *end; + uint64_t count; + unsigned long long parsed; + + errno = 0; + parsed = strtoull(value, &end, 10); + if (errno != 0 || end == value) + die("invalid count: %s", value); + count = (uint64_t) parsed; + if (*end != '\0' && end[1] != '\0') + die("invalid count suffix: %s", value); + if (*end == 'k' || *end == 'K') + count *= 1000; + else if (*end == 'm' || *end == 'M') + count *= 1000 * 1000; + else if (*end == 'g' || *end == 'G') + count *= 1000 * 1000 * 1000; + else if (*end != '\0') + die("invalid count suffix: %s", value); + if (count == 0) + die("count must be greater than zero"); + return count; +} + +__attribute__((__noreturn__)) static void +usage(int status) +{ + FILE *out; + + out = status == 0 ? stdout : stderr; + fprintf(out, "usage: history-bench [-k] [-d dir] [-n entries] " + "[-r random-lookups] [-s sync-every] [method ...]\n"); + fprintf(out, "\n"); + fprintf(out, "Default: -n 100M -r 1M -s 10K hisv6 hissqlite\n"); + fprintf(out, "Counts accept K, M, and G decimal suffixes.\n"); + fprintf(out, "Use -s 0 to disable periodic HISsync during writes.\n"); + fprintf(out, "Benchmark data is removed unless -k is given.\n"); + exit(status); +} + +static char * +join_path(const char *base, const char *name) +{ + size_t len; + char *path; + + len = strlen(base) + strlen(name) + 2; + path = (char *) xmalloc(len); + snprintf(path, len, "%s/%s", base, name); + return path; +} + +static void +make_msgid(char *buf, size_t len, uint64_t n) +{ + snprintf(buf, len, "<%020llu@history-bench.invalid>", + (unsigned long long) n); +} + +static void +make_token(TOKEN *token, uint64_t n) +{ + /* 64-bit golden-ratio constant (floor(2^64 / phi)); used here purely as a + fixed XOR mask so the two halves of the synthetic token differ rather + than both being the counter. */ + const uint64_t golden_ratio = UINT64_C(0x9e3779b97f4a7c15); + uint64_t hi, lo; + + memset(token, 0, sizeof(*token)); + token->type = 1; + token->class = (STORAGECLASS) (n & 0xff); + hi = n; + lo = n ^ golden_ratio; + memcpy(token->token, &hi, sizeof(hi)); + memcpy(token->token + sizeof(hi), &lo, sizeof(lo)); +} + +static void +print_progress(const char *method, const char *phase, uint64_t done, + uint64_t total) +{ + double pct; + + if (total == 0) + return; + pct = ((double) done * 100.0) / (double) total; + fprintf(stderr, "%s %-12s %6.2f%% (%llu/%llu)\n", method, phase, pct, + (unsigned long long) done, (unsigned long long) total); +} + +static bool +progress_due(uint64_t i, uint64_t total) +{ + uint64_t step; + + if (total < PROGRESS_STEPS) + step = 1; + else + step = total / PROGRESS_STEPS; + return ((i + 1) == total || ((i + 1) % step) == 0); +} + +static uint64_t +lcg_next(uint64_t *state) +{ + /* Knuth's MMIX 64-bit linear congruential generator (also the PCG + defaults): state = state * multiplier + increment. A cheap, adequate + PRNG for the benchmark's pseudo-random key sequence. */ + const uint64_t multiplier = UINT64_C(6364136223846793005); + const uint64_t increment = UINT64_C(1442695040888963407); + + *state = *state * multiplier + increment; + return *state; +} + +static void +sum_tree(const char *path, uint64_t *bytes) +{ + struct stat st; + DIR *dir; + struct dirent *entry; + + if (lstat(path, &st) < 0) + return; + if (S_ISREG(st.st_mode)) { + if (st.st_size > 0) + *bytes += (uint64_t) st.st_size; + return; + } + if (!S_ISDIR(st.st_mode)) + return; + dir = opendir(path); + if (dir == NULL) + return; + while ((entry = readdir(dir)) != NULL) { + char *child; + + if (strcmp(entry->d_name, ".") == 0 + || strcmp(entry->d_name, "..") == 0) + continue; + child = join_path(path, entry->d_name); + sum_tree(child, bytes); + free(child); + } + closedir(dir); +} + +static void +remove_tree(const char *path) +{ + struct stat st; + DIR *dir; + struct dirent *entry; + + if (lstat(path, &st) < 0) + return; + if (S_ISDIR(st.st_mode)) { + dir = opendir(path); + if (dir != NULL) { + while ((entry = readdir(dir)) != NULL) { + char *child; + + if (strcmp(entry->d_name, ".") == 0 + || strcmp(entry->d_name, "..") == 0) + continue; + child = join_path(path, entry->d_name); + remove_tree(child); + free(child); + } + closedir(dir); + } + if (rmdir(path) < 0) + syswarn("cannot remove %s", path); + } else if (unlink(path) < 0) { + syswarn("cannot remove %s", path); + } +} + +static struct history * +create_history(const char *method, const char *histpath, uint64_t entries) +{ + struct history *h; + size_t npairs; + + h = HISopen(NULL, method, HIS_CREAT | HIS_RDWR | HIS_INCORE_HINT); + if (h == NULL) + return NULL; + npairs = (size_t) entries; + if ((uint64_t) npairs != entries) + npairs = (size_t) -1; + if (!HISctl(h, HISCTLS_NPAIRS, &npairs)) + die("%s: cannot set HISCTLS_NPAIRS: %s", method, HISerror(h)); + if (!HISctl(h, HISCTLS_PATH, (void *) histpath)) + die("%s: cannot set history path %s: %s", method, histpath, + HISerror(h)); + return h; +} + +/* HISwalk callback: count every entry. HISwalk is the full-history scan that + feeds the expireover bloom-filter rebuild and the migration converter. */ +static bool +walk_count(void *cookie, const HASH *hash UNUSED, time_t arrived UNUSED, + time_t posted UNUSED, time_t expires UNUSED, const TOKEN *token UNUSED) +{ + ++*(uint64_t *) cookie; + return true; +} + +static void +benchmark_method(const struct bench_config *config, const char *method, + struct bench_result *result) +{ + char *method_dir; + char *histpath; + struct history *h; + TOKEN token, got; + char msgid[64]; + uint64_t i, misses; + uint64_t state; + double start; + + /* Arbitrary but fixed PRNG seeds (any value works): a fixed seed makes + each phase replay the same pseudo-random order on every run and for + every method, so the comparison is apples-to-apples; the two phases use + distinct seeds so they probe in different orders. */ + const uint64_t lookup_seed = UINT64_C(0x123456789abcdef0); + const uint64_t missing_seed = UINT64_C(0xfedcba9876543210); + + memset(result, 0, sizeof(*result)); + result->method = method; + result->entries = config->entries; + result->random_lookups = config->random_lookups; + + method_dir = join_path(config->root, method); + histpath = join_path(method_dir, "history"); + if (mkdir(method_dir, 0777) < 0) + sysdie("cannot create %s", method_dir); + result->path = method_dir; + + fprintf(stderr, "%s create/write %llu entries at %s\n", method, + (unsigned long long) config->entries, histpath); + start = now_seconds(); + h = create_history(method, histpath, config->entries); + if (h == NULL) { + fprintf(stderr, "%s skipped: %s\n", method, + HISerror(h) != NULL ? HISerror(h) : "method unavailable"); + result->skipped = true; + free(histpath); + return; + } + for (i = 0; i < config->entries; i++) { + make_msgid(msgid, sizeof(msgid), i); + make_token(&token, i); + if (!HISwrite(h, msgid, BASE_TIME + (time_t) i, BASE_TIME + (time_t) i, + 0, &token)) + die("%s: HISwrite %llu failed: %s", method, (unsigned long long) i, + HISerror(h)); + if (config->sync_every != 0 && (i + 1) % config->sync_every == 0) + if (!HISsync(h)) + die("%s: periodic HISsync at %llu failed: %s", method, + (unsigned long long) (i + 1), HISerror(h)); + if (progress_due(i, config->entries)) + print_progress(method, "write", i + 1, config->entries); + } + result->write_seconds = now_seconds() - start; + + start = now_seconds(); + if (!HISsync(h)) + die("%s: HISsync failed: %s", method, HISerror(h)); + result->sync_seconds = now_seconds() - start; + + start = now_seconds(); + if (!HISclose(h)) + die("%s: HISclose failed", method); + result->close_seconds = now_seconds() - start; + + start = now_seconds(); + h = HISopen(histpath, method, HIS_RDONLY | HIS_INCORE_HINT); + if (h == NULL) + die("%s: read-only reopen failed", method); + result->open_ro_seconds = now_seconds() - start; + + fprintf(stderr, "%s sequential check %llu entries\n", method, + (unsigned long long) config->entries); + misses = 0; + start = now_seconds(); + for (i = 0; i < config->entries; i++) { + make_msgid(msgid, sizeof(msgid), i); + if (!HIScheck(h, msgid)) + misses++; + if (progress_due(i, config->entries)) + print_progress(method, "check", i + 1, config->entries); + } + result->check_seconds = now_seconds() - start; + if (misses != 0) + die("%s: sequential check missed %llu entries", method, + (unsigned long long) misses); + + fprintf(stderr, "%s walk %llu entries\n", method, + (unsigned long long) config->entries); + { + uint64_t walked = 0; + + /* NULL reason: a read-only walk must not pause innd. hisv6 calls + ICCpause(reason) at EOF when reason is non-NULL, to catch stragglers + during a rebuild, there is no server here, and hissqlite ignores + reason regardless. */ + start = now_seconds(); + if (!HISwalk(h, NULL, &walked, walk_count)) + die("%s: HISwalk failed: %s", method, HISerror(h)); + result->walk_seconds = now_seconds() - start; + if (walked != config->entries) + die("%s: walk visited %llu of %llu entries", method, + (unsigned long long) walked, + (unsigned long long) config->entries); + } + + fprintf(stderr, "%s random lookup %llu entries\n", method, + (unsigned long long) config->random_lookups); + state = lookup_seed; + misses = 0; + start = now_seconds(); + for (i = 0; i < config->random_lookups; i++) { + uint64_t n; + + n = lcg_next(&state) % config->entries; + make_msgid(msgid, sizeof(msgid), n); + memset(&got, 0, sizeof(got)); + if (!HISlookup(h, msgid, NULL, NULL, NULL, &got) + || got.type == TOKEN_EMPTY) + misses++; + if (progress_due(i, config->random_lookups)) + print_progress(method, "lookup", i + 1, config->random_lookups); + } + result->lookup_seconds = now_seconds() - start; + if (misses != 0) + die("%s: random lookup missed %llu entries", method, + (unsigned long long) misses); + + fprintf(stderr, "%s missing check %llu entries\n", method, + (unsigned long long) config->random_lookups); + state = missing_seed; + misses = 0; + start = now_seconds(); + for (i = 0; i < config->random_lookups; i++) { + uint64_t n; + + n = config->entries + (lcg_next(&state) % config->entries); + make_msgid(msgid, sizeof(msgid), n); + if (HIScheck(h, msgid)) + misses++; + if (progress_due(i, config->random_lookups)) + print_progress(method, "missing", i + 1, config->random_lookups); + } + result->missing_seconds = now_seconds() - start; + if (misses != 0) + die("%s: missing check found %llu unexpected entries", method, + (unsigned long long) misses); + + if (!HISclose(h)) + die("%s: final HISclose failed", method); + + result->bytes = 0; + sum_tree(method_dir, &result->bytes); + free(histpath); +} + +static double +rate(uint64_t count, double seconds) +{ + if (seconds <= 0.0) + return 0.0; + return (double) count / seconds; +} + +static void +print_result(const struct bench_result *result) +{ + if (result->skipped) { + printf("%-10s skipped\n", result->method); + return; + } + printf("%-10s entries=%llu bytes=%llu path=%s\n", result->method, + (unsigned long long) result->entries, + (unsigned long long) result->bytes, result->path); + printf(" write: %8.3fs %12.0f entries/s\n", result->write_seconds, + rate(result->entries, result->write_seconds)); + printf(" sync: %8.3fs\n", result->sync_seconds); + printf(" close: %8.3fs\n", result->close_seconds); + printf(" open-ro: %8.3fs\n", result->open_ro_seconds); + printf(" check: %8.3fs %12.0f entries/s\n", result->check_seconds, + rate(result->entries, result->check_seconds)); + printf(" walk: %8.3fs %12.0f entries/s\n", result->walk_seconds, + rate(result->entries, result->walk_seconds)); + printf(" lookup: %8.3fs %12.0f entries/s\n", result->lookup_seconds, + rate(result->random_lookups, result->lookup_seconds)); + printf(" missing: %8.3fs %12.0f entries/s\n", result->missing_seconds, + rate(result->random_lookups, result->missing_seconds)); +} + +static void +print_csv_header(void) +{ + printf("\nmethod,entries,random_lookups,bytes,write_s,write_per_s,sync_s," + "close_s,open_ro_s,check_s,check_per_s,walk_s,walk_per_s," + "lookup_s,lookup_per_s,missing_s,missing_per_s,path\n"); +} + +static void +print_csv(const struct bench_result *result) +{ + if (result->skipped) + return; + printf( + "%s,%llu,%llu,%llu,%.6f,%.2f,%.6f,%.6f,%.6f,%.6f,%.2f,%.6f,%.2f," + "%.6f,%.2f,%.6f,%.2f,%s\n", + result->method, (unsigned long long) result->entries, + (unsigned long long) result->random_lookups, + (unsigned long long) result->bytes, result->write_seconds, + rate(result->entries, result->write_seconds), result->sync_seconds, + result->close_seconds, result->open_ro_seconds, result->check_seconds, + rate(result->entries, result->check_seconds), result->walk_seconds, + rate(result->entries, result->walk_seconds), result->lookup_seconds, + rate(result->random_lookups, result->lookup_seconds), + result->missing_seconds, + rate(result->random_lookups, result->missing_seconds), result->path); +} + +int +main(int argc, char **argv) +{ + static const char *default_methods[] = {"hisv6", "hissqlite"}; + struct bench_config config; + struct bench_result *results; + char root_template[] = "history-bench-XXXXXX"; + int i; + size_t j; + + /* Best-effort: load inn.conf so both backends use the operator's real + tuning (hissqlite{cachesize,mmapsize,pagesize}, dbz nfs hints). If no + inn.conf is reachable, innconf stays NULL and each backend falls back to + its built-in defaults, which match the inn.conf defaults, so the + comparison stays fair either way. */ + if (!innconf_read(NULL)) { + innconf = NULL; + warn("no inn.conf found; using built-in history defaults for both" + " methods"); + } + + memset(&config, 0, sizeof(config)); + config.entries = DEFAULT_ENTRIES; + config.random_lookups = DEFAULT_RANDOM; + config.sync_every = DEFAULT_SYNC; + config.methods = default_methods; + config.method_count = 2; + + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + usage(0); + } else if (strcmp(argv[i], "-k") == 0) { + config.keep = true; + } else if (strcmp(argv[i], "-n") == 0) { + if (++i >= argc) + usage(1); + config.entries = parse_count(argv[i]); + } else if (strcmp(argv[i], "-r") == 0) { + if (++i >= argc) + usage(1); + config.random_lookups = parse_count(argv[i]); + } else if (strcmp(argv[i], "-s") == 0) { + if (++i >= argc) + usage(1); + if (strcmp(argv[i], "0") == 0) + config.sync_every = 0; + else + config.sync_every = parse_count(argv[i]); + } else if (strcmp(argv[i], "-d") == 0) { + if (++i >= argc) + usage(1); + config.root = argv[i]; + } else if (argv[i][0] == '-') { + usage(1); + } else { + config.methods = (const char **) &argv[i]; + config.method_count = (size_t) (argc - i); + break; + } + } + if (config.random_lookups > config.entries) + config.random_lookups = config.entries; + if (config.root == NULL) { + if (mkdtemp(root_template) == NULL) + sysdie("cannot create benchmark directory"); + config.root = root_template; + } else if (mkdir(config.root, 0777) < 0) { + sysdie("cannot create %s", config.root); + } + + printf("history benchmark root: %s\n", config.root); + printf("entries: %llu, random lookups: %llu, sync every: %llu\n\n", + (unsigned long long) config.entries, + (unsigned long long) config.random_lookups, + (unsigned long long) config.sync_every); + fflush(stdout); + + results = + (struct bench_result *) xcalloc(config.method_count, sizeof(*results)); + for (j = 0; j < config.method_count; j++) { + benchmark_method(&config, config.methods[j], &results[j]); + print_result(&results[j]); + printf("\n"); + fflush(stdout); + } + + print_csv_header(); + for (j = 0; j < config.method_count; j++) + print_csv(&results[j]); + + if (config.keep) { + printf("\nkept benchmark data in %s\n", config.root); + } else { + remove_tree(config.root); + printf("\nremoved benchmark data from %s\n", config.root); + } + free(results); + return 0; +}