diff --git a/.gitignore b/.gitignore index 30cc70949..1e07d1326 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,14 @@ /history/buildconfig /history/hismethods.c /history/hismethods.h +/history/hissqlite/hissqlite-convert +/history/hissqlite/hissqlite-init.c +/history/hissqlite/hissqlite-init.h +/history/hissqlite/hissqlite-main.c +/history/hissqlite/hissqlite-main.h +/history/hissqlite/hissqlite-read.c +/history/hissqlite/hissqlite-read.h +/history/hissqlite/hissqlite-util /include/config.h /include/config.h.in /include/inn/paths.h @@ -143,6 +151,7 @@ /innfeed/imapfeed /innfeed/innfeed /innfeed/procbatch +/lib/sqlite-helper-gen /nnrpd/nnrpd /perl/INN/Config.pm /perl/INN/Utils/Shlock.pm @@ -216,6 +225,9 @@ /tests/lib/hashtab.t /tests/lib/headers.t /tests/lib/hex.t +/tests/lib/hissqlite-convert.t +/tests/lib/hissqlite.t +/tests/lib/history-bench /tests/lib/inet_aton.t /tests/lib/inet_ntoa.t /tests/lib/inet_ntop.t diff --git a/MANIFEST b/MANIFEST index 3ed2818fb..12b0b6f53 100644 --- a/MANIFEST +++ b/MANIFEST @@ -157,6 +157,9 @@ doc/man/fastrm.1 Manpage for fastrm utility doc/man/gencancel.1 Manpage for gencancel program doc/man/getlist.1 Manpage for getlist frontend doc/man/grephistory.1 Manpage for grephistory +doc/man/hissqlite-convert.8 Manpage for hissqlite-convert +doc/man/hissqlite-util.8 Manpage for hissqlite-util +doc/man/hissqlite.5 Manpage for the hissqlite history method doc/man/history.5 Manpage for history database doc/man/ident.8 Manpage for ident resolver doc/man/incoming.conf.5 Manpage for incoming.conf config file @@ -276,6 +279,9 @@ doc/pod/gencancel.pod Master file for gencancel.1 doc/pod/getlist.pod Master file for getlist.1 doc/pod/grephistory.pod Master file for grephistory.1 doc/pod/hacking.pod Master file for HACKING +doc/pod/hissqlite-convert.pod Master file for hissqlite-convert.8 +doc/pod/hissqlite-util.pod Master file for hissqlite-util.8 +doc/pod/hissqlite.pod Master file for hissqlite.5 doc/pod/history.pod Master file for history.5 doc/pod/hook-perl.pod Master file for doc/hook-perl doc/pod/hook-python.pod Master file for doc/hook-python @@ -402,6 +408,23 @@ history/his.c History API glue implementation history/hisinterface.h History API interface history/hismethods.c Generated table of history methods history/hismethods.h Generated interface to history methods +history/hissqlite hissqlite history method (Directory) +history/hissqlite/hismethod.config buildconfig definition for hissqlite +history/hissqlite/hismethod.mk Make rules for hissqlite +history/hissqlite/hissqlite-convert.c Convert a history database to hissqlite +history/hissqlite/hissqlite-init.c Generated database setup implementation +history/hissqlite/hissqlite-init.h Generated database setup interface +history/hissqlite/hissqlite-init.sql SQLite code for database setup +history/hissqlite/hissqlite-main.c Generated daily operation implementation +history/hissqlite/hissqlite-main.h Generated daily operation interface +history/hissqlite/hissqlite-main.sql SQLite code for daily operation +history/hissqlite/hissqlite-private.h Private header for hissqlite +history/hissqlite/hissqlite-read.c Generated read-only query implementation +history/hissqlite/hissqlite-read.h Generated read-only query interface +history/hissqlite/hissqlite-read.sql SQLite code for direct reader queries +history/hissqlite/hissqlite-util.in Utility program for hissqlite +history/hissqlite/hissqlite.c hissqlite history method +history/hissqlite/hissqlite.h Header for hissqlite history history/hisv6 History v6 method (Directory) history/hisv6/hismethod.config hisbuildconfig definition history/hisv6/hisv6-private.h Private header file for hisv6 @@ -440,6 +463,7 @@ include/inn/paths.h.in Header file for paths include/inn/qio.h Header file for quick I/O package include/inn/secrets.h Header file for the secrets struct include/inn/sequence.h Header file for sequence space arithmetic +include/inn/sqlite-helper.h SQLite code package interface include/inn/storage.h Header file for storage API include/inn/timer.h Header file for generic timers include/inn/tombstone.h Header file for cancel tombstone log @@ -585,6 +609,8 @@ lib/setenv.c setenv replacement lib/seteuid.c seteuid replacement lib/setproctitle.c setproctitle replacement lib/snprintf.c snprintf and vsnprintf replacement +lib/sqlite-helper-gen.in Package SQLite code for convenient use +lib/sqlite-helper.c SQLite code package implementation lib/strcasecmp.c Case-insensitive string comparison (BSD) lib/strlcat.c strlcat replacement lib/strlcpy.c strlcpy replacement @@ -798,9 +824,6 @@ storage/ovsqlite/sql-main.sql SQLite code for daily operation storage/ovsqlite/sql-read.c Generated read-only query implementation storage/ovsqlite/sql-read.h Generated read-only query interface storage/ovsqlite/sql-read.sql SQLite code for direct reader queries -storage/ovsqlite/sqlite-helper-gen.in Package SQLite code for convenient use -storage/ovsqlite/sqlite-helper.c SQLite code package implementation -storage/ovsqlite/sqlite-helper.h SQLite code package interface storage/timecaf timecaf storage method (Directory) storage/timecaf/README.CAF README the CAF file format storage/timecaf/caf.c CAF file implementation @@ -969,6 +992,9 @@ tests/lib/hash-t.c Tests for lib/hash.c tests/lib/hashtab-t.c Tests for lib/hashtab.c tests/lib/headers-t.c Tests for lib/headers.c tests/lib/hex-t.c Tests for lib/hex.c +tests/lib/hissqlite-convert-t.c Tests for the hissqlite-convert tool +tests/lib/hissqlite-t.c Tests for the hissqlite history method +tests/lib/history-bench.c Benchmark for the history backends tests/lib/inet_aton-t.c Tests for lib/inet_aton.c tests/lib/inet_ntoa-t.c Tests for lib/inet_ntoa.c tests/lib/inet_ntop-t.c Tests for lib/inet_ntop.c diff --git a/doc/pod/Makefile b/doc/pod/Makefile index 29a8b2604..403ffdd02 100644 --- a/doc/pod/Makefile +++ b/doc/pod/Makefile @@ -23,6 +23,7 @@ MAN3 = ../man/libinn.3 ../man/libinn_clientlib.3 ../man/libinn_dbz.3 \ MAN5 = ../man/active.5 ../man/active.times.5 ../man/buffindexed.conf.5 \ ../man/control.ctl.5 ../man/cycbuff.conf.5 ../man/distrib.pats.5 \ ../man/distributions.5 ../man/expire.ctl.5 ../man/history.5 \ + ../man/hissqlite.5 \ ../man/incoming.conf.5 ../man/inn.conf.5 ../man/innfeed.conf.5 \ ../man/innreport.conf.5 ../man/inn-secrets.conf.5 \ ../man/innwatch.ctl.5 ../man/moderators.5 \ @@ -38,7 +39,9 @@ MAN8 = ../man/actsync.8 ../man/archive.8 ../man/auth_krb5.8 \ ../man/controlchan.8 ../man/ctlinnd.8 ../man/cvtbatch.8 \ ../man/docheckgroups.8 \ ../man/domain.8 ../man/expire.8 ../man/expireover.8 \ - ../man/expirerm.8 ../man/ident.8 \ + ../man/expirerm.8 \ + ../man/hissqlite-convert.8 ../man/hissqlite-util.8 \ + ../man/ident.8 \ ../man/innbind.8 ../man/inncheck.8 ../man/innd.8 ../man/inndf.8 \ ../man/innfeed.8 ../man/innreport.8 ../man/innstat.8 \ ../man/innupgrade.8 \ @@ -108,6 +111,7 @@ maintclean: distclean ../man/distributions.5: distributions.pod ; $(POD2MAN) -s 5 $? > $@ ../man/expire.ctl.5: expire.ctl.pod ; $(POD2MAN) -s 5 $? > $@ ../man/history.5: history.pod ; $(POD2MAN) -s 5 $? > $@ +../man/hissqlite.5: hissqlite.pod ; $(POD2MAN) -s 5 $? > $@ ../man/incoming.conf.5: incoming.conf.pod ; $(POD2MAN) -s 5 $? > $@ ../man/inn.conf.5: inn.conf.pod ; $(POD2MAN) -s 5 $? > $@ ../man/innfeed.conf.5: innfeed.conf.pod ; $(POD2MAN) -s 5 $? > $@ @@ -145,6 +149,8 @@ maintclean: distclean ../man/expire.8: expire.pod ; $(POD2MAN) -s 8 $? > $@ ../man/expireover.8: expireover.pod ; $(POD2MAN) -s 8 $? > $@ ../man/expirerm.8: expirerm.pod ; $(POD2MAN) -s 8 $? > $@ +../man/hissqlite-convert.8: hissqlite-convert.pod ; $(POD2MAN) -s 8 $? > $@ +../man/hissqlite-util.8: hissqlite-util.pod ; $(POD2MAN) -s 8 $? > $@ ../man/ident.8: ident.pod ; $(POD2MAN) -s 8 $? > $@ ../man/innbind.8: innbind.pod ; $(POD2MAN) -s 8 $? > $@ ../man/inncheck.8: inncheck.pod ; $(POD2MAN) -s 8 $? > $@ diff --git a/doc/pod/expire.pod b/doc/pod/expire.pod index 29456e146..bc8bd0909 100644 --- a/doc/pod/expire.pod +++ b/doc/pod/expire.pod @@ -49,6 +49,15 @@ be better to only use self-expiring backends like CNFS, as the history entries will still get cleaned up by B when it detects that an article no longer exists in that backend. +When the C history method is in use, B updates the +history database in place instead of building a new file and swapping it. +It does not pause B (the two coordinate through the database). The +B<-d> and B<-f> options, which write a rebuilt history to a side file, are +rejected; use B<-t> for a dry run (it reports what would be removed without +changing the history or the spool). B<-x> on its own is also rejected, +because suppressing the history rewrite while still removing articles would +leave history entries for deleted articles. See hissqlite(5). + =head1 OPTIONS =over 4 @@ -193,7 +202,7 @@ POD by Julien Elie. =head1 SEE ALSO -ctlinnd(8), expire.ctl(5), expireover(8), history(5), inn.conf(5), innd(8), -libinn_dbz(3), libinn_inndcomm(3), news.daily(8). +ctlinnd(8), expire.ctl(5), expireover(8), hissqlite(5), history(5), +inn.conf(5), innd(8), libinn_dbz(3), libinn_inndcomm(3), news.daily(8). =cut diff --git a/doc/pod/hissqlite-convert.pod b/doc/pod/hissqlite-convert.pod new file mode 100644 index 000000000..c6b518b26 --- /dev/null +++ b/doc/pod/hissqlite-convert.pod @@ -0,0 +1,74 @@ +=head1 NAME + +hissqlite-convert - Convert an INN history database to the hissqlite method + +=head1 SYNOPSIS + +B [B<-m> I] I I + +=head1 DESCRIPTION + +B performs a faithful, one-time migration of an existing +INN history database into a new hissqlite (SQLite) database. It reads every +entry from the source history through the standard history interface and +bulk-loads it into a freshly created hissqlite database. + +The conversion is keyed by hash: the history records only the MD5 of each +Message-ID, not the Message-ID itself, so the original Message-IDs cannot be +recovered. Unlike a from-spool rebuild with B, this preserves +both the exact arrival, posting, and expiration timestamps and the +I (token-less) entries, which a rebuild cannot reconstruct. + +The new database is built in a temporary file and atomically renamed into +place only on full success, so an interrupted or failed run never leaves a +usable-but-incomplete destination. B refuses to run if +the destination already exists. + +B must not be writing the source history while it is being converted. +After a successful conversion, set I to C in +F and restart INN; see hissqlite(5). + +=head1 OPTIONS + +=over 4 + +=item B<-m> I + +The history method of the source database. The default is C. + +=back + +=head1 ARGUMENTS + +=over 4 + +=item I + +The path to the source history (for C, the F file without +any extension), as given by the I parameter in F. + +=item I + +The base path of the hissqlite database to create. A F<.sqlite> suffix is +appended, so I should match I (the resulting file is +F in I). + +=back + +=head1 EXAMPLES + +Convert the default C history into a hissqlite database alongside it: + + hissqlite-convert /news/db/history /news/db/history + +This creates F. + +=head1 HISTORY + +Written by Kevin Bowling for InterNetNews. + +=head1 SEE ALSO + +hissqlite(5), hissqlite-util(8), history(5), inn.conf(5), makehistory(8). + +=cut diff --git a/doc/pod/hissqlite-util.pod b/doc/pod/hissqlite-util.pod new file mode 100644 index 000000000..34db0f1bf --- /dev/null +++ b/doc/pod/hissqlite-util.pod @@ -0,0 +1,92 @@ +=head1 NAME + +hissqlite-util - History manipulation utility for hissqlite + +=head1 SYNOPSIS + +B [B<-AcdhV>] [B<-p> I] + +=head1 DESCRIPTION + +B is an administrative interface to the hissqlite history +method for INN. It only works on hissqlite history databases, not on any +other type of INN history. It allows the administrator to audit the +database for problems, dump its contents, and report entry counts and the +schema version. + +All of its actions are read-only, so it is safe to run while INN is running. +B opens the database directly; because B holds the +write lock for only a single statement at a time, it waits at most briefly +for any in-progress write. + +=head1 REQUIREMENTS + +The C Perl module with the C driver must be installed for +this program to work. They are present in most distributions as a package +named like C, C or C. +They can also be directly installed from CPAN (C and C +distributions available at L and +L). + +Naturally, a hissqlite history database also needs being in use, as detailed +in the hissqlite(5) manual page. + +=head1 OPTIONS + +Only one action may be given at a time. With no action, B<-A> is assumed. + +=over 4 + +=item B<-A> + +Audit the history database for problems and report them to standard error, +without trying to fix them. This reports the schema version and entry +counts, checks for rows with a missing arrival time, and runs SQLite's +C. This is the default action. + +=item B<-c> + +Print the total, real (token-bearing), and remembered (token-less) entry +counts. + +=item B<-d> + +Dump all history records as text, one per line: the hash in hexadecimal, +the arrival, posting, and expiration timestamps (in seconds since epoch), +and the storage API token in hexadecimal (or C for a token-less +entry). + +=item B<-h> + +Print a usage message and exit. + +=item B<-p> I + +Act on the F database in I instead of the default +I directory from F. + +=item B<-V> + +Print the version of this tool and the database schema version. + +=back + +=head1 EXAMPLES + +Audit the history database for problems: + + hissqlite-util -A + +Print entry counts for a database in F: + + hissqlite-util -c -p /news/db + +=head1 HISTORY + +Written by Kevin Bowling for InterNetNews. + +=head1 SEE ALSO + +hissqlite(5), hissqlite-convert(8), history(5), inn.conf(5). + +=cut diff --git a/doc/pod/hissqlite.pod b/doc/pod/hissqlite.pod new file mode 100644 index 000000000..b357e0cdd --- /dev/null +++ b/doc/pod/hissqlite.pod @@ -0,0 +1,162 @@ +=head1 NAME + +hissqlite - SQLite-based history storage method for INN + +=head1 DESCRIPTION + +This method uses SQLite to store the history database, as an alternative to +the default C method. It requires version 3.8.2 or later of the +SQLite library (3.20.0+ recommended). + +SQLite source, documentation, etc. are available at +L. One of the stated goals of the SQLite file +format is long-term stability and compatibility, which makes it a good +choice for a history database. + +Unlike C, hissqlite keeps everything in a single SQLite database +file: there is no separate F text file and no I index files. +Consequently the B and B text-file tools do not +apply, and the I path simply names the database file (see +L). Each entry is keyed on the 16-byte MD5 hash of its +Message-ID and is in one of three states: a real article (with a storage +API token), a I Message-ID (no token, recorded so a cancelled +or expired article is not re-offered), or absent. + +Only one schema version of the hissqlite method currently exists, +implemented since S. + +This method comes with a migration tool (B) and a +support utility (B). See their manual pages for details. + +=head1 INSTALLATION + +The configure script will automatically enable SQLite support if it finds +the SQLite library. If the library isn't installed in a standard location, +you may have to specify the B<--with-sqlite3> option to help configure find +it. For complicated cases, you can use separate B<--with-sqlite3-include> +and B<--with-sqlite3-lib> options. Finally, if you I want SQLite +support even when your system has the SQLite library, you can use the +B<--without-sqlite3> option. + +=head1 CONFIGURATION + +To select hissqlite as your history method, set the I parameter +in F to C. The database is stored at the location +named by the I parameter with a F<.sqlite> suffix appended +(by default, F in I). After changing I, +migrate or rebuild your history (see L) and restart INN. + +The following F parameters tune the database; their defaults suit +most sites. + +=over 4 + +=item I + +The SQLite page size in bytes, applied when the database is created (it +cannot be changed afterwards without rebuilding). Must be a power of two +between 512 and 65536. The default is C<4096>, matching the common +filesystem block size so that a random single-row insert or point lookup +on the random-MD5 keys touches exactly one block (no read-modify-write). +For best results the page size should match the underlying storage block +size: on ZFS, create the history dataset with C. A small +page on a coarse-recordsize ZFS dataset (the 128k default) forces +read-modify-write on every random write and will perform worse, not +better. + +=item I + +The page cache size in kilobytes for the B writer connection. The +default is C<65536> (64 MB). + +=item I + +The page cache size in kilobytes for each B reader process, which +opens the database directly for read-only access. Each reader has its own +cache, so this is kept small; history lookups are not B's hot path +(overview is), and readers share the operating system page cache. The +default is C<2000> (about 2 MB). + +=item I + +The maximum number of bytes of the database that SQLite memory-maps, for both +the writer and the direct readers. When enabled, memory-mapped pages are +served from the shared operating system page cache, which can keep the B-tree +hot and lets reader processes share physical memory. This is of little +benefit on ZFS and other systems without a unified buffer cache, where the +pages would simply be cached twice, and memory-mapped I/O turns a read error +into a fatal signal (B) rather than a recoverable error. The default +value of C<0> therefore disables it, matching ovsqlite(5). A good starting +value, if you enable it after benchmarking, is the size of the database file +or the amount of available memory, whichever is smaller. + +=back + +The database is opened in WAL journal mode with C set to +C (see L). + +=head1 DURABILITY + +hissqlite uses WAL (Write-Ahead Logging) journal mode with C +set to C. A committed write is a buffered append to the WAL: it is +durable across an application crash (for instance, if B is killed), +but a sudden power loss or kernel panic can lose the transactions written +since the last checkpoint. The database itself is never corrupted. This +matches the durability contract of the C/I backend, where +history updates are likewise periodically flushed rather than synchronously +committed; in both cases the affected articles are simply re-offered by +peers. The WAL is checkpointed in the background and truncated on a clean +shutdown. + +B WAL mode requires shared-memory support and does not work on +network filesystems such as NFS if duties are split across multiple machines. +Keep I on local storage or only use a single instance to run +B and other programs like B. + +=head1 MIGRATION + +To convert an existing C history into a hissqlite database without +losing remembered entries or exact timestamps, use B; +see hissqlite-convert(8). Alternatively, you can rebuild the history from +the article spool with B, though a from-spool rebuild cannot +reconstruct remembered Message-IDs. + +=head1 RUNNING + +hissqlite has no separate server daemon. B writes the database +directly and is the steady-state writer; the only other writer is B, +which runs in place (see L) rather than rebuilding and swapping +files, so B is not paused for it. SQLite's WAL locking and busy +timeout serialize the two writers, and because writes autocommit there is no +broker process to funnel them through (unlike ovsqlite). Reader processes +and the offline tools open the same WAL database read-only. + +While B is running, B can safely read the database: +B holds the write lock for only a single statement at a time, so it +waits at most briefly. (B is a one-time migration tool +that reads a I source history, which must not be written while it +runs; see hissqlite-convert(8).) + +=head1 EXPIRATION + +B runs against the live database in place: real articles past their +retention are transitioned to remembered entries (their token is cleared), +and remembered entries older than the F threshold in +F are deleted. Because hissqlite expires in place, B +cannot write a rebuilt history to a side file, so its B<-d> and B<-f> options +are rejected. Use B<-t> for a dry run: it lists the tokens that would be +removed and changes neither the history nor the spool (no F file +is produced, unlike with C). B<-x> on its own is also rejected, since +suppressing the history rewrite while still removing articles would leave the +history pointing at deleted articles. See expire(8). + +=head1 HISTORY + +Written by Kevin Bowling for InterNetNews. + +=head1 SEE ALSO + +expire(8), expire.ctl(5), hissqlite-convert(8), hissqlite-util(8), +history(5), inn.conf(5), makehistory(8), ovsqlite(5). + +=cut diff --git a/doc/pod/history.pod b/doc/pod/history.pod index 1e796222a..cd6a19705 100644 --- a/doc/pod/history.pod +++ b/doc/pod/history.pod @@ -9,6 +9,11 @@ stored in the news system, as well as those that have been received but since expired. In a typical production environment, this file will be many megabytes. +The text format described here applies to the default C history +method. The C method stores the same information in a single +SQLite database instead of this text file and its I indexes; see +hissqlite(5). + The file consists of text lines. Each line corresponds to one article. The file is normally kept sorted in the order in which articles are received, although this is not a requirement. B appends a new @@ -59,6 +64,7 @@ into POD by Julien Elie. =head1 SEE ALSO -expire(8), inn.conf(5), innd(8), libinn_dbz(3), makehistory(8). +expire(8), hissqlite(5), inn.conf(5), innd(8), libinn_dbz(3), +makehistory(8). =cut diff --git a/doc/pod/inn.conf.pod b/doc/pod/inn.conf.pod index dd5182c41..ab5ad05b6 100644 --- a/doc/pod/inn.conf.pod +++ b/doc/pod/inn.conf.pod @@ -484,15 +484,15 @@ is C<10> and probably doesn't need to be changed. =head2 History Settings -The following parameter affect the history database. +The following parameters affect the history database. =over 4 =item I -Which history storage method to use. The only currently supported -value is C. There is no default value; this parameter must -be set. +Which history storage method to use. The supported values are C +and, when INN is built with SQLite, C. There is no default +value; this parameter must be set. =over 4 @@ -504,8 +504,42 @@ v6 format, or tagged hash format, depending on the build options. Separation of these two is a project which has not yet been undertaken. +=item C + +Stores history data in a single SQLite database file, as an alternative +to C. This requires INN to be built with SQLite support. See +hissqlite(5). + =back +=item I + +The SQLite page cache size in kilobytes for the B writer connection +when the C history method is used. The default value is C<65536> +(64 MB). See hissqlite(5). + +=item I + +The maximum number of bytes of the C history database that SQLite +will memory-map, for both the writer and the direct readers. The default +value of C<0> disables memory-mapped I/O (as with ovsqlite); see hissqlite(5) +for the trade-offs before enabling it. + +=item I + +The SQLite page size in bytes for a newly created C history +database; it has no effect on an existing one. Must be a power of two +between 512 and 65536. The default value is C<4096>, which matches the +common filesystem block size; for best results pair it with the storage +block size (on ZFS, a C dataset). See hissqlite(5). + +=item I + +The SQLite page cache size in kilobytes for each B reader process +using the C history method. Each reader has its own cache, so +this is typically kept small. The default value is C<2000> (about 2 MB). +See hissqlite(5). + =back =head2 Article Storage diff --git a/doc/pod/makehistory.pod b/doc/pod/makehistory.pod index fbd786f3e..3c535558a 100644 --- a/doc/pod/makehistory.pod +++ b/doc/pod/makehistory.pod @@ -28,6 +28,13 @@ If a malformed article is found in the news spool, in a way which prevents its integration into the history or overview data, a log line will be output and the malformed article will just be skipped. +When the C history method is configured, B writes the +new history into a hissqlite database just as it does for C, and the +makedbz(8) step above does not apply. A from-spool rebuild cannot recover +I Message-IDs, so to migrate an existing C history into +hissqlite without losing them, use hissqlite-convert(8) instead. See +hissqlite(5). + =head1 OVERVIEW REBUILD I: If you're trying to rebuild the overview database, be sure to @@ -283,7 +290,8 @@ updated by various other people since. =head1 SEE ALSO -active(5), ctlinnd(8), history(5), inn.conf(5), innd(8), libinn_dbz(3), -makedbz(8), ovdb_init(8), overchan(8), ovsqlite-server(8). +active(5), ctlinnd(8), hissqlite(5), hissqlite-convert(8), history(5), +inn.conf(5), innd(8), libinn_dbz(3), makedbz(8), ovdb_init(8), overchan(8), +ovsqlite-server(8). =cut diff --git a/expire/expire.c b/expire/expire.c index 42082d425..c6a5946ba 100644 --- a/expire/expire.c +++ b/expire/expire.c @@ -710,6 +710,7 @@ main(int ac, char *av[]) bool Writing; bool UnlinkFile; bool val; + bool inplace; time_t TimeWarp; size_t Size = 0; @@ -880,6 +881,56 @@ main(int ac, char *av[]) HISctl(History, HISCTLS_NPAIRS, &Size); } + /* In-place backends (e.g. hissqlite) mutate the live database via a writer + * rather than rebuilding a new file and swapping it, so they must be opened + * read/write (even for a dry run: the expire scan runs on the writer + * connection). Backends that do not implement HISCTLG_INPLACEEXPIRE leave + * the flag untouched, so the hisv6 rebuild-and-swap path is unchanged. + * + * The modes that decouple the history rewrite from article removal need + * care here, because a run that does one but not the other corrupts the + * spool<->history correspondence (e.g. delete articles but keep their + * history rows). All exits go through CleanupAndExit, not die(), so the + * server reservation taken above is released. + * + * -t (tracing) is the dry run: EXPremove() already suppresses the + * SMcancel(), and we additionally force Writing off so the live history + * is not mutated -- it lists what would be removed and changes nothing. + * (Unlike hisv6 it produces no history.n side file.) + * + * -d/-f ask for a rebuilt history written to a side file, which an + * in-place backend cannot produce, so they are refused. + * + * -x on its own (suppress the history rewrite, but still remove articles) + * is exactly the inconsistent run the dry run replaces, so it is refused + * too; -t is the supported way to preview an expire. */ + inplace = false; + HISctl(History, HISCTLG_INPLACEEXPIRE, &inplace); + if (inplace) { + if (EXPtracing) { + Writing = false; + } else if (NHistory != NULL) { + warn("%s expires in place and cannot write a separate history file" + " (-d/-f); use -t for a dry run", + innconf->hismethod); + CleanupAndExit(Server, false, 1); + } else if (!Writing) { + warn("%s expires in place: -x without -t would remove articles but" + " leave their history entries; use -t for a dry run", + innconf->hismethod); + CleanupAndExit(Server, false, 1); + } + HISclose(History); + History = HISopen(HistoryText, innconf->hismethod, HIS_RDWR); + if (!History) { + warn("cannot reopen history read/write for in-place expire"); + CleanupAndExit(Server, false, 1); + } + HISctl(History, HISCTLS_IGNOREOLD, &IgnoreOld); + if (Size != 0) + HISctl(History, HISCTLS_NPAIRS, &Size); + } + val = true; if (!SMsetup(SM_RDWR, (void *) &val) || !SMsetup(SM_PREOPEN, (void *) &val)) { diff --git a/history/Makefile b/history/Makefile index 0cc3b6c04..a60e873a3 100644 --- a/history/Makefile +++ b/history/Makefile @@ -9,7 +9,7 @@ include ../Makefile.global LTVERSION = 3:9:0 top = .. -CFLAGS = $(GCFLAGS) -I. +CFLAGS = $(GCFLAGS) -I. $(SQLITE3_CPPFLAGS) SOURCES = his.c hismethods.c $(METHOD_SOURCES) OBJECTS = $(SOURCES:.c=.o) @@ -28,6 +28,9 @@ warnings: install: all $(LI_LPUB) libinnhist.$(EXTLIB) $D$(PATHLIB)/libinnhist.$(EXTLIB) + for F in $(PROGRAMS) ; do \ + $(LI_XPRI) $$F $D$(PATHBIN)/`basename $$F` ; \ + done bootstrap: Make.methods diff --git a/history/hissqlite/hismethod.config b/history/hissqlite/hismethod.config new file mode 100644 index 000000000..8b64b2f6d --- /dev/null +++ b/history/hissqlite/hismethod.config @@ -0,0 +1,5 @@ +name = hissqlite +number = 1 +sources = hissqlite.c hissqlite-init.c hissqlite-main.c hissqlite-read.c +extra-sources = hissqlite-convert.c +programs = hissqlite-convert hissqlite-util diff --git a/history/hissqlite/hismethod.mk b/history/hissqlite/hismethod.mk new file mode 100644 index 000000000..30b0aaa74 --- /dev/null +++ b/history/hissqlite/hismethod.mk @@ -0,0 +1,32 @@ +## Build rules for the hissqlite history method. Included into +## history/Make.methods by buildconfig. The .c/.h are generated from the +## .sql files by the shared sqlite-helper-gen, which lives in lib/ (built +## before history, so it is available here). + +hissqlite/hissqlite-init.c: hissqlite/hissqlite-init.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen hissqlite/hissqlite-init.sql +hissqlite/hissqlite-init.h: hissqlite/hissqlite-init.c ; + +hissqlite/hissqlite-main.c: hissqlite/hissqlite-main.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen hissqlite/hissqlite-main.sql +hissqlite/hissqlite-main.h: hissqlite/hissqlite-main.c ; + +hissqlite/hissqlite-read.c: hissqlite/hissqlite-read.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen hissqlite/hissqlite-read.sql +hissqlite/hissqlite-read.h: hissqlite/hissqlite-read.c ; + +## hissqlite.c and the generated translation units include the generated +## headers; make that explicit so a parallel build orders correctly. +hissqlite/hissqlite.o hissqlite/hissqlite.lo: \ + hissqlite/hissqlite-init.h hissqlite/hissqlite-main.h \ + hissqlite/hissqlite-read.h + +## Programs: the migration converter (C) and the inspect/dump utility (Perl). +hissqlite/hissqlite-convert: hissqlite/hissqlite-convert.o libinnhist.la \ + $(LIBSTORAGE) $(LIBINN) + $(LIBLD) $(LDFLAGS) $(SQLITE3_LDFLAGS) -o $@ \ + hissqlite/hissqlite-convert.o libinnhist.la \ + $(LIBSTORAGE) $(LIBINN) $(STORAGE_LIBS) $(LIBS) + +hissqlite/hissqlite-util: hissqlite/hissqlite-util.in $(FIXSCRIPT) + $(FIX) hissqlite/hissqlite-util.in diff --git a/history/hissqlite/hissqlite-convert.c b/history/hissqlite/hissqlite-convert.c new file mode 100644 index 000000000..f0fac2c19 --- /dev/null +++ b/history/hissqlite/hissqlite-convert.c @@ -0,0 +1,223 @@ +/* +** Convert an existing history database to the hissqlite (SQLite) backend. +** +** Usage: hissqlite-convert [-m srcmethod] +** +** Reads the source history (default method "hisv6") via HISwalk and bulk-loads +** every entry into a freshly created hissqlite database keyed by hash. This +** is a faithful migration (the makedbz analog): it preserves exact +** timestamps AND remembered (token-less) entries, which a from-spool rebuild +** (makehistory) cannot reconstruct. +** +** Insertion is by HASH, not Message-ID: the history text stores only the MD5, +** and the original Message-ID is unrecoverable, so HISwrite (which re-hashes a +** Message-ID) cannot be used. We therefore create the canonical schema via the +** hissqlite backend, then bulk-insert by hash through a raw SQLite connection. +** +** Written by Kevin Bowling in 2026. +*/ + +#include "portable/system.h" + +#include "inn/history.h" +#include "inn/innconf.h" +#include "inn/libinn.h" +#include "inn/messages.h" +#include "inn/storage.h" + +#ifndef HAVE_SQLITE3 + +int +main(void) +{ + die("INN was built without SQLite support"); +} + +#else + +# include +# include + +/* Commit the load transaction every this many rows. */ +# define COMMIT_EVERY 50000 + +struct convstate { + sqlite3 *db; + sqlite3_stmt *insert; + unsigned long count; /* rows successfully inserted */ + unsigned long dups; /* duplicate-hash rows skipped (anomalous) */ +}; + +static void +checked(sqlite3 *db, int status, const char *what) +{ + if (status != SQLITE_OK) + die("%s: %s", what, sqlite3_errmsg(db)); +} + +static void +exec(sqlite3 *db, const char *sql) +{ + char *errmsg = NULL; + + if (sqlite3_exec(db, sql, NULL, NULL, &errmsg) != SQLITE_OK) + die("%s: %s", sql, errmsg ? errmsg : "(unknown)"); +} + +/* +** HISwalk callback: insert one source entry into the hissqlite DB by hash. +** token == NULL marks a remembered entry (stored with a NULL token column). +*/ +static bool +convert_cb(void *cookie, const HASH *hash, time_t arrived, time_t posted, + time_t expires, const TOKEN *token) +{ + struct convstate *st = cookie; + sqlite3_stmt *ins = st->insert; + int rc; + + sqlite3_bind_blob(ins, 1, hash, sizeof(HASH), SQLITE_STATIC); + sqlite3_bind_int64(ins, 2, (sqlite3_int64) arrived); + sqlite3_bind_int64(ins, 3, (sqlite3_int64) posted); + sqlite3_bind_int64(ins, 4, (sqlite3_int64) expires); + if (token == NULL) + sqlite3_bind_null(ins, 5); + else + sqlite3_bind_blob(ins, 5, token, sizeof(TOKEN), SQLITE_STATIC); + + rc = sqlite3_step(ins); + sqlite3_reset(ins); + if (rc == SQLITE_DONE) { + st->count++; + } else if ((rc & 0xFF) == SQLITE_CONSTRAINT) { + /* Duplicate hash in the source: anomalous for a valid history, but not + fatal -- count it rather than silently dropping it. */ + st->dups++; + return true; + } else { + /* A real error (I/O, disk full, corruption): abort. The half-built + database is a temp file that is never renamed into place. */ + die("insert failed after %lu entries: %s", st->count, + sqlite3_errmsg(st->db)); + } + + if (st->count % COMMIT_EVERY == 0) { + exec(st->db, "commit"); + exec(st->db, "begin"); + if (st->count % (COMMIT_EVERY * 20) == 0) + notice("%lu entries...", st->count); + } + return true; +} + +int +main(int argc, char *argv[]) +{ + const char *srcmethod = "hisv6"; + const char *srcpath, *dstpath; + char *finaldb, *tempbase, *tempdb; + struct history *src, *tmp; + struct convstate st; + struct stat sb; + int opt; + + message_program_name = "hissqlite-convert"; + + while ((opt = getopt(argc, argv, "m:")) != -1) { + switch (opt) { + case 'm': + srcmethod = optarg; + break; + default: + die("usage: hissqlite-convert [-m srcmethod] "); + } + } + if (argc - optind != 2) + die("usage: hissqlite-convert [-m srcmethod] "); + srcpath = argv[optind]; + dstpath = argv[optind + 1]; + + /* The created database honors the hissqlite* tunables (page size in + particular) from inn.conf, matching what innd will later use. */ + if (!innconf_read(NULL)) + exit(1); + + /* Refuse to overwrite an existing destination: a re-run or a typo must not + clobber or merge into a live history. */ + finaldb = concat(dstpath, ".sqlite", (char *) NULL); + if (stat(finaldb, &sb) == 0) + die("destination %s already exists; refusing to overwrite", finaldb); + + /* Build into a temp file and rename it into place only on full success, so + a crash or error never leaves a usable-but-incomplete destination (this + is also what makes journal_mode=off below safe). Clear any stale temp + from a previous failed run. */ + tempbase = concat(dstpath, ".new", (char *) NULL); + tempdb = concat(tempbase, ".sqlite", (char *) NULL); + unlink(tempdb); + + /* Create the temp DB with the canonical hissqlite schema, then reopen the + raw SQLite file for a fast by-hash bulk load. */ + tmp = HISopen(tempbase, "hissqlite", HIS_CREAT | HIS_RDWR); + if (tmp == NULL) + die("cannot create destination %s", tempdb); + HISclose(tmp); + + memset(&st, 0, sizeof(st)); + if (sqlite3_open(tempdb, &st.db) != SQLITE_OK) + die("cannot open %s: %s", tempdb, sqlite3_errmsg(st.db)); + + /* Bulk-load tuning: no per-row durability (safe -- a failure leaves the + temp file, never the final), and drop the secondary index so it is + built once at the end. (The CREATE INDEX statement mirrors + hissqlite-init.sql.) */ + exec(st.db, "pragma synchronous = off"); + exec(st.db, "pragma journal_mode = off"); + /* A large page cache keeps the random-MD5 B-tree hot during the load, so + inserts don't constantly evict and re-fetch leaf pages. (The first innd + open flips the finished DB to WAL.) */ + exec(st.db, "pragma cache_size = -262144"); /* 256 MiB (negative = KiB) */ + exec(st.db, "drop index if exists hist_remember"); + + /* Plain INSERT (not OR IGNORE): a duplicate hash returns SQLITE_CONSTRAINT + so convert_cb can count it instead of silently dropping the row. */ + checked(st.db, + sqlite3_prepare_v2(st.db, + "insert into hist" + "(hash, arrived, posted, expires, token)" + " values(?1, ?2, ?3, ?4, ?5)", + -1, &st.insert, NULL), + "prepare insert"); + + src = HISopen(srcpath, srcmethod, HIS_RDONLY); + if (src == NULL) + die("cannot open source %s (method %s)", srcpath, srcmethod); + + exec(st.db, "begin"); + if (!HISwalk(src, NULL, &st, convert_cb)) + die("HISwalk failed: %s", HISerror(src)); + exec(st.db, "commit"); + HISclose(src); + + sqlite3_finalize(st.insert); + notice("rebuilding index..."); + exec(st.db, + "create index hist_remember on hist(posted, arrived)" + " where token is null"); + if (sqlite3_close(st.db) != SQLITE_OK) + die("cannot close %s", tempdb); + + /* Atomically publish the completed database. */ + if (rename(tempdb, finaldb) != 0) + sysdie("cannot rename %s to %s", tempdb, finaldb); + + if (st.dups > 0) + warn("%lu duplicate hashes in the source were skipped", st.dups); + notice("converted %lu entries into %s", st.count, dstpath); + free(finaldb); + free(tempbase); + free(tempdb); + return 0; +} + +#endif /* HAVE_SQLITE3 */ diff --git a/history/hissqlite/hissqlite-init.sql b/history/hissqlite/hissqlite-init.sql new file mode 100644 index 000000000..fdc052c19 --- /dev/null +++ b/history/hissqlite/hissqlite-init.sql @@ -0,0 +1,53 @@ +-- hissqlite schema version 1 +-- +-- Run once at database creation. Schema 1: a single WITHOUT ROWID table +-- clustered on the 16-byte MD5 of the Message-ID, so a Message-ID -> token +-- lookup is one clustered-leaf access (the token lives in the leaf). +-- +-- Three logical states of a row: +-- token IS NOT NULL -> a real, stored article +-- token IS NULL -> a "remembered" entry (article seen/rejected or +-- expired; kept so it is not re-accepted) +-- no row -> never seen +-- +-- The token-NULL state is a first-class lifecycle stage, NOT a tombstone for +-- deletion: when a real article reaches its retention it is UPDATEd to +-- remembered (token=NULL), and only DELETEd once the remembered entry passes +-- the /remember/ posting-time threshold. See hissqlite.c hissqlite_expire(). + +-- Page size (pragma page_size) is applied by hissqlite.c from the +-- hissqlitepagesize inn.conf parameter before this DDL runs: it must be set +-- before any table exists (changing it later needs a VACUUM). The default, +-- 4096, matches the common filesystem block (and a ZFS recordsize=4k dataset), +-- so a random single-row insert or a point lookup touches exactly one block -- +-- no read-modify-write, no super-block read amplification. Benchmarking the +-- random-MD5-key workload measured 4k beating 8k on both writes and reads at +-- equal footprint. + +create table hist ( + hash blob not null primary key, -- HashMessageID(): 16-byte MD5 + arrived integer not null, -- time_t + posted integer, -- time_t; 0/NULL if unknown + expires integer, -- time_t article expiry; 0/NULL = none + token blob -- raw TOKEN (sizeof TOKEN); NULL = remembered +) without rowid; + +-- Find remember-threshold delete candidates (remembered -> gone): a partial +-- index over only the token-NULL rows, so expire pass 2 scans the remembered +-- entries without touching the real articles. (Real-article expiry needs no +-- index: pass 1 evaluates every token-bearing row via the policy callback, in +-- clustered hash order -- see hissqlite_expire().) +create index hist_remember on hist(posted, arrived) where token is null; + +-- Schema/version marker, mirroring ovsqlite's misc table. +create table misc ( + key text not null primary key, + value not null +) without rowid; + +-- .get_version +select value from misc where key = 'version'; + +-- .set_version +insert into misc(key, value) values('version', ?1) + on conflict(key) do update set value = excluded.value; diff --git a/history/hissqlite/hissqlite-main.sql b/history/hissqlite/hissqlite-main.sql new file mode 100644 index 000000000..911a48992 --- /dev/null +++ b/history/hissqlite/hissqlite-main.sql @@ -0,0 +1,107 @@ +-- Writer prepared statements for hissqlite (used by innd and the offline +-- tools: makehistory, prunehistory, the migration converter). +-- +-- Implementation follows ovsqlite's limits/patterns; the operator-facing +-- durability contract matches dbz (lose recent on power loss, never corrupt, +-- peer resends). WAL + synchronous=NORMAL: a commit is durable across an +-- application crash; a power loss can lose transactions since the last +-- checkpoint, but the DB is never corrupted -- parity with dbz's periodic +-- msync, not a regression. + +pragma journal_mode = wal; +pragma synchronous = normal; +pragma busy_timeout = 10000; +pragma foreign_keys = off; + +-- Writer cache + mmap (pragma cache_size / mmap_size) are applied by +-- hissqlite.c from the hissqlitecachesize and hissqlitemmapsize inn.conf +-- parameters after this section runs. innd is a single process, so a larger +-- per-connection page cache is affordable; mmap lets reads come from the shared +-- OS page cache and keeps the random-MD5 B-tree hot. + +-- Writes are NOT batched: each statement autocommits (one implicit transaction +-- per write). Under synchronous=NORMAL a commit is a buffered WAL append, not +-- an fsync, so there is no fsync cost to amortise; batching would only hold the +-- write lock across the batch and starve the other writer (expire) in the +-- two-writer model. Batching is reserved for the offline bulk load +-- (hissqlite-convert.c). See hissqlite.c. + +-- HISlookup: Message-ID -> token. A row with token IS NULL (remembered) +-- returns found=true with an empty token; ARTopenbyid treats TOKEN_EMPTY as +-- "no article". +-- .lookup +select arrived, posted, expires, token + from hist + where hash = ?1; + +-- HIScheck: existence only (real OR remembered both count -> refuse re-offers). +-- his.c keeps an in-memory existence cache in front of this; this is the +-- authoritative fallback. +-- .check +select 1 from hist where hash = ?1; + +-- HISwrite: a real, stored article. The accept path runs HIScheck first, so +-- this is normally a fresh insert. DO NOTHING on a hash conflict keeps the +-- existing row rather than failing: a duplicate Message-ID during a makehistory +-- rebuild must warn and continue, not abort the whole rebuild, as in hisv6 +-- (hissqlite.c detects the no-op via sqlite3_changes()). +-- .write +insert into hist(hash, arrived, posted, expires, token) + values(?1, ?2, ?3, ?4, ?5) + on conflict(hash) do nothing; + +-- HISremember: record a seen/rejected Message-ID with no token. Must NEVER +-- downgrade an existing real article to remembered, hence DO NOTHING. +-- .remember +insert into hist(hash, arrived, posted, expires, token) + values(?1, ?2, ?3, 0, null) + on conflict(hash) do nothing; + +-- HISreplace: the only token-changing op (prunehistory real->remembered by +-- passing a null token; remembered->real upgrade). Full upsert. +-- .replace +insert into hist(hash, arrived, posted, expires, token) + values(?1, ?2, ?3, ?4, ?5) + on conflict(hash) do update set + arrived = excluded.arrived, + posted = excluded.posted, + expires = excluded.expires, + token = excluded.token; + +-- HISwalk: full iteration (feeds the bloom build, dump/migration). Full +-- clustered B-tree scan: each leaf page read once, semi-sequential. +-- .walk +select hash, arrived, posted, expires, token from hist; + +-- HISexpire pass 1: iterate token-bearing entries so the policy callback can +-- decide keep / transition-to-remembered. (expire.ctl needs every entry +-- evaluated, as hisv6 does.) Keyset-paginated by the hash PK: ?1 is the last +-- hash of the previous page (empty blob for the first), ?2 the page size. The +-- caller streams pages so the read snapshot is released between them; the PK +-- range scan is in hash order, and rows transitioned to token=NULL drop out. +-- .expire_scan +select hash, arrived, posted, expires, token + from hist + where token is not null + and hash > ?1 + order by hash + limit ?2; + +-- HISexpire pass 1 action: transition a real article to remembered. +-- .transition_remember +update hist set token = null, expires = 0 where hash = ?1; + +-- HISexpire pass 1 action: callback kept the article but rewrote the token. +-- .update_token +update hist set token = ?2, expires = ?3 where hash = ?1; + +-- HISexpire pass 2: delete remembered entries older than the /remember/ +-- posting-time threshold (indexed by hist_remember). Bounded to a chunk per +-- step (LIMIT) so each autocommit holds the write lock only briefly; the caller +-- loops until a step deletes nothing. +-- .expire_remembered +delete from hist where hash in ( + select hash from hist + where token is null + and (case when posted > 0 then posted else arrived end) < ?1 + limit 10000); diff --git a/history/hissqlite/hissqlite-private.h b/history/hissqlite/hissqlite-private.h new file mode 100644 index 000000000..892f3d2c4 --- /dev/null +++ b/history/hissqlite/hissqlite-private.h @@ -0,0 +1,65 @@ +/* +** Private declarations for the hissqlite history backend. +*/ + +#ifndef HISSQLITE_PRIVATE_H +#define HISSQLITE_PRIVATE_H + +#include "config.h" +#include "inn/history.h" +#include "inn/libinn.h" +#include "inn/storage.h" + +#include + +/* +** Generated by sqlite-helper-gen from the .sql files at build time: +** hissqlite-init.{c,h} -> hissqlite_init_t / hissqlite_init_helper +** hissqlite-main.{c,h} -> hissqlite_main_t / hissqlite_main_helper +** hissqlite-read.{c,h} -> hissqlite_read_t / hissqlite_read_helper +** sqlite-helper.{c,h} and sqlite-helper-gen are shared with the ovsqlite +** overview backend (lib/ and include/inn/). +*/ +#include "hissqlite-init.h" +#include "hissqlite-main.h" +#include "hissqlite-read.h" + +#define HISSQLITE_SCHEMA_VERSION 1 + +/* Checkpoint the WAL once it has grown past this many frames (mirrors + ovsqlite's threshold). Done as a non-blocking PASSIVE checkpoint from + HISsync; TRUNCATE (disk reclaim) is left to close. */ +#define HISSQLITE_WAL_CKPT_PAGES 1000 + +/* Expire streams pass 1 in hash-keyset pages of this many rows: bounds the + in-memory action buffer and releases the read snapshot between pages so a + long expire does not pin the WAL. */ +#define HISSQLITE_EXPIRE_BATCH 10000 + +/* Fallback values for the inn.conf tunables (hissqlitepagesize, + hissqlitecachesize, hissqlitemmapsize), used when inn.conf has not been read + -- offline tools and unit tests that open history without innconf_read(). + When inn.conf is loaded these are superseded by the configured values (whose + own defaults, set in lib/innconf.c, match these). */ +#define HISSQLITE_DEF_PAGE_SIZE 4096 /* bytes; before first table write */ +#define HISSQLITE_DEF_CACHE_SIZE 65536 /* writer page cache, kB */ +#define HISSQLITE_DEF_MMAP_SIZE 0 /* bytes; 0 disables mmap (as ovsqlite) */ +#define HISSQLITE_DEF_RCACHE_SIZE 2000 /* per-nnrpd reader page cache, kB */ + +struct hissqlite { + char *path; /* SQLite database file path. */ + struct history *history; /* For his_seterror(). */ + sqlite3 *db; + int flags; /* HIS_RDONLY / HIS_RDWR / HIS_CREAT / ... */ + + /* Writer side (NULL for a pure read-only direct reader). Writes + autocommit -- there is no batching state (see hissqlite.c). */ + hissqlite_main_t main; + int wal_pages; /* WAL frame count, updated by the wal_hook. */ + + /* Reader side (used when opened HIS_RDONLY in WAL mode). */ + hissqlite_read_t read; + bool direct_reader; +}; + +#endif /* HISSQLITE_PRIVATE_H */ diff --git a/history/hissqlite/hissqlite-read.sql b/history/hissqlite/hissqlite-read.sql new file mode 100644 index 000000000..9d3a071b3 --- /dev/null +++ b/history/hissqlite/hissqlite-read.sql @@ -0,0 +1,24 @@ +-- Read-only prepared statements for hissqlite (nnrpd and other readers). +-- +-- In WAL mode readers open the SQLite file read-only and query it directly, +-- bypassing any writer process. + +pragma query_only = 1; +pragma busy_timeout = 10000; + +-- Reader cache + mmap (pragma cache_size / mmap_size) are applied by +-- hissqlite.c from the hissqlitereadercachesize and hissqlitemmapsize inn.conf +-- parameters after this section runs. nnrpd runs one process PER connection, +-- so the per-connection cache is kept small by default; readers lean on the +-- shared OS page cache instead. + +-- .lookup +select arrived, posted, expires, token + from hist + where hash = ?1; + +-- .check +select 1 from hist where hash = ?1; + +-- .walk +select hash, arrived, posted, expires, token from hist; diff --git a/history/hissqlite/hissqlite-util.in b/history/hissqlite/hissqlite-util.in new file mode 100644 index 000000000..efb440419 --- /dev/null +++ b/history/hissqlite/hissqlite-util.in @@ -0,0 +1,230 @@ +#! /usr/bin/perl +# fixscript will replace this line with code to load INN::Config + +## History manipulation utility for hissqlite. +## +## Initial version written in 2026, modeled on ovsqlite-util. + +use strict; +use warnings; + +use Getopt::Std; +use POSIX qw(strftime); ## no critic (ProhibitUnusedImport) + +$0 =~ s!.*/!!; + +# Version of this tool. +my $VERSION = "1.0"; + +# Bail out if the needed DBI Perl module is not installed. +eval { + require DBI; + require DBD::SQLite; + 1; +} or die "DBI Perl module with SQLite driver needed" + . " (usually packaged as libdbd-sqlite3-perl, perl-DBD-SQLite," + . " or p5-DBD-SQLite)"; + +# Name of the database file. +my $dbfile = "history.sqlite"; + +my $usage = "Usage: + $0 [-AcdhV] [-p path] + +Options: + -A Audit the history database for problems, and report them to + standard error, without trying to fix them. This is the + default action. + -c Print total, real, and remembered entry counts. + -d Dump all history records as text, one per line. + -h Print this help message. + -p path Read $dbfile database file in path directory instead of + default \$INN::Config::pathdb directory. + -V Print the version of this tool and the database schema + version. +"; + +sub HELP_MESSAGE { + print $usage; + exit(0); +} + +my %opt; +getopts("AcdhVp:", \%opt) || die $usage; + +HELP_MESSAGE() if defined($opt{'h'}); + +my $modes = 0; +$modes++ if defined($opt{'A'}); +$modes++ if defined($opt{'c'}); +$modes++ if defined($opt{'d'}); +$modes++ if defined($opt{'V'}); + +die "Only one action allowed at the same time\n\n$usage" + if $modes > 1; + +my $dbdir = $opt{'p'} || $INN::Config::pathdb; +my $datasource = "dbi:SQLite:dbname=$dbdir/$dbfile"; + +# All implemented modes are read-only, so open the database read-only. +# SQLITE_OPEN_READONLY is 0x00000001. +my $SQLITE_OPEN_READONLY = 0x00000001; + +# Open the connection. The username and password fields are left empty. +# Enabling RaiseError permits not checking every return error codes. +my $dbh = DBI->connect( + $datasource, '', '', + { + PrintError => 0, + RaiseError => 1, + AutoCommit => 1, + sqlite_open_flags => $SQLITE_OPEN_READONLY, + }, +); + +# Extra safety net against any accidental write in the read-only modes. +$dbh->do("pragma query_only = 1;"); + +# Raise the busy timeout value to a huge number of milliseconds to prevent +# timeouts because of database locks while innd holds the writer. +$dbh->sqlite_busy_timeout(999999999); + +if (defined($opt{'V'})) { + print_version(); +} elsif (defined($opt{'c'})) { + count_entries(); +} elsif (defined($opt{'d'})) { + dump_history(); +} else { + # Default action: audit. + audit_history(); +} + +# Close the connection properly. +$dbh->disconnect; + +exit(0); + +# Return the schema version stored in the misc table, or undef if absent. +sub get_version { + my $statement = $dbh->prepare("select value from misc where key = ?;"); + my ($version) = $dbh->selectrow_array($statement, undef, "version"); + return $version; +} + +# Return a list of (total, real, remembered) entry counts. +sub get_counts { + my ($total) = $dbh->selectrow_array("select count(*) from hist;"); + my ($real) + = $dbh->selectrow_array("select count(*) from hist" + . " where token is not null;"); + my ($remembered) + = $dbh->selectrow_array("select count(*) from hist" + . " where token is null;"); + return ($total, $real, $remembered); +} + +# Print the tool version and the database schema version (-V option). +sub print_version { + my $version = get_version(); + $version = defined($version) ? $version : "unknown"; + print "$0 version $VERSION\n"; + print "hissqlite database schema version $version\n"; +} + +# Print total, real, and remembered entry counts (-c option). +sub count_entries { + my ($total, $real, $remembered) = get_counts(); + print "total $total\n"; + print "real $real\n"; + print "remembered $remembered\n"; +} + +# Dump all history records (-d option). +sub dump_history { + my $statement; + + # quote(token) renders a BLOB as X'...' uppercase hex, or NULL for a + # remembered entry; hex(hash) renders the hash as uppercase hex. + $statement = $dbh->prepare( + q{ + select hex(hash), arrived, posted, expires, quote(token) + from hist; + }, + ); + $statement->execute(); + + while (my @row = $statement->fetchrow_array()) { + my ($hash, $arrived, $posted, $expires, $token) = @row; + $posted = defined($posted) ? $posted : '-'; + $expires = defined($expires) ? $expires : '-'; + if (!defined($token) or uc($token) eq 'NULL') { + $token = 'remembered'; + } else { + # quote() returns the token in the form X'...'; keep the hex part. + $token = uc(substr($token, 2, -1)); + } + print "$hash $arrived $posted $expires $token\n"; + } +} + +# Audit the history database for problems and report them (-A option, default). +sub audit_history { + my $problems = 0; + + # Schema version. + my $version = get_version(); + if (defined($version)) { + print "Schema version: $version\n"; + } else { + print STDERR "No schema version found in misc table\n"; + $problems++; + } + + # Entry counts. + my ($total, $real, $remembered) = get_counts(); + print "Total entries: $total\n"; + print "Real entries (token not null): $real\n"; + print "Remembered entries (token null): $remembered\n"; + + # Arrived time range. + my ($minarrived, $maxarrived) + = $dbh->selectrow_array( + "select min(arrived), max(arrived) from hist;"); + if (defined($minarrived) and defined($maxarrived)) { + printf "Arrived range: %s .. %s\n", + strftime('%Y-%m-%d %H:%M:%S', localtime($minarrived)), + strftime('%Y-%m-%d %H:%M:%S', localtime($maxarrived)); + } else { + print "Arrived range: (empty database)\n"; + } + + # Rows with a NULL arrived value (arrived is declared not null, so any + # such row indicates corruption). + my ($nullarrived) + = $dbh->selectrow_array( + "select count(*) from hist where arrived is null;"); + if (defined($nullarrived) and $nullarrived > 0) { + print STDERR "$nullarrived row(s) with NULL arrived value\n"; + $problems++; + } + + # SQLite low-level integrity check. + my $statement = $dbh->prepare("pragma integrity_check;"); + $statement->execute(); + while (my @row = $statement->fetchrow_array()) { + next if $row[0] eq 'ok'; + print STDERR "integrity_check: $row[0]\n"; + $problems++; + } + + if ($problems > 0) { + printf STDERR ( + "%d problem%s found\n", + $problems, ($problems > 1) ? "s" : "", + ); + exit(1); + } else { + print "No problems found\n"; + } +} diff --git a/history/hissqlite/hissqlite.c b/history/hissqlite/hissqlite.c new file mode 100644 index 000000000..e0f331a2a --- /dev/null +++ b/history/hissqlite/hissqlite.c @@ -0,0 +1,885 @@ +/* +** The SQLite history method (hissqlite). +** +** A transactional replacement for hisv6/dbz. Schema 1: a single WITHOUT +** ROWID table clustered on the 16-byte MD5 of the Message-ID, so a +** Message-ID -> token lookup is one clustered-leaf access. WAL + +** synchronous=NORMAL, autocommit writes; readers use the WAL direct path. +** +** innd owns its write connection directly and is the steady-state writer; the +** only other writer is expire, which runs in place (it reopens the live DB +** read/write instead of rebuilding and swapping a file, so innd is not +** paused). SQLite's WAL write lock and busy_timeout serialize the two without +** a broker server process (unlike ovsqlite): this works because writes +** autocommit, so the lock is held only per statement and the periodic, light +** expire does not contend with innd for long. See hissqlite(5). +** +** Written by Kevin Bowling in 2026. +*/ + +#include "portable/system.h" + +#include "inn/messages.h" +#include "../hisinterface.h" +#include "hissqlite.h" + +#ifdef HAVE_SQLITE3 + +#include "inn/innconf.h" +#include "hissqlite-private.h" + +/* +** Record a SQLite error against the history handle. +*/ +static void +hissqlite_seterror(struct hissqlite *h, const char *context) +{ + if (h->history != NULL) + his_seterror(h->history, + concat("hissqlite: ", context, ": ", + h->db ? sqlite3_errmsg(h->db) : "(no db)", NULL)); +} + +/* +** Bind the MD5 of a Message-ID as the hash key (parameter 1). +*/ +static void +bind_key(sqlite3_stmt *stmt, const char *key) +{ + HASH hash = HashMessageID(key); + + sqlite3_bind_blob(stmt, 1, &hash, sizeof(HASH), SQLITE_TRANSIENT); +} + +/* +** Copy a column blob into a fixed-size destination, verifying its length +** first. A wrong-length blob means a corrupt row; report it (so corruption +** is visible) and return false rather than over-reading or silently producing +** a bogus value. +*/ +static bool +copy_blob(struct hissqlite *h, sqlite3_stmt *stmt, int col, void *dst, + size_t size, const char *what) +{ + if (sqlite3_column_bytes(stmt, col) != (int) size) { + /* A length mismatch is the error itself; the generic seterror would + append an unrelated sqlite3_errmsg, so report "what" on its own. */ + if (h->history != NULL) + his_seterror(h->history, concat("hissqlite: ", what, (char *) NULL)); + return false; + } + memcpy(dst, sqlite3_column_blob(stmt, col), size); + return true; +} + +/* +** Writes are NOT batched: each HISwrite/HISremember/HISreplace autocommits +** (one implicit transaction per statement). With synchronous=NORMAL a commit +** is a buffered WAL append, not an fsync, so there is no fsync cost to +** amortise; batching would buy only a little WAL overhead, at the price of +** holding the write lock across the batch. Holding the lock is exactly wrong +** for the two-writer model (innd + a separate expire process): a batch +** would starve the other writer. Autocommit holds the lock for a single +** statement, so innd and expire interleave cleanly, and there is no open +** transaction to leave dangling on error or to wrap the expire scan. +** Batching only makes sense for the single-writer, offline bulk load -- see +** hissqlite-convert.c. +*/ + +/* +** Track the WAL frame count so HISsync can decide when to checkpoint. +*/ +static int +hissqlite_wal_hook(void *arg, sqlite3 *db UNUSED, const char *name UNUSED, + int pages) +{ + ((struct hissqlite *) arg)->wal_pages = pages; + return SQLITE_OK; +} + +/* +** A PASSIVE checkpoint never blocks: it writes back as many WAL frames as the +** oldest reader allows and lets the WAL be reused (not grown) on the next +** write. Safe to call from innd's HISsync timer without stalling accepts. +** Disk reclaim (TRUNCATE) is left to close, where readers are expected gone. +*/ +static void +hissqlite_wal_passive(struct hissqlite *h) +{ + int log = 0, ckpt = 0, rc; + + rc = sqlite3_wal_checkpoint_v2(h->db, NULL, SQLITE_CHECKPOINT_PASSIVE, &log, + &ckpt); + if (rc != SQLITE_OK) { + /* A real failure (e.g. I/O error): surface it so a WAL that stops + shrinking -- and could fill the disk -- is visible, not silent. */ + warn("hissqlite: WAL checkpoint failed: %s", sqlite3_errmsg(h->db)); + return; + } + if (log == ckpt) + h->wal_pages = 0; /* Fully flushed; the WAL resets on the next write. */ + /* else a reader still pins older frames; wal_pages stays high and we retry + on the next HISsync (not logged per-call -- that is normal, transient). */ +} + +/* +** Has the schema already been created? HIS_CREAT must initialise a brand-new +** database but must NOT re-run the DDL (or re-stamp the version) on an existing +** one -- otherwise opening an existing DB with HIS_CREAT fails on "table hist +** already exists" and would mask a version mismatch. +*/ +static bool +hissqlite_initialized(struct hissqlite *h) +{ + sqlite3_stmt *stmt; + bool exists = false; + + if (sqlite3_prepare_v2( + h->db, + "select 1 from sqlite_master where type = 'table' and name = 'hist'", + -1, &stmt, NULL) + == SQLITE_OK) { + exists = (sqlite3_step(stmt) == SQLITE_ROW); + sqlite3_finalize(stmt); + } + return exists; +} + +/* +** Refuse to open a database whose schema version we do not understand, so a +** mismatch fails cleanly rather than as obscure later query errors. +*/ +static bool +hissqlite_check_version(struct hissqlite *h) +{ + sqlite3_stmt *stmt; + long version = -1; + char buf[160]; + + if (sqlite3_prepare_v2(h->db, + "select value from misc where key = 'version'", -1, + &stmt, NULL) + != SQLITE_OK) { + hissqlite_seterror(h, "prepare version check"); + return false; + } + if (sqlite3_step(stmt) == SQLITE_ROW) + version = (long) sqlite3_column_int64(stmt, 0); + sqlite3_finalize(stmt); + if (version == HISSQLITE_SCHEMA_VERSION) + return true; + snprintf(buf, sizeof(buf), + "hissqlite: %s has schema version %ld, expected %d", h->path, + version, HISSQLITE_SCHEMA_VERSION); + his_seterror(h->history, xstrdup(buf)); + return false; +} + +/* +** Is the database in WAL journal mode? A direct reader requires it; a +** non-WAL reader would take SHARED locks that conflict with the writer's +** commits. +*/ +static bool +hissqlite_is_wal(struct hissqlite *h) +{ + sqlite3_stmt *stmt; + bool wal = false; + + if (sqlite3_prepare_v2(h->db, "pragma journal_mode", -1, &stmt, NULL) + == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + const char *mode = (const char *) sqlite3_column_text(stmt, 0); + wal = (mode != NULL && strcmp(mode, "wal") == 0); + } + sqlite3_finalize(stmt); + } + return wal; +} + +/* +** Open (or create) the database named by h->path and prepare the statement +** sets, using h->flags. Factored out of hissqlite_open so the deferred-path +** pattern -- HISopen(NULL, ...) followed by HISCTLS_PATH, as makehistory uses +** -- can drive the real open once the path is known. On failure it reports +** the error, tears down any half-open database, and returns false; the handle +** itself is left for the caller to free or (for HISCTLS_PATH) retry. +*/ +static bool +hissqlite_doopen(struct hissqlite *h) +{ + char *errmsg = NULL; + char pragma[64]; + unsigned long pagesize = HISSQLITE_DEF_PAGE_SIZE; + unsigned long cachesize = HISSQLITE_DEF_CACHE_SIZE; + unsigned long mmapsize = HISSQLITE_DEF_MMAP_SIZE; + unsigned long rcachesize = HISSQLITE_DEF_RCACHE_SIZE; + int oflags; + + /* Performance tunables come from inn.conf when it has been read (innd, + expire, makehistory, the converter); offline callers and unit tests that + open history without innconf_read() fall back to the built-in defaults. */ + if (innconf != NULL) { + pagesize = innconf->hissqlitepagesize; + cachesize = innconf->hissqlitecachesize; + mmapsize = innconf->hissqlitemmapsize; + rcachesize = innconf->hissqlitereadercachesize; + } + + if (!(h->flags & HIS_RDWR)) { + /* Read-only open (HIS_RDONLY is 0, but callers may OR in hint flags + such as HIS_ONDISK, so test for the absence of HIS_RDWR rather than + exact equality). WAL direct reader: open read-only, verify WAL, + prepare read stmts. */ + if (sqlite3_open_v2(h->path, &h->db, SQLITE_OPEN_READONLY, NULL) + != SQLITE_OK) { + hissqlite_seterror(h, "open read-only"); + goto fail; + } + if (!hissqlite_is_wal(h)) { + hissqlite_seterror(h, "history is not in WAL mode; direct reader" + " unavailable (NFS history needs a server)"); + goto fail; + } + if (sqlite_helper_init(&hissqlite_read_helper, + (sqlite3_stmt **) &h->read, h->db, + SQLITE_PREPARE_PERSISTENT, &errmsg) + != SQLITE_OK) { + hissqlite_seterror(h, "prepare read statements"); + goto fail; + } + if (!hissqlite_check_version(h)) { + sqlite_helper_term(&hissqlite_read_helper, + (sqlite3_stmt **) &h->read); + goto fail; + } + /* Reader cache + mmap from inn.conf. Each nnrpd has its own cache, so + it is kept small by default (history is not nnrpd's hot path -- + overview is); readers lean on the shared OS page cache instead. */ + snprintf(pragma, sizeof(pragma), "pragma cache_size = -%lu;", + rcachesize); + sqlite3_exec(h->db, pragma, NULL, NULL, NULL); + snprintf(pragma, sizeof(pragma), "pragma mmap_size = %lu;", mmapsize); + sqlite3_exec(h->db, pragma, NULL, NULL, NULL); + h->direct_reader = true; + return true; + } + + oflags = SQLITE_OPEN_READWRITE; + if (h->flags & HIS_CREAT) + oflags |= SQLITE_OPEN_CREATE; + if (sqlite3_open_v2(h->path, &h->db, oflags, NULL) != SQLITE_OK) { + hissqlite_seterror(h, "open read/write"); + goto fail; + } + sqlite3_extended_result_codes(h->db, 1); + + if ((h->flags & HIS_CREAT) && !hissqlite_initialized(h)) { + /* Brand-new database: run the DDL (unnamed init section of + hissqlite-init.sql) and stamp the schema version. Skipped for an + existing DB so HIS_CREAT is idempotent. Mirrors ovsqlite-server.c. */ + hissqlite_init_t init; + + /* page_size only takes effect before the first table is written, so + apply it here, ahead of the DDL. (innconf validates it.) */ + snprintf(pragma, sizeof(pragma), "pragma page_size = %lu;", pagesize); + sqlite3_exec(h->db, pragma, NULL, NULL, NULL); + + if (sqlite_helper_init(&hissqlite_init_helper, + (sqlite3_stmt **) &init, h->db, 0, &errmsg) + != SQLITE_OK) { + hissqlite_seterror(h, "create schema"); + goto fail; + } + sqlite3_bind_int64(init.set_version, 1, HISSQLITE_SCHEMA_VERSION); + if (sqlite3_step(init.set_version) != SQLITE_DONE) + hissqlite_seterror(h, "set version"); + sqlite3_reset(init.set_version); + sqlite_helper_term(&hissqlite_init_helper, (sqlite3_stmt **) &init); + } + + /* Prepare writer statements; the unnamed init section applies the WAL / + synchronous / busy_timeout pragmas. PERSISTENT: these are long-lived. */ + if (sqlite_helper_init(&hissqlite_main_helper, (sqlite3_stmt **) &h->main, + h->db, SQLITE_PREPARE_PERSISTENT, &errmsg) + != SQLITE_OK) { + hissqlite_seterror(h, "prepare main statements"); + goto fail; + } + if (!hissqlite_check_version(h)) { + sqlite_helper_term(&hissqlite_main_helper, (sqlite3_stmt **) &h->main); + goto fail; + } + /* Track WAL growth so HISsync can keep it bounded (the .sql init section + has already put us in WAL mode). */ + sqlite3_wal_hook(h->db, hissqlite_wal_hook, h); + + /* Writer cache + mmap from inn.conf, applied after the .sql WAL pragmas. */ + snprintf(pragma, sizeof(pragma), "pragma cache_size = -%lu;", cachesize); + sqlite3_exec(h->db, pragma, NULL, NULL, NULL); + snprintf(pragma, sizeof(pragma), "pragma mmap_size = %lu;", mmapsize); + sqlite3_exec(h->db, pragma, NULL, NULL, NULL); + return true; + +fail: + if (errmsg != NULL) + sqlite3_free(errmsg); + if (h->db != NULL) { + sqlite3_close_v2(h->db); /* _v2 zombie-closes even if a stmt lingers */ + h->db = NULL; + } + return false; +} + +void * +hissqlite_open(const char *path, int flags, struct history *history) +{ + struct hissqlite *h; + + h = xcalloc(1, sizeof(*h)); + h->history = history; + h->flags = flags; + + /* makehistory opens with a NULL path and supplies the real one afterwards + via HISCTLS_PATH; defer the database open until then. Until the path is + set the handle is unusable, exactly as with hisv6. */ + if (path == NULL) + return h; + + h->path = concat(path, ".sqlite", (char *) NULL); + if (!hissqlite_doopen(h)) { + free(h->path); + free(h); + return NULL; + } + return h; +} + +bool +hissqlite_sync(void *history) +{ + struct hissqlite *h = history; + + /* HISsync is deliberately NOT a durability barrier, matching dbz: hisv6_sync + does fflush(3) + msync(MS_ASYNC), neither of which waits for the disk, so + dbz history is only kernel-buffered after a sync (survives an innd crash, + lost on power loss). Under synchronous=NORMAL each autocommit already + write(2)s the WAL into the kernel page cache, so hissqlite sits + continuously in that same state: there is nothing application-buffered + for HISsync to flush. (Per-commit durability would be synchronous=FULL, a + possible future knob, not an fsync here.) innd calls HISsync on a timer, + so it is simply the convenient place to keep the WAL bounded: once it has + grown past the threshold, do a non-blocking PASSIVE checkpoint. */ + if (!h->direct_reader && h->wal_pages >= HISSQLITE_WAL_CKPT_PAGES) + hissqlite_wal_passive(h); + return true; +} + +bool +hissqlite_close(void *history) +{ + struct hissqlite *h = history; + bool ok = true; + + /* A deferred handle (HISopen(NULL) whose HISCTLS_PATH was never set or + failed) never opened a database, so there is nothing to checkpoint or + close; just free it. Skipping this would call sqlite3_wal_checkpoint_v2() + on a NULL connection, which is not a no-op like sqlite3_close_v2(NULL). */ + if (h->db == NULL) { + free(h->path); + free(h); + return true; + } + + /* Flush and reclaim the WAL on a clean shutdown: PASSIVE first (never + blocks, even if direct-reader processes still hold the DB open), then + TRUNCATE only if every frame was checkpointed (no readers pinning it) so + we don't hang waiting for readers to disconnect. Mirrors ovsqlite. */ + if (!h->direct_reader) { + int log = 0, ckpt = 0; + + if (sqlite3_wal_checkpoint_v2(h->db, NULL, SQLITE_CHECKPOINT_PASSIVE, + &log, &ckpt) + == SQLITE_OK + && log == ckpt) + sqlite3_wal_checkpoint_v2(h->db, NULL, SQLITE_CHECKPOINT_TRUNCATE, + NULL, NULL); + sqlite_helper_term(&hissqlite_main_helper, (sqlite3_stmt **) &h->main); + } else { + sqlite_helper_term(&hissqlite_read_helper, (sqlite3_stmt **) &h->read); + } + + if (sqlite3_close_v2(h->db) != SQLITE_OK) + ok = false; + free(h->path); + free(h); + return ok; +} + +bool +hissqlite_lookup(void *history, const char *key, time_t *arrived, + time_t *posted, time_t *expires, struct token *token) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->direct_reader ? h->read.lookup : h->main.lookup; + bool found = false; + int status; + + bind_key(stmt, key); + status = sqlite3_step(stmt); + if (status == SQLITE_ROW) { + found = true; + if (arrived != NULL) + *arrived = (time_t) sqlite3_column_int64(stmt, 0); + if (posted != NULL) + *posted = (time_t) sqlite3_column_int64(stmt, 1); + if (expires != NULL) + *expires = (time_t) sqlite3_column_int64(stmt, 2); + if (token != NULL) { + if (sqlite3_column_type(stmt, 3) == SQLITE_NULL) { + /* Remembered entry: found, but no article. */ + memset(token, 0, sizeof(TOKEN)); + token->type = TOKEN_EMPTY; + } else if (!copy_blob(h, stmt, 3, token, sizeof(TOKEN), + "corrupt token blob in lookup")) { + /* Wrong-length token = corruption. Treat as a lookup failure + (reported above) rather than masking it as a no-article. */ + found = false; + } + } + } else if (status != SQLITE_DONE) { + hissqlite_seterror(h, "lookup"); + } + sqlite3_reset(stmt); + return found; +} + +bool +hissqlite_check(void *history, const char *key) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->direct_reader ? h->read.check : h->main.check; + bool found; + int status; + + /* Existence only: real OR remembered both count (refuse re-offers). + his.c keeps an in-memory existence cache in front of this. */ + bind_key(stmt, key); + status = sqlite3_step(stmt); + found = (status == SQLITE_ROW); + if (status != SQLITE_ROW && status != SQLITE_DONE) + hissqlite_seterror(h, "check"); + sqlite3_reset(stmt); + return found; +} + +bool +hissqlite_write(void *history, const char *key, time_t arrived, time_t posted, + time_t expires, const struct token *token) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->main.write; + bool ok; + + bind_key(stmt, key); + sqlite3_bind_int64(stmt, 2, (sqlite3_int64) arrived); + sqlite3_bind_int64(stmt, 3, (sqlite3_int64) posted); + sqlite3_bind_int64(stmt, 4, (sqlite3_int64) expires); + sqlite3_bind_blob(stmt, 5, token, sizeof(TOKEN), SQLITE_TRANSIENT); + ok = (sqlite3_step(stmt) == SQLITE_DONE); + if (!ok) + hissqlite_seterror(h, "write"); + else if (sqlite3_changes(h->db) == 0 && h->history != NULL) + /* The .write INSERT used ON CONFLICT DO NOTHING, so a duplicate hash + changed nothing. Report it (his_seterror logs a warning) but still + return success, so a makehistory rebuild over a spool with duplicate + Message-IDs warns and continues instead of aborting -- as hisv6 does + (hisv6.c hisv6_writedbz, DBZSTORE_EXISTS). */ + his_seterror(h->history, + concat("hissqlite: duplicate message-id, write ignored: ", + key, (char *) NULL)); + sqlite3_reset(stmt); + return ok; +} + +bool +hissqlite_remember(void *history, const char *key, time_t arrived, + time_t posted) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->main.remember; + bool ok; + + /* INSERT ... ON CONFLICT DO NOTHING: never downgrade a real article. */ + bind_key(stmt, key); + sqlite3_bind_int64(stmt, 2, (sqlite3_int64) arrived); + sqlite3_bind_int64(stmt, 3, (sqlite3_int64) posted); + ok = (sqlite3_step(stmt) == SQLITE_DONE); + if (!ok) + hissqlite_seterror(h, "remember"); + sqlite3_reset(stmt); + return ok; +} + +bool +hissqlite_replace(void *history, const char *key, time_t arrived, + time_t posted, time_t expires, const struct token *token) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->main.replace; + bool ok; + + /* The only token-changing op. A NULL token downgrades real->remembered + (prunehistory); a non-NULL token upgrades remembered->real. */ + bind_key(stmt, key); + sqlite3_bind_int64(stmt, 2, (sqlite3_int64) arrived); + sqlite3_bind_int64(stmt, 3, (sqlite3_int64) posted); + sqlite3_bind_int64(stmt, 4, (sqlite3_int64) expires); + if (token == NULL) + sqlite3_bind_null(stmt, 5); + else + sqlite3_bind_blob(stmt, 5, token, sizeof(TOKEN), SQLITE_TRANSIENT); + ok = (sqlite3_step(stmt) == SQLITE_DONE); + if (!ok) + hissqlite_seterror(h, "replace"); + sqlite3_reset(stmt); + return ok; +} + +bool +hissqlite_walk(void *history, const char *reason UNUSED, void *cookie, + bool (*callback)(void *, const HASH *, time_t, time_t, time_t, + const TOKEN *)) +{ + struct hissqlite *h = history; + sqlite3_stmt *stmt = h->direct_reader ? h->read.walk : h->main.walk; + bool ok = true; + int status; + + /* WAL gives a consistent read snapshot, unlike hisv6, no + ICCpause/straggler re-scan is needed even while innd writes. */ + while ((status = sqlite3_step(stmt)) == SQLITE_ROW) { + HASH hash; + TOKEN token, *tp; + time_t arrived, posted, expires; + + if (!copy_blob(h, stmt, 0, &hash, sizeof(HASH), + "corrupt hash blob in walk")) { + ok = false; + break; + } + arrived = (time_t) sqlite3_column_int64(stmt, 1); + posted = (time_t) sqlite3_column_int64(stmt, 2); + expires = (time_t) sqlite3_column_int64(stmt, 3); + if (sqlite3_column_type(stmt, 4) == SQLITE_NULL) { + tp = NULL; /* Remembered entry: callback sees no token. */ + } else { + if (!copy_blob(h, stmt, 4, &token, sizeof(TOKEN), + "corrupt token blob in walk")) { + ok = false; + break; + } + tp = &token; + } + if (!(*callback)(cookie, &hash, arrived, posted, expires, tp)) { + ok = false; + break; + } + } + if (status != SQLITE_DONE && ok) { + hissqlite_seterror(h, "walk"); + ok = false; + } + sqlite3_reset(stmt); + return ok; +} + +/* A pending in-place action collected during the pass-1 scan, so we do not + mutate the table through the same cursor we are scanning. */ +struct expire_action { + HASH hash; + TOKEN token; + time_t expires; + bool transition; /* true: real -> remembered; false: keep, rewrite token */ +}; + +bool +hissqlite_expire(void *history, const char *path UNUSED, + const char *reason UNUSED, bool writing, void *cookie, + time_t threshold, + bool (*decide)(void *, time_t, time_t, time_t, struct token *)) +{ + struct hissqlite *h = history; + struct expire_action *acts; + HASH last; + bool ok = true, more = true, first = true; + + /* + * hisv6 rewrites the whole file; we mutate in place. Pass 1: evaluate + * every token-bearing entry via the policy callback. A real article whose + * retention has passed is UPDATEd to remembered (token=NULL) NOT deleted + * so the Message-ID is not re-accepted. Pass 2: delete remembered + * entries older than the /remember/ posting-time threshold. + * + * The keep/remove decision is made entirely by the caller's callback + * (expire.c's EXPdoline), including the groupbaseexpiry case where article + * retention is driven by expireover plus storage existence and the tombstone + * log rather than expire.ctl: this backend is decision-agnostic and only + * applies the verdict in place. + * + * Concurrency: expire is a separate writer process with its own r/w + * connection, coordinating with innd via the WAL write-lock + busy_timeout + * (the two-writer model described in the file header). + * Nothing is wrapped in a transaction: each apply autocommits a single + * UPDATE/DELETE so the lock is held one statement at a time. Pass 1 streams + * the scan in hash-keyset pages: each page reads under its own short read + * snapshot which is reset before the page is applied, so a long expire never + * pins the WAL (which would block innd's checkpoints) and RAM is bounded to + * one page rather than the whole change set. + */ + + memset(&last, 0, sizeof(last)); + acts = xmalloc(HISSQLITE_EXPIRE_BATCH * sizeof(*acts)); + + while (ok && more) { + sqlite3_stmt *scan = h->main.expire_scan; + HASH maxhash; + size_t n = 0; /* actions collected this page */ + int rows = 0; /* rows read this page */ + int status; + + memset(&maxhash, 0, sizeof(maxhash)); + /* Resume after the previous page's last hash (an empty blob sorts + before every hash, so the first page starts at the beginning). */ + if (first) + sqlite3_bind_blob(scan, 1, "", 0, SQLITE_STATIC); + else + sqlite3_bind_blob(scan, 1, &last, sizeof(HASH), SQLITE_TRANSIENT); + sqlite3_bind_int(scan, 2, HISSQLITE_EXPIRE_BATCH); + + while ((status = sqlite3_step(scan)) == SQLITE_ROW) { + TOKEN token, ltoken; + time_t arrived, posted, expires; + bool keep; + + /* maxhash tracks every row read (rows come in hash order), so + after the page it is the resume point. */ + if (!copy_blob(h, scan, 0, &maxhash, sizeof(HASH), + "corrupt hash blob in expire scan")) { + ok = false; + break; + } + arrived = (time_t) sqlite3_column_int64(scan, 1); + posted = (time_t) sqlite3_column_int64(scan, 2); + expires = (time_t) sqlite3_column_int64(scan, 3); + if (!copy_blob(h, scan, 4, &token, sizeof(TOKEN), + "corrupt token blob in expire scan")) { + ok = false; + break; + } + rows++; + ltoken = token; + + keep = (*decide)(cookie, arrived, posted, expires, <oken); + if (keep && memcmp(<oken, &token, sizeof(TOKEN)) == 0) + continue; /* Unchanged: nothing to do. */ + + acts[n].hash = maxhash; + acts[n].transition = !keep; + acts[n].token = ltoken; + acts[n].expires = keep ? expires : 0; + n++; + } + if (status != SQLITE_DONE && ok) { + hissqlite_seterror(h, "expire scan"); + ok = false; + } + sqlite3_reset(scan); /* release the read snapshot before applying */ + + /* Apply this page (only when writing; the expire(8) -t dry run passes + writing=false, so it just scans and reports via the callback). Each + statement autocommits, holding the write lock one row at a time so + innd's accepts interleave. */ + for (size_t i = 0; writing && ok && i < n; i++) { + sqlite3_stmt *stmt; + + if (acts[i].transition) { + stmt = h->main.transition_remember; + sqlite3_bind_blob(stmt, 1, &acts[i].hash, sizeof(HASH), + SQLITE_TRANSIENT); + } else { + stmt = h->main.update_token; + sqlite3_bind_blob(stmt, 1, &acts[i].hash, sizeof(HASH), + SQLITE_TRANSIENT); + sqlite3_bind_blob(stmt, 2, &acts[i].token, sizeof(TOKEN), + SQLITE_TRANSIENT); + sqlite3_bind_int64(stmt, 3, (sqlite3_int64) acts[i].expires); + } + if (sqlite3_step(stmt) != SQLITE_DONE) { + hissqlite_seterror(h, "expire apply"); + ok = false; + } + sqlite3_reset(stmt); + } + + if (rows < HISSQLITE_EXPIRE_BATCH) + more = false; /* short page -> end of table */ + else + last = maxhash; /* resume after the last row read */ + first = false; + } + free(acts); + + /* Pass 2: delete remembered entries past the /remember/ threshold, in + bounded chunks (the statement deletes up to a LIMIT per step). Each step + autocommits, so the write lock is released between chunks and innd's + accepts interleave; loop until a chunk deletes nothing. */ + if (writing && ok) { + sqlite3_stmt *stmt = h->main.expire_remembered; + + do { + sqlite3_bind_int64(stmt, 1, (sqlite3_int64) threshold); + if (sqlite3_step(stmt) != SQLITE_DONE) { + hissqlite_seterror(h, "expire remembered"); + ok = false; + } + sqlite3_reset(stmt); + } while (ok && sqlite3_changes(h->db) > 0); + } + return ok; +} + +bool +hissqlite_ctl(void *history, int selector, void *val) +{ + struct hissqlite *h = history; + + switch (selector) { + case HISCTLG_PATH: + *(char **) val = h->path; + return true; + case HISCTLS_SYNCCOUNT: + /* No-op: writes autocommit, so there is no batch size to set. */ + return true; + case HISCTLS_STATINTERVAL: + /* No-op: WAL readers always see a consistent snapshot; there is no + swapped-inode poll as in hisv6. */ + return true; + case HISCTLS_NPAIRS: + case HISCTLS_IGNOREOLD: + /* No-op: the B-tree grows itself; there is no fixed-size table to + presize and no rebuild-to-grow. */ + return true; + case HISCTLG_INPLACEEXPIRE: + /* hissqlite expires in place (UPDATE/DELETE on the live DB), so + expire(8) must open it read/write, not the hisv6 rebuild-and-swap + way. See hissqlite_expire(). */ + *(bool *) val = true; + return true; + case HISCTLS_PATH: + /* Deferred-open path (makehistory): HISopen(NULL) left the database + unopened; set the real path now and perform the open/create. Refuse + a second path set, as hisv6 does. */ + if (h->path != NULL) { + his_seterror(h->history, + concat("hissqlite: path already set in handle", + (char *) NULL)); + return false; + } + h->path = concat((char *) val, ".sqlite", (char *) NULL); + if (!hissqlite_doopen(h)) { + free(h->path); + h->path = NULL; + return false; + } + return true; + default: + return false; + } +} + +#else /* ! HAVE_SQLITE3 */ + +void * +hissqlite_open(const char *path UNUSED, int flags UNUSED, + struct history *history UNUSED) +{ + warn("hissqlite: SQLite support not enabled"); + return NULL; +} + +bool +hissqlite_close(void *history UNUSED) +{ + return false; +} + +bool +hissqlite_sync(void *history UNUSED) +{ + return false; +} + +bool +hissqlite_lookup(void *history UNUSED, const char *key UNUSED, + time_t *arrived UNUSED, time_t *posted UNUSED, + time_t *expires UNUSED, struct token *token UNUSED) +{ + return false; +} + +bool +hissqlite_check(void *history UNUSED, const char *key UNUSED) +{ + return false; +} + +bool +hissqlite_write(void *history UNUSED, const char *key UNUSED, + time_t arrived UNUSED, time_t posted UNUSED, + time_t expires UNUSED, const struct token *token UNUSED) +{ + return false; +} + +bool +hissqlite_replace(void *history UNUSED, const char *key UNUSED, + time_t arrived UNUSED, time_t posted UNUSED, + time_t expires UNUSED, const struct token *token UNUSED) +{ + return false; +} + +bool +hissqlite_expire(void *history UNUSED, const char *path UNUSED, + const char *reason UNUSED, bool writing UNUSED, + void *cookie UNUSED, time_t threshold UNUSED, + bool (*decide)(void *, time_t, time_t, time_t, + struct token *) UNUSED) +{ + return false; +} + +bool +hissqlite_walk(void *history UNUSED, const char *reason UNUSED, + void *cookie UNUSED, + bool (*callback)(void *, const HASH *, time_t, time_t, time_t, + const struct token *) UNUSED) +{ + return false; +} + +bool +hissqlite_remember(void *history UNUSED, const char *key UNUSED, + time_t arrived UNUSED, time_t posted UNUSED) +{ + return false; +} + +bool +hissqlite_ctl(void *history UNUSED, int selector UNUSED, void *val UNUSED) +{ + return false; +} + +#endif /* ! HAVE_SQLITE3 */ diff --git a/history/hissqlite/hissqlite.h b/history/hissqlite/hissqlite.h new file mode 100644 index 000000000..4c9e7da56 --- /dev/null +++ b/history/hissqlite/hissqlite.h @@ -0,0 +1,48 @@ +/* +** Internal history API interface for the SQLite (hissqlite) backend. +** +** Exposes the HIS_METHOD vtable functions to history/hismethods.c (which is +** auto-generated by buildconfig from the per-method hismethod.config files). +** See hissqlite(5) for the design rationale. +*/ + +#ifndef HISSQLITE_H +#define HISSQLITE_H + +#include "inn/libinn.h" + +struct token; +struct history; + +void *hissqlite_open(const char *path, int flags, struct history *); + +bool hissqlite_close(void *); + +bool hissqlite_sync(void *); + +bool hissqlite_lookup(void *, const char *key, time_t *arrived, time_t *posted, + time_t *expires, struct token *token); + +bool hissqlite_check(void *, const char *key); + +bool hissqlite_write(void *, const char *key, time_t arrived, time_t posted, + time_t expires, const struct token *token); + +bool hissqlite_replace(void *, const char *key, time_t arrived, time_t posted, + time_t expires, const struct token *token); + +bool hissqlite_expire(void *, const char *, const char *, bool, void *, + time_t threshold, + bool (*exists)(void *, time_t, time_t, time_t, + struct token *)); + +bool hissqlite_walk(void *, const char *, void *, + bool (*)(void *, const HASH *, time_t, time_t, time_t, + const struct token *)); + +bool hissqlite_remember(void *, const char *key, time_t arrived, + time_t posted); + +bool hissqlite_ctl(void *, int, void *); + +#endif /* HISSQLITE_H */ diff --git a/include/inn/history.h b/include/inn/history.h index 895242aa5..290722b77 100644 --- a/include/inn/history.h +++ b/include/inn/history.h @@ -83,7 +83,15 @@ enum { /* (time_t) interval, in s, between stats of the history database * for * detecting a replacement, or 0 to disable (no checks); * defaults {hisv6, taggedhash} */ - HISCTLS_STATINTERVAL + HISCTLS_STATINTERVAL, + + /* (bool) does this backend expire IN PLACE (mutating the live database + * via a writer) rather than rebuilding a new file and swapping it? + * Backends that do not implement this selector leave the value untouched, + * so the caller must initialise it to false. expire(8) uses this to + * decide whether to open the history read/write (in place) or read-only + * (rebuild-and-swap, the hisv6 model). */ + HISCTLG_INPLACEEXPIRE }; struct history *HISopen(const char *, const char *, int); diff --git a/include/inn/innconf.h b/include/inn/innconf.h index 91a40376a..a1703efff 100644 --- a/include/inn/innconf.h +++ b/include/inn/innconf.h @@ -72,7 +72,12 @@ struct innconf { unsigned long wipexpire; /* How long to keep pending article record */ /* History settings */ - char *hismethod; /* Which history method to use */ + char *hismethod; /* Which history method to use */ + unsigned long hissqlitecachesize; /* hissqlite writer page cache, in kB */ + unsigned long hissqlitemmapsize; /* hissqlite mmap size in bytes; 0 = off */ + unsigned long hissqlitepagesize; /* hissqlite database page size, in bytes */ + unsigned long + hissqlitereadercachesize; /* hissqlite per-nnrpd reader cache, in kB */ /* Article Storage */ unsigned long cnfscheckfudgesize; /* Additional CNFS integrity checking */ diff --git a/storage/ovsqlite/sqlite-helper.h b/include/inn/sqlite-helper.h similarity index 100% rename from storage/ovsqlite/sqlite-helper.h rename to include/inn/sqlite-helper.h diff --git a/lib/Makefile b/lib/Makefile index 7299cd71b..f8890e882 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -6,7 +6,7 @@ include ../Makefile.global # recover from make update. We can't use .OLD extensions for the shared # library since ldconfig will think .OLD sorts after the regular library and # will point the binaries at the old library. -LTVERSION = 9:2:0 +LTVERSION = 9:3:0 top = .. CFLAGS = $(GCFLAGS) @@ -24,7 +24,7 @@ SOURCES = argparse.c artnumber.c bloom.c buffer.c cleanfrom.c \ remopen.c reservedfd.c resource.c secrets.c sendarticle.c \ sendpass.c \ sequence.c timer.c tst.c uwildmat.c vector.c wire.c \ - xfopena.c xmalloc.c xsignal.c xwrite.c + xfopena.c xmalloc.c xsignal.c xwrite.c sqlite-helper.c # Sources for additional functions only built to replace missing system ones. EXTRA_SOURCES = asprintf.c daemon.c fseeko.c ftello.c getaddrinfo.c \ @@ -39,7 +39,7 @@ LOBJECTS = $(OBJECTS:.o=.lo) .SUFFIXES: .lo -all: libinn.$(EXTLIB) canlock.o perl.o +all: libinn.$(EXTLIB) canlock.o perl.o sqlite-helper-gen warnings: $(MAKE) COPT="$(COPT) $(WARNINGS)" all @@ -52,11 +52,15 @@ bootstrap: clean clobber distclean maintclean: rm -f *.o *.lo libinn.la libinn.a rm -f profiled perl$(PROFSUFFIX).o libinn$(PROFSUFFIX).a - rm -f libinn_pure_*.a .pure + rm -f libinn_pure_*.a .pure sqlite-helper-gen rm -rf .libs +# libinn includes sqlite-helper.lo, so it must record the SQLite dependency +# (a no-op, empty libs, when configured --without-sqlite3) -- otherwise every +# executable linking libinn would need to name -lsqlite3 itself. libinn.la: $(OBJECTS) $(LOBJECTS) - $(LIBLD) $(LDFLAGS) -o $@ $(LOBJECTS) $(LIBS) \ + $(LIBLD) $(LDFLAGS) -o $@ $(LOBJECTS) \ + $(SQLITE3_LDFLAGS) $(SQLITE3_LIBS) $(LIBS) \ -rpath $(PATHLIB) -version-info $(LTVERSION) libinn.a: $(OBJECTS) @@ -72,6 +76,18 @@ canlock.o: canlock.c perl.o: perl.c $(CC) $(CFLAGS) $(PERL_CPPFLAGS) -c perl.c +# sqlite-helper.c needs the SQLite headers; scope the flags to just this object +# (the file is a no-op when built without HAVE_SQLITE3). +sqlite-helper.o sqlite-helper.lo: sqlite-helper.c ../include/inn/sqlite-helper.h + $(LIBCC) $(CFLAGS) $(SQLITE3_CPPFLAGS) -c sqlite-helper.c + +# Shared SQL prepared-statement codegen, used by the ovsqlite overview method +# and the hissqlite history method. +FIX = $(SHELL) $(FIXSCRIPT) + +sqlite-helper-gen: sqlite-helper-gen.in $(FIXSCRIPT) + $(FIX) -i sqlite-helper-gen.in + ../include/inn/system.h: (cd ../include && $(MAKE)) diff --git a/lib/innconf.c b/lib/innconf.c index 52d18c2d2..8335cca7c 100644 --- a/lib/innconf.c +++ b/lib/innconf.c @@ -226,6 +226,10 @@ static const struct config config_table[] = { /* The following settings are specific to the history subsystem. */ {K(hismethod), STRING(NULL) }, + {K(hissqlitecachesize), UNUMBER(65536) }, + {K(hissqlitemmapsize), UNUMBER(0) }, + {K(hissqlitepagesize), UNUMBER(4096) }, + {K(hissqlitereadercachesize), UNUMBER(2000) }, /* The following settings are specific to rc.news. */ {K(docnfsstat), BOOL(false) }, @@ -491,6 +495,17 @@ innconf_validate(struct config_group *group) innconf->datamovethreshold = 1024 * 1024; } + /* hissqlitepagesize must be a power of two between 512 and 65536, as + required by SQLite's pragma page_size. */ + if (innconf->hissqlitepagesize < 512 || innconf->hissqlitepagesize > 65536 + || (innconf->hissqlitepagesize & (innconf->hissqlitepagesize - 1)) + != 0) { + config_error_param(group, "hissqlitepagesize", + "hissqlitepagesize must be a power of 2 between 512" + " and 65536"); + innconf->hissqlitepagesize = 4096; + } + if (innconf->docancels != NULL && strcasecmp(innconf->docancels, "require-auth") != 0 && strcasecmp(innconf->docancels, "auth") != 0 diff --git a/storage/ovsqlite/sqlite-helper-gen.in b/lib/sqlite-helper-gen.in similarity index 99% rename from storage/ovsqlite/sqlite-helper-gen.in rename to lib/sqlite-helper-gen.in index 87688aa64..550d92043 100644 --- a/storage/ovsqlite/sqlite-helper-gen.in +++ b/lib/sqlite-helper-gen.in @@ -108,7 +108,7 @@ $preamble #ifndef $guard #define $guard 1 -#include "sqlite-helper.h" +#include "inn/sqlite-helper.h" #ifdef HAVE_SQLITE3 diff --git a/storage/ovsqlite/sqlite-helper.c b/lib/sqlite-helper.c similarity index 98% rename from storage/ovsqlite/sqlite-helper.c rename to lib/sqlite-helper.c index 607b04117..300daea8c 100644 --- a/storage/ovsqlite/sqlite-helper.c +++ b/lib/sqlite-helper.c @@ -1,4 +1,4 @@ -#include "sqlite-helper.h" +#include "inn/sqlite-helper.h" #ifdef HAVE_SQLITE3 diff --git a/storage/Makefile b/storage/Makefile index 02f787bd3..e9b7544db 100644 --- a/storage/Makefile +++ b/storage/Makefile @@ -283,13 +283,9 @@ ovsqlite/ovsqlite.o: ovsqlite/ovsqlite.c ovsqlite/ovsqlite.h ../include/config.h ../include/inn/confparse.h ../include/inn/fdflag.h \ ../include/inn/portable-socket.h ../include/inn/innconf.h \ ../include/inn/newsuser.h ../include/inn/paths.h \ - ovsqlite/../ovinterface.h ovsqlite/sql-read.h ovsqlite/sqlite-helper.h + ovsqlite/../ovinterface.h ovsqlite/sql-read.h ../include/inn/sqlite-helper.h ovsqlite/sql-read.o: ovsqlite/sql-read.c ovsqlite/sql-read.h \ - ovsqlite/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ - ../include/inn/portable-macros.h ../include/inn/options.h \ - ../include/inn/system.h ../include/portable/stdbool.h -ovsqlite/sqlite-helper.o: ovsqlite/sqlite-helper.c ovsqlite/sqlite-helper.h \ - ../include/config.h ../include/inn/macros.h \ + ../include/inn/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ ../include/inn/portable-macros.h ../include/inn/options.h \ ../include/inn/system.h ../include/portable/stdbool.h timecaf/caf.o: timecaf/caf.c ../include/portable/system.h ../include/config.h \ @@ -418,13 +414,13 @@ ovsqlite/ovsqlite-server.o: ovsqlite/ovsqlite-server.c ../include/config.h \ ../include/inn/innconf.h ../include/inn/libinn.h \ ../include/inn/xmalloc.h ../include/inn/system.h ../include/inn/xwrite.h \ ../include/inn/storage.h ../include/inn/options.h ovsqlite/sql-init.h \ - ovsqlite/sqlite-helper.h ovsqlite/sql-main.h + ../include/inn/sqlite-helper.h ovsqlite/sql-main.h ovsqlite/sql-init.o: ovsqlite/sql-init.c ovsqlite/sql-init.h \ - ovsqlite/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ + ../include/inn/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ ../include/inn/portable-macros.h ../include/inn/options.h \ ../include/inn/system.h ../include/portable/stdbool.h ovsqlite/sql-main.o: ovsqlite/sql-main.c ovsqlite/sql-main.h \ - ovsqlite/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ + ../include/inn/sqlite-helper.h ../include/config.h ../include/inn/macros.h \ ../include/inn/portable-macros.h ../include/inn/options.h \ ../include/inn/system.h ../include/portable/stdbool.h tradindexed/tdx-util.o: tradindexed/tdx-util.c ../include/portable/system.h \ diff --git a/storage/ovsqlite/ovmethod.config b/storage/ovsqlite/ovmethod.config index 5ba6f5612..31d7aa914 100644 --- a/storage/ovsqlite/ovmethod.config +++ b/storage/ovsqlite/ovmethod.config @@ -1,7 +1,6 @@ name = ovsqlite number = 5 -sources = ovsqlite.c ovsqlite-private.c sqlite-helper.c sql-read.c +sources = ovsqlite.c ovsqlite-private.c sql-read.c extra-sources = ovsqlite-server.c sql-main.c sql-init.c programs = ovsqlite-server ovsqlite-util -clean = sqlite-helper-gen maintclean = sql-init.c sql-init.h sql-main.c sql-main.h sql-read.c sql-read.h diff --git a/storage/ovsqlite/ovmethod.mk b/storage/ovsqlite/ovmethod.mk index 6fc0f0044..fa53b971b 100644 --- a/storage/ovsqlite/ovmethod.mk +++ b/storage/ovsqlite/ovmethod.mk @@ -11,21 +11,20 @@ ovsqlite/ovsqlite-server: $(OVSQLITEOBJECTS) libinnstorage.$(EXTLIB) ovsqlite/ovsqlite-util: ovsqlite/ovsqlite-util.in $(FIXSCRIPT) $(FIX) ovsqlite/ovsqlite-util.in -ovsqlite/sqlite-helper-gen: ovsqlite/sqlite-helper-gen.in $(FIXSCRIPT) - $(FIX) -i ovsqlite/sqlite-helper-gen.in +## sqlite-helper-gen now lives in lib/ (shared with the hissqlite history +## method); lib is built before storage so it is available here. -ovsqlite/sql-main.c: ovsqlite/sql-main.sql ovsqlite/sqlite-helper-gen - ovsqlite/sqlite-helper-gen ovsqlite/sql-main.sql +ovsqlite/sql-main.c: ovsqlite/sql-main.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen ovsqlite/sql-main.sql ovsqlite/sql-main.h: ovsqlite/sql-main.c ; -ovsqlite/sql-init.c: ovsqlite/sql-init.sql ovsqlite/sqlite-helper-gen - ovsqlite/sqlite-helper-gen ovsqlite/sql-init.sql +ovsqlite/sql-init.c: ovsqlite/sql-init.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen ovsqlite/sql-init.sql ovsqlite/sql-init.h: ovsqlite/sql-init.c ; -ovsqlite/sql-read.c: ovsqlite/sql-read.sql ovsqlite/sqlite-helper-gen - ovsqlite/sqlite-helper-gen ovsqlite/sql-read.sql +ovsqlite/sql-read.c: ovsqlite/sql-read.sql ../lib/sqlite-helper-gen + ../lib/sqlite-helper-gen ovsqlite/sql-read.sql ovsqlite/sql-read.h: ovsqlite/sql-read.c ; - diff --git a/storage/ovsqlite/ovsqlite.c b/storage/ovsqlite/ovsqlite.c index eae8cd6b6..964ffb633 100644 --- a/storage/ovsqlite/ovsqlite.c +++ b/storage/ovsqlite/ovsqlite.c @@ -37,7 +37,7 @@ # include "../ovinterface.h" # include "sql-read.h" -# include "sqlite-helper.h" +# include "inn/sqlite-helper.h" # include # ifdef HAVE_ZLIB diff --git a/support/mkmanifest b/support/mkmanifest index a8f73377e..da1a940d1 100755 --- a/support/mkmanifest +++ b/support/mkmanifest @@ -275,7 +275,12 @@ storage/buffindexed/buffindexed_d storage/buildconfig storage/ovsqlite/ovsqlite-server storage/ovsqlite/ovsqlite-util -storage/ovsqlite/sqlite-helper-gen +lib/sqlite-helper-gen +history/hissqlite/hissqlite-convert +history/hissqlite/hissqlite-util +tests/lib/hissqlite.t +tests/lib/hissqlite-convert.t +tests/lib/history-bench storage/tradindexed/tdx-util support/fixconfig support/fixscript diff --git a/tests/Makefile b/tests/Makefile index d2d3004c6..74084b491 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -19,6 +19,7 @@ LIBM_LDFLAGS = '-lm' TESTS = authprogs/ident.t expire/tombstone.t expire/tombstone-hisexpire.t \ innd/artparse.t innd/chan.t lib/artnumber.t \ lib/asprintf.t lib/bloom.t lib/bloom-hiswalk.t lib/buffer.t \ + lib/hissqlite.t lib/hissqlite-convert.t \ lib/canlock.t lib/concat.t lib/conffile.t \ lib/confparse.t lib/daemon.t lib/date.t \ lib/dispatch.t lib/fdflag.t \ @@ -41,17 +42,24 @@ EXTRA = runtests clients/server-list docs/pod.t lib/xmalloc \ overview/ovsqlite-read.t overview/ovsqlite-write.t \ perl/minimum-version.t +BENCHMARKS = lib/history-bench + all check test tests: $(TESTS) $(EXTRA) ./runtests -l TESTS build: $(TESTS) $(EXTRA) +benchmarks: $(BENCHMARKS) + +benchmark-history: $(BENCHMARKS) + ./lib/history-bench -n 100M + warnings: $(MAKE) COPT="$(COPT) $(WARNINGS)" build clean clobber distclean maintclean: rm -f *.o *.lo */*.o */*.lo */*/*.o */*/*.o \ - .pure */.pure */*/.pure $(TESTS) $(EXTRA) + .pure */.pure */*/.pure $(TESTS) $(EXTRA) $(BENCHMARKS) rm -rf .libs */.libs */*/.libs $(FIXSCRIPT): @@ -123,6 +131,15 @@ lib/bloom.t: lib/bloom-t.o tap/basic.o $(LIBINN) lib/bloom-hiswalk.t: lib/bloom-hiswalk-t.o tap/basic.o $(STORAGEDEPS) $(LINKDEPS) lib/bloom-hiswalk-t.o tap/basic.o $(STORAGELIBS) $(LIBS) +lib/hissqlite.t: lib/hissqlite-t.o tap/basic.o $(STORAGEDEPS) + $(LINKDEPS) lib/hissqlite-t.o tap/basic.o $(STORAGELIBS) $(LIBS) + +lib/hissqlite-convert.t: lib/hissqlite-convert-t.o tap/basic.o $(STORAGEDEPS) + $(LINKDEPS) lib/hissqlite-convert-t.o tap/basic.o $(STORAGELIBS) $(LIBS) + +lib/history-bench: lib/history-bench.o $(STORAGEDEPS) + $(LINKDEPS) lib/history-bench.o $(STORAGELIBS) $(LIBS) + lib/buffer.t: lib/buffer-t.o tap/basic.o $(LIBINN) $(LINK) lib/buffer-t.o tap/basic.o $(LIBINN) diff --git a/tests/TESTS b/tests/TESTS index 1455d2c38..7cecb2506 100644 --- a/tests/TESTS +++ b/tests/TESTS @@ -27,6 +27,8 @@ lib/hash lib/hashtab lib/headers lib/hex +lib/hissqlite +lib/hissqlite-convert lib/inet_aton lib/inet_ntoa lib/inet_ntop diff --git a/tests/lib/hissqlite-convert-t.c b/tests/lib/hissqlite-convert-t.c new file mode 100644 index 000000000..a3ba527ee --- /dev/null +++ b/tests/lib/hissqlite-convert-t.c @@ -0,0 +1,202 @@ +/* +** End-to-end test for the hissqlite-convert migration tool. +** +** Seeds a small hisv6 history (real + remembered entries, distinct +** timestamps), runs the hissqlite-convert binary against it, and verifies the +** resulting hissqlite database through the public HIS API: the faithful +** migration must preserve tokens, timestamps and -- crucially -- the +** remembered (token-less) entries that a from-spool rebuild cannot +** reconstruct. Also checks the refuse-to-overwrite guard and that the +** temporary build file is cleaned up. +** +** This drives the real binary (via system()) rather than the library, so it +** needs inn.conf: a minimal one is written to the temp dir and pointed at with +** INNCONF, exactly as the shell integration tests do. +** +** Written by Kevin Bowling in 2026. +*/ + +#include "portable/system.h" + +#include "inn/history.h" +#include "inn/innconf.h" +#include "inn/libinn.h" +#include "inn/messages.h" +#include "inn/storage.h" +#include "tap/basic.h" + +#ifndef HAVE_SQLITE3 + +int +main(void) +{ + skip_all("not built with SQLite"); + return 0; +} + +#else + +# include +# include + +# define BASE ((time_t) 1600000000) + +static const char convert[] = "../../history/hissqlite/hissqlite-convert"; + +/* Run " " with output suppressed; return its exit code, + or -1 if it could not be run. */ +static int +run_convert(const char *src, const char *dst) +{ + char cmd[768]; /* Should provide a PATH_MAX fill in build somewhere */ + int status; + + snprintf(cmd, sizeof(cmd), "%s %s %s >/dev/null 2>&1", convert, src, dst); + status = system(cmd); + if (status == -1 || !WIFEXITED(status)) + return -1; + return WEXITSTATUS(status); +} + +int +main(void) +{ + char tmpdir[64], conf[128], src[128], dst[128], dstdb[160], tempdb[160]; + struct history *h; + FILE *f; + TOKEN t1, t2, got; + bool reals_ok, remembered_ok, absent_ok; + struct stat sb; + + /* runtests runs us from tests/; move into our own directory so the + relative path to the binary resolves */ + if (access("hissqlite-convert.t", F_OK) < 0) + if (access("lib/hissqlite-convert.t", F_OK) == 0) + if (chdir("lib") < 0) + sysbail("cannot chdir to lib"); + if (access(convert, X_OK) != 0) + skip_all("hissqlite-convert not built"); + + test_init(8); + + strlcpy(tmpdir, "hissqlite-conv-XXXXXX", sizeof(tmpdir)); + if (mkdtemp(tmpdir) == NULL) + sysbail("can't create temp directory"); + snprintf(conf, sizeof(conf), "%s/inn.conf", tmpdir); + snprintf(src, sizeof(src), "%s/src", tmpdir); + snprintf(dst, sizeof(dst), "%s/dst", tmpdir); + snprintf(dstdb, sizeof(dstdb), "%s/dst.sqlite", tmpdir); + snprintf(tempdb, sizeof(tempdb), "%s/dst.new.sqlite", tmpdir); + + /* Minimal inn.conf so both this process and the convert binary can read a + valid configuration (convert calls innconf_read). */ + f = fopen(conf, "w"); + if (f == NULL) + sysbail("can't write %s", conf); + fprintf(f, + "domain: news.example.com\n" + "mta: \"/bin/true %%s\"\n" + "hismethod: hisv6\n" + "enableoverview: false\n" + "wireformat: true\n" + "pathnews: %s\n" + "pathdb: %s\n" + "pathetc: %s\n", + tmpdir, tmpdir, tmpdir); + fclose(f); + if (setenv("INNCONF", conf, 1) != 0) + sysbail("can't set INNCONF"); + if (!innconf_read(NULL)) + bail("can't read test inn.conf"); + + /* Seed a hisv6 source: two real articles with distinct timestamps/tokens + and two remembered (token-less) entries. */ + h = HISopen(src, "hisv6", HIS_CREAT | HIS_RDWR); + if (h == NULL) + bail("can't create hisv6 source"); + memset(&t1, 0, sizeof(t1)); + t1.type = 1; + memcpy(t1.token, "0123456789abcdef", sizeof(t1.token)); + memset(&t2, 0, sizeof(t2)); + t2.type = 2; + memcpy(t2.token, "fedcba9876543210", sizeof(t2.token)); + if (!HISwrite(h, "", BASE + 1, BASE + 1, BASE + 5000, &t1) + || !HISwrite(h, "", BASE + 2, BASE + 7, (time_t) 0, &t2) + || !HISremember(h, "", BASE + 3, BASE + 3) + || !HISremember(h, "", BASE + 4, BASE + 4)) + bail("can't seed hisv6 source"); + HISsync(h); + HISclose(h); + + /* Convert hisv6 -> hissqlite. */ + ok(1, run_convert(src, dst) == 0); + + /* Verify the result. The converter leaves the database in a non-WAL + journal mode (the first innd open flips it to WAL), so open read/write + rather than via the WAL-only direct reader. */ + h = HISopen(dst, "hissqlite", HIS_RDWR); + if (h == NULL) + bail("can't open converted hissqlite database"); + + { + time_t a = 0, p = 0, e = -1; + bool r1, r2; + + memset(&got, 0xff, sizeof(got)); + r1 = HISlookup(h, "", &a, &p, &e, &got) && got.type == 1 + && memcmp(got.token, t1.token, sizeof(t1.token)) == 0 + && a == BASE + 1 && p == BASE + 1 && e == BASE + 5000; + memset(&got, 0xff, sizeof(got)); + r2 = HISlookup(h, "", &a, &p, &e, &got) && got.type == 2 + && memcmp(got.token, t2.token, sizeof(t2.token)) == 0 + && a == BASE + 2 && p == BASE + 7; + reals_ok = r1 && r2; + } + ok(2, reals_ok); /* real tokens and timestamps round-trip */ + + { + TOKEN g1, g2; + bool f1, f2; + + memset(&g1, 0xff, sizeof(g1)); + memset(&g2, 0xff, sizeof(g2)); + /* Remembered entries are found but carry no token (TOKEN_EMPTY). */ + f1 = HISlookup(h, "", NULL, NULL, NULL, &g1) + && g1.type == TOKEN_EMPTY; + f2 = HISlookup(h, "", NULL, NULL, NULL, &g2) + && g2.type == TOKEN_EMPTY; + remembered_ok = f1 && f2; + } + ok(3, remembered_ok); /* remembered entries survived the migration */ + + absent_ok = !HIScheck(h, ""); + ok(4, absent_ok); + + { + bool counts; + /* Existence of every seeded entry, nothing spurious: 2 real + 2 + remembered, all present and distinguishable. */ + counts = HIScheck(h, "") && HIScheck(h, "") + && HIScheck(h, "") && HIScheck(h, ""); + ok(5, counts); + } + HISclose(h); + + /* Refuse to overwrite an existing destination: a second run must fail and + leave the database in place. */ + ok(6, run_convert(src, dst) != 0); + ok(7, stat(dstdb, &sb) == 0); /* existing database left in place */ + + /* The temporary build file must not survive a successful conversion. */ + ok(8, access(tempdb, F_OK) != 0); + + { + char cmd[160]; + snprintf(cmd, sizeof(cmd), "/bin/rm -rf %s", tmpdir); + if (system(cmd) < 0) + sysdiag("can't clean up %s", tmpdir); + } + return 0; +} + +#endif /* HAVE_SQLITE3 */ diff --git a/tests/lib/hissqlite-t.c b/tests/lib/hissqlite-t.c new file mode 100644 index 000000000..21f1209fb --- /dev/null +++ b/tests/lib/hissqlite-t.c @@ -0,0 +1,390 @@ +/* +** Runtime test for the hissqlite history backend. +** +** Exercises the full HIS_METHOD vtable through the public HIS API against a +** real SQLite database: open/create, write, remember, check, lookup, replace, +** walk, the two-horizon expire (real->remembered transition + remember-delete), +** sync, close, and persistence across reopen. +** +** Written by Kevin Bowling in 2026. +*/ + +#include "portable/system.h" + +#include "inn/history.h" +#include "inn/libinn.h" +#include "inn/messages.h" +#include "inn/storage.h" +#include "tap/basic.h" + +#ifndef HAVE_SQLITE3 + +int +main(void) +{ + skip_all("not built with SQLite"); + return 0; +} + +#else + +# include +# include + +# define N_TOKEN 100 /* entries with a storage token */ +# define N_REMEMBER 20 /* remembered (token-less) entries */ +# define BASE ((time_t) 1600000000) + +/* msgid 0..N_TOKEN-1 are token entries; N_TOKEN..N_TOKEN+N_REMEMBER-1 are + remembered. arrived == posted == BASE + i for entry i. */ +static char * +make_msgid(unsigned long n) +{ + char buf[64]; + snprintf(buf, sizeof(buf), "", n); + return xstrdup(buf); +} + +struct walkcount { + unsigned long total; + unsigned long with_token; +}; + +static bool +walk_cb(void *cookie, const HASH *hash UNUSED, time_t arrived UNUSED, + time_t posted UNUSED, time_t expires UNUSED, const TOKEN *token) +{ + struct walkcount *w = cookie; + w->total++; + if (token != NULL) + w->with_token++; + return true; +} + +/* Expire policy: keep token entries with arrived >= cutoff, drop (->remember) + the older ones. */ +static time_t expire_cutoff; +static bool +decide_drop_old(void *cookie UNUSED, time_t arrived, time_t posted UNUSED, + time_t expires UNUSED, TOKEN *token UNUSED) +{ + return arrived >= expire_cutoff; +} + +static bool +decide_keep_all(void *cookie UNUSED, time_t arrived UNUSED, time_t posted UNUSED, + time_t expires UNUSED, TOKEN *token UNUSED) +{ + return true; +} + +/* Does HISlookup find this msgid, and (optionally) does it carry a token? */ +static bool +present_with_token(struct history *h, unsigned long n, bool *has_token, + TOKEN *out) +{ + char *msgid = make_msgid(n); + TOKEN token; + bool found; + + memset(&token, 0, sizeof(token)); + found = HISlookup(h, msgid, NULL, NULL, NULL, &token); + if (has_token != NULL) + *has_token = (found && token.type != TOKEN_EMPTY); + if (out != NULL) + *out = token; + free(msgid); + return found; +} + +int +main(void) +{ + struct history *h; + char tmpdir[64], histpath[128]; + TOKEN token, t2; + unsigned long i; + struct walkcount wc; + bool has_token; + + test_init(27); + + strlcpy(tmpdir, "hissqlite-XXXXXX", sizeof(tmpdir)); + if (mkdtemp(tmpdir) == NULL) + sysbail("can't create temp directory"); + snprintf(histpath, sizeof(histpath), "%s/history", tmpdir); + + /* create + populate */ + h = HISopen(histpath, "hissqlite", HIS_CREAT | HIS_RDWR); + ok(1, h != NULL); + if (h == NULL) + bail("can't create hissqlite history at %s", histpath); + + memset(&token, 0, sizeof(token)); + token.type = 1; + memcpy(token.token, "0123456789abcdef", sizeof(token.token)); + + for (i = 0; i < N_TOKEN; i++) { + char *msgid = make_msgid(i); + if (!HISwrite(h, msgid, BASE + i, BASE + i, (time_t) 0, &token)) + bail("HISwrite %lu failed: %s", i, HISerror(h)); + free(msgid); + } + for (i = N_TOKEN; i < N_TOKEN + N_REMEMBER; i++) { + char *msgid = make_msgid(i); + if (!HISremember(h, msgid, BASE + i, BASE + i)) + bail("HISremember %lu failed: %s", i, HISerror(h)); + free(msgid); + } + ok(2, HISsync(h)); + ok(3, HISclose(h)); + + /* reopen read/write and verify */ + h = HISopen(histpath, "hissqlite", HIS_RDWR); + ok(4, h != NULL); + if (h == NULL) + bail("can't reopen hissqlite history"); + + /* check: real present, remembered present, absent absent */ + { + char *m_real = make_msgid(5); + char *m_rem = make_msgid(N_TOKEN + 1); + char *m_abs = make_msgid(99999); + ok(5, HIScheck(h, m_real)); + ok(6, HIScheck(h, m_rem)); + ok(7, !HIScheck(h, m_abs)); + free(m_real); + free(m_rem); + free(m_abs); + } + + /* lookup: real -> token round-trips; remembered -> empty token */ + { + char *m = make_msgid(5); + time_t arrived = 0, posted = 0, expires = -1; + memset(&t2, 0xff, sizeof(t2)); + ok(8, HISlookup(h, m, &arrived, &posted, &expires, &t2) + && arrived == BASE + 5 && posted == BASE + 5 + && t2.type == 1 + && memcmp(t2.token, token.token, sizeof(token.token)) == 0); + free(m); + } + ok(9, present_with_token(h, N_TOKEN + 2, &has_token, NULL) && !has_token); + ok(10, !present_with_token(h, 99999, NULL, NULL)); + + /* walk (before mutations): total and token counts */ + wc.total = wc.with_token = 0; + HISwalk(h, NULL, &wc, walk_cb); + ok(11, wc.total == N_TOKEN + N_REMEMBER); + ok(12, wc.with_token == N_TOKEN); + + /* replace: change a token; downgrade real->remembered */ + memset(&t2, 0, sizeof(t2)); + t2.type = 2; + memcpy(t2.token, "fedcba9876543210", sizeof(t2.token)); + { + char *m = make_msgid(0); + TOKEN got; + HISreplace(h, m, BASE, BASE, 0, &t2); + present_with_token(h, 0, &has_token, &got); + ok(13, has_token && got.type == 2 + && memcmp(got.token, t2.token, sizeof(t2.token)) == 0); + free(m); + } + { + char *m = make_msgid(1); /* real -> remembered (NULL token) */ + HISreplace(h, m, BASE + 1, BASE + 1, 0, NULL); + ok(14, present_with_token(h, 1, &has_token, NULL) && !has_token); + free(m); + } + + /* expire pass 1: drop old token entries -> remembered */ + expire_cutoff = BASE + N_TOKEN / 2; /* keep i>=50, drop i<50 */ + ok(15, + HISexpire(h, NULL, NULL, true, NULL, (time_t) (BASE - 1), decide_drop_old)); + /* #10 (arrived BASE+10 < cutoff) -> now remembered; #90 still a token */ + { + bool r10 = present_with_token(h, 10, &has_token, NULL); + bool t10 = has_token; + bool r90 = present_with_token(h, 90, &has_token, NULL); + bool t90 = has_token; + ok(16, r10 && !t10 && r90 && t90); + } + + /* expire pass 2: remember-delete past the threshold */ + ok(17, HISexpire(h, NULL, NULL, true, NULL, BASE + 1000000, decide_keep_all)); + /* original remembered (#N_TOKEN+5) gone; surviving token (#90) stays */ + { + char *m = make_msgid(N_TOKEN + 5); + bool rem_gone = !HIScheck(h, m); + bool tok_stays = present_with_token(h, 90, &has_token, NULL) && has_token; + ok(18, rem_gone && tok_stays); + free(m); + } + + /* HISCTLG_INPLACEEXPIRE capability contract (expire.c relies on it) */ + { + bool inplace = false; + ok(19, HISctl(h, HISCTLG_INPLACEEXPIRE, &inplace) && inplace); + } + HISclose(h); + + /* A backend that does not implement the selector must leave the flag + untouched (hisv6 -> stays false, i.e. rebuild-and-swap). */ + { + struct history *hv; + bool inplace = false; + char hv6path[160]; + + snprintf(hv6path, sizeof(hv6path), "%s/hv6", tmpdir); + hv = HISopen(hv6path, "hisv6", HIS_CREAT | HIS_RDWR); + if (hv != NULL) + HISctl(hv, HISCTLG_INPLACEEXPIRE, &inplace); + ok(20, hv != NULL && !inplace); + if (hv != NULL) + HISclose(hv); + } + + /* corrupt token -> reported, and schema-version mismatch -> refused. + Both need DB-level tampering there is no HIS API for, so reach in with + SQLite directly (this whole test is already HAVE_SQLITE3-only). */ + { + char vpath[160], vdb[176]; + struct history *vh; + sqlite3 *raw; + TOKEN tk; + + snprintf(vpath, sizeof(vpath), "%s/v", tmpdir); + snprintf(vdb, sizeof(vdb), "%s.sqlite", vpath); + vh = HISopen(vpath, "hissqlite", HIS_CREAT | HIS_RDWR); + memset(&tk, 0, sizeof(tk)); + tk.type = 1; + HISwrite(vh, "", BASE, BASE, 0, &tk); + HISsync(vh); + HISclose(vh); + + /* Truncate the stored token to a wrong length: lookup must report the + entry as not-found rather than mask the corruption (MI2). */ + if (sqlite3_open(vdb, &raw) == SQLITE_OK) { + sqlite3_exec(raw, "update hist set token = x'0011'", NULL, NULL, + NULL); + sqlite3_close(raw); + } + vh = HISopen(vpath, "hissqlite", HIS_RDONLY); + if (vh != NULL) { + TOKEN got; + memset(&got, 0, sizeof(got)); + ok(21, !HISlookup(vh, "", NULL, NULL, NULL, &got)); + HISclose(vh); + } else { + ok(21, false); + } + + /* Bump the stored schema version: reopen must refuse cleanly (MA2). */ + if (sqlite3_open(vdb, &raw) == SQLITE_OK) { + sqlite3_exec(raw, + "update misc set value = 99 where key = 'version'", + NULL, NULL, NULL); + sqlite3_close(raw); + } + vh = HISopen(vpath, "hissqlite", HIS_RDONLY); + ok(22, vh == NULL); + if (vh != NULL) + HISclose(vh); + } + + /* deferred-open path: HISopen(NULL) then HISCTLS_PATH, as makehistory + drives it. The handle must defer the open until the path is set, then + create the real database at that path (not an anonymous temp DB), and a + write must land in that file. */ + { + char dpath[160], ddb[176]; + struct history *dh; + struct stat sb; + TOKEN tk; + + snprintf(dpath, sizeof(dpath), "%s/deferred", tmpdir); + snprintf(ddb, sizeof(ddb), "%s.sqlite", dpath); + dh = HISopen(NULL, "hissqlite", HIS_CREAT | HIS_RDWR); + ok(23, dh != NULL && HISctl(dh, HISCTLS_PATH, dpath) + && stat(ddb, &sb) == 0); + if (dh != NULL) { + memset(&tk, 0, sizeof(tk)); + tk.type = 1; + HISwrite(dh, "", BASE, BASE, 0, &tk); + HISsync(dh); + HISclose(dh); + } + /* Reopen the real file and confirm the entry persisted there, proving + the write did not go to a throwaway temporary database. */ + dh = HISopen(dpath, "hissqlite", HIS_RDWR); + ok(24, dh != NULL && HIScheck(dh, "")); + if (dh != NULL) + HISclose(dh); + + /* Closing a deferred handle whose path was never set (so no database + was ever opened) must be a clean no-op, not a NULL-connection crash. */ + dh = HISopen(NULL, "hissqlite", HIS_CREAT | HIS_RDWR); + ok(25, dh != NULL && HISclose(dh)); + } + + /* duplicate HISwrite is tolerated (makehistory over a spool with a + duplicate Message-ID must warn and continue, not abort): both calls + succeed and the entry exists. */ + { + char wpath[160]; + struct history *wh; + TOKEN w1, w2; + bool first, second; + + snprintf(wpath, sizeof(wpath), "%s/dup", tmpdir); + wh = HISopen(wpath, "hissqlite", HIS_CREAT | HIS_RDWR); + memset(&w1, 0, sizeof(w1)); + w1.type = 1; + memset(&w2, 0, sizeof(w2)); + w2.type = 2; + first = wh != NULL && HISwrite(wh, "", BASE, BASE, 0, &w1); + second = wh != NULL && HISwrite(wh, "", BASE, BASE, 0, &w2); + ok(26, first && second && wh != NULL && HIScheck(wh, "")); + if (wh != NULL) + HISclose(wh); + } + + /* a dry-run expire (writing == false, as expire(8) -t now passes for an + in-place backend) scans but must change nothing: an entry the policy + would drop stays a real token. */ + { + char epath[160]; + struct history *eh; + TOKEN et, got; + bool kept; + + snprintf(epath, sizeof(epath), "%s/dry", tmpdir); + eh = HISopen(epath, "hissqlite", HIS_CREAT | HIS_RDWR); + memset(&et, 0, sizeof(et)); + et.type = 1; + if (eh != NULL) + HISwrite(eh, "", BASE, BASE, 0, &et); + expire_cutoff = BASE + 100; /* decide_drop_old would drop arrived=BASE */ + if (eh != NULL) + HISexpire(eh, NULL, NULL, false, NULL, BASE + 1000000, + decide_drop_old); + memset(&got, 0, sizeof(got)); + kept = eh != NULL + && HISlookup(eh, "", NULL, NULL, NULL, &got) + && got.type == 1; /* still real, not transitioned to remembered */ + ok(27, kept); + if (eh != NULL) + HISclose(eh); + } + + { + char cmd[160]; + snprintf(cmd, sizeof(cmd), "/bin/rm -rf %s", tmpdir); + if (system(cmd) < 0) + sysdiag("can't clean up %s", tmpdir); + } + return 0; +} + +#endif /* HAVE_SQLITE3 */ diff --git a/tests/lib/history-bench.c b/tests/lib/history-bench.c new file mode 100644 index 000000000..7770557c5 --- /dev/null +++ b/tests/lib/history-bench.c @@ -0,0 +1,649 @@ +/* +** Benchmark hisv6 and hissqlite history backends. +** +** This is intentionally not part of the TAP test suite. The default run is +** sized for large local performance testing (100M entries) and can consume a +** lot of time and disk. +** +** Fairness / representativeness: +** +** Both methods are opened exactly as innd opens them, including the in-core +** index hint (HIS_MMAP when INND_DBZINCORE, else HIS_ONDISK). Without it, +** hisv6 would run dbz fully on disk (INCORE_NO plus writethrough), slower +** than production, while hissqlite, which ignores the storage hint flags, +** would be unaffected. +** +** Tuning comes from inn.conf, so each backend uses the operator's real +** settings. Point at a config with the INNCONF environment variable: +** +** INNCONF=/path/to/inn.conf ./history-bench -n 100M +** +** If neither $INNCONF nor the compiled default inn.conf is present, both +** backends fall back to their built-in defaults (which equal the inn.conf +** defaults), so the comparison stays fair either way. The knobs that most +** affect these results: +** +** hissqlitecachesize writer page cache (KB). The default (64 MB) is +** far smaller than a 100M-entry B-tree, so +** random-key inserts thrash; raise it to isolate +** "SQLite is slower at random-key writes" from +** "under-cached". +** hissqlitemmapsize SQLite mmap window (bytes). Default 0 (no mmap) +** reads the DB via pread; set it to compare in-core +** against hisv6's mmap'd index on equal footing. +** hissqlitepagesize B-tree page size (bytes); affects tree depth and +** per-insert WAL write amplification. +** +** Note the write phase models innd's steady-state, one-autocommit-per-article +** path, not a makehistory rebuild: bulk loading uses the batched converter +** (hissqlite-convert.c) and is faster than the per-row figure here. +** +** Written by Kevin Bowling in 2026. +*/ + +#include "portable/system.h" + +#include "inn/history.h" +#include "inn/innconf.h" +#include "inn/libinn.h" +#include "inn/messages.h" +#include "inn/options.h" + +#if defined(__clang__) +# pragma clang diagnostic ignored "-Wc++-keyword" +#endif +#include "inn/storage.h" + +#ifdef HAVE_SYS_TIME_H +# include +#endif +#include +#include +#include + +#define DEFAULT_ENTRIES ((uint64_t) 100000000) +#define DEFAULT_RANDOM ((uint64_t) 1000000) +#define DEFAULT_SYNC ((uint64_t) 10000) +#define PROGRESS_STEPS 20 +#define BASE_TIME ((time_t) 1700000000) + +/* +** Open hisv6 the way innd does, not with a bare HIS_RDWR/HIS_RDONLY. Without +** an in-core hint, hisv6 hands dbz pag_incore = exists_incore = INCORE_NO +** with writethrough on, its slowest fully-on-disk index mode, so every check +** and lookup hits the disk and writes thrash the on-disk index. innd instead +** opens with HIS_MMAP (INND_DBZINCORE defaults to 1; HIS_ONDISK otherwise), +** keeping the index mmap()ed in core. hissqlite ignores these storage hint +** flags (its journal/cache behaviour is fixed), so passing them does not +** change hissqlite; it only stops the benchmark from unfairly crippling +** hisv6. +*/ +#define HIS_INCORE_HINT (INND_DBZINCORE ? HIS_MMAP : HIS_ONDISK) + +struct bench_config { + uint64_t entries; + uint64_t random_lookups; + uint64_t sync_every; + const char *root; + bool keep; + const char **methods; + size_t method_count; +}; + +struct bench_result { + const char *method; + uint64_t entries; + uint64_t random_lookups; + double write_seconds; + double sync_seconds; + double close_seconds; + double open_ro_seconds; + double check_seconds; + double walk_seconds; + double lookup_seconds; + double missing_seconds; + uint64_t bytes; + const char *path; + bool skipped; +}; + +static double +now_seconds(void) +{ + struct timeval tv; + + if (gettimeofday(&tv, NULL) < 0) + sysdie("gettimeofday failed"); + return (double) tv.tv_sec + ((double) tv.tv_usec / 1000000.0); +} + +static uint64_t +parse_count(const char *value) +{ + char *end; + uint64_t count; + unsigned long long parsed; + + errno = 0; + parsed = strtoull(value, &end, 10); + if (errno != 0 || end == value) + die("invalid count: %s", value); + count = (uint64_t) parsed; + if (*end != '\0' && end[1] != '\0') + die("invalid count suffix: %s", value); + if (*end == 'k' || *end == 'K') + count *= 1000; + else if (*end == 'm' || *end == 'M') + count *= 1000 * 1000; + else if (*end == 'g' || *end == 'G') + count *= 1000 * 1000 * 1000; + else if (*end != '\0') + die("invalid count suffix: %s", value); + if (count == 0) + die("count must be greater than zero"); + return count; +} + +__attribute__((__noreturn__)) static void +usage(int status) +{ + FILE *out; + + out = status == 0 ? stdout : stderr; + fprintf(out, "usage: history-bench [-k] [-d dir] [-n entries] " + "[-r random-lookups] [-s sync-every] [method ...]\n"); + fprintf(out, "\n"); + fprintf(out, "Default: -n 100M -r 1M -s 10K hisv6 hissqlite\n"); + fprintf(out, "Counts accept K, M, and G decimal suffixes.\n"); + fprintf(out, "Use -s 0 to disable periodic HISsync during writes.\n"); + fprintf(out, "Benchmark data is removed unless -k is given.\n"); + exit(status); +} + +static char * +join_path(const char *base, const char *name) +{ + size_t len; + char *path; + + len = strlen(base) + strlen(name) + 2; + path = (char *) xmalloc(len); + snprintf(path, len, "%s/%s", base, name); + return path; +} + +static void +make_msgid(char *buf, size_t len, uint64_t n) +{ + snprintf(buf, len, "<%020llu@history-bench.invalid>", + (unsigned long long) n); +} + +static void +make_token(TOKEN *token, uint64_t n) +{ + /* 64-bit golden-ratio constant (floor(2^64 / phi)); used here purely as a + fixed XOR mask so the two halves of the synthetic token differ rather + than both being the counter. */ + const uint64_t golden_ratio = UINT64_C(0x9e3779b97f4a7c15); + uint64_t hi, lo; + + memset(token, 0, sizeof(*token)); + token->type = 1; + token->class = (STORAGECLASS) (n & 0xff); + hi = n; + lo = n ^ golden_ratio; + memcpy(token->token, &hi, sizeof(hi)); + memcpy(token->token + sizeof(hi), &lo, sizeof(lo)); +} + +static void +print_progress(const char *method, const char *phase, uint64_t done, + uint64_t total) +{ + double pct; + + if (total == 0) + return; + pct = ((double) done * 100.0) / (double) total; + fprintf(stderr, "%s %-12s %6.2f%% (%llu/%llu)\n", method, phase, pct, + (unsigned long long) done, (unsigned long long) total); +} + +static bool +progress_due(uint64_t i, uint64_t total) +{ + uint64_t step; + + if (total < PROGRESS_STEPS) + step = 1; + else + step = total / PROGRESS_STEPS; + return ((i + 1) == total || ((i + 1) % step) == 0); +} + +static uint64_t +lcg_next(uint64_t *state) +{ + /* Knuth's MMIX 64-bit linear congruential generator (also the PCG + defaults): state = state * multiplier + increment. A cheap, adequate + PRNG for the benchmark's pseudo-random key sequence. */ + const uint64_t multiplier = UINT64_C(6364136223846793005); + const uint64_t increment = UINT64_C(1442695040888963407); + + *state = *state * multiplier + increment; + return *state; +} + +static void +sum_tree(const char *path, uint64_t *bytes) +{ + struct stat st; + DIR *dir; + struct dirent *entry; + + if (lstat(path, &st) < 0) + return; + if (S_ISREG(st.st_mode)) { + if (st.st_size > 0) + *bytes += (uint64_t) st.st_size; + return; + } + if (!S_ISDIR(st.st_mode)) + return; + dir = opendir(path); + if (dir == NULL) + return; + while ((entry = readdir(dir)) != NULL) { + char *child; + + if (strcmp(entry->d_name, ".") == 0 + || strcmp(entry->d_name, "..") == 0) + continue; + child = join_path(path, entry->d_name); + sum_tree(child, bytes); + free(child); + } + closedir(dir); +} + +static void +remove_tree(const char *path) +{ + struct stat st; + DIR *dir; + struct dirent *entry; + + if (lstat(path, &st) < 0) + return; + if (S_ISDIR(st.st_mode)) { + dir = opendir(path); + if (dir != NULL) { + while ((entry = readdir(dir)) != NULL) { + char *child; + + if (strcmp(entry->d_name, ".") == 0 + || strcmp(entry->d_name, "..") == 0) + continue; + child = join_path(path, entry->d_name); + remove_tree(child); + free(child); + } + closedir(dir); + } + if (rmdir(path) < 0) + syswarn("cannot remove %s", path); + } else if (unlink(path) < 0) { + syswarn("cannot remove %s", path); + } +} + +static struct history * +create_history(const char *method, const char *histpath, uint64_t entries) +{ + struct history *h; + size_t npairs; + + h = HISopen(NULL, method, HIS_CREAT | HIS_RDWR | HIS_INCORE_HINT); + if (h == NULL) + return NULL; + npairs = (size_t) entries; + if ((uint64_t) npairs != entries) + npairs = (size_t) -1; + if (!HISctl(h, HISCTLS_NPAIRS, &npairs)) + die("%s: cannot set HISCTLS_NPAIRS: %s", method, HISerror(h)); + if (!HISctl(h, HISCTLS_PATH, (void *) histpath)) + die("%s: cannot set history path %s: %s", method, histpath, + HISerror(h)); + return h; +} + +/* HISwalk callback: count every entry. HISwalk is the full-history scan that + feeds the expireover bloom-filter rebuild and the migration converter. */ +static bool +walk_count(void *cookie, const HASH *hash UNUSED, time_t arrived UNUSED, + time_t posted UNUSED, time_t expires UNUSED, const TOKEN *token UNUSED) +{ + ++*(uint64_t *) cookie; + return true; +} + +static void +benchmark_method(const struct bench_config *config, const char *method, + struct bench_result *result) +{ + char *method_dir; + char *histpath; + struct history *h; + TOKEN token, got; + char msgid[64]; + uint64_t i, misses; + uint64_t state; + double start; + + /* Arbitrary but fixed PRNG seeds (any value works): a fixed seed makes + each phase replay the same pseudo-random order on every run and for + every method, so the comparison is apples-to-apples; the two phases use + distinct seeds so they probe in different orders. */ + const uint64_t lookup_seed = UINT64_C(0x123456789abcdef0); + const uint64_t missing_seed = UINT64_C(0xfedcba9876543210); + + memset(result, 0, sizeof(*result)); + result->method = method; + result->entries = config->entries; + result->random_lookups = config->random_lookups; + + method_dir = join_path(config->root, method); + histpath = join_path(method_dir, "history"); + if (mkdir(method_dir, 0777) < 0) + sysdie("cannot create %s", method_dir); + result->path = method_dir; + + fprintf(stderr, "%s create/write %llu entries at %s\n", method, + (unsigned long long) config->entries, histpath); + start = now_seconds(); + h = create_history(method, histpath, config->entries); + if (h == NULL) { + fprintf(stderr, "%s skipped: %s\n", method, + HISerror(h) != NULL ? HISerror(h) : "method unavailable"); + result->skipped = true; + free(histpath); + return; + } + for (i = 0; i < config->entries; i++) { + make_msgid(msgid, sizeof(msgid), i); + make_token(&token, i); + if (!HISwrite(h, msgid, BASE_TIME + (time_t) i, BASE_TIME + (time_t) i, + 0, &token)) + die("%s: HISwrite %llu failed: %s", method, (unsigned long long) i, + HISerror(h)); + if (config->sync_every != 0 && (i + 1) % config->sync_every == 0) + if (!HISsync(h)) + die("%s: periodic HISsync at %llu failed: %s", method, + (unsigned long long) (i + 1), HISerror(h)); + if (progress_due(i, config->entries)) + print_progress(method, "write", i + 1, config->entries); + } + result->write_seconds = now_seconds() - start; + + start = now_seconds(); + if (!HISsync(h)) + die("%s: HISsync failed: %s", method, HISerror(h)); + result->sync_seconds = now_seconds() - start; + + start = now_seconds(); + if (!HISclose(h)) + die("%s: HISclose failed", method); + result->close_seconds = now_seconds() - start; + + start = now_seconds(); + h = HISopen(histpath, method, HIS_RDONLY | HIS_INCORE_HINT); + if (h == NULL) + die("%s: read-only reopen failed", method); + result->open_ro_seconds = now_seconds() - start; + + fprintf(stderr, "%s sequential check %llu entries\n", method, + (unsigned long long) config->entries); + misses = 0; + start = now_seconds(); + for (i = 0; i < config->entries; i++) { + make_msgid(msgid, sizeof(msgid), i); + if (!HIScheck(h, msgid)) + misses++; + if (progress_due(i, config->entries)) + print_progress(method, "check", i + 1, config->entries); + } + result->check_seconds = now_seconds() - start; + if (misses != 0) + die("%s: sequential check missed %llu entries", method, + (unsigned long long) misses); + + fprintf(stderr, "%s walk %llu entries\n", method, + (unsigned long long) config->entries); + { + uint64_t walked = 0; + + /* NULL reason: a read-only walk must not pause innd. hisv6 calls + ICCpause(reason) at EOF when reason is non-NULL, to catch stragglers + during a rebuild, there is no server here, and hissqlite ignores + reason regardless. */ + start = now_seconds(); + if (!HISwalk(h, NULL, &walked, walk_count)) + die("%s: HISwalk failed: %s", method, HISerror(h)); + result->walk_seconds = now_seconds() - start; + if (walked != config->entries) + die("%s: walk visited %llu of %llu entries", method, + (unsigned long long) walked, + (unsigned long long) config->entries); + } + + fprintf(stderr, "%s random lookup %llu entries\n", method, + (unsigned long long) config->random_lookups); + state = lookup_seed; + misses = 0; + start = now_seconds(); + for (i = 0; i < config->random_lookups; i++) { + uint64_t n; + + n = lcg_next(&state) % config->entries; + make_msgid(msgid, sizeof(msgid), n); + memset(&got, 0, sizeof(got)); + if (!HISlookup(h, msgid, NULL, NULL, NULL, &got) + || got.type == TOKEN_EMPTY) + misses++; + if (progress_due(i, config->random_lookups)) + print_progress(method, "lookup", i + 1, config->random_lookups); + } + result->lookup_seconds = now_seconds() - start; + if (misses != 0) + die("%s: random lookup missed %llu entries", method, + (unsigned long long) misses); + + fprintf(stderr, "%s missing check %llu entries\n", method, + (unsigned long long) config->random_lookups); + state = missing_seed; + misses = 0; + start = now_seconds(); + for (i = 0; i < config->random_lookups; i++) { + uint64_t n; + + n = config->entries + (lcg_next(&state) % config->entries); + make_msgid(msgid, sizeof(msgid), n); + if (HIScheck(h, msgid)) + misses++; + if (progress_due(i, config->random_lookups)) + print_progress(method, "missing", i + 1, config->random_lookups); + } + result->missing_seconds = now_seconds() - start; + if (misses != 0) + die("%s: missing check found %llu unexpected entries", method, + (unsigned long long) misses); + + if (!HISclose(h)) + die("%s: final HISclose failed", method); + + result->bytes = 0; + sum_tree(method_dir, &result->bytes); + free(histpath); +} + +static double +rate(uint64_t count, double seconds) +{ + if (seconds <= 0.0) + return 0.0; + return (double) count / seconds; +} + +static void +print_result(const struct bench_result *result) +{ + if (result->skipped) { + printf("%-10s skipped\n", result->method); + return; + } + printf("%-10s entries=%llu bytes=%llu path=%s\n", result->method, + (unsigned long long) result->entries, + (unsigned long long) result->bytes, result->path); + printf(" write: %8.3fs %12.0f entries/s\n", result->write_seconds, + rate(result->entries, result->write_seconds)); + printf(" sync: %8.3fs\n", result->sync_seconds); + printf(" close: %8.3fs\n", result->close_seconds); + printf(" open-ro: %8.3fs\n", result->open_ro_seconds); + printf(" check: %8.3fs %12.0f entries/s\n", result->check_seconds, + rate(result->entries, result->check_seconds)); + printf(" walk: %8.3fs %12.0f entries/s\n", result->walk_seconds, + rate(result->entries, result->walk_seconds)); + printf(" lookup: %8.3fs %12.0f entries/s\n", result->lookup_seconds, + rate(result->random_lookups, result->lookup_seconds)); + printf(" missing: %8.3fs %12.0f entries/s\n", result->missing_seconds, + rate(result->random_lookups, result->missing_seconds)); +} + +static void +print_csv_header(void) +{ + printf("\nmethod,entries,random_lookups,bytes,write_s,write_per_s,sync_s," + "close_s,open_ro_s,check_s,check_per_s,walk_s,walk_per_s," + "lookup_s,lookup_per_s,missing_s,missing_per_s,path\n"); +} + +static void +print_csv(const struct bench_result *result) +{ + if (result->skipped) + return; + printf( + "%s,%llu,%llu,%llu,%.6f,%.2f,%.6f,%.6f,%.6f,%.6f,%.2f,%.6f,%.2f," + "%.6f,%.2f,%.6f,%.2f,%s\n", + result->method, (unsigned long long) result->entries, + (unsigned long long) result->random_lookups, + (unsigned long long) result->bytes, result->write_seconds, + rate(result->entries, result->write_seconds), result->sync_seconds, + result->close_seconds, result->open_ro_seconds, result->check_seconds, + rate(result->entries, result->check_seconds), result->walk_seconds, + rate(result->entries, result->walk_seconds), result->lookup_seconds, + rate(result->random_lookups, result->lookup_seconds), + result->missing_seconds, + rate(result->random_lookups, result->missing_seconds), result->path); +} + +int +main(int argc, char **argv) +{ + static const char *default_methods[] = {"hisv6", "hissqlite"}; + struct bench_config config; + struct bench_result *results; + char root_template[] = "history-bench-XXXXXX"; + int i; + size_t j; + + /* Best-effort: load inn.conf so both backends use the operator's real + tuning (hissqlite{cachesize,mmapsize,pagesize}, dbz nfs hints). If no + inn.conf is reachable, innconf stays NULL and each backend falls back to + its built-in defaults, which match the inn.conf defaults, so the + comparison stays fair either way. */ + if (!innconf_read(NULL)) { + innconf = NULL; + warn("no inn.conf found; using built-in history defaults for both" + " methods"); + } + + memset(&config, 0, sizeof(config)); + config.entries = DEFAULT_ENTRIES; + config.random_lookups = DEFAULT_RANDOM; + config.sync_every = DEFAULT_SYNC; + config.methods = default_methods; + config.method_count = 2; + + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + usage(0); + } else if (strcmp(argv[i], "-k") == 0) { + config.keep = true; + } else if (strcmp(argv[i], "-n") == 0) { + if (++i >= argc) + usage(1); + config.entries = parse_count(argv[i]); + } else if (strcmp(argv[i], "-r") == 0) { + if (++i >= argc) + usage(1); + config.random_lookups = parse_count(argv[i]); + } else if (strcmp(argv[i], "-s") == 0) { + if (++i >= argc) + usage(1); + if (strcmp(argv[i], "0") == 0) + config.sync_every = 0; + else + config.sync_every = parse_count(argv[i]); + } else if (strcmp(argv[i], "-d") == 0) { + if (++i >= argc) + usage(1); + config.root = argv[i]; + } else if (argv[i][0] == '-') { + usage(1); + } else { + config.methods = (const char **) &argv[i]; + config.method_count = (size_t) (argc - i); + break; + } + } + if (config.random_lookups > config.entries) + config.random_lookups = config.entries; + if (config.root == NULL) { + if (mkdtemp(root_template) == NULL) + sysdie("cannot create benchmark directory"); + config.root = root_template; + } else if (mkdir(config.root, 0777) < 0) { + sysdie("cannot create %s", config.root); + } + + printf("history benchmark root: %s\n", config.root); + printf("entries: %llu, random lookups: %llu, sync every: %llu\n\n", + (unsigned long long) config.entries, + (unsigned long long) config.random_lookups, + (unsigned long long) config.sync_every); + fflush(stdout); + + results = + (struct bench_result *) xcalloc(config.method_count, sizeof(*results)); + for (j = 0; j < config.method_count; j++) { + benchmark_method(&config, config.methods[j], &results[j]); + print_result(&results[j]); + printf("\n"); + fflush(stdout); + } + + print_csv_header(); + for (j = 0; j < config.method_count; j++) + print_csv(&results[j]); + + if (config.keep) { + printf("\nkept benchmark data in %s\n", config.root); + } else { + remove_tree(config.root); + printf("\nremoved benchmark data from %s\n", config.root); + } + free(results); + return 0; +}