From e84c431866c4bba6df063756b70fbb302064b838 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:50:35 -0500 Subject: [PATCH 01/19] rewrite part 1 --- .cargo/config.toml | 2 + .containerignore | 9 +- .gitignore | 6 +- Cargo.lock | 598 ++++++++++++++++-- Cargo.toml | 43 +- Containerfile | 15 +- README.md | 60 +- benchmarks/Cargo.toml | 15 + benchmarks/benches/generate.rs | 29 + grainpit/Cargo.toml | 12 + src/markov3.txt => grainpit/data/config.txt | 0 src/markov1.txt => grainpit/data/html.txt | 8 +- src/markov4.txt => grainpit/data/url.txt | 0 src/markov2.txt => grainpit/data/url_name.txt | 0 grainpit/src/lib.rs | 1 + grainpit/src/markov.rs | 90 +++ grainpit/src/markov/chain.rs | 252 ++++++++ grainpit_proxy/Cargo.toml | 6 + grainpit_proxy/src/main.rs | 3 + grainpit_webserver/Cargo.toml | 11 + grainpit_webserver/src/main.rs | 59 ++ src/main.rs | 164 ----- src/markov.rs | 146 ----- 23 files changed, 1130 insertions(+), 399 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 benchmarks/Cargo.toml create mode 100644 benchmarks/benches/generate.rs create mode 100644 grainpit/Cargo.toml rename src/markov3.txt => grainpit/data/config.txt (100%) rename src/markov1.txt => grainpit/data/html.txt (99%) rename src/markov4.txt => grainpit/data/url.txt (100%) rename src/markov2.txt => grainpit/data/url_name.txt (100%) create mode 100644 grainpit/src/lib.rs create mode 100644 grainpit/src/markov.rs create mode 100644 grainpit/src/markov/chain.rs create mode 100644 grainpit_proxy/Cargo.toml create mode 100644 grainpit_proxy/src/main.rs create mode 100644 grainpit_webserver/Cargo.toml create mode 100644 grainpit_webserver/src/main.rs delete mode 100644 src/main.rs delete mode 100644 src/markov.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..ddff440 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags = ["-C", "target-cpu=native"] diff --git a/.containerignore b/.containerignore index d301b0a..9aeb401 100644 --- a/.containerignore +++ b/.containerignore @@ -1,2 +1,7 @@ -!src/ -!Cargo.* +README.md +.gitignore +compose.yml +Containerfile +example.png +LICENSE +target/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index a685039..a90c756 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ /target -perf.data \ No newline at end of file + +# profiling bs +perf.data* +flamegraph.svg +callgrind.out* \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 92be5ed..a04907c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,12 +11,45 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "atomic-waker" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "axum" version = "0.8.9" @@ -69,11 +102,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "benchmarks" +version = "0.1.0" +dependencies = [ + "criterion", + "grainpit", +] + [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" @@ -81,6 +128,22 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -98,6 +161,58 @@ dependencies = [ "rand_core", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cpufeatures" version = "0.3.0" @@ -107,6 +222,78 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + [[package]] name = "errno" version = "0.3.14" @@ -117,6 +304,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -128,30 +321,30 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-core", "futures-task", @@ -173,17 +366,42 @@ dependencies = [ [[package]] name = "grainpit" -version = "0.2.0" +version = "0.1.0" dependencies = [ - "axum", + "arrayvec", "rand", + "rand_xoshiro", "regex", "rustc-hash", + "tracing", +] + +[[package]] +name = "grainpit_proxy" +version = "0.1.0" + +[[package]] +name = "grainpit_webserver" +version = "0.1.0" +dependencies = [ + "axum", + "grainpit", "tokio", "tracing", "tracing-subscriber", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "http" version = "1.4.2" @@ -196,9 +414,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -206,9 +424,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -231,9 +449,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -264,12 +482,32 @@ dependencies = [ "tower-service", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -278,9 +516,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "lock_api" @@ -297,6 +535,15 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -317,9 +564,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -335,12 +582,37 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -376,20 +648,48 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -417,6 +717,35 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_xoshiro" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "662effc7698e08ea324d3acccf8d9d7f7bf79b9785e270a174ea36e56900c91d" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -428,9 +757,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -440,9 +769,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -461,12 +790,27 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "ryu" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -475,38 +819,39 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", + "serde_derive", ] [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -547,6 +892,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -571,9 +922,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys", @@ -581,9 +932,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -598,18 +960,28 @@ checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -624,13 +996,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -681,7 +1053,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -711,10 +1083,14 @@ version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex-automata", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] @@ -731,12 +1107,108 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" @@ -752,8 +1224,28 @@ dependencies = [ "windows-link", ] +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index ebf8d89..1a37581 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,16 +1,41 @@ -[package] -name = "grainpit" +[workspace] +resolver = "3" +members = ["benchmarks", "grainpit", "grainpit_proxy", "grainpit_webserver"] + +[workspace.package] version = "0.2.0" edition = "2024" description = "tarpit to trap malicious scrapers" license = "CC0-1.0" repository = "https://github.com/GrainWare/grainpit" -[dependencies] -axum = "0.8.9" -rand = "0.10.2" -regex = "1.13.0" -rustc-hash = "2.1.3" -tokio = { version = "1.52.3", features = ["full"] } +[workspace.dependencies] tracing = "0.1.44" -tracing-subscriber = "0.3.23" +tokio = { version = "1.53.0", features = ["full"] } + +[profile.dev] +opt-level = 3 +debug = true +strip = "none" +debug-assertions = true +overflow-checks = true +lto = "thin" +panic = "unwind" +incremental = true +codegen-units = 16 +rpath = false + +[profile.release] +opt-level = 3 +debug = false +strip = "none" +debug-assertions = false +overflow-checks = false +lto = "thin" # increased performance by ~20% in benchmarks +panic = "unwind" +incremental = false +codegen-units = 16 +rpath = false + +[profile.bench] +inherits = "release" diff --git a/Containerfile b/Containerfile index d7fb47f..c8c661d 100644 --- a/Containerfile +++ b/Containerfile @@ -1,23 +1,14 @@ FROM rust AS build -# blazing fast and optimized!!!!!!!!!1111!!! -RUN mkdir -p .cargo && printf '\ - [profile.release]\n\ - lto = true\n\ - codegen-units = 1\n\ - opt-level = '3'\n\ - panic = "abort"\n\ - strip = true\n' > .cargo/config.toml - WORKDIR /app COPY . . -RUN cargo install --root /tmp --path . +RUN cargo install --root /tmp --path grainpit_webserver/ FROM gcr.io/distroless/cc-debian13:nonroot AS runtime -COPY --from=build --chown=nonroot:nonroot /tmp/bin/grainpit . +COPY --from=build --chown=nonroot:nonroot /tmp/bin/grainpit_webserver . USER nonroot ENV GRAINPIT_ADDR="0.0.0.0:5000" EXPOSE 5000 -ENTRYPOINT ["./grainpit"] +ENTRYPOINT ["./grainpit_webserver"] diff --git a/README.md b/README.md index 10b9ada..d4eb9c5 100644 --- a/README.md +++ b/README.md @@ -4,23 +4,42 @@ markov tarpit but grain -keep in mind that this works different than other markov tarpits where a link to the tarpit is injected into the html, for this you need to setup reverse proxy rules to get the user here - -some example rules for haproxy are included here, if you want to contribute more examples it would be appreciated +> [!IMPORTANT] +> grainpit is under heavy development currently! please keep in mind that right now if you want stability you should use the `webserver` version! ![image of the tarpit](example.png) +## backends + +grainpit itself is just a markov library and some extra utilities, there are 3 different backends you can use to actually add functionality + +### `webserver` + +this only exposes a webserver which you have to setup rules on your reverse proxy to actually get to, currently this is the most stable + +### `proxy` (in development) + +this is under heavy development and is more complex, but proxies the traffic through grainpit which allows injecting hidden links to the tarpit and more configuration on grainpits side + +this is planned to be the preferred way to use grainpit once its completed + +currently this is empty in the repo because the general project structure for it is still being figured out + +### cgi script (unofficial) + +see + ## features -- fast and lightweight, on my low resource proxy vps (2 cores, e5-2680) it manages to hover around 2-3ms per request with 21mb ram usage while handling ~15 requests per second continuously to the tarpit without consuming enough cpu time where regular requests slow down substantially +- fast and lightweight, on my low resource proxy vps (2 cores, e5-2680) it manages to hover around 2-3ms per request with 21mb ram usage while handling ~30 requests per second continuously to the tarpit without consuming enough cpu time where regular requests slow down substantially - batshit insane default training data that (somehow) manages to put out html/css - able to generate fake config files as well for the malicious credential scanning bots -- effective, as of writing this GPTBot has been sending requests to the tarpit for over 15 hours straight +- effective, as of writing this GPTBot has been sending requests to the tarpit since july 13 as well as amazons scraper and claudes scraper more recently ## reasons you might want to use a different tarpit - not as many features as other alternatives have -- requires reverse proxy config to send bots here +- requires reverse proxy config to send bots here (a proxy is currently in progress so you dont need this) ## configuration @@ -37,7 +56,7 @@ in order to add these there are a few variables you can configure ### systemd/bare metal -install this using `cargo install grainpit` +install this using `cargo install grainpit_webserver` example systemd service: @@ -47,7 +66,7 @@ Description=Tarpit After=network.target [Service] -ExecStart=/root/.cargo/bin/grainpit +ExecStart=/root/.cargo/bin/grainpit_webserver Restart=always [Install] @@ -100,3 +119,28 @@ backend evil_bot_punishment_zone ``` i also recommend combining this with to provide better matching + +## development + +> [!WARNING] +> this section is a work in progress + +### getting started + +make sure you have rust installed with rustup already and this repo cloned + +once you have those installed you can develop grainpit by running `cargo run --bin ` (e.g. grainpit_webserver for the webserver version) + +> [!TIP] +> for development the default `dev` profile has been configured to be almost as optimized as the release profile (and more optimized than the default rust release profile) + +### benchmarks + +if you are contributing changes that may affect performance its appreciated (but not required) if you can include a benchmark comparison of before and after by doing this: + +```bash +git checkout main +cargo bench -p benchmarks -- --save-baseline main +git checkout +cargo bench -p benchmarks -- --baseline main +``` diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml new file mode 100644 index 0000000..a56e668 --- /dev/null +++ b/benchmarks/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "benchmarks" +version = "0.1.0" +publish = false +edition.workspace = true + +[dependencies] +grainpit = { path = "../grainpit" } + +[dev-dependencies] +criterion = "0.8.2" + +[[bench]] +name = "generate" +harness = false diff --git a/benchmarks/benches/generate.rs b/benchmarks/benches/generate.rs new file mode 100644 index 0000000..228a4bc --- /dev/null +++ b/benchmarks/benches/generate.rs @@ -0,0 +1,29 @@ +use grainpit::markov::Markov; + +use criterion::{Criterion, Throughput, criterion_group, criterion_main}; + +fn bench(c: &mut Criterion) { + let markov = Markov::new(); + + c.bench_function("gen_html", |b| b.iter(|| markov.gen_html())); + + let mut group = c.benchmark_group("generate"); + let length = 2048; + group.throughput(Throughput::Elements(length as u64)); + group.bench_function("html-chain", |b| { + b.iter(|| markov.html_chain.generate(length)) + }); + group.bench_function("config-chain", |b| { + b.iter(|| markov.config_chain.generate(length)) + }); + group.bench_function("url-chain", |b| { + b.iter(|| markov.url_chain.generate(length)) + }); + group.bench_function("url-name-chain", |b| { + b.iter(|| markov.url_name_chain.generate(length)) + }); + group.finish(); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/grainpit/Cargo.toml b/grainpit/Cargo.toml new file mode 100644 index 0000000..cb029d5 --- /dev/null +++ b/grainpit/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "grainpit" +version = "0.1.0" +edition = "2024" + +[dependencies] +rand = "0.10.2" +regex = "1.13.0" +tracing.workspace = true +rustc-hash = "2.1.3" +arrayvec = "0.7.8" +rand_xoshiro = "0.8.1" diff --git a/src/markov3.txt b/grainpit/data/config.txt similarity index 100% rename from src/markov3.txt rename to grainpit/data/config.txt diff --git a/src/markov1.txt b/grainpit/data/html.txt similarity index 99% rename from src/markov1.txt rename to grainpit/data/html.txt index d7e80ec..b06f883 100644 --- a/src/markov1.txt +++ b/grainpit/data/html.txt @@ -348,7 +348,7 @@ you need to stop looking at the grainware wiki and get a job "> - accurate linux graphs + griggy microsoft.com @@ -615,7 +615,7 @@ you need to stop looking at the grainware wiki and get a job - easter egg + easter egg via accuracy linux graphs @@ -677,7 +677,7 @@ you need to stop looking at the grainware wiki and get a job - Fizzbuzz Generator + Linux Graph Fizzbuzz Generator @@ -789,7 +789,7 @@ you need to stop looking at the grainware wiki and get a job - ✿✧･ﾟ accuwate winux gwaphs wite ✿✧･ﾟ + ✿✧･ﾟ accuwate winux gwaphs wite ✿✧･ﾟ Kawaii Linux Graphs Accuracy Linux Microsoft diff --git a/src/markov4.txt b/grainpit/data/url.txt similarity index 100% rename from src/markov4.txt rename to grainpit/data/url.txt diff --git a/src/markov2.txt b/grainpit/data/url_name.txt similarity index 100% rename from src/markov2.txt rename to grainpit/data/url_name.txt diff --git a/grainpit/src/lib.rs b/grainpit/src/lib.rs new file mode 100644 index 0000000..706c4a7 --- /dev/null +++ b/grainpit/src/lib.rs @@ -0,0 +1 @@ +pub mod markov; diff --git a/grainpit/src/markov.rs b/grainpit/src/markov.rs new file mode 100644 index 0000000..c240bc4 --- /dev/null +++ b/grainpit/src/markov.rs @@ -0,0 +1,90 @@ +use rand::seq::IteratorRandom; + +use crate::markov::chain::Chain; +use regex::regex; + +pub mod chain; + +#[derive(Debug, Clone)] +pub struct Markov { + pub html_chain: Chain, + pub url_name_chain: Chain, + pub url_chain: Chain, + pub config_chain: Chain, +} + +impl Markov { + pub fn new() -> Self { + Self { + html_chain: Chain::new(include_str!("../data/html.txt")), + url_name_chain: Chain::new(include_str!("../data/url_name.txt")), + url_chain: Chain::new(include_str!("../data/url.txt")), + config_chain: Chain::new(include_str!("../data/config.txt")), + } + } + + pub fn gen_html(&self) -> String { + let href_regex = regex!(r#"href="(.*?)""#); + let mut generated = self.html_chain.generate(2048); + for i in href_regex.find_iter(&generated.clone()) { + generated = generated.replace(i.as_str(), &self.random_link()); + } + + let text_content_regex = regex!(r">([^<]+)<"); + let url_replacements: Vec = text_content_regex + .captures_iter(&generated) + .flat_map(|caps| caps.get(0).unwrap().as_str().split_whitespace()) + .sample(&mut rand::rng(), 15) + .into_iter() + .map(|v| v.to_string()) + .collect(); + + for r in url_replacements { + generated = generated.replacen( + &r, + &format!("{}", self.random_link(), r), + 1, + ); + } + + format!("

{}

", generated) + } + + pub fn random_link(&self) -> String { + let start_path = if std::env::var("GRAINPIT_EXTRAURLS").is_ok() { + if rand::random_bool( + std::env::var("GRAINPIT_EXTRAURLS_CHANCE") + .unwrap_or("5".to_owned()) + .parse::() + .unwrap() as f64 + / 100.0, + ) { + let rng = &mut rand::rng(); + std::env::var("GRAINPIT_EXTRAURLS") + .unwrap() + .split(',') + .choose(rng) + .unwrap() + .to_owned() + } else { + "/".to_string() + } + } else { + "/".to_string() + }; + + format!( + "{}{}/{}/{}.html", + start_path, + self.url_chain.generate(4), + self.url_chain.generate(4), + self.url_chain.generate(24) + ) + } +} + +impl Default for Markov { + fn default() -> Self { + Markov::new() + } +} diff --git a/grainpit/src/markov/chain.rs b/grainpit/src/markov/chain.rs new file mode 100644 index 0000000..0d69c91 --- /dev/null +++ b/grainpit/src/markov/chain.rs @@ -0,0 +1,252 @@ +use std::ops::Deref; + +use arrayvec::ArrayVec; +use rand::{RngExt, SeedableRng}; +use rand_xoshiro::Xoshiro256Plus; +use rustc_hash::FxHashMap; + +const MAX_CONTEXT_SIZE: usize = 3; + +#[derive(Clone, PartialEq, Debug, Default)] +pub struct Chain { + #[allow(clippy::type_complexity)] // Pain and misery + tokens: FxHashMap, (Vec<(u32, usize)>, usize)>, + tokenizer: Vec, +} + +impl Chain { + #[tracing::instrument(skip_all)] + pub fn new(text: &str) -> Self { + // this whole thing is pretty ass and can probably be optimized but its only ran at start so as long as its not like 100ms+ wtv + let mut chain = Self::default(); + let mut context: Vec = Vec::with_capacity(MAX_CONTEXT_SIZE + 1); + let (tokenizer, tokens) = Self::make_tokenizer(text); + chain.tokenizer = tokenizer; + let mut temp_tokens: FxHashMap, FxHashMap> = + FxHashMap::default(); + for token in tokens { + for cs in 0..=context.len() { + let context_key: ArrayVec = context + [(context.len() - cs)..context.len()] + .try_into() + .unwrap(); + + *temp_tokens + .entry(context_key) + .or_default() + .entry(token) + .or_default() += 1; + } + + context.push(token); + + if context.len() > MAX_CONTEXT_SIZE { + context.remove(0); + } + } + + chain.tokens = temp_tokens + .into_iter() + .map(|(k, v)| { + ( + k, + ( + v.clone().into_iter().collect(), + v.clone().into_values().sum(), + ), + ) + }) + .collect(); + + chain.tokens.values_mut().for_each(|v| { + v.0.sort_by_key(|k| k.1); + v.0.reverse(); + v.0.shrink_to_fit(); + }); + + chain + } + + pub fn generate(&self, length: usize) -> String { + let mut out: Vec = Vec::with_capacity(length); + let mut rng = Xoshiro256Plus::from_rng(&mut rand::rng()); + + while out.len() < length { + let mut next_token = None; + + for cs in (0..=MAX_CONTEXT_SIZE).rev() { + if cs > out.len() { + continue; + } + + let context: ArrayVec = + out[(out.len() - cs)..out.len()].try_into().unwrap(); + + if let Some(next_tokens) = self.tokens.get(&context) { + let mut remaining_distance = rng.random_range::(..next_tokens.1); + for (t, c) in &next_tokens.0 { + remaining_distance = remaining_distance.saturating_sub(*c); + if remaining_distance == 0 { + next_token = Some(*t); + break; + } + } + + break; + } + } + + if let Some(next_token) = next_token { + out.push(next_token); + } else { + break; + } + } + + out.iter() + .map(|x| self.tokenizer.get(*x as usize).unwrap()) + .map(|s| s.deref()) + .collect::>() + .join("") // probably a better way to do this but seems to still be faster than the previous method + } + + fn make_tokenizer(s: &str) -> (Vec, Vec) { + // TODO: replace this with nom or something that isnt this complete unreadable mess + let len = s.len(); + let bytes = s.as_bytes(); + let mut pos = 0; + let mut tokens: Vec<&str> = Vec::new(); + + while pos < len { + let start = pos; + let ch = s[pos..].chars().next().unwrap(); + + if ch.is_whitespace() { + pos += ch.len_utf8(); + while pos < len { + let c = s[pos..].chars().next().unwrap(); + if c.is_whitespace() { + pos += c.len_utf8(); + } else { + break; + } + } + tokens.push(&s[start..pos]); + continue; + } + + if ch == '<' && bytes.get(pos + 1) == Some(&b'?') { + pos += 2; + tokens.push(&s[start..pos]); + continue; + } + if ch == '?' && bytes.get(pos + 1) == Some(&b'>') { + pos += 2; + tokens.push(&s[start..pos]); + continue; + } + + if ch == '<' { + pos += ch.len_utf8(); + while pos < len { + let c = s[pos..].chars().next().unwrap(); + pos += c.len_utf8(); + if c == '>' { + break; + } + } + tokens.push(&s[start..pos]); + continue; + } + + if ch == '\'' || ch == '"' { + pos += ch.len_utf8(); + while pos < len { + let c = s[pos..].chars().next().unwrap(); + if c == '\n' { + break; + } + pos += c.len_utf8(); + if c == ch { + break; + } + } + tokens.push(&s[start..pos]); + continue; + } + + if ch == '/' && bytes.get(pos + 1) == Some(&b'/') { + while pos < len { + let c = s[pos..].chars().next().unwrap(); + if c == '\n' { + break; + } + pos += c.len_utf8(); + } + tokens.push(&s[start..pos]); + continue; + } + + if ch == '$' { + pos += ch.len_utf8(); + while pos < len { + let c = s[pos..].chars().next().unwrap(); + if c.is_alphanumeric() || c == '_' { + pos += c.len_utf8(); + } else { + break; + } + } + tokens.push(&s[start..pos]); + continue; + } + + if ch == '/' { + pos += ch.len_utf8(); + while pos < len { + let c = s[pos..].chars().next().unwrap(); + if c.is_alphanumeric() || matches!(c, '/' | '.' | '-' | '_') { + pos += c.len_utf8(); + } else { + break; + } + } + tokens.push(&s[start..pos]); + continue; + } + + if ch.is_alphanumeric() || matches!(ch, '#' | '%') { + pos += ch.len_utf8(); + while pos < len { + let c = s[pos..].chars().next().unwrap(); + if c.is_alphanumeric() || matches!(c, '.' | '-' | '#' | '%') { + pos += c.len_utf8(); + } else { + break; + } + } + tokens.push(&s[start..pos]); + continue; + } + + pos += ch.len_utf8(); + tokens.push(&s[start..pos]); + } + + let mut vocab: FxHashMap<&str, u32> = FxHashMap::default(); + let mut id_to_str: Vec = Vec::default(); + + let token_ids: Vec = tokens + .iter() + .map(|&t| { + *vocab.entry(t).or_insert_with(|| { + let id = id_to_str.len() as u32; + id_to_str.push(t.to_string()); + id + }) + }) + .collect(); + + (id_to_str, token_ids) + } +} diff --git a/grainpit_proxy/Cargo.toml b/grainpit_proxy/Cargo.toml new file mode 100644 index 0000000..85927a6 --- /dev/null +++ b/grainpit_proxy/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "grainpit_proxy" +version = "0.1.0" +edition = "2024" + +[dependencies] diff --git a/grainpit_proxy/src/main.rs b/grainpit_proxy/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/grainpit_proxy/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/grainpit_webserver/Cargo.toml b/grainpit_webserver/Cargo.toml new file mode 100644 index 0000000..44ce50e --- /dev/null +++ b/grainpit_webserver/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "grainpit_webserver" +version = "0.1.0" +edition = "2024" + +[dependencies] +axum = "0.8.9" +tokio.workspace = true +tracing.workspace = true +tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } +grainpit = { path = "../grainpit" } diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs new file mode 100644 index 0000000..d28e529 --- /dev/null +++ b/grainpit_webserver/src/main.rs @@ -0,0 +1,59 @@ +use std::sync::Arc; + +use axum::{ + Router, + extract::{Path, State}, + response::{Html, IntoResponse, Response}, + routing::get, +}; +use grainpit::markov::Markov; +use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt}; + +#[derive(Debug)] +struct AppState { + m: Markov, +} + +#[tokio::main] +async fn main() { + tracing_subscriber::registry() + .with( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { + format!("{}=debug,grainpit=debug", env!("CARGO_CRATE_NAME")).into() + }), + ) + .with(tracing_subscriber::fmt::layer().with_span_events(FmtSpan::CLOSE)) + .init(); + + let shared_state = Arc::new(AppState { m: Markov::new() }); + + let app = Router::new() + .route("/", get(handler)) + .route("/{*wildcard}", get(wildcard_handler)) + .with_state(shared_state); + + let listener = tokio::net::TcpListener::bind( + std::env::var("GRAINPIT_ADDR").unwrap_or("127.0.0.1:5000".to_string()), + ) + .await + .unwrap(); + println!("listening on {}", listener.local_addr().unwrap()); + axum::serve(listener, app).await.unwrap(); +} + +async fn wildcard_handler( + State(state): State>, + Path(path): Path, +) -> Response { + if path.contains(".html") { + handler(State(state)).await.into_response() + } else { + let mut a = state.m.config_chain.generate(512); + a = a.trim_start().to_owned(); + a.into_response() + } +} + +async fn handler(State(state): State>) -> Html { + Html(state.m.gen_html()) +} diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 07d8da0..0000000 --- a/src/main.rs +++ /dev/null @@ -1,164 +0,0 @@ -pub mod markov; - -use std::sync::Arc; - -use crate::markov::Chain; -use axum::{ - Router, - extract::{Path, State}, - response::{Html, IntoResponse, Response}, - routing::get, -}; -use rand::seq::IteratorRandom; -use regex::regex; -use tracing::info_span; -use tracing_subscriber::fmt::format::FmtSpan; - -#[derive(Debug)] -struct AppState { - markov1: Chain, - markov2: Chain, - markov3: Chain, - markov4: Chain, -} - -#[tokio::main] -async fn main() { - tracing_subscriber::fmt() - .with_span_events(FmtSpan::CLOSE) - .with_max_level(tracing::Level::DEBUG) - .init(); - - let mut chain1 = Chain::default(); - let mut chain2 = Chain::default(); - let mut chain3 = Chain::default(); - let mut chain4 = Chain::default(); - - info_span!("train_markov1").in_scope(|| { - let mut lines = include_str!("markov1.txt").lines(); - while let Some(_) = lines.next() { - let mut buffer = String::new(); - for _ in 0..512 { - if let Some(next_line) = lines.next() { - buffer = buffer + "\n" + next_line; - } - } - chain1.train(&buffer.to_owned()); - } - }); - - info_span!("train_markov2").in_scope(|| { - for line in include_str!("markov2.txt").lines() { - chain2.train(line); - } - }); - - info_span!("train_markov3").in_scope(|| { - let mut lines = include_str!("markov3.txt").lines(); - while let Some(_) = lines.next() { - let mut buffer = String::new(); - for _ in 0..50 { - if let Some(next_line) = lines.next() { - buffer = buffer + "\n" + next_line; - } - } - chain3.train(&buffer.to_owned()); - } - }); - - info_span!("train_markov4").in_scope(|| { - for line in include_str!("markov4.txt").lines() { - chain4.train(line); - } - }); - - let shared_state = Arc::new(AppState { - markov1: chain1, - markov2: chain2, - markov3: chain3, - markov4: chain4, - }); - let app = Router::new() - .route("/", get(handler)) - .route("/{*wildcard}", get(wildcard_handler)) - .with_state(shared_state); - - // run it - let listener = tokio::net::TcpListener::bind( - std::env::var("GRAINPIT_ADDR").unwrap_or("127.0.0.1:5000".to_string()), - ) - .await - .unwrap(); - println!("listening on {}", listener.local_addr().unwrap()); - axum::serve(listener, app).await.unwrap(); -} - -async fn wildcard_handler( - State(state): State>, - Path(path): Path, -) -> Response { - if path.contains(".html") { - handler(State(state)).await.into_response() - } else { - info_span!("wildcard_handler").in_scope(|| { - let mut a = state.markov3.generate(512); - a = a.trim_start().to_owned(); - a.into_response() - }) - } -} - -#[tracing::instrument(skip_all)] -async fn handler(State(state): State>) -> Html { - let mut fakeurls = String::new(); - for _ in 0..16 { - fakeurls.push_str(&format!( - "\n{}
", - random_link(&state.markov4), - state.markov2.generate(16) - )); - } - - let href_regex = regex!(r#"href="(.*?)""#); - let mut generated = state.markov1.generate(4096); - for i in href_regex.find_iter(&generated.clone()) { - generated = generated.replace(i.as_str(), &random_link(&state.markov4)); - } - - Html(format!( - "

This is my website and it is Amazing!!

\n{}\n

{}

", - fakeurls, generated - )) -} - -fn random_link(markov4: &Chain) -> String { - let start_path = if std::env::var("GRAINPIT_EXTRAURLS").is_ok() { - if rand::random_bool( - std::env::var("GRAINPIT_EXTRAURLS_CHANCE") - .unwrap_or("5".to_owned()) - .parse::() - .unwrap() as f64 - / 100.0, - ) { - let rng = &mut rand::rng(); - std::env::var("GRAINPIT_EXTRAURLS") - .unwrap() - .split(",") - .choose(rng) - .unwrap() - .to_owned() - } else { - "/".to_string() - } - } else { - "/".to_string() - }; - - format!( - "{}{}/{}/{}.html", - start_path, - markov4.generate(4), - markov4.generate(4), - markov4.generate(24) - ) -} diff --git a/src/markov.rs b/src/markov.rs deleted file mode 100644 index 0c0f85b..0000000 --- a/src/markov.rs +++ /dev/null @@ -1,146 +0,0 @@ -use std::iter; - -use rand::seq::IndexedRandom; -use rustc_hash::FxHashMap; - -#[derive(Clone, PartialEq, Debug, Default)] -pub struct Chain { - tokens: FxHashMap, FxHashMap>, - vocab: FxHashMap, - vocab_rev: Vec, -} - -impl Chain { - const MAX_CONTEXT_SIZE: usize = 5; - - fn intern(&mut self, s: &str) -> u32 { - if let Some(&id) = self.vocab.get(s) { - return id; - } - let id = self.vocab_rev.len() as u32; - self.vocab_rev.push(s.to_string()); - self.vocab.insert(s.to_string(), id); - id - } - - pub fn train(&mut self, text: &str) { - let mut context: Vec = Vec::with_capacity(Self::MAX_CONTEXT_SIZE + 1); - - for token in Self::tokenize(text) { - let token_id = self.intern(token); - - for cs in 0..=context.len() { - let context_key: Vec = context[(context.len() - cs)..context.len()].to_vec(); - - *self - .tokens - .entry(context_key) - .or_default() - .entry(token_id) - .or_default() += 1; - } - - context.push(token_id); - - if context.len() > Self::MAX_CONTEXT_SIZE { - context.remove(0); - } - } - } - - pub fn generate(&self, length: usize) -> String { - let mut out: Vec = Vec::with_capacity(length); - let mut rng = rand::rng(); - - while out.len() < length { - let mut next_token = None; - - for cs in (0..=Self::MAX_CONTEXT_SIZE).rev() { - if cs > out.len() { - continue; - } - - let context: &[u32] = &out[(out.len() - cs)..out.len()]; - - if let Some(next_tokens) = self.tokens.get(context) { - let next_tokens: Vec<_> = next_tokens.iter().collect(); - - next_token = Some( - *next_tokens - .choose_weighted(&mut rng, |(_token, frequency)| *frequency) - .unwrap() - .0, - ); - - break; - } - } - - if let Some(next_token) = next_token { - out.push(next_token); - } else { - break; - } - } - - out.iter() - .map(|id| self.vocab_rev[*id as usize].as_str()) - .collect() - } - - fn tokenize(s: &str) -> impl Iterator { - let mut chars = s.char_indices().peekable(); - - iter::from_fn(move || { - let (idx, ch) = chars.next()?; - - if ch.is_alphanumeric() { - let idx_from = idx; - let mut idx_to = idx + ch.len_utf8(); - - while let Some((idx, ch)) = chars.peek() { - if ch.is_alphanumeric() { - idx_to = idx + ch.len_utf8(); - chars.next(); - } else { - break; - } - } - - Some(&s[idx_from..idx_to]) - } else if ch.is_whitespace() { - let idx_from = idx; - let mut idx_to = idx + ch.len_utf8(); - - while let Some((idx, ch)) = chars.peek() { - if ch.is_whitespace() { - idx_to = idx + ch.len_utf8(); - chars.next(); - } else { - break; - } - } - - Some(&s[idx_from..idx_to]) - } else if ch == '<' { - let idx_from = idx; - let mut idx_to = idx + ch.len_utf8(); - - while let Some(&(idx, ch)) = chars.peek() { - idx_to = idx + ch.len_utf8(); - chars.next(); - if ch == '>' { - break; - } - } - - Some(&s[idx_from..idx_to]) - } else { - let idx_from = idx; - let idx_to = idx + ch.len_utf8(); - - Some(&s[idx_from..idx_to]) - } - }) - } -} From 74055761055ffb65796157eb5b60073e582b3752 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:13:05 -0500 Subject: [PATCH 02/19] fix the single equals character i forgot and graig slightly saner --- grainpit/src/markov/chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grainpit/src/markov/chain.rs b/grainpit/src/markov/chain.rs index 0d69c91..5639140 100644 --- a/grainpit/src/markov/chain.rs +++ b/grainpit/src/markov/chain.rs @@ -83,7 +83,7 @@ impl Chain { out[(out.len() - cs)..out.len()].try_into().unwrap(); if let Some(next_tokens) = self.tokens.get(&context) { - let mut remaining_distance = rng.random_range::(..next_tokens.1); + let mut remaining_distance = rng.random_range::(..=next_tokens.1); for (t, c) in &next_tokens.0 { remaining_distance = remaining_distance.saturating_sub(*c); if remaining_distance == 0 { From 70c3b6d968487299cb9b923beac0e2d002a5ce10 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:21:57 -0500 Subject: [PATCH 03/19] testing something --- grainpit/src/markov/chain.rs | 4 +--- grainpit_webserver/src/g.txt.gz | Bin 0 -> 1042077 bytes grainpit_webserver/src/main.rs | 11 +++++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 grainpit_webserver/src/g.txt.gz diff --git a/grainpit/src/markov/chain.rs b/grainpit/src/markov/chain.rs index 5639140..f1ee5e7 100644 --- a/grainpit/src/markov/chain.rs +++ b/grainpit/src/markov/chain.rs @@ -1,5 +1,3 @@ -use std::ops::Deref; - use arrayvec::ArrayVec; use rand::{RngExt, SeedableRng}; use rand_xoshiro::Xoshiro256Plus; @@ -105,7 +103,7 @@ impl Chain { out.iter() .map(|x| self.tokenizer.get(*x as usize).unwrap()) - .map(|s| s.deref()) + .map(|s| s.as_str()) .collect::>() .join("") // probably a better way to do this but seems to still be faster than the previous method } diff --git a/grainpit_webserver/src/g.txt.gz b/grainpit_webserver/src/g.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..d7eb09d9cb2b7ae98c9448c776e3f9da507ea8c2 GIT binary patch literal 1042077 zcmeI&p$^09ea2y@HaI#`Z3m>pVAj~k(13|0n;0A9VArTxr4$`77Ef8wu47;q5HwI7 z7E+u{8ps6{1q8YB&X;>A_@w8LK4$+O{Qkp#{_w*;{pnx-_?z?(|NQg6{ha*umw!)w zy`}!Yzy0=i|NUclI|2j<5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV45yf#rWM z=`IA++hZ7ias&tvAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!8B(0&|gPVGFRV zQ`N^F)d?T^vt2p~X!009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjY`mvk4#0QL45MjQkP5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7dX^a68{XJHGltW(uTUn7730RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zxVB9hwgAcde9CYTAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0?`W`KCkY( z1-Ngw{W1C+0R#vTAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!CvWlJ3G7pxz$C zh=TwD0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBm_USKZrENlUmb*lR4YXlG= zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk*S0Cc79e?_PZ-KSrM;fB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oQK)(p?w>)Z1eiaS$LtfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009Eg3(Q5Hg)P9cPE{X$jQ|1!2oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkL{+BRj_0wnMADZ@d4009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXFL@#joyt?le;J)4V$LMne5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RqoUx(j1~dV35b4gv%S5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV45`fw{=D zumxDwsp_Mz5kPb_fm`*zzOqt6jQfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72s|(8E{p-{?J|u5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7csf#)UNg)u0*BA5`)&d5+iic0K1Ton0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBly@VunEFb1f%$1vg`K!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1fmz1i#!WkfMuPkKKdE~1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfWWnF%CH4U-se+>g8%^n1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oQ)~;P82M-z~s>yX}wB=LjG`fB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C7o|kkN#sKy97)Bff2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+K=cB0 zk!N8Gu&h(nM_(g=009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5V*EY8MXk) z`+Ul95FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0Rqtr96qn^y9Ky!xBW5t z903Ff5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7e?^OEku7@*!B!-#_b0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&Uh+beW@+@otmUXK7=xYQJAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0@t=F!xkWUpHCSM0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyKp=X7!{^m~w*dF;wm(LnBY*$_0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBm#Uea9{1Jv7N7;z9FK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk(F@E)o`o&IvQAYWeT@JD1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZ;Mz82*a9T)^C`nYfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72t+S% z_`JIB7T~_!_Q&XR1P~xVfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009EeOS%hV zfO>ljBMt%t2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkJxdV#sfv#g_R%I0z6RK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=E$ z1?D2p!WLjzr>c*>MgRc<1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNA}ZJRP| z0h0Iml;I#ifB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7q8B)PUfp*KaNlnG zWAr%!2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+0DE z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5Fik}z+B{6*a9r;RQ1u<2p~X!009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjZBZBvFVK=MAHG8_a55FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7dX^a6*^tNU&N?%QpDj6O#I0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAn?4TyD$c*x5qHzAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0tBKLn2S6MTYzPqsy_M}0R#vTAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!CutZOX6(NZ#jDhJyeB0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBm_ zUf}R~b>A((eY@?C(dP&tK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1fG|4 z7sdef_83MS1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfI##DbCG9Z3$UzH z)kj|=fB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oSioO&PWT$@_fDa1bCs zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009Eg3miVL?z;uJZ@2w1`WyiS2oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+!1I#s!Wf|59>a)(009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5Qtu2F7hmF0hV>D`six}5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0Rq>yDZ>^Zd7n=i4gv%S5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV45`fy3w3eYXJj?Y2KgpCf<(0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UcwW+77z5PXV;FG|AV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0?`Z1MV^H%z_LzNAAOAg0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK;YUo zW!M5F@AE0cL4W`O0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PDYgaQM8s?-t;` z-S)@ma|93|K!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk&r7-sV}N>l3?mK# z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZAbNqh$g{8oSk|fPqpuM_fB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72wdBy3|oNYeLiJ42oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+0D6VZ=dz009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXFL@zKGc^0++%Q{tk^fdwq5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7csfot28VGEGF&!-Fr0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAP~L4;q&UgTY&p^+aIIP5kPd|usm3vl0V`(yMu0tgTw zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=E#CEbNFK)pSN5eESR1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oN9;y}(@LS=a(B>s0m8*9ag$fB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C7u5D9>EkN= z90UjuAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!8B>0&|gPVGFRVQ`JXbBY*$_ z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBngwoMtf0LlA&%5V@MK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk(F+_tukO1AxNo=pG5Q<<1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfWY&T?!p+L-X6nSHf7iXB=7Sn z!$E)m0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&|FL3z0y6+a?zTNi6=yL=R zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0?$jj3uAzKdkiBE0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyKp=X7xyZAy1z6Up>Z7j_K!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PEN)rVLwvE2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5Fik}z~S@i zzFUC%cH1AL&k;a?009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXFJTK`ki~;KH zF^o6}5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7csf#?P1BG1AWU|FZCkG@6# z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAn;#pQ-&=-`5*uD=U@Ku5T1X> Q8`FRJ@%MlE^#k4i14F&QfdBvi literal 0 HcmV?d00001 diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs index d28e529..5b4fd90 100644 --- a/grainpit_webserver/src/main.rs +++ b/grainpit_webserver/src/main.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use axum::{ Router, extract::{Path, State}, + http::{HeaderMap, header}, response::{Html, IntoResponse, Response}, routing::get, }; @@ -44,7 +45,17 @@ async fn main() { async fn wildcard_handler( State(state): State>, Path(path): Path, + headers: HeaderMap, ) -> Response { + if let Some(header) = headers.get("Accept-Encoding") + && header.to_str().unwrap_or("").contains("gzip") + { + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_TYPE, "text/plain".parse().unwrap()); + headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap()); + return (headers, include_bytes!("./g.txt.gz")).into_response(); + } + if path.contains(".html") { handler(State(state)).await.into_response() } else { From 893213f88e248050e0b18d17a9f3b72ea131703c Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:36:07 -0500 Subject: [PATCH 04/19] Oops --- grainpit_webserver/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs index 5b4fd90..0ec4bf3 100644 --- a/grainpit_webserver/src/main.rs +++ b/grainpit_webserver/src/main.rs @@ -47,6 +47,7 @@ async fn wildcard_handler( Path(path): Path, headers: HeaderMap, ) -> Response { + /* if let Some(header) = headers.get("Accept-Encoding") && header.to_str().unwrap_or("").contains("gzip") { @@ -54,7 +55,7 @@ async fn wildcard_handler( headers.insert(header::CONTENT_TYPE, "text/plain".parse().unwrap()); headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap()); return (headers, include_bytes!("./g.txt.gz")).into_response(); - } + }*/ if path.contains(".html") { handler(State(state)).await.into_response() From 0a1b5451e6e3f22a41c571cd6884499acfb09636 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:48:20 -0500 Subject: [PATCH 05/19] rejoice --- grainpit_webserver/src/g.txt.br | Bin 0 -> 4201 bytes grainpit_webserver/src/g.txt.gz | Bin 1042077 -> 0 bytes grainpit_webserver/src/main.rs | 9 ++++----- 3 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 grainpit_webserver/src/g.txt.br delete mode 100644 grainpit_webserver/src/g.txt.gz diff --git a/grainpit_webserver/src/g.txt.br b/grainpit_webserver/src/g.txt.br new file mode 100644 index 0000000000000000000000000000000000000000..3a6d4c9898796bfca7643c278d2c2bb8ff349f27 GIT binary patch literal 4201 zcmX^A|9|}t)pL(FIw*YqH~;_t{~x>=jwCWRy!&ep2j=+XDbtCRla= literal 0 HcmV?d00001 diff --git a/grainpit_webserver/src/g.txt.gz b/grainpit_webserver/src/g.txt.gz deleted file mode 100644 index d7eb09d9cb2b7ae98c9448c776e3f9da507ea8c2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1042077 zcmeI&p$^09ea2y@HaI#`Z3m>pVAj~k(13|0n;0A9VArTxr4$`77Ef8wu47;q5HwI7 z7E+u{8ps6{1q8YB&X;>A_@w8LK4$+O{Qkp#{_w*;{pnx-_?z?(|NQg6{ha*umw!)w zy`}!Yzy0=i|NUclI|2j<5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV45yf#rWM z=`IA++hZ7ias&tvAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!8B(0&|gPVGFRV zQ`N^F)d?T^vt2p~X!009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjY`mvk4#0QL45MjQkP5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7dX^a68{XJHGltW(uTUn7730RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zxVB9hwgAcde9CYTAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0?`W`KCkY( z1-Ngw{W1C+0R#vTAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!CvWlJ3G7pxz$C zh=TwD0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBm_USKZrENlUmb*lR4YXlG= zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk*S0Cc79e?_PZ-KSrM;fB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oQK)(p?w>)Z1eiaS$LtfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009Eg3(Q5Hg)P9cPE{X$jQ|1!2oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkL{+BRj_0wnMADZ@d4009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXFL@#joyt?le;J)4V$LMne5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RqoUx(j1~dV35b4gv%S5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV45`fw{=D zumxDwsp_Mz5kPb_fm`*zzOqt6jQfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72s|(8E{p-{?J|u5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7csf#)UNg)u0*BA5`)&d5+iic0K1Ton0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBly@VunEFb1f%$1vg`K!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1fmz1i#!WkfMuPkKKdE~1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfWWnF%CH4U-se+>g8%^n1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oQ)~;P82M-z~s>yX}wB=LjG`fB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C7o|kkN#sKy97)Bff2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+K=cB0 zk!N8Gu&h(nM_(g=009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5V*EY8MXk) z`+Ul95FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0Rqtr96qn^y9Ky!xBW5t z903Ff5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7e?^OEku7@*!B!-#_b0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&Uh+beW@+@otmUXK7=xYQJAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0@t=F!xkWUpHCSM0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyKp=X7!{^m~w*dF;wm(LnBY*$_0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBm#Uea9{1Jv7N7;z9FK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk(F@E)o`o&IvQAYWeT@JD1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZ;Mz82*a9T)^C`nYfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72t+S% z_`JIB7T~_!_Q&XR1P~xVfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009EeOS%hV zfO>ljBMt%t2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkJxdV#sfv#g_R%I0z6RK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=E$ z1?D2p!WLjzr>c*>MgRc<1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNA}ZJRP| z0h0Iml;I#ifB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7q8B)PUfp*KaNlnG zWAr%!2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+0DE z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5Fik}z+B{6*a9r;RQ1u<2p~X!009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjZBZBvFVK=MAHG8_a55FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7dX^a6*^tNU&N?%QpDj6O#I0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAn?4TyD$c*x5qHzAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0tBKLn2S6MTYzPqsy_M}0R#vTAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!CutZOX6(NZ#jDhJyeB0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBm_ zUf}R~b>A((eY@?C(dP&tK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1fG|4 z7sdef_83MS1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfI##DbCG9Z3$UzH z)kj|=fB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oSioO&PWT$@_fDa1bCs zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009Eg3miVL?z;uJZ@2w1`WyiS2oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+!1I#s!Wf|59>a)(009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5Qtu2F7hmF0hV>D`six}5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0Rq>yDZ>^Zd7n=i4gv%S5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV45`fy3w3eYXJj?Y2KgpCf<(0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UcwW+77z5PXV;FG|AV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0?`Z1MV^H%z_LzNAAOAg0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK;YUo zW!M5F@AE0cL4W`O0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PDYgaQM8s?-t;` z-S)@ma|93|K!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk&r7-sV}N>l3?mK# z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZAbNqh$g{8oSk|fPqpuM_fB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72wdBy3|oNYeLiJ42oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+0D6VZ=dz009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXFL@zKGc^0++%Q{tk^fdwq5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7csfot28VGEGF&!-Fr0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAP~L4;q&UgTY&p^+aIIP5kPd|usm3vl0V`(yMu0tgTw zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=E#CEbNFK)pSN5eESR1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oN9;y}(@LS=a(B>s0m8*9ag$fB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C7u5D9>EkN= z90UjuAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!8B>0&|gPVGFRVQ`JXbBY*$_ z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBngwoMtf0LlA&%5V@MK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk(F+_tukO1AxNo=pG5Q<<1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfWY&T?!p+L-X6nSHf7iXB=7Sn z!$E)m0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&|FL3z0y6+a?zTNi6=yL=R zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0?$jj3uAzKdkiBE0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyKp=X7xyZAy1z6Up>Z7j_K!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PEN)rVLwvE2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5Fik}z~S@i zzFUC%cH1AL&k;a?009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXFJTK`ki~;KH zF^o6}5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7csf#?P1BG1AWU|FZCkG@6# z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAn;#pQ-&=-`5*uD=U@Ku5T1X> Q8`FRJ@%MlE^#k4i14F&QfdBvi diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs index 0ec4bf3..71db252 100644 --- a/grainpit_webserver/src/main.rs +++ b/grainpit_webserver/src/main.rs @@ -47,15 +47,14 @@ async fn wildcard_handler( Path(path): Path, headers: HeaderMap, ) -> Response { - /* if let Some(header) = headers.get("Accept-Encoding") - && header.to_str().unwrap_or("").contains("gzip") + && header.to_str().unwrap_or("").contains("br") { let mut headers = HeaderMap::new(); headers.insert(header::CONTENT_TYPE, "text/plain".parse().unwrap()); - headers.insert(header::CONTENT_ENCODING, "gzip".parse().unwrap()); - return (headers, include_bytes!("./g.txt.gz")).into_response(); - }*/ + headers.insert(header::CONTENT_ENCODING, "br".parse().unwrap()); + return (headers, include_bytes!("./g.txt.br")).into_response(); + } if path.contains(".html") { handler(State(state)).await.into_response() From 0dea617112a3c9684ae90fe2e20b095f903b76d4 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:59:52 -0500 Subject: [PATCH 06/19] experimental compression --- Cargo.lock | 195 +++++++++++++++++++++++++++++++++ README.md | 2 + grainpit_webserver/Cargo.toml | 9 ++ grainpit_webserver/src/main.rs | 10 ++ 4 files changed, 216 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index a04907c..58febd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -11,6 +17,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "alloca" version = "0.4.0" @@ -38,6 +59,18 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -116,6 +149,27 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -141,6 +195,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -213,6 +269,26 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "cpufeatures" version = "0.3.0" @@ -222,6 +298,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.8.2" @@ -310,6 +395,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -334,6 +429,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + [[package]] name = "futures-task" version = "0.3.33" @@ -387,6 +488,7 @@ dependencies = [ "axum", "grainpit", "tokio", + "tower-http", "tracing", "tracing-subscriber", ] @@ -497,6 +599,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + [[package]] name = "js-sys" version = "0.3.103" @@ -562,6 +674,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -648,6 +770,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "plotters" version = "0.3.7" @@ -908,6 +1036,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "slab" version = "0.4.12" @@ -1005,6 +1139,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "tower" version = "0.5.3" @@ -1021,6 +1168,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -1249,3 +1416,31 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/README.md b/README.md index d4eb9c5..c85d374 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ grainpit itself is just a markov library and some extra utilities, there are 3 d this only exposes a webserver which you have to setup rules on your reverse proxy to actually get to, currently this is the most stable +you can enable content compression for this proxy with the `compress` feature incase network load is a concern for you + ### `proxy` (in development) this is under heavy development and is more complex, but proxies the traffic through grainpit which allows injecting hidden links to the tarpit and more configuration on grainpits side diff --git a/grainpit_webserver/Cargo.toml b/grainpit_webserver/Cargo.toml index 44ce50e..55c5e0b 100644 --- a/grainpit_webserver/Cargo.toml +++ b/grainpit_webserver/Cargo.toml @@ -9,3 +9,12 @@ tokio.workspace = true tracing.workspace = true tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } grainpit = { path = "../grainpit" } +tower-http = { version = "0.7.0", features = [ + "compression-br", + "compression-gzip", + "compression-deflate", + "compression-zstd", +], optional = true } + +[features] +compress = ["dep:tower-http"] diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs index 71db252..b2de413 100644 --- a/grainpit_webserver/src/main.rs +++ b/grainpit_webserver/src/main.rs @@ -10,6 +10,8 @@ use axum::{ use grainpit::markov::Markov; use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt}; +#[cfg(feature = "compress")] +use tower_http::compression::CompressionLayer; #[derive(Debug)] struct AppState { m: Markov, @@ -28,11 +30,19 @@ async fn main() { let shared_state = Arc::new(AppState { m: Markov::new() }); + #[cfg(not(feature = "compress"))] let app = Router::new() .route("/", get(handler)) .route("/{*wildcard}", get(wildcard_handler)) .with_state(shared_state); + #[cfg(feature = "compress")] + let app = Router::new() + .route("/", get(handler)) + .route("/{*wildcard}", get(wildcard_handler)) + .layer(CompressionLayer::new()) + .with_state(shared_state); + let listener = tokio::net::TcpListener::bind( std::env::var("GRAINPIT_ADDR").unwrap_or("127.0.0.1:5000".to_string()), ) From 121312cfdb25427636b36545fccf3d7766a12647 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:09:30 -0500 Subject: [PATCH 07/19] remove deflate/gzip --- Cargo.lock | 43 ----------------------------------- grainpit_webserver/Cargo.toml | 2 -- 2 files changed, 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58febd6..ae5bd3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "aho-corasick" version = "1.1.4" @@ -277,8 +271,6 @@ checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "brotli", "compression-core", - "flate2", - "memchr", "zstd", "zstd-safe", ] @@ -298,15 +290,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - [[package]] name = "criterion" version = "0.8.2" @@ -395,16 +378,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "flate2" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -674,16 +647,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", - "simd-adler32", -] - [[package]] name = "mio" version = "1.2.2" @@ -1036,12 +999,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simd-adler32" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" - [[package]] name = "slab" version = "0.4.12" diff --git a/grainpit_webserver/Cargo.toml b/grainpit_webserver/Cargo.toml index 55c5e0b..dd288c3 100644 --- a/grainpit_webserver/Cargo.toml +++ b/grainpit_webserver/Cargo.toml @@ -11,8 +11,6 @@ tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } grainpit = { path = "../grainpit" } tower-http = { version = "0.7.0", features = [ "compression-br", - "compression-gzip", - "compression-deflate", "compression-zstd", ], optional = true } From e4d10e081604f3a2a3aa281740d5b1e4146303f7 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:12:15 -0500 Subject: [PATCH 08/19] improve compression quality --- Cargo.lock | 43 ++++++++++++++++++++++++++++++++++ grainpit_webserver/Cargo.toml | 2 ++ grainpit_webserver/src/main.rs | 2 +- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index ae5bd3f..58febd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -271,6 +277,8 @@ checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "brotli", "compression-core", + "flate2", + "memchr", "zstd", "zstd-safe", ] @@ -290,6 +298,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.8.2" @@ -378,6 +395,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -647,6 +674,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -999,6 +1036,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "slab" version = "0.4.12" diff --git a/grainpit_webserver/Cargo.toml b/grainpit_webserver/Cargo.toml index dd288c3..55c5e0b 100644 --- a/grainpit_webserver/Cargo.toml +++ b/grainpit_webserver/Cargo.toml @@ -11,6 +11,8 @@ tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } grainpit = { path = "../grainpit" } tower-http = { version = "0.7.0", features = [ "compression-br", + "compression-gzip", + "compression-deflate", "compression-zstd", ], optional = true } diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs index b2de413..f176dba 100644 --- a/grainpit_webserver/src/main.rs +++ b/grainpit_webserver/src/main.rs @@ -40,7 +40,7 @@ async fn main() { let app = Router::new() .route("/", get(handler)) .route("/{*wildcard}", get(wildcard_handler)) - .layer(CompressionLayer::new()) + .layer(CompressionLayer::new().quality(tower_http::CompressionLevel::Best)) .with_state(shared_state); let listener = tokio::net::TcpListener::bind( From f334801c4aad9d72b6fd20c4f1354266cb7bafea Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:14:38 -0500 Subject: [PATCH 09/19] unsurprisingly that was a bad idea --- grainpit_webserver/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs index f176dba..2708754 100644 --- a/grainpit_webserver/src/main.rs +++ b/grainpit_webserver/src/main.rs @@ -40,7 +40,7 @@ async fn main() { let app = Router::new() .route("/", get(handler)) .route("/{*wildcard}", get(wildcard_handler)) - .layer(CompressionLayer::new().quality(tower_http::CompressionLevel::Best)) + .layer(CompressionLayer::new().quality(tower_http::CompressionLevel::Precise(9))) .with_state(shared_state); let listener = tokio::net::TcpListener::bind( From 5e54e82e7410b679da600b25475febb1303b9afd Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:22:20 -0500 Subject: [PATCH 10/19] make random url generation sometimes return txt links --- grainpit/src/markov.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/grainpit/src/markov.rs b/grainpit/src/markov.rs index c240bc4..138c114 100644 --- a/grainpit/src/markov.rs +++ b/grainpit/src/markov.rs @@ -1,4 +1,4 @@ -use rand::seq::IteratorRandom; +use rand::{random_bool, seq::IteratorRandom}; use crate::markov::chain::Chain; use regex::regex; @@ -74,11 +74,16 @@ impl Markov { }; format!( - "{}{}/{}/{}.html", + "{}{}/{}/{}{}", start_path, self.url_chain.generate(4), self.url_chain.generate(4), - self.url_chain.generate(24) + self.url_chain.generate(24), + if rand::random_bool(0.95) { + ".html" + } else { + ".txt" + } ) } } From 6dd9d4cc412ff204734851709ee8bfd3aa7c3266 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:25:06 -0500 Subject: [PATCH 11/19] ok this might be more effective --- grainpit/src/markov.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/grainpit/src/markov.rs b/grainpit/src/markov.rs index 138c114..72f08bd 100644 --- a/grainpit/src/markov.rs +++ b/grainpit/src/markov.rs @@ -79,11 +79,7 @@ impl Markov { self.url_chain.generate(4), self.url_chain.generate(4), self.url_chain.generate(24), - if rand::random_bool(0.95) { - ".html" - } else { - ".txt" - } + if rand::random_bool(0.95) { ".html" } else { "" } ) } } From aea0138458d3a2cbeb77e01ea352bb23d4ed5c34 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:31:49 -0500 Subject: [PATCH 12/19] fake image generation --- Cargo.lock | 69 +++++++++++++++++++ benchmarks/Cargo.toml | 6 +- benchmarks/benches/images.rs | 10 +++ benchmarks/benches/{generate.rs => markov.rs} | 0 grainpit/Cargo.toml | 3 + grainpit/src/image.rs | 31 +++++++++ grainpit/src/lib.rs | 1 + grainpit/src/markov.rs | 30 +++++--- grainpit_webserver/src/main.rs | 23 ++++--- 9 files changed, 154 insertions(+), 19 deletions(-) create mode 100644 benchmarks/benches/images.rs rename benchmarks/benches/{generate.rs => markov.rs} (100%) create mode 100644 grainpit/src/image.rs diff --git a/Cargo.lock b/Cargo.lock index 58febd6..2f85a95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,6 +176,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + [[package]] name = "bytes" version = "1.12.1" @@ -475,6 +481,9 @@ dependencies = [ "regex", "rustc-hash", "tracing", + "zune-core", + "zune-image", + "zune-imageprocs", ] [[package]] @@ -609,6 +618,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-encoder" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0370574b86f7eca156b9f298392b5e69a23f8c86f3f865add60bbc2e79467a6" + [[package]] name = "js-sys" version = "0.3.103" @@ -695,6 +710,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -813,6 +838,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "quote" version = "1.0.47" @@ -1444,3 +1475,41 @@ dependencies = [ "cc", "pkg-config", ] + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-image" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d67f66717c2ca4f5fe18894e4724c7d22076dea8f245ea5386a05a4b6b8f4a53" +dependencies = [ + "bytemuck", + "jpeg-encoder", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "zune-imageprocs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915f2f441a4fe4a113839e6e0832c91516f09137f6421b3ce834d3db3ff1b067" +dependencies = [ + "moxcms", + "zune-core", + "zune-image", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index a56e668..21f0d96 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -11,5 +11,9 @@ grainpit = { path = "../grainpit" } criterion = "0.8.2" [[bench]] -name = "generate" +name = "markov" +harness = false + +[[bench]] +name = "images" harness = false diff --git a/benchmarks/benches/images.rs b/benchmarks/benches/images.rs new file mode 100644 index 0000000..5854b84 --- /dev/null +++ b/benchmarks/benches/images.rs @@ -0,0 +1,10 @@ +use grainpit::image::gen_image; + +use criterion::{Criterion, criterion_group, criterion_main}; + +fn bench(c: &mut Criterion) { + c.bench_function("gen_image", |b| b.iter(|| gen_image())); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/benchmarks/benches/generate.rs b/benchmarks/benches/markov.rs similarity index 100% rename from benchmarks/benches/generate.rs rename to benchmarks/benches/markov.rs diff --git a/grainpit/Cargo.toml b/grainpit/Cargo.toml index cb029d5..c469bca 100644 --- a/grainpit/Cargo.toml +++ b/grainpit/Cargo.toml @@ -10,3 +10,6 @@ tracing.workspace = true rustc-hash = "2.1.3" arrayvec = "0.7.8" rand_xoshiro = "0.8.1" +zune-image = { version = "0.5", default-features = false, features = ["jpeg"] } +zune-core = "0.5.1" +zune-imageprocs = "0.5.1" diff --git a/grainpit/src/image.rs b/grainpit/src/image.rs new file mode 100644 index 0000000..e103e9c --- /dev/null +++ b/grainpit/src/image.rs @@ -0,0 +1,31 @@ +use rand::RngExt; +use zune_core::options::EncoderOptions; +use zune_image::{codecs::jpeg::JpegEncoder, traits::OperationsTrait}; +use zune_imageprocs::{invert::Invert, sobel::Sobel}; + +pub fn gen_image() -> Vec { + let img_x = 88; + let img_y = 31; + let encoder = JpegEncoder::new_with_options(EncoderOptions::default().set_quality(5)); + + let mut pixels = [0; (88 * 31) * 3]; + rand::rng().fill(&mut pixels); + + let mut image = zune_image::image::Image::from_u8( + &pixels, + img_x, + img_y, + zune_core::colorspace::ColorSpace::RGB, + ); + + let sobel = Sobel::new(); + sobel.execute(&mut image).unwrap(); + if rand::random_bool(0.5) { + let invert = Invert::new(); + invert.execute(&mut image).unwrap(); + } + + let mut output = vec![]; + image.write_with_encoder(encoder, &mut output).unwrap(); + output +} diff --git a/grainpit/src/lib.rs b/grainpit/src/lib.rs index 706c4a7..6a6452f 100644 --- a/grainpit/src/lib.rs +++ b/grainpit/src/lib.rs @@ -1 +1,2 @@ +pub mod image; pub mod markov; diff --git a/grainpit/src/markov.rs b/grainpit/src/markov.rs index 72f08bd..fd8aa95 100644 --- a/grainpit/src/markov.rs +++ b/grainpit/src/markov.rs @@ -1,4 +1,4 @@ -use rand::{random_bool, seq::IteratorRandom}; +use rand::seq::IteratorRandom; use crate::markov::chain::Chain; use regex::regex; @@ -27,7 +27,7 @@ impl Markov { let href_regex = regex!(r#"href="(.*?)""#); let mut generated = self.html_chain.generate(2048); for i in href_regex.find_iter(&generated.clone()) { - generated = generated.replace(i.as_str(), &self.random_link()); + generated = generated.replace(i.as_str(), &self.random_link(false)); } let text_content_regex = regex!(r">([^<]+)<"); @@ -40,17 +40,25 @@ impl Markov { .collect(); for r in url_replacements { - generated = generated.replacen( - &r, - &format!("{}", self.random_link(), r), - 1, - ); + if rand::random_bool(0.9) { + generated = generated.replacen( + &r, + &format!("{}", self.random_link(false), r), + 1, + ); + } else { + generated = generated.replacen( + &r, + &format!("{}", self.random_link(true), r), + 1, + ); + } } format!("

{}

", generated) } - pub fn random_link(&self) -> String { + pub fn random_link(&self, image: bool) -> String { let start_path = if std::env::var("GRAINPIT_EXTRAURLS").is_ok() { if rand::random_bool( std::env::var("GRAINPIT_EXTRAURLS_CHANCE") @@ -79,7 +87,11 @@ impl Markov { self.url_chain.generate(4), self.url_chain.generate(4), self.url_chain.generate(24), - if rand::random_bool(0.95) { ".html" } else { "" } + if rand::random_bool(0.95) { + if image { ".jpg" } else { ".html" } + } else { + if image { ".png" } else { "" } + } ) } } diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs index 2708754..13b52cb 100644 --- a/grainpit_webserver/src/main.rs +++ b/grainpit_webserver/src/main.rs @@ -57,18 +57,23 @@ async fn wildcard_handler( Path(path): Path, headers: HeaderMap, ) -> Response { - if let Some(header) = headers.get("Accept-Encoding") - && header.to_str().unwrap_or("").contains("br") - { - let mut headers = HeaderMap::new(); - headers.insert(header::CONTENT_TYPE, "text/plain".parse().unwrap()); - headers.insert(header::CONTENT_ENCODING, "br".parse().unwrap()); - return (headers, include_bytes!("./g.txt.br")).into_response(); - } - if path.contains(".html") { handler(State(state)).await.into_response() + } else if path.contains(".jpg") || path.contains(".png") { + let output = grainpit::image::gen_image(); + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_TYPE, "image/jpeg".parse().unwrap()); + (headers, output).into_response() } else { + if let Some(header) = headers.get("Accept-Encoding") + && header.to_str().unwrap_or("").contains("br") + { + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_TYPE, "text/plain".parse().unwrap()); + headers.insert(header::CONTENT_ENCODING, "br".parse().unwrap()); + return (headers, include_bytes!("./g.txt.br")).into_response(); + } + let mut a = state.m.config_chain.generate(512); a = a.trim_start().to_owned(); a.into_response() From fef8731b9c3ed9de5368cdffe84dc3e7ac739de3 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:37:13 -0500 Subject: [PATCH 13/19] send the 5gb of compressed data on images too --- grainpit_webserver/src/main.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs index 13b52cb..f22f2bc 100644 --- a/grainpit_webserver/src/main.rs +++ b/grainpit_webserver/src/main.rs @@ -60,6 +60,15 @@ async fn wildcard_handler( if path.contains(".html") { handler(State(state)).await.into_response() } else if path.contains(".jpg") || path.contains(".png") { + if let Some(header) = headers.get("Accept-Encoding") + && header.to_str().unwrap_or("").contains("br") + { + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_TYPE, "image/jpeg".parse().unwrap()); + headers.insert(header::CONTENT_ENCODING, "br".parse().unwrap()); + return (headers, include_bytes!("./g.txt.br")).into_response(); + } + let output = grainpit::image::gen_image(); let mut headers = HeaderMap::new(); headers.insert(header::CONTENT_TYPE, "image/jpeg".parse().unwrap()); From 5d852879509c556fa3a3f6cc486107ed6901c4ea Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:39:44 -0500 Subject: [PATCH 14/19] disable multithreading for fake image generation (improved performance) --- grainpit/src/image.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/grainpit/src/image.rs b/grainpit/src/image.rs index e103e9c..ba3dc85 100644 --- a/grainpit/src/image.rs +++ b/grainpit/src/image.rs @@ -6,7 +6,8 @@ use zune_imageprocs::{invert::Invert, sobel::Sobel}; pub fn gen_image() -> Vec { let img_x = 88; let img_y = 31; - let encoder = JpegEncoder::new_with_options(EncoderOptions::default().set_quality(5)); + let encoder = + JpegEncoder::new_with_options(EncoderOptions::default().set_quality(5).set_num_threads(0)); let mut pixels = [0; (88 * 31) * 3]; rand::rng().fill(&mut pixels); From bed7f9a0e4b7ebbb1b344ac2e7c803886e5d421b Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Wed, 29 Jul 2026 01:12:41 -0500 Subject: [PATCH 15/19] its now 25gb instead of 5g --- grainpit_webserver/src/g.txt.br | Bin 4201 -> 42076 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/grainpit_webserver/src/g.txt.br b/grainpit_webserver/src/g.txt.br index 3a6d4c9898796bfca7643c278d2c2bb8ff349f27..62d39437cda02eb9b34ac9c7c1b097521a2dd54e 100644 GIT binary patch literal 42076 zcmeI*F;7!b7zW_e3bBR+(80;1UD!xi+34;cU^7mvkhqyhjRS6uuKWaCO^7j=IG7mz zfX3kBKolmL2z+oa;r19h*r7Kid49nq?OV=s-*fJFzeKF`7rq|6>vm4g-$ZO4Jx-sV z-tYZBYe&S7x9Rm>GkyEJavt$9OK(BAOT6_yqT zVW{p&g+UmE!3tZpRm@OWdLqNnN-c`qSmolfV4#qhPFI>V$Dn%GvzcVsEAe<`O(iX5DG)kW$etHaO2 zAPmCbmclJX^`M6^3=acEQY1y%s7k|GDA zB{UAh8wa?9JGi?=xU204OOX^wkrYXh6lqMv)wDX30Zaxk8DNx$!6}j=DUu>7k|HUR zA_wE5O`a=fwYsJFk6Q|gq)3XSNQ$ILilj)2oSiX(VAK%=6iJa3Ns$yukrYXh6iJbn z?=c*4yg5ZuBt=prMN%Y1QY1xEBt>4n=jwCWRy!&ep2j=+XDbtCRla= From 7b8b8c11751fa157084087ad29e9667c8e252a13 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:49:44 -0500 Subject: [PATCH 16/19] disable the 25gb compression thing for image requests --- grainpit_webserver/src/main.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs index f22f2bc..13b52cb 100644 --- a/grainpit_webserver/src/main.rs +++ b/grainpit_webserver/src/main.rs @@ -60,15 +60,6 @@ async fn wildcard_handler( if path.contains(".html") { handler(State(state)).await.into_response() } else if path.contains(".jpg") || path.contains(".png") { - if let Some(header) = headers.get("Accept-Encoding") - && header.to_str().unwrap_or("").contains("br") - { - let mut headers = HeaderMap::new(); - headers.insert(header::CONTENT_TYPE, "image/jpeg".parse().unwrap()); - headers.insert(header::CONTENT_ENCODING, "br".parse().unwrap()); - return (headers, include_bytes!("./g.txt.br")).into_response(); - } - let output = grainpit::image::gen_image(); let mut headers = HeaderMap::new(); headers.insert(header::CONTENT_TYPE, "image/jpeg".parse().unwrap()); From a2e407608b5a679eac40734e83303535cd10c2dd Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:11:53 -0500 Subject: [PATCH 17/19] git hub action --- .github/workflows/rust.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..7396e36 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,24 @@ +name: Rust + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: build with release mode + run: cargo build --release + - uses: actions/upload-artifact@v7 + with: + path: target/release/grainpit_webserver + archive: false From 4f3a9d13d4ab30b84e2597a689807fc17ed7ddca Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:13:16 -0500 Subject: [PATCH 18/19] i forgot to add thsi --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7396e36..4d9fe6d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: build with release mode - run: cargo build --release + run: RUSTFLAGS="-Ctarget-cpu=x86-64" cargo build --release - uses: actions/upload-artifact@v7 with: path: target/release/grainpit_webserver From c85faf1e7a3b07dc013d52d97cedfaefe77bf1c9 Mon Sep 17 00:00:00 2001 From: electron271 <66094410+electron271@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:45:53 -0500 Subject: [PATCH 19/19] update readme and fix warning --- README.md | 2 +- benchmarks/benches/images.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c85d374..a95fe2b 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ see ## features -- fast and lightweight, on my low resource proxy vps (2 cores, e5-2680) it manages to hover around 2-3ms per request with 21mb ram usage while handling ~30 requests per second continuously to the tarpit without consuming enough cpu time where regular requests slow down substantially +- fast and lightweight, on my low resource proxy vps (2 cores, e5-2680) it manages to hover around ~800µs per request with 13mb ram usage while handling ~30 requests per second continuously to the tarpit without consuming enough cpu time where regular requests slow down substantially - batshit insane default training data that (somehow) manages to put out html/css - able to generate fake config files as well for the malicious credential scanning bots - effective, as of writing this GPTBot has been sending requests to the tarpit since july 13 as well as amazons scraper and claudes scraper more recently diff --git a/benchmarks/benches/images.rs b/benchmarks/benches/images.rs index 5854b84..904d694 100644 --- a/benchmarks/benches/images.rs +++ b/benchmarks/benches/images.rs @@ -3,7 +3,7 @@ use grainpit::image::gen_image; use criterion::{Criterion, criterion_group, criterion_main}; fn bench(c: &mut Criterion) { - c.bench_function("gen_image", |b| b.iter(|| gen_image())); + c.bench_function("gen_image", |b| b.iter(gen_image)); } criterion_group!(benches, bench);