diff --git a/.cargo/config.toml b/.cargo/config.toml
new file mode 100644
index 0000000..ddff440
--- /dev/null
+++ b/.cargo/config.toml
@@ -0,0 +1,2 @@
+[build]
+rustflags = ["-C", "target-cpu=native"]
diff --git a/.containerignore b/.containerignore
index d301b0a..9aeb401 100644
--- a/.containerignore
+++ b/.containerignore
@@ -1,2 +1,7 @@
-!src/
-!Cargo.*
+README.md
+.gitignore
+compose.yml
+Containerfile
+example.png
+LICENSE
+target/
\ No newline at end of file
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
new file mode 100644
index 0000000..4d9fe6d
--- /dev/null
+++ b/.github/workflows/rust.yml
@@ -0,0 +1,24 @@
+name: Rust
+
+on:
+ push:
+ branches: ["main"]
+ pull_request:
+ branches: ["main"]
+
+env:
+ CARGO_TERM_COLOR: always
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions-rust-lang/setup-rust-toolchain@v1
+ - name: build with release mode
+ run: RUSTFLAGS="-Ctarget-cpu=x86-64" cargo build --release
+ - uses: actions/upload-artifact@v7
+ with:
+ path: target/release/grainpit_webserver
+ archive: false
diff --git a/.gitignore b/.gitignore
index a685039..a90c756 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,6 @@
/target
-perf.data
\ No newline at end of file
+
+# profiling bs
+perf.data*
+flamegraph.svg
+callgrind.out*
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 92be5ed..2f85a95 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,12 @@
# It is not intended for manual editing.
version = 4
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
[[package]]
name = "aho-corasick"
version = "1.1.4"
@@ -11,12 +17,72 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "alloc-no-stdlib"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
+
+[[package]]
+name = "alloc-stdlib"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195"
+dependencies = [
+ "alloc-no-stdlib",
+]
+
+[[package]]
+name = "alloca"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
+[[package]]
+name = "anstyle"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
+
+[[package]]
+name = "arrayvec"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
+
+[[package]]
+name = "async-compression"
+version = "0.4.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac"
+dependencies = [
+ "compression-codecs",
+ "compression-core",
+ "pin-project-lite",
+ "tokio",
+]
+
[[package]]
name = "atomic-waker"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
[[package]]
name = "axum"
version = "0.8.9"
@@ -69,11 +135,52 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "benchmarks"
+version = "0.1.0"
+dependencies = [
+ "criterion",
+ "grainpit",
+]
+
[[package]]
name = "bitflags"
-version = "2.13.0"
+version = "2.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
+
+[[package]]
+name = "brotli"
+version = "8.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "5.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+
+[[package]]
+name = "bytemuck"
+version = "1.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
+checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
[[package]]
name = "bytes"
@@ -81,6 +188,24 @@ version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
+[[package]]
+name = "cc"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9"
+dependencies = [
+ "find-msvc-tools",
+ "jobserver",
+ "libc",
+ "shlex",
+]
+
[[package]]
name = "cfg-if"
version = "1.0.4"
@@ -98,6 +223,78 @@ dependencies = [
"rand_core",
]
+[[package]]
+name = "ciborium"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
+[[package]]
+name = "clap"
+version = "4.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
+dependencies = [
+ "clap_builder",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
+dependencies = [
+ "anstyle",
+ "clap_lex",
+]
+
+[[package]]
+name = "clap_lex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
+
+[[package]]
+name = "compression-codecs"
+version = "0.4.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf"
+dependencies = [
+ "brotli",
+ "compression-core",
+ "flate2",
+ "memchr",
+ "zstd",
+ "zstd-safe",
+]
+
+[[package]]
+name = "compression-core"
+version = "0.4.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789"
+
[[package]]
name = "cpufeatures"
version = "0.3.0"
@@ -107,6 +304,87 @@ dependencies = [
"libc",
]
+[[package]]
+name = "crc32fast"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "criterion"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3"
+dependencies = [
+ "alloca",
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "itertools",
+ "num-traits",
+ "oorandom",
+ "page_size",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
+
+[[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
+[[package]]
+name = "either"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
+
[[package]]
name = "errno"
version = "0.3.14"
@@ -117,6 +395,22 @@ dependencies = [
"windows-sys",
]
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+
+[[package]]
+name = "flate2"
+version = "1.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@@ -128,30 +422,36 @@ dependencies = [
[[package]]
name = "futures-channel"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
+checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
dependencies = [
"futures-core",
]
[[package]]
name = "futures-core"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
+
+[[package]]
+name = "futures-sink"
+version = "0.3.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
[[package]]
name = "futures-task"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
+checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
[[package]]
name = "futures-util"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
+checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
dependencies = [
"futures-core",
"futures-task",
@@ -173,17 +473,46 @@ dependencies = [
[[package]]
name = "grainpit"
-version = "0.2.0"
+version = "0.1.0"
dependencies = [
- "axum",
+ "arrayvec",
"rand",
+ "rand_xoshiro",
"regex",
"rustc-hash",
+ "tracing",
+ "zune-core",
+ "zune-image",
+ "zune-imageprocs",
+]
+
+[[package]]
+name = "grainpit_proxy"
+version = "0.1.0"
+
+[[package]]
+name = "grainpit_webserver"
+version = "0.1.0"
+dependencies = [
+ "axum",
+ "grainpit",
"tokio",
+ "tower-http",
"tracing",
"tracing-subscriber",
]
+[[package]]
+name = "half"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+ "zerocopy",
+]
+
[[package]]
name = "http"
version = "1.4.2"
@@ -196,9 +525,9 @@ dependencies = [
[[package]]
name = "http-body"
-version = "1.0.1"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
dependencies = [
"bytes",
"http",
@@ -206,9 +535,9 @@ dependencies = [
[[package]]
name = "http-body-util"
-version = "0.1.3"
+version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
+checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
dependencies = [
"bytes",
"futures-core",
@@ -231,9 +560,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "hyper"
-version = "1.10.1"
+version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
+checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72"
dependencies = [
"atomic-waker",
"bytes",
@@ -264,12 +593,48 @@ dependencies = [
"tower-service",
]
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+[[package]]
+name = "jobserver"
+version = "0.1.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
+dependencies = [
+ "getrandom",
+ "libc",
+]
+
+[[package]]
+name = "jpeg-encoder"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0370574b86f7eca156b9f298392b5e69a23f8c86f3f865add60bbc2e79467a6"
+
+[[package]]
+name = "js-sys"
+version = "0.3.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "wasm-bindgen",
+]
+
[[package]]
name = "lazy_static"
version = "1.5.0"
@@ -278,9 +643,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libc"
-version = "0.2.186"
+version = "0.2.189"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
[[package]]
name = "lock_api"
@@ -297,6 +662,15 @@ version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+[[package]]
+name = "matchers"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
+dependencies = [
+ "regex-automata",
+]
+
[[package]]
name = "matchit"
version = "0.8.4"
@@ -315,17 +689,37 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+[[package]]
+name = "miniz_oxide"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
[[package]]
name = "mio"
-version = "1.2.1"
+version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
+checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
dependencies = [
"libc",
"wasi",
"windows-sys",
]
+[[package]]
+name = "moxcms"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
+dependencies = [
+ "num-traits",
+ "pxfm",
+]
+
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
@@ -335,12 +729,37 @@ dependencies = [
"windows-sys",
]
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
[[package]]
name = "once_cell"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+[[package]]
+name = "oorandom"
+version = "11.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
+
+[[package]]
+name = "page_size"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
[[package]]
name = "parking_lot"
version = "0.12.5"
@@ -376,20 +795,60 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+[[package]]
+name = "pkg-config"
+version = "0.3.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
+
+[[package]]
+name = "plotters"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
+dependencies = [
+ "plotters-backend",
+]
+
[[package]]
name = "proc-macro2"
-version = "1.0.106"
+version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
dependencies = [
"unicode-ident",
]
+[[package]]
+name = "pxfm"
+version = "0.1.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
+
[[package]]
name = "quote"
-version = "1.0.46"
+version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [
"proc-macro2",
]
@@ -417,6 +876,35 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
+[[package]]
+name = "rand_xoshiro"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "662effc7698e08ea324d3acccf8d9d7f7bf79b9785e270a174ea36e56900c91d"
+dependencies = [
+ "rand_core",
+]
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -428,9 +916,9 @@ dependencies = [
[[package]]
name = "regex"
-version = "1.13.0"
+version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
dependencies = [
"aho-corasick",
"memchr",
@@ -440,9 +928,9 @@ dependencies = [
[[package]]
name = "regex-automata"
-version = "0.4.15"
+version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db"
+checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
dependencies = [
"aho-corasick",
"memchr",
@@ -461,12 +949,27 @@ version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
+[[package]]
+name = "rustversion"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
+
[[package]]
name = "ryu"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
[[package]]
name = "scopeguard"
version = "1.2.0"
@@ -475,38 +978,39 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "serde"
-version = "1.0.228"
+version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
dependencies = [
"serde_core",
+ "serde_derive",
]
[[package]]
name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 3.0.3",
]
[[package]]
name = "serde_json"
-version = "1.0.150"
+version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
"itoa",
"memchr",
@@ -547,6 +1051,12 @@ dependencies = [
"lazy_static",
]
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
[[package]]
name = "signal-hook-registry"
version = "1.4.8"
@@ -557,6 +1067,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "simd-adler32"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
+
[[package]]
name = "slab"
version = "0.4.12"
@@ -571,9 +1087,9 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
[[package]]
name = "socket2"
-version = "0.6.4"
+version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [
"libc",
"windows-sys",
@@ -581,9 +1097,20 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.118"
+version = "2.0.119"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
dependencies = [
"proc-macro2",
"quote",
@@ -598,18 +1125,28 @@ checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
[[package]]
name = "thread_local"
-version = "1.1.9"
+version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
+checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070"
dependencies = [
"cfg-if",
]
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "tokio"
-version = "1.52.3"
+version = "1.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
dependencies = [
"bytes",
"libc",
@@ -624,13 +1161,26 @@ dependencies = [
[[package]]
name = "tokio-macros"
-version = "2.7.0"
+version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
+checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "tokio-util"
+version = "0.7.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
]
[[package]]
@@ -649,6 +1199,26 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "tower-http"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233"
+dependencies = [
+ "async-compression",
+ "bitflags",
+ "bytes",
+ "futures-core",
+ "http",
+ "http-body",
+ "percent-encoding",
+ "pin-project-lite",
+ "tokio",
+ "tokio-util",
+ "tower-layer",
+ "tower-service",
+]
+
[[package]]
name = "tower-layer"
version = "0.3.3"
@@ -681,7 +1251,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -711,10 +1281,14 @@ version = "0.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
dependencies = [
+ "matchers",
"nu-ansi-term",
+ "once_cell",
+ "regex-automata",
"sharded-slab",
"smallvec",
"thread_local",
+ "tracing",
"tracing-core",
"tracing-log",
]
@@ -731,12 +1305,108 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "web-sys"
+version = "0.3.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
[[package]]
name = "windows-link"
version = "0.2.1"
@@ -752,8 +1422,94 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "zerocopy"
+version = "0.8.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.55"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
[[package]]
name = "zmij"
-version = "1.0.21"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
+
+[[package]]
+name = "zstd"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.16+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
+[[package]]
+name = "zune-core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"
+
+[[package]]
+name = "zune-image"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
+checksum = "d67f66717c2ca4f5fe18894e4724c7d22076dea8f245ea5386a05a4b6b8f4a53"
+dependencies = [
+ "bytemuck",
+ "jpeg-encoder",
+ "zune-core",
+ "zune-jpeg",
+]
+
+[[package]]
+name = "zune-imageprocs"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "915f2f441a4fe4a113839e6e0832c91516f09137f6421b3ce834d3db3ff1b067"
+dependencies = [
+ "moxcms",
+ "zune-core",
+ "zune-image",
+]
+
+[[package]]
+name = "zune-jpeg"
+version = "0.5.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
+dependencies = [
+ "zune-core",
+]
diff --git a/Cargo.toml b/Cargo.toml
index ebf8d89..1a37581 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,16 +1,41 @@
-[package]
-name = "grainpit"
+[workspace]
+resolver = "3"
+members = ["benchmarks", "grainpit", "grainpit_proxy", "grainpit_webserver"]
+
+[workspace.package]
version = "0.2.0"
edition = "2024"
description = "tarpit to trap malicious scrapers"
license = "CC0-1.0"
repository = "https://github.com/GrainWare/grainpit"
-[dependencies]
-axum = "0.8.9"
-rand = "0.10.2"
-regex = "1.13.0"
-rustc-hash = "2.1.3"
-tokio = { version = "1.52.3", features = ["full"] }
+[workspace.dependencies]
tracing = "0.1.44"
-tracing-subscriber = "0.3.23"
+tokio = { version = "1.53.0", features = ["full"] }
+
+[profile.dev]
+opt-level = 3
+debug = true
+strip = "none"
+debug-assertions = true
+overflow-checks = true
+lto = "thin"
+panic = "unwind"
+incremental = true
+codegen-units = 16
+rpath = false
+
+[profile.release]
+opt-level = 3
+debug = false
+strip = "none"
+debug-assertions = false
+overflow-checks = false
+lto = "thin" # increased performance by ~20% in benchmarks
+panic = "unwind"
+incremental = false
+codegen-units = 16
+rpath = false
+
+[profile.bench]
+inherits = "release"
diff --git a/Containerfile b/Containerfile
index d7fb47f..c8c661d 100644
--- a/Containerfile
+++ b/Containerfile
@@ -1,23 +1,14 @@
FROM rust AS build
-# blazing fast and optimized!!!!!!!!!1111!!!
-RUN mkdir -p .cargo && printf '\
- [profile.release]\n\
- lto = true\n\
- codegen-units = 1\n\
- opt-level = '3'\n\
- panic = "abort"\n\
- strip = true\n' > .cargo/config.toml
-
WORKDIR /app
COPY . .
-RUN cargo install --root /tmp --path .
+RUN cargo install --root /tmp --path grainpit_webserver/
FROM gcr.io/distroless/cc-debian13:nonroot AS runtime
-COPY --from=build --chown=nonroot:nonroot /tmp/bin/grainpit .
+COPY --from=build --chown=nonroot:nonroot /tmp/bin/grainpit_webserver .
USER nonroot
ENV GRAINPIT_ADDR="0.0.0.0:5000"
EXPOSE 5000
-ENTRYPOINT ["./grainpit"]
+ENTRYPOINT ["./grainpit_webserver"]
diff --git a/README.md b/README.md
index 1898bbf..a95fe2b 100644
--- a/README.md
+++ b/README.md
@@ -4,26 +4,44 @@
markov tarpit but grain
-keep in mind that this works different than other markov tarpits where a link to the tarpit is injected into the html, for this you need to setup reverse proxy rules to get the user here
-
-some example rules for haproxy are included here, if you want to contribute more examples it would be appreciated
+> [!IMPORTANT]
+> grainpit is under heavy development currently! please keep in mind that right now if you want stability you should use the `webserver` version!
-> [!TIP]
-> if you want to help test the latest features build from https://github.com/GrainWare/grainpit/pull/2
+## backends
+
+grainpit itself is just a markov library and some extra utilities, there are 3 different backends you can use to actually add functionality
+
+### `webserver`
+
+this only exposes a webserver which you have to setup rules on your reverse proxy to actually get to, currently this is the most stable
+
+you can enable content compression for this proxy with the `compress` feature incase network load is a concern for you
+
+### `proxy` (in development)
+
+this is under heavy development and is more complex, but proxies the traffic through grainpit which allows injecting hidden links to the tarpit and more configuration on grainpits side
+
+this is planned to be the preferred way to use grainpit once its completed
+
+currently this is empty in the repo because the general project structure for it is still being figured out
+
+### cgi script (unofficial)
+
+see
## features
-- fast and lightweight, on my low resource proxy vps (2 cores, e5-2680) it manages to hover around 2-3ms per request with 21mb ram usage while handling ~15 requests per second continuously to the tarpit without consuming enough cpu time where regular requests slow down substantially
+- fast and lightweight, on my low resource proxy vps (2 cores, e5-2680) it manages to hover around ~800µs per request with 13mb ram usage while handling ~30 requests per second continuously to the tarpit without consuming enough cpu time where regular requests slow down substantially
- batshit insane default training data that (somehow) manages to put out html/css
- able to generate fake config files as well for the malicious credential scanning bots
-- effective, as of writing this GPTBot has been sending requests to the tarpit for over 15 hours straight
+- effective, as of writing this GPTBot has been sending requests to the tarpit since july 13 as well as amazons scraper and claudes scraper more recently
## reasons you might want to use a different tarpit
- not as many features as other alternatives have
-- requires reverse proxy config to send bots here
+- requires reverse proxy config to send bots here (a proxy is currently in progress so you dont need this)
## configuration
@@ -40,7 +58,7 @@ in order to add these there are a few variables you can configure
### systemd/bare metal
-install this using `cargo install grainpit`
+install this using `cargo install grainpit_webserver`
example systemd service:
@@ -50,7 +68,7 @@ Description=Tarpit
After=network.target
[Service]
-ExecStart=/root/.cargo/bin/grainpit
+ExecStart=/root/.cargo/bin/grainpit_webserver
Restart=always
[Install]
@@ -103,3 +121,28 @@ backend evil_bot_punishment_zone
```
i also recommend combining this with to provide better matching
+
+## development
+
+> [!WARNING]
+> this section is a work in progress
+
+### getting started
+
+make sure you have rust installed with rustup already and this repo cloned
+
+once you have those installed you can develop grainpit by running `cargo run --bin ` (e.g. grainpit_webserver for the webserver version)
+
+> [!TIP]
+> for development the default `dev` profile has been configured to be almost as optimized as the release profile (and more optimized than the default rust release profile)
+
+### benchmarks
+
+if you are contributing changes that may affect performance its appreciated (but not required) if you can include a benchmark comparison of before and after by doing this:
+
+```bash
+git checkout main
+cargo bench -p benchmarks -- --save-baseline main
+git checkout
+cargo bench -p benchmarks -- --baseline main
+```
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
new file mode 100644
index 0000000..21f0d96
--- /dev/null
+++ b/benchmarks/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "benchmarks"
+version = "0.1.0"
+publish = false
+edition.workspace = true
+
+[dependencies]
+grainpit = { path = "../grainpit" }
+
+[dev-dependencies]
+criterion = "0.8.2"
+
+[[bench]]
+name = "markov"
+harness = false
+
+[[bench]]
+name = "images"
+harness = false
diff --git a/benchmarks/benches/images.rs b/benchmarks/benches/images.rs
new file mode 100644
index 0000000..904d694
--- /dev/null
+++ b/benchmarks/benches/images.rs
@@ -0,0 +1,10 @@
+use grainpit::image::gen_image;
+
+use criterion::{Criterion, criterion_group, criterion_main};
+
+fn bench(c: &mut Criterion) {
+ c.bench_function("gen_image", |b| b.iter(gen_image));
+}
+
+criterion_group!(benches, bench);
+criterion_main!(benches);
diff --git a/benchmarks/benches/markov.rs b/benchmarks/benches/markov.rs
new file mode 100644
index 0000000..228a4bc
--- /dev/null
+++ b/benchmarks/benches/markov.rs
@@ -0,0 +1,29 @@
+use grainpit::markov::Markov;
+
+use criterion::{Criterion, Throughput, criterion_group, criterion_main};
+
+fn bench(c: &mut Criterion) {
+ let markov = Markov::new();
+
+ c.bench_function("gen_html", |b| b.iter(|| markov.gen_html()));
+
+ let mut group = c.benchmark_group("generate");
+ let length = 2048;
+ group.throughput(Throughput::Elements(length as u64));
+ group.bench_function("html-chain", |b| {
+ b.iter(|| markov.html_chain.generate(length))
+ });
+ group.bench_function("config-chain", |b| {
+ b.iter(|| markov.config_chain.generate(length))
+ });
+ group.bench_function("url-chain", |b| {
+ b.iter(|| markov.url_chain.generate(length))
+ });
+ group.bench_function("url-name-chain", |b| {
+ b.iter(|| markov.url_name_chain.generate(length))
+ });
+ group.finish();
+}
+
+criterion_group!(benches, bench);
+criterion_main!(benches);
diff --git a/grainpit/Cargo.toml b/grainpit/Cargo.toml
new file mode 100644
index 0000000..c469bca
--- /dev/null
+++ b/grainpit/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "grainpit"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+rand = "0.10.2"
+regex = "1.13.0"
+tracing.workspace = true
+rustc-hash = "2.1.3"
+arrayvec = "0.7.8"
+rand_xoshiro = "0.8.1"
+zune-image = { version = "0.5", default-features = false, features = ["jpeg"] }
+zune-core = "0.5.1"
+zune-imageprocs = "0.5.1"
diff --git a/src/markov3.txt b/grainpit/data/config.txt
similarity index 100%
rename from src/markov3.txt
rename to grainpit/data/config.txt
diff --git a/src/markov1.txt b/grainpit/data/html.txt
similarity index 99%
rename from src/markov1.txt
rename to grainpit/data/html.txt
index d7e80ec..b06f883 100644
--- a/src/markov1.txt
+++ b/grainpit/data/html.txt
@@ -348,7 +348,7 @@ you need to stop looking at the grainware wiki and get a job
">
- accurate linux graphs
+ griggy microsoft.com
@@ -615,7 +615,7 @@ you need to stop looking at the grainware wiki and get a job
- easter egg
+ easter egg via accuracy linux graphs
@@ -677,7 +677,7 @@ you need to stop looking at the grainware wiki and get a job
- Fizzbuzz Generator
+ Linux Graph Fizzbuzz Generator
@@ -789,7 +789,7 @@ you need to stop looking at the grainware wiki and get a job
- ✿✧･ﾟ accuwate winux gwaphs wite ✿✧･ﾟ
+ ✿✧･ﾟ accuwate winux gwaphs wite ✿✧･ﾟ Kawaii Linux Graphs Accuracy Linux Microsoft
diff --git a/src/markov4.txt b/grainpit/data/url.txt
similarity index 100%
rename from src/markov4.txt
rename to grainpit/data/url.txt
diff --git a/src/markov2.txt b/grainpit/data/url_name.txt
similarity index 100%
rename from src/markov2.txt
rename to grainpit/data/url_name.txt
diff --git a/grainpit/src/image.rs b/grainpit/src/image.rs
new file mode 100644
index 0000000..ba3dc85
--- /dev/null
+++ b/grainpit/src/image.rs
@@ -0,0 +1,32 @@
+use rand::RngExt;
+use zune_core::options::EncoderOptions;
+use zune_image::{codecs::jpeg::JpegEncoder, traits::OperationsTrait};
+use zune_imageprocs::{invert::Invert, sobel::Sobel};
+
+pub fn gen_image() -> Vec {
+ let img_x = 88;
+ let img_y = 31;
+ let encoder =
+ JpegEncoder::new_with_options(EncoderOptions::default().set_quality(5).set_num_threads(0));
+
+ let mut pixels = [0; (88 * 31) * 3];
+ rand::rng().fill(&mut pixels);
+
+ let mut image = zune_image::image::Image::from_u8(
+ &pixels,
+ img_x,
+ img_y,
+ zune_core::colorspace::ColorSpace::RGB,
+ );
+
+ let sobel = Sobel::new();
+ sobel.execute(&mut image).unwrap();
+ if rand::random_bool(0.5) {
+ let invert = Invert::new();
+ invert.execute(&mut image).unwrap();
+ }
+
+ let mut output = vec![];
+ image.write_with_encoder(encoder, &mut output).unwrap();
+ output
+}
diff --git a/grainpit/src/lib.rs b/grainpit/src/lib.rs
new file mode 100644
index 0000000..6a6452f
--- /dev/null
+++ b/grainpit/src/lib.rs
@@ -0,0 +1,2 @@
+pub mod image;
+pub mod markov;
diff --git a/grainpit/src/markov.rs b/grainpit/src/markov.rs
new file mode 100644
index 0000000..fd8aa95
--- /dev/null
+++ b/grainpit/src/markov.rs
@@ -0,0 +1,103 @@
+use rand::seq::IteratorRandom;
+
+use crate::markov::chain::Chain;
+use regex::regex;
+
+pub mod chain;
+
+#[derive(Debug, Clone)]
+pub struct Markov {
+ pub html_chain: Chain,
+ pub url_name_chain: Chain,
+ pub url_chain: Chain,
+ pub config_chain: Chain,
+}
+
+impl Markov {
+ pub fn new() -> Self {
+ Self {
+ html_chain: Chain::new(include_str!("../data/html.txt")),
+ url_name_chain: Chain::new(include_str!("../data/url_name.txt")),
+ url_chain: Chain::new(include_str!("../data/url.txt")),
+ config_chain: Chain::new(include_str!("../data/config.txt")),
+ }
+ }
+
+ pub fn gen_html(&self) -> String {
+ let href_regex = regex!(r#"href="(.*?)""#);
+ let mut generated = self.html_chain.generate(2048);
+ for i in href_regex.find_iter(&generated.clone()) {
+ generated = generated.replace(i.as_str(), &self.random_link(false));
+ }
+
+ let text_content_regex = regex!(r">([^<]+)<");
+ let url_replacements: Vec = text_content_regex
+ .captures_iter(&generated)
+ .flat_map(|caps| caps.get(0).unwrap().as_str().split_whitespace())
+ .sample(&mut rand::rng(), 15)
+ .into_iter()
+ .map(|v| v.to_string())
+ .collect();
+
+ for r in url_replacements {
+ if rand::random_bool(0.9) {
+ generated = generated.replacen(
+ &r,
+ &format!("{}", self.random_link(false), r),
+ 1,
+ );
+ } else {
+ generated = generated.replacen(
+ &r,
+ &format!("
{}", self.random_link(true), r),
+ 1,
+ );
+ }
+ }
+
+ format!("{}
", generated)
+ }
+
+ pub fn random_link(&self, image: bool) -> String {
+ let start_path = if std::env::var("GRAINPIT_EXTRAURLS").is_ok() {
+ if rand::random_bool(
+ std::env::var("GRAINPIT_EXTRAURLS_CHANCE")
+ .unwrap_or("5".to_owned())
+ .parse::()
+ .unwrap() as f64
+ / 100.0,
+ ) {
+ let rng = &mut rand::rng();
+ std::env::var("GRAINPIT_EXTRAURLS")
+ .unwrap()
+ .split(',')
+ .choose(rng)
+ .unwrap()
+ .to_owned()
+ } else {
+ "/".to_string()
+ }
+ } else {
+ "/".to_string()
+ };
+
+ format!(
+ "{}{}/{}/{}{}",
+ start_path,
+ self.url_chain.generate(4),
+ self.url_chain.generate(4),
+ self.url_chain.generate(24),
+ if rand::random_bool(0.95) {
+ if image { ".jpg" } else { ".html" }
+ } else {
+ if image { ".png" } else { "" }
+ }
+ )
+ }
+}
+
+impl Default for Markov {
+ fn default() -> Self {
+ Markov::new()
+ }
+}
diff --git a/grainpit/src/markov/chain.rs b/grainpit/src/markov/chain.rs
new file mode 100644
index 0000000..f1ee5e7
--- /dev/null
+++ b/grainpit/src/markov/chain.rs
@@ -0,0 +1,250 @@
+use arrayvec::ArrayVec;
+use rand::{RngExt, SeedableRng};
+use rand_xoshiro::Xoshiro256Plus;
+use rustc_hash::FxHashMap;
+
+const MAX_CONTEXT_SIZE: usize = 3;
+
+#[derive(Clone, PartialEq, Debug, Default)]
+pub struct Chain {
+ #[allow(clippy::type_complexity)] // Pain and misery
+ tokens: FxHashMap, (Vec<(u32, usize)>, usize)>,
+ tokenizer: Vec,
+}
+
+impl Chain {
+ #[tracing::instrument(skip_all)]
+ pub fn new(text: &str) -> Self {
+ // this whole thing is pretty ass and can probably be optimized but its only ran at start so as long as its not like 100ms+ wtv
+ let mut chain = Self::default();
+ let mut context: Vec = Vec::with_capacity(MAX_CONTEXT_SIZE + 1);
+ let (tokenizer, tokens) = Self::make_tokenizer(text);
+ chain.tokenizer = tokenizer;
+ let mut temp_tokens: FxHashMap, FxHashMap> =
+ FxHashMap::default();
+ for token in tokens {
+ for cs in 0..=context.len() {
+ let context_key: ArrayVec = context
+ [(context.len() - cs)..context.len()]
+ .try_into()
+ .unwrap();
+
+ *temp_tokens
+ .entry(context_key)
+ .or_default()
+ .entry(token)
+ .or_default() += 1;
+ }
+
+ context.push(token);
+
+ if context.len() > MAX_CONTEXT_SIZE {
+ context.remove(0);
+ }
+ }
+
+ chain.tokens = temp_tokens
+ .into_iter()
+ .map(|(k, v)| {
+ (
+ k,
+ (
+ v.clone().into_iter().collect(),
+ v.clone().into_values().sum(),
+ ),
+ )
+ })
+ .collect();
+
+ chain.tokens.values_mut().for_each(|v| {
+ v.0.sort_by_key(|k| k.1);
+ v.0.reverse();
+ v.0.shrink_to_fit();
+ });
+
+ chain
+ }
+
+ pub fn generate(&self, length: usize) -> String {
+ let mut out: Vec = Vec::with_capacity(length);
+ let mut rng = Xoshiro256Plus::from_rng(&mut rand::rng());
+
+ while out.len() < length {
+ let mut next_token = None;
+
+ for cs in (0..=MAX_CONTEXT_SIZE).rev() {
+ if cs > out.len() {
+ continue;
+ }
+
+ let context: ArrayVec =
+ out[(out.len() - cs)..out.len()].try_into().unwrap();
+
+ if let Some(next_tokens) = self.tokens.get(&context) {
+ let mut remaining_distance = rng.random_range::(..=next_tokens.1);
+ for (t, c) in &next_tokens.0 {
+ remaining_distance = remaining_distance.saturating_sub(*c);
+ if remaining_distance == 0 {
+ next_token = Some(*t);
+ break;
+ }
+ }
+
+ break;
+ }
+ }
+
+ if let Some(next_token) = next_token {
+ out.push(next_token);
+ } else {
+ break;
+ }
+ }
+
+ out.iter()
+ .map(|x| self.tokenizer.get(*x as usize).unwrap())
+ .map(|s| s.as_str())
+ .collect::>()
+ .join("") // probably a better way to do this but seems to still be faster than the previous method
+ }
+
+ fn make_tokenizer(s: &str) -> (Vec, Vec) {
+ // TODO: replace this with nom or something that isnt this complete unreadable mess
+ let len = s.len();
+ let bytes = s.as_bytes();
+ let mut pos = 0;
+ let mut tokens: Vec<&str> = Vec::new();
+
+ while pos < len {
+ let start = pos;
+ let ch = s[pos..].chars().next().unwrap();
+
+ if ch.is_whitespace() {
+ pos += ch.len_utf8();
+ while pos < len {
+ let c = s[pos..].chars().next().unwrap();
+ if c.is_whitespace() {
+ pos += c.len_utf8();
+ } else {
+ break;
+ }
+ }
+ tokens.push(&s[start..pos]);
+ continue;
+ }
+
+ if ch == '<' && bytes.get(pos + 1) == Some(&b'?') {
+ pos += 2;
+ tokens.push(&s[start..pos]);
+ continue;
+ }
+ if ch == '?' && bytes.get(pos + 1) == Some(&b'>') {
+ pos += 2;
+ tokens.push(&s[start..pos]);
+ continue;
+ }
+
+ if ch == '<' {
+ pos += ch.len_utf8();
+ while pos < len {
+ let c = s[pos..].chars().next().unwrap();
+ pos += c.len_utf8();
+ if c == '>' {
+ break;
+ }
+ }
+ tokens.push(&s[start..pos]);
+ continue;
+ }
+
+ if ch == '\'' || ch == '"' {
+ pos += ch.len_utf8();
+ while pos < len {
+ let c = s[pos..].chars().next().unwrap();
+ if c == '\n' {
+ break;
+ }
+ pos += c.len_utf8();
+ if c == ch {
+ break;
+ }
+ }
+ tokens.push(&s[start..pos]);
+ continue;
+ }
+
+ if ch == '/' && bytes.get(pos + 1) == Some(&b'/') {
+ while pos < len {
+ let c = s[pos..].chars().next().unwrap();
+ if c == '\n' {
+ break;
+ }
+ pos += c.len_utf8();
+ }
+ tokens.push(&s[start..pos]);
+ continue;
+ }
+
+ if ch == '$' {
+ pos += ch.len_utf8();
+ while pos < len {
+ let c = s[pos..].chars().next().unwrap();
+ if c.is_alphanumeric() || c == '_' {
+ pos += c.len_utf8();
+ } else {
+ break;
+ }
+ }
+ tokens.push(&s[start..pos]);
+ continue;
+ }
+
+ if ch == '/' {
+ pos += ch.len_utf8();
+ while pos < len {
+ let c = s[pos..].chars().next().unwrap();
+ if c.is_alphanumeric() || matches!(c, '/' | '.' | '-' | '_') {
+ pos += c.len_utf8();
+ } else {
+ break;
+ }
+ }
+ tokens.push(&s[start..pos]);
+ continue;
+ }
+
+ if ch.is_alphanumeric() || matches!(ch, '#' | '%') {
+ pos += ch.len_utf8();
+ while pos < len {
+ let c = s[pos..].chars().next().unwrap();
+ if c.is_alphanumeric() || matches!(c, '.' | '-' | '#' | '%') {
+ pos += c.len_utf8();
+ } else {
+ break;
+ }
+ }
+ tokens.push(&s[start..pos]);
+ continue;
+ }
+
+ pos += ch.len_utf8();
+ tokens.push(&s[start..pos]);
+ }
+
+ let mut vocab: FxHashMap<&str, u32> = FxHashMap::default();
+ let mut id_to_str: Vec = Vec::default();
+
+ let token_ids: Vec = tokens
+ .iter()
+ .map(|&t| {
+ *vocab.entry(t).or_insert_with(|| {
+ let id = id_to_str.len() as u32;
+ id_to_str.push(t.to_string());
+ id
+ })
+ })
+ .collect();
+
+ (id_to_str, token_ids)
+ }
+}
diff --git a/grainpit_proxy/Cargo.toml b/grainpit_proxy/Cargo.toml
new file mode 100644
index 0000000..85927a6
--- /dev/null
+++ b/grainpit_proxy/Cargo.toml
@@ -0,0 +1,6 @@
+[package]
+name = "grainpit_proxy"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
diff --git a/grainpit_proxy/src/main.rs b/grainpit_proxy/src/main.rs
new file mode 100644
index 0000000..e7a11a9
--- /dev/null
+++ b/grainpit_proxy/src/main.rs
@@ -0,0 +1,3 @@
+fn main() {
+ println!("Hello, world!");
+}
diff --git a/grainpit_webserver/Cargo.toml b/grainpit_webserver/Cargo.toml
new file mode 100644
index 0000000..55c5e0b
--- /dev/null
+++ b/grainpit_webserver/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "grainpit_webserver"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+axum = "0.8.9"
+tokio.workspace = true
+tracing.workspace = true
+tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
+grainpit = { path = "../grainpit" }
+tower-http = { version = "0.7.0", features = [
+ "compression-br",
+ "compression-gzip",
+ "compression-deflate",
+ "compression-zstd",
+], optional = true }
+
+[features]
+compress = ["dep:tower-http"]
diff --git a/grainpit_webserver/src/g.txt.br b/grainpit_webserver/src/g.txt.br
new file mode 100644
index 0000000..62d3943
Binary files /dev/null and b/grainpit_webserver/src/g.txt.br differ
diff --git a/grainpit_webserver/src/main.rs b/grainpit_webserver/src/main.rs
new file mode 100644
index 0000000..13b52cb
--- /dev/null
+++ b/grainpit_webserver/src/main.rs
@@ -0,0 +1,85 @@
+use std::sync::Arc;
+
+use axum::{
+ Router,
+ extract::{Path, State},
+ http::{HeaderMap, header},
+ response::{Html, IntoResponse, Response},
+ routing::get,
+};
+use grainpit::markov::Markov;
+use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
+
+#[cfg(feature = "compress")]
+use tower_http::compression::CompressionLayer;
+#[derive(Debug)]
+struct AppState {
+ m: Markov,
+}
+
+#[tokio::main]
+async fn main() {
+ tracing_subscriber::registry()
+ .with(
+ tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| {
+ format!("{}=debug,grainpit=debug", env!("CARGO_CRATE_NAME")).into()
+ }),
+ )
+ .with(tracing_subscriber::fmt::layer().with_span_events(FmtSpan::CLOSE))
+ .init();
+
+ let shared_state = Arc::new(AppState { m: Markov::new() });
+
+ #[cfg(not(feature = "compress"))]
+ let app = Router::new()
+ .route("/", get(handler))
+ .route("/{*wildcard}", get(wildcard_handler))
+ .with_state(shared_state);
+
+ #[cfg(feature = "compress")]
+ let app = Router::new()
+ .route("/", get(handler))
+ .route("/{*wildcard}", get(wildcard_handler))
+ .layer(CompressionLayer::new().quality(tower_http::CompressionLevel::Precise(9)))
+ .with_state(shared_state);
+
+ let listener = tokio::net::TcpListener::bind(
+ std::env::var("GRAINPIT_ADDR").unwrap_or("127.0.0.1:5000".to_string()),
+ )
+ .await
+ .unwrap();
+ println!("listening on {}", listener.local_addr().unwrap());
+ axum::serve(listener, app).await.unwrap();
+}
+
+async fn wildcard_handler(
+ State(state): State>,
+ Path(path): Path,
+ headers: HeaderMap,
+) -> Response {
+ if path.contains(".html") {
+ handler(State(state)).await.into_response()
+ } else if path.contains(".jpg") || path.contains(".png") {
+ let output = grainpit::image::gen_image();
+ let mut headers = HeaderMap::new();
+ headers.insert(header::CONTENT_TYPE, "image/jpeg".parse().unwrap());
+ (headers, output).into_response()
+ } else {
+ if let Some(header) = headers.get("Accept-Encoding")
+ && header.to_str().unwrap_or("").contains("br")
+ {
+ let mut headers = HeaderMap::new();
+ headers.insert(header::CONTENT_TYPE, "text/plain".parse().unwrap());
+ headers.insert(header::CONTENT_ENCODING, "br".parse().unwrap());
+ return (headers, include_bytes!("./g.txt.br")).into_response();
+ }
+
+ let mut a = state.m.config_chain.generate(512);
+ a = a.trim_start().to_owned();
+ a.into_response()
+ }
+}
+
+async fn handler(State(state): State>) -> Html {
+ Html(state.m.gen_html())
+}
diff --git a/src/main.rs b/src/main.rs
deleted file mode 100644
index 07d8da0..0000000
--- a/src/main.rs
+++ /dev/null
@@ -1,164 +0,0 @@
-pub mod markov;
-
-use std::sync::Arc;
-
-use crate::markov::Chain;
-use axum::{
- Router,
- extract::{Path, State},
- response::{Html, IntoResponse, Response},
- routing::get,
-};
-use rand::seq::IteratorRandom;
-use regex::regex;
-use tracing::info_span;
-use tracing_subscriber::fmt::format::FmtSpan;
-
-#[derive(Debug)]
-struct AppState {
- markov1: Chain,
- markov2: Chain,
- markov3: Chain,
- markov4: Chain,
-}
-
-#[tokio::main]
-async fn main() {
- tracing_subscriber::fmt()
- .with_span_events(FmtSpan::CLOSE)
- .with_max_level(tracing::Level::DEBUG)
- .init();
-
- let mut chain1 = Chain::default();
- let mut chain2 = Chain::default();
- let mut chain3 = Chain::default();
- let mut chain4 = Chain::default();
-
- info_span!("train_markov1").in_scope(|| {
- let mut lines = include_str!("markov1.txt").lines();
- while let Some(_) = lines.next() {
- let mut buffer = String::new();
- for _ in 0..512 {
- if let Some(next_line) = lines.next() {
- buffer = buffer + "\n" + next_line;
- }
- }
- chain1.train(&buffer.to_owned());
- }
- });
-
- info_span!("train_markov2").in_scope(|| {
- for line in include_str!("markov2.txt").lines() {
- chain2.train(line);
- }
- });
-
- info_span!("train_markov3").in_scope(|| {
- let mut lines = include_str!("markov3.txt").lines();
- while let Some(_) = lines.next() {
- let mut buffer = String::new();
- for _ in 0..50 {
- if let Some(next_line) = lines.next() {
- buffer = buffer + "\n" + next_line;
- }
- }
- chain3.train(&buffer.to_owned());
- }
- });
-
- info_span!("train_markov4").in_scope(|| {
- for line in include_str!("markov4.txt").lines() {
- chain4.train(line);
- }
- });
-
- let shared_state = Arc::new(AppState {
- markov1: chain1,
- markov2: chain2,
- markov3: chain3,
- markov4: chain4,
- });
- let app = Router::new()
- .route("/", get(handler))
- .route("/{*wildcard}", get(wildcard_handler))
- .with_state(shared_state);
-
- // run it
- let listener = tokio::net::TcpListener::bind(
- std::env::var("GRAINPIT_ADDR").unwrap_or("127.0.0.1:5000".to_string()),
- )
- .await
- .unwrap();
- println!("listening on {}", listener.local_addr().unwrap());
- axum::serve(listener, app).await.unwrap();
-}
-
-async fn wildcard_handler(
- State(state): State>,
- Path(path): Path,
-) -> Response {
- if path.contains(".html") {
- handler(State(state)).await.into_response()
- } else {
- info_span!("wildcard_handler").in_scope(|| {
- let mut a = state.markov3.generate(512);
- a = a.trim_start().to_owned();
- a.into_response()
- })
- }
-}
-
-#[tracing::instrument(skip_all)]
-async fn handler(State(state): State>) -> Html {
- let mut fakeurls = String::new();
- for _ in 0..16 {
- fakeurls.push_str(&format!(
- "\n{}
",
- random_link(&state.markov4),
- state.markov2.generate(16)
- ));
- }
-
- let href_regex = regex!(r#"href="(.*?)""#);
- let mut generated = state.markov1.generate(4096);
- for i in href_regex.find_iter(&generated.clone()) {
- generated = generated.replace(i.as_str(), &random_link(&state.markov4));
- }
-
- Html(format!(
- "This is my website and it is Amazing!!
\n{}\n{}
",
- fakeurls, generated
- ))
-}
-
-fn random_link(markov4: &Chain) -> String {
- let start_path = if std::env::var("GRAINPIT_EXTRAURLS").is_ok() {
- if rand::random_bool(
- std::env::var("GRAINPIT_EXTRAURLS_CHANCE")
- .unwrap_or("5".to_owned())
- .parse::()
- .unwrap() as f64
- / 100.0,
- ) {
- let rng = &mut rand::rng();
- std::env::var("GRAINPIT_EXTRAURLS")
- .unwrap()
- .split(",")
- .choose(rng)
- .unwrap()
- .to_owned()
- } else {
- "/".to_string()
- }
- } else {
- "/".to_string()
- };
-
- format!(
- "{}{}/{}/{}.html",
- start_path,
- markov4.generate(4),
- markov4.generate(4),
- markov4.generate(24)
- )
-}
diff --git a/src/markov.rs b/src/markov.rs
deleted file mode 100644
index 0c0f85b..0000000
--- a/src/markov.rs
+++ /dev/null
@@ -1,146 +0,0 @@
-use std::iter;
-
-use rand::seq::IndexedRandom;
-use rustc_hash::FxHashMap;
-
-#[derive(Clone, PartialEq, Debug, Default)]
-pub struct Chain {
- tokens: FxHashMap, FxHashMap>,
- vocab: FxHashMap,
- vocab_rev: Vec,
-}
-
-impl Chain {
- const MAX_CONTEXT_SIZE: usize = 5;
-
- fn intern(&mut self, s: &str) -> u32 {
- if let Some(&id) = self.vocab.get(s) {
- return id;
- }
- let id = self.vocab_rev.len() as u32;
- self.vocab_rev.push(s.to_string());
- self.vocab.insert(s.to_string(), id);
- id
- }
-
- pub fn train(&mut self, text: &str) {
- let mut context: Vec = Vec::with_capacity(Self::MAX_CONTEXT_SIZE + 1);
-
- for token in Self::tokenize(text) {
- let token_id = self.intern(token);
-
- for cs in 0..=context.len() {
- let context_key: Vec = context[(context.len() - cs)..context.len()].to_vec();
-
- *self
- .tokens
- .entry(context_key)
- .or_default()
- .entry(token_id)
- .or_default() += 1;
- }
-
- context.push(token_id);
-
- if context.len() > Self::MAX_CONTEXT_SIZE {
- context.remove(0);
- }
- }
- }
-
- pub fn generate(&self, length: usize) -> String {
- let mut out: Vec = Vec::with_capacity(length);
- let mut rng = rand::rng();
-
- while out.len() < length {
- let mut next_token = None;
-
- for cs in (0..=Self::MAX_CONTEXT_SIZE).rev() {
- if cs > out.len() {
- continue;
- }
-
- let context: &[u32] = &out[(out.len() - cs)..out.len()];
-
- if let Some(next_tokens) = self.tokens.get(context) {
- let next_tokens: Vec<_> = next_tokens.iter().collect();
-
- next_token = Some(
- *next_tokens
- .choose_weighted(&mut rng, |(_token, frequency)| *frequency)
- .unwrap()
- .0,
- );
-
- break;
- }
- }
-
- if let Some(next_token) = next_token {
- out.push(next_token);
- } else {
- break;
- }
- }
-
- out.iter()
- .map(|id| self.vocab_rev[*id as usize].as_str())
- .collect()
- }
-
- fn tokenize(s: &str) -> impl Iterator- {
- let mut chars = s.char_indices().peekable();
-
- iter::from_fn(move || {
- let (idx, ch) = chars.next()?;
-
- if ch.is_alphanumeric() {
- let idx_from = idx;
- let mut idx_to = idx + ch.len_utf8();
-
- while let Some((idx, ch)) = chars.peek() {
- if ch.is_alphanumeric() {
- idx_to = idx + ch.len_utf8();
- chars.next();
- } else {
- break;
- }
- }
-
- Some(&s[idx_from..idx_to])
- } else if ch.is_whitespace() {
- let idx_from = idx;
- let mut idx_to = idx + ch.len_utf8();
-
- while let Some((idx, ch)) = chars.peek() {
- if ch.is_whitespace() {
- idx_to = idx + ch.len_utf8();
- chars.next();
- } else {
- break;
- }
- }
-
- Some(&s[idx_from..idx_to])
- } else if ch == '<' {
- let idx_from = idx;
- let mut idx_to = idx + ch.len_utf8();
-
- while let Some(&(idx, ch)) = chars.peek() {
- idx_to = idx + ch.len_utf8();
- chars.next();
- if ch == '>' {
- break;
- }
- }
-
- Some(&s[idx_from..idx_to])
- } else {
- let idx_from = idx;
- let idx_to = idx + ch.len_utf8();
-
- Some(&s[idx_from..idx_to])
- }
- })
- }
-}