diff --git a/.gitignore b/.gitignore index 4dd9b3c..6164529 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ # Maven build artefacts target/ +# JMEOS native dependency — generated by build-jmeos.sh, never committed +.build-jmeos/ +kafka-streams-app/jar/ +kafka-streams-app/lib/ +*.jar +*.so + # IDE .idea/ .vscode/ diff --git a/README.md b/README.md index d4cedeb..9238699 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,24 @@ Spatial predicates today use pure-Java great-circle (`Haversine`) and planar seg ## Build and run +### MEOS native dependency + +The spatial predicates route through MEOS via the [JMEOS](https://github.com/MobilityDB/JMEOS) +bridge, so the build needs the JMEOS jar and the native `libmeos.so`. Neither is +committed to this repository — generate them from source with the helper script, +which clones MobilityDB and JMEOS at pinned, immutable refs, builds `libmeos.so`, +builds the jar, installs the jar into the local Maven repository, and stages +`libmeos.so` for the runtime: + +``` +./build-jmeos.sh +``` + +Run it once (re-run it only to bump the pinned MobilityDB/JMEOS refs at the top of +the script). After it succeeds, JMEOS resolves as an ordinary Maven dependency. + +### Build the app + ``` cd kafka-streams-app mvn -q clean package -DskipTests diff --git a/build-jmeos.sh b/build-jmeos.sh new file mode 100755 index 0000000..5d792d5 --- /dev/null +++ b/build-jmeos.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# +# build-jmeos.sh — build the JMEOS jar and the native libmeos.so from source and +# install them locally, so the repository never has to carry the binaries. +# +# This is the downstream generation chain for the JVM streaming tools: +# +# MobilityDB/MEOS (deliverable PRs) -> JMEOS (FFI facade + jar) -> MobilityKafka +# +# (The MEOS-API meos-idl.json step is pre-materialized in the JMEOS branch's +# committed codegen/input/meos-idl.json, so this script only has to build the +# two endpoints.) +# +# What it does: +# 1. Clones MobilityDB at the pinned ref and builds libmeos.so (cmake -DMEOS=ON). +# 2. Clones JMEOS at the pinned ref, drops libmeos.so in, and builds JMEOS.jar. +# 3. Registers the jar in the local Maven repository via +# `mvn install:install-file` under the coordinates the kafka-streams-app +# pom depends on (com.mobilitydb:jmeos:1.4.0 by default). +# 4. Copies libmeos.so into kafka-streams-app/lib/ for the test/runtime +# LD_LIBRARY_PATH. +# +# After running this once, `cd kafka-streams-app && mvn test` resolves JMEOS as +# an ordinary dependency — no committed jar/so required. +# +# The refs below point at the DELIVERABLE pull requests that provide the surface +# this project consumes — NOT at an ecosystem-pin tag. A pin is the ecosystem's +# benchmark / evidence vehicle, not a source of truth, so a binding must never +# depend on one. The dependency is expressed as the PRs' immutable head commit +# SHAs (overridable env vars): +# * MobilityDB PR #1148 — the *_tgeoarr_tgeoarr set-set spatial-join MEOS symbols. +# * JMEOS PR #25 — the org.mobilitydb.meos facade incl. MeosSetSetJoin. +# A head SHA is immutable: a later rebase/force-push of either PR creates a NEW +# SHA and leaves the pinned one unchanged. Once both PRs merge upstream, repoint +# MOBILITYDB_REF -> MobilityDB/MobilityDB master and JMEOS_REF -> MobilityDB/JMEOS +# main (or a release tag) — that is the only change needed. +# +set -euo pipefail + +# --------------------------------------------------------------------------- +# Pinned sources (override any of these via the environment). +# --------------------------------------------------------------------------- +# MobilityDB PR #1148 (set-set spatial join, rebased onto #1162 clean/geo so the +# set-set join inherits the geodetic-disjoint surface) — immutable head SHA. +MOBILITYDB_REPO="${MOBILITYDB_REPO:-https://github.com/estebanzimanyi/MobilityDB.git}" +MOBILITYDB_REF="${MOBILITYDB_REF:-fab7025b24f9b2b45db26d6bd0a3118058d5b55c}" # PR #1148 head + +# JMEOS PR #25 (org.mobilitydb.meos facade + MeosSetSetJoin) — immutable head SHA. +JMEOS_REPO="${JMEOS_REPO:-https://github.com/estebanzimanyi/JMEOS.git}" +JMEOS_REF="${JMEOS_REF:-f921d8608f18574a5a824b837af1a6b82c985fc2}" # PR #25 head + +# Maven coordinates the jar is installed under (must match kafka-streams-app/pom.xml). +JMEOS_GROUP_ID="${JMEOS_GROUP_ID:-com.mobilitydb}" +JMEOS_ARTIFACT_ID="${JMEOS_ARTIFACT_ID:-jmeos}" +JMEOS_VERSION="${JMEOS_VERSION:-1.4.0}" + +# --------------------------------------------------------------------------- +# Layout. +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +APP_DIR="${SCRIPT_DIR}/kafka-streams-app" +WORK_DIR="${WORK_DIR:-${SCRIPT_DIR}/.build-jmeos}" +JOBS="${JOBS:-$(nproc 2>/dev/null || echo 4)}" + +log() { printf '\n\033[1;34m==>\033[0m %s\n' "$*"; } + +# --------------------------------------------------------------------------- +# Preconditions. +# --------------------------------------------------------------------------- +for tool in git cmake make mvn; do + command -v "$tool" >/dev/null 2>&1 || { echo "error: '$tool' is required but not on PATH" >&2; exit 1; } +done + +mkdir -p "${WORK_DIR}" + +# clone_at +# Clones (or reuses) and checks out the exact . may be a tag, +# branch or commit SHA; PR-head SHAs are fetched from the pull ref namespace if +# they are not reachable from the default branches. +clone_at() { + local repo="$1" ref="$2" dest="$3" + if [ ! -d "${dest}/.git" ]; then + log "Cloning ${repo}" + git clone "${repo}" "${dest}" + fi + git -C "${dest}" fetch --quiet --tags origin + if ! git -C "${dest}" cat-file -e "${ref}^{commit}" 2>/dev/null; then + # SHA only reachable through an open PR — fetch every PR head, then retry. + git -C "${dest}" fetch --quiet origin '+refs/pull/*/head:refs/remotes/origin/pr/*' || true + fi + log "Checking out ${ref}" + git -C "${dest}" -c advice.detachedHead=false checkout --quiet "${ref}" +} + +# --------------------------------------------------------------------------- +# 1. Build libmeos.so from MobilityDB. +# --------------------------------------------------------------------------- +MDB_DIR="${WORK_DIR}/MobilityDB" +clone_at "${MOBILITYDB_REPO}" "${MOBILITYDB_REF}" "${MDB_DIR}" + +# Enable the MEOS type families the streaming app exercises: circular buffers, +# network points (default ON) and geoposes (which auto-enables rigid geometries). +# The facade smoke tests link these symbols, so they must be in libmeos.so. +# H3 and POINTCLOUD are left OFF — the app does not use them and they require +# extra system libraries (libh3, libpointcloud); enable them via MEOS_CMAKE_ARGS +# if a downstream consumer ever needs them. +MEOS_CMAKE_ARGS="${MEOS_CMAKE_ARGS:--DCBUFFER=ON -DNPOINT=ON -DPOSE=ON}" + +log "Building libmeos.so (MEOS=ON ${MEOS_CMAKE_ARGS})" +rm -rf "${MDB_DIR}/build" +cmake -S "${MDB_DIR}" -B "${MDB_DIR}/build" -DMEOS=ON ${MEOS_CMAKE_ARGS} >/dev/null +cmake --build "${MDB_DIR}/build" --target meos -j "${JOBS}" + +LIBMEOS_SO="$(find "${MDB_DIR}/build" -name 'libmeos.so' -print -quit)" +[ -n "${LIBMEOS_SO}" ] || { echo "error: libmeos.so not produced by the MEOS build" >&2; exit 1; } +log "Built ${LIBMEOS_SO}" + +# --------------------------------------------------------------------------- +# 2. Build JMEOS.jar against that libmeos.so. +# --------------------------------------------------------------------------- +JMEOS_DIR="${WORK_DIR}/JMEOS" +clone_at "${JMEOS_REPO}" "${JMEOS_REF}" "${JMEOS_DIR}" + +# JMEOS' build bundles src/libmeos.so into the jar and JarLibraryLoader extracts it. +cp -f "${LIBMEOS_SO}" "${JMEOS_DIR}/jmeos-core/src/libmeos.so" + +log "Building JMEOS.jar" +# FunctionsGenerator lives in the codegen module, which jmeos-core does not +# declare as a Maven dependency — so '-am' will not build it. Compile it first +# so jmeos-core's build-time facade generation can find it. Use +# 'maven.test.skip' (not 'skipTests'): the jmeos-core pom hardcodes +# false, which overrides -DskipTests but not this. +mvn -f "${JMEOS_DIR}/pom.xml" -q -pl codegen compile +mvn -f "${JMEOS_DIR}/pom.xml" -q -pl jmeos-core -am -Dmaven.test.skip=true package + +JMEOS_JAR="${JMEOS_DIR}/jar/JMEOS.jar" +[ -f "${JMEOS_JAR}" ] || { echo "error: ${JMEOS_JAR} was not produced" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# 3. Install the jar into the local Maven repository. +# --------------------------------------------------------------------------- +log "Installing ${JMEOS_GROUP_ID}:${JMEOS_ARTIFACT_ID}:${JMEOS_VERSION} into the local Maven repo" +mvn -q install:install-file \ + -Dfile="${JMEOS_JAR}" \ + -DgroupId="${JMEOS_GROUP_ID}" \ + -DartifactId="${JMEOS_ARTIFACT_ID}" \ + -Dversion="${JMEOS_VERSION}" \ + -Dpackaging=jar + +# --------------------------------------------------------------------------- +# 4. Stage libmeos.so for the kafka-streams-app runtime (LD_LIBRARY_PATH). +# --------------------------------------------------------------------------- +mkdir -p "${APP_DIR}/lib" +cp -f "${LIBMEOS_SO}" "${APP_DIR}/lib/libmeos.so" + +log "Done." +cat < [--max N] [--only Q3-continuous] +``` + +## Figures + +Real BerlinMOD corpus (216,075 instants, 5 vehicles, ~11 days, EPSG:3857), +single-broker `EmbeddedKafkaCluster`, one stream thread per cell, Java 21, +16-core host; libmeos built with `-DMEOS/CBUFFER/NPOINT/POSE/RGEO=ON`. + +| Cell | Events in | Output rows | Wall (ms) | Throughput (ev/s) | +|---|---:|---:|---:|---:| +| Q1-continuous | 216075 | 5 | 1899 | 113,785 | +| Q1-snapshot | 216075 | 703 | 1701 | 127,029 | +| Q1-windowed | 216075 | 86 | 1650 | 130,956 | +| Q2-continuous | 216075 | 61170 | 1289 | 167,631 | +| Q2-snapshot | 216075 | 140 | 1683 | 128,388 | +| Q2-windowed | 216075 | 50 | 1704 | 126,806 | +| Q3-continuous | 216075 | 216075 | 4618 | 46,790 | +| Q3-snapshot | 216075 | 97 | 2607 | 82,883 | +| Q3-windowed | 216075 | 50 | 4188 | 51,594 | +| Q4-continuous | 216075 | 62 | 9356 | 23,095 | +| Q4-snapshot | 216075 | 4685 | 10499 | 20,581 | +| Q4-windowed | 216075 | 98 | 9741 | 22,182 | +| Q5-continuous | 216075 | 60577 | 22889 | 9,440 | +| Q5-snapshot | 216075 | 34 | 2006 | 107,715 | +| Q5-windowed | 216075 | 6 | 2896 | 74,612 | +| Q6-continuous | 216075 | 216075 | 8708 | 24,814 | +| Q6-snapshot | 216075 | 698 | 7258 | 29,771 | +| Q6-windowed | 216075 | 203 | 6268 | 34,473 | +| Q7-continuous | 216075 | 5 | 4321 | 50,006 | +| Q7-snapshot | 216075 | 632 | 7487 | 28,860 | +| Q7-windowed | 216075 | 53 | 11625 | 18,587 | +| Q8-continuous | 216075 | 216075 | 4807 | 44,950 | +| Q8-snapshot | 216075 | 281 | 2825 | 76,487 | +| Q8-windowed | 216075 | 77 | 5946 | 36,340 | +| Q9-continuous | 216075 | 107870 | 4561 | 47,375 | +| Q9-snapshot | 216075 | 140 | 2281 | 94,729 | +| Q9-windowed | 216075 | 22 | 2384 | 90,636 | + +The per-event MEOS-predicate cells that emit one row per input event +(Q3/Q8/Q9-continuous, 216,075 output rows through `edwithin_tgeo_geo` / +`eintersects_tgeo_geo`) sustain 45,000–47,000 ev/s. In the cross-platform +streaming catalog these cells run below MobilityFlink's in-JVM mini-cluster on +the same corpus, because Kafka Streams routes every record through the broker; +the per-cell shape matches across both engines. Q5-continuous is the O(V²) +all-pairs-meeting outlier (9,440 ev/s). Non-spatial cells (Q1/Q2) reach +110,000–168,000 ev/s. The `TopologyTestDriver` correctness harness +([`BerlinMODBenchmark`](../src/main/java/berlinmod/BerlinMODBenchmark.java)) +reports output cardinality and correctness, not throughput: its per-event +bookkeeping dominates wall-clock, so a no-predicate cell already runs far below a +real broker's rate. diff --git a/kafka-streams-app/pom.xml b/kafka-streams-app/pom.xml index 4ea8465..d399dfb 100644 --- a/kafka-streams-app/pom.xml +++ b/kafka-streams-app/pom.xml @@ -16,9 +16,26 @@ 2.18.2 2.0.16 5.11.4 + true + ${project.basedir}/lib + + + com.mobilitydb + jmeos + 1.4.0 + + + com.github.jnr + jnr-ffi + 2.1.10 + org.apache.kafka kafka-streams @@ -61,6 +78,44 @@ ${junit.version} test + + + + org.apache.kafka + kafka_2.13 + ${kafka.version} + test + + + org.apache.kafka + kafka_2.13 + ${kafka.version} + test-jartesttest + + + org.apache.kafka + kafka-streams + ${kafka.version} + test-jartesttest + + + org.apache.kafka + kafka-clients + ${kafka.version} + test-jartesttest + + + org.apache.kafka + kafka-server-common + ${kafka.version} + test-jartesttest + + + org.hamcrest + hamcrest + 2.2 + test + @@ -76,6 +131,16 @@ maven-surefire-plugin 3.2.5 + + false + + ${meos.enabled} + + + ${meos.lib.dir} + + --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + maven-shade-plugin diff --git a/kafka-streams-app/src/main/java/berlinmod/BerlinMODCorpus.java b/kafka-streams-app/src/main/java/berlinmod/BerlinMODCorpus.java new file mode 100644 index 0000000..cc74be5 --- /dev/null +++ b/kafka-streams-app/src/main/java/berlinmod/BerlinMODCorpus.java @@ -0,0 +1,167 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package berlinmod; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.TreeSet; +import java.util.stream.Stream; + +/** + * Corpus loader and query-parameter derivation for the BerlinMOD streaming + * benchmark. + * + *

Supplies either a deterministic synthetic corpus or the real BerlinMOD + * instants corpus read from the {@code berlinmod_instants.csv} produced by the + * BerlinMOD generator. Real instants are stored in EPSG:3857; they are + * reprojected to EPSG:4326 through MEOS {@code geo_transform} at load — the + * loader holds no projection mathematics of its own. + * + *

{@link Params} fixes the per-query parameters from the corpus itself (its + * centroid, bounding box, vehicle ids, and time span) so every spatial cell is + * selective and the windowing granularity yields a comparable number of windows + * regardless of the corpus time span. + */ +public final class BerlinMODCorpus { + + private static final DateTimeFormatter TS = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss") + .optionalStart().appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true).optionalEnd() + .appendOffset("+HH", "Z") + .toFormatter(); + + private BerlinMODCorpus() { /* utility */ } + + /** Query parameters derived from a corpus. */ + public static final class Params { + public final double pLon, pLat, radiusMetres, dMeetMetres; + public final double xmin, ymin, xmax, ymax; + public final double s1Lon, s1Lat, s2Lon, s2Lat; + public final List pois; + public final int targetId, xId, yId; + public final long windowSeconds, snapshotTickMillis; + + Params(double pLon, double pLat, double radiusMetres, double dMeetMetres, + double xmin, double ymin, double xmax, double ymax, + double s1Lon, double s1Lat, double s2Lon, double s2Lat, + List pois, int targetId, int xId, int yId, + long windowSeconds, long snapshotTickMillis) { + this.pLon = pLon; this.pLat = pLat; this.radiusMetres = radiusMetres; this.dMeetMetres = dMeetMetres; + this.xmin = xmin; this.ymin = ymin; this.xmax = xmax; this.ymax = ymax; + this.s1Lon = s1Lon; this.s1Lat = s1Lat; this.s2Lon = s2Lon; this.s2Lat = s2Lat; + this.pois = pois; this.targetId = targetId; this.xId = xId; this.yId = yId; + this.windowSeconds = windowSeconds; this.snapshotTickMillis = snapshotTickMillis; + } + } + + /** Real BerlinMOD instants from {@code berlinmod_instants.csv} + * (columns {@code tripid,vehid,day,seqno,geom,t}), reprojected 3857→4326 + * through MEOS, sorted by timestamp. {@code maxRows <= 0} loads all rows. */ + public static List fromInstantsCsv(String path, int maxRows) throws Exception { + ensureMeos(); + List events = new ArrayList<>(); + try (Stream lines = Files.lines(Paths.get(path))) { + java.util.Iterator it = lines.iterator(); + if (it.hasNext()) { + it.next(); // header + } + while (it.hasNext() && (maxRows <= 0 || events.size() < maxRows)) { + String[] f = it.next().split(","); + int vid = Integer.parseInt(f[1].trim()); + long ms = OffsetDateTime.parse(f[5].trim(), TS).toInstant().toEpochMilli(); + Pointer g4326 = GeneratedFunctions.geo_transform( + GeneratedFunctions.geom_in(f[4].trim(), -1), 4326); + String txt = GeneratedFunctions.geo_as_text(g4326, 7); // POINT(lon lat) + String[] xy = txt.substring(txt.indexOf('(') + 1, txt.indexOf(')')).trim().split("\\s+"); + events.add(make(vid, ms, Double.parseDouble(xy[0]), Double.parseDouble(xy[1]))); + } + } + events.sort((a, b) -> Long.compare(a.getTimestamp(), b.getTimestamp())); + return events; + } + + /** Derive selective per-query parameters and a window/tick granularity that + * yields ~200 windows over the corpus time span. */ + public static Params derive(List corpus) { + double sumLon = 0, sumLat = 0, minLon = Double.MAX_VALUE, minLat = Double.MAX_VALUE, + maxLon = -Double.MAX_VALUE, maxLat = -Double.MAX_VALUE, minT = Double.MAX_VALUE, maxT = -Double.MAX_VALUE; + TreeSet ids = new TreeSet<>(); + for (BerlinMODTrip t : corpus) { + sumLon += t.getLon(); sumLat += t.getLat(); + minLon = Math.min(minLon, t.getLon()); maxLon = Math.max(maxLon, t.getLon()); + minLat = Math.min(minLat, t.getLat()); maxLat = Math.max(maxLat, t.getLat()); + minT = Math.min(minT, t.getTimestamp()); maxT = Math.max(maxT, t.getTimestamp()); + ids.add(t.getVehicleId()); + } + int n = corpus.size(); + double cLon = sumLon / n, cLat = sumLat / n; + double exLon = maxLon - minLon, exLat = maxLat - minLat; + List idList = new ArrayList<>(ids); + int targetId = idList.get(idList.size() / 2); + int xId = idList.get(0); + int yId = idList.get(Math.min(idList.size() - 1, idList.size() / 2)); + long span = (long) (maxT - minT); + long windowSeconds = Math.max(1L, span / 1000 / 200); + long tickMillis = Math.max(1000L, windowSeconds * 1000L / 2); + List pois = Arrays.asList( + new PointOfInterest(1, cLon, cLat, 2_000.0), + new PointOfInterest(2, cLon + 0.1 * exLon, cLat + 0.1 * exLat, 1_000.0), + new PointOfInterest(3, cLon - 0.1 * exLon, cLat - 0.1 * exLat, 2_000.0)); + return new Params(cLon, cLat, 5_000.0, 5_000.0, + cLon - 0.25 * exLon, cLat - 0.25 * exLat, cLon + 0.25 * exLon, cLat + 0.25 * exLat, + minLon + 0.25 * exLon, cLat, maxLon - 0.25 * exLon, cLat, + pois, targetId, xId, yId, windowSeconds, tickMillis); + } + + private static BerlinMODTrip make(int vid, long t, double lon, double lat) { + BerlinMODTrip trip = new BerlinMODTrip(); + trip.setVehicleId(vid); + trip.setTimestamp(t); + trip.setLon(lon); + trip.setLat(lat); + return trip; + } + + private static final ThreadLocal MEOS_INIT = ThreadLocal.withInitial(() -> Boolean.FALSE); + private static void ensureMeos() { + if (!MEOS_INIT.get()) { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + MEOS_INIT.set(Boolean.TRUE); + } + } + +} diff --git a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTopology.java b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTopology.java index 3f0a88d..a0babb1 100644 --- a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTopology.java +++ b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTopology.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.common.serialization.Serdes; @@ -58,10 +83,9 @@ public final class BerlinMODTopology { // ---------- Q3 ---------- public static final String Q3_CONTINUOUS_OUTPUT = "berlinmod-q3-continuous"; - // Query params below are anchored to the canonical BerlinMOD sample's - // per-vehicle geometry (vehicles 1-5, centroids 7-18 km apart); radii use - // km-scale margins so the dwithin partition is identical under the - // pure-Java Haversine and the MEOS geodetic engines. + // Query params anchored to the canonical BerlinMOD sample's per-vehicle + // geometry (vehicles 1-5, 7-18 km apart); km-scale radii keep the dwithin + // partition identical under pure-Java Haversine and the MEOS geodetic engine. public static final double Q3_P_LON = 4.4322; // near vehicle 1 public static final double Q3_P_LAT = 50.7670; public static final double Q3_RADIUS_METRES = 5_000.0; @@ -120,165 +144,184 @@ public final class BerlinMODTopology { private BerlinMODTopology() {} - public static Topology build() { + public static Topology build() { return build(defaultParams()); } + public static Topology build(BerlinMODCorpus.Params p) { return buildInternal(p, null); } + public static Topology buildCell(BerlinMODCorpus.Params p, String cell) { return buildInternal(p, cell); } + + private static BerlinMODCorpus.Params defaultParams() { + return new BerlinMODCorpus.Params(Q3_P_LON, Q3_P_LAT, Q3_RADIUS_METRES, Q5_D_MEET_METRES, + Q4_XMIN, Q4_YMIN, Q4_XMAX, Q4_YMAX, Q8_S1_LON, Q8_S1_LAT, Q8_S2_LON, Q8_S2_LAT, + Q7_POIS, Q2_TARGET_VEHICLE_ID, Q9_X_VEHICLE_ID, Q9_Y_VEHICLE_ID, + WINDOW_SIZE_MILLIS / 1000L, SNAPSHOT_TICK_MILLIS); + } + + private static Topology buildInternal(BerlinMODCorpus.Params p, String only) { StreamsBuilder builder = new StreamsBuilder(); + final long WIN_MS = p.windowSeconds * 1000L; BerlinMODTripSerde tripSerde = new BerlinMODTripSerde(); - - // ---- continuous-form state stores ---- - addStore(builder, Q1_SEEN_STORE, Serdes.Integer(), Serdes.Boolean()); - addStore(builder, Q4_WAS_INSIDE_STORE, Serdes.Integer(), Serdes.Boolean()); - addStore(builder, Q5_LAST_POS_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q6_STATE_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q7_FIRST_PASSED_STORE, Serdes.Integer(), Serdes.Long()); - addStore(builder, Q9_STATE_STORE, Serdes.Integer(), Serdes.String()); - - // ---- windowed-form state stores ---- - addStore(builder, Q1_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q2_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q3_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q4_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q5_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q6_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q7_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q8_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q9_WIN_STORE, Serdes.Long(), Serdes.String()); - - // ---- snapshot-form state stores (separate to avoid co-write conflicts with continuous) ---- - addStore(builder, Q1_SNAP_STORE, Serdes.Integer(), Serdes.Long()); - addStore(builder, Q2_SNAP_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q3_SNAP_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q4_SNAP_WAS_INSIDE_STORE, Serdes.Integer(), Serdes.Boolean()); - addStore(builder, Q4_SNAP_ENTRIES_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q5_SNAP_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q6_SNAP_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q7_SNAP_STORE, Serdes.Integer(), Serdes.Long()); - addStore(builder, Q8_SNAP_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q9_SNAP_STORE, Serdes.Integer(), Serdes.String()); - - // ---- streams ---- KStream trips = builder.stream(INPUT_TOPIC, Consumed.with(Serdes.Integer(), tripSerde)); - - // Re-keyed by constant for the shared-state snapshot/multi-vehicle processors KStream tripsK0 = trips.selectKey((k, v) -> 0); - // ====== continuous form ====== - trips.process(() -> new Q1ContinuousProcessor(Q1_SEEN_STORE), Q1_SEEN_STORE) - .to(Q1_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); - - trips.process(() -> new Q2ContinuousProcessor(Q2_TARGET_VEHICLE_ID)) - .to(Q2_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), tripSerde)); - - trips.process(() -> new Q3ContinuousProcessor(Q3_P_LON, Q3_P_LAT, Q3_RADIUS_METRES)) - .to(Q3_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Boolean())); - - trips.process(() -> new Q4ContinuousProcessor(Q4_WAS_INSIDE_STORE, Q4_XMIN, Q4_YMIN, Q4_XMAX, Q4_YMAX), - Q4_WAS_INSIDE_STORE) - .to(Q4_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); - - tripsK0.process(() -> new Q5ContinuousProcessor(Q5_LAST_POS_STORE, - Q5_P_LON, Q5_P_LAT, Q5_D_P_METRES, Q5_D_MEET_METRES), - Q5_LAST_POS_STORE) - .to(Q5_CONTINUOUS_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); - - trips.process(() -> new Q6ContinuousProcessor(Q6_STATE_STORE), Q6_STATE_STORE) - .to(Q6_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Double())); - - trips.process(() -> new Q7ContinuousProcessor(Q7_FIRST_PASSED_STORE, Q7_POIS), - Q7_FIRST_PASSED_STORE) - .to(Q7_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); - - trips.process(() -> new Q8ContinuousProcessor(Q8_S1_LON, Q8_S1_LAT, Q8_S2_LON, Q8_S2_LAT, Q8_RADIUS_METRES)) - .to(Q8_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Boolean())); - - tripsK0.process(() -> new Q9ContinuousProcessor(Q9_STATE_STORE, Q9_X_VEHICLE_ID, Q9_Y_VEHICLE_ID), - Q9_STATE_STORE) - .to(Q9_CONTINUOUS_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); - - // ====== windowed form (distinct-count per tumbling window for Q1/Q3/Q8) ====== - tripsK0.process(() -> new Q1WindowedProcessor(Q1_WIN_STORE, WINDOW_SIZE_MILLIS), Q1_WIN_STORE) - .to(Q1_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); - - tripsK0.process(() -> new Q3WindowedProcessor(Q3_WIN_STORE, - Q3_P_LON, Q3_P_LAT, Q3_RADIUS_METRES, - WINDOW_SIZE_MILLIS), - Q3_WIN_STORE) - .to(Q3_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); - - tripsK0.process(() -> new Q8WindowedProcessor(Q8_WIN_STORE, - Q8_S1_LON, Q8_S1_LAT, Q8_S2_LON, Q8_S2_LAT, - Q8_RADIUS_METRES, WINDOW_SIZE_MILLIS), - Q8_WIN_STORE) - .to(Q8_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); - - tripsK0.process(() -> new Q2WindowedProcessor(Q2_WIN_STORE, Q2_TARGET_VEHICLE_ID, WINDOW_SIZE_MILLIS), - Q2_WIN_STORE) - .to(Q2_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q4WindowedProcessor(Q4_WIN_STORE, - Q4_XMIN, Q4_YMIN, Q4_XMAX, Q4_YMAX, - WINDOW_SIZE_MILLIS), - Q4_WIN_STORE) - .to(Q4_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q5WindowedProcessor(Q5_WIN_STORE, - Q5_P_LON, Q5_P_LAT, Q5_D_P_METRES, Q5_D_MEET_METRES, - WINDOW_SIZE_MILLIS), - Q5_WIN_STORE) - .to(Q5_WINDOWED_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); - - tripsK0.process(() -> new Q6WindowedProcessor(Q6_WIN_STORE, WINDOW_SIZE_MILLIS), Q6_WIN_STORE) - .to(Q6_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q7WindowedProcessor(Q7_WIN_STORE, Q7_POIS, WINDOW_SIZE_MILLIS), Q7_WIN_STORE) - .to(Q7_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q9WindowedProcessor(Q9_WIN_STORE, - Q9_X_VEHICLE_ID, Q9_Y_VEHICLE_ID, - WINDOW_SIZE_MILLIS), - Q9_WIN_STORE) - .to(Q9_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); - - // ====== snapshot form (all via constant key, with STREAM_TIME punctuators) ====== - tripsK0.process(() -> new Q1SnapshotProcessor(Q1_SNAP_STORE, SNAPSHOT_TICK_MILLIS), Q1_SNAP_STORE) - .to(Q1_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); - - tripsK0.process(() -> new Q2SnapshotProcessor(Q2_SNAP_STORE, Q2_TARGET_VEHICLE_ID, SNAPSHOT_TICK_MILLIS), - Q2_SNAP_STORE) - .to(Q2_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q3SnapshotProcessor(Q3_SNAP_STORE, - Q3_P_LON, Q3_P_LAT, Q3_RADIUS_METRES, SNAPSHOT_TICK_MILLIS), - Q3_SNAP_STORE) - .to(Q3_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); - - tripsK0.process(() -> new Q4SnapshotProcessor(Q4_SNAP_WAS_INSIDE_STORE, Q4_SNAP_ENTRIES_STORE, - Q4_XMIN, Q4_YMIN, Q4_XMAX, Q4_YMAX, SNAPSHOT_TICK_MILLIS), - Q4_SNAP_WAS_INSIDE_STORE, Q4_SNAP_ENTRIES_STORE) - .to(Q4_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q5SnapshotProcessor(Q5_SNAP_STORE, - Q5_P_LON, Q5_P_LAT, Q5_D_P_METRES, Q5_D_MEET_METRES, - SNAPSHOT_TICK_MILLIS), - Q5_SNAP_STORE) - .to(Q5_SNAPSHOT_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); - - tripsK0.process(() -> new Q6SnapshotProcessor(Q6_SNAP_STORE, SNAPSHOT_TICK_MILLIS), Q6_SNAP_STORE) - .to(Q6_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q7SnapshotProcessor(Q7_SNAP_STORE, Q7_POIS, SNAPSHOT_TICK_MILLIS), Q7_SNAP_STORE) - .to(Q7_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q8SnapshotProcessor(Q8_SNAP_STORE, - Q8_S1_LON, Q8_S1_LAT, Q8_S2_LON, Q8_S2_LAT, - Q8_RADIUS_METRES, SNAPSHOT_TICK_MILLIS), - Q8_SNAP_STORE) - .to(Q8_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); - - tripsK0.process(() -> new Q9SnapshotProcessor(Q9_SNAP_STORE, - Q9_X_VEHICLE_ID, Q9_Y_VEHICLE_ID, SNAPSHOT_TICK_MILLIS), - Q9_SNAP_STORE) - .to(Q9_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); + if (only == null || only.equals("Q1-continuous")) { + addStore(builder, Q1_SEEN_STORE, Serdes.Integer(), Serdes.Boolean()); + trips.process(() -> new Q1ContinuousProcessor(Q1_SEEN_STORE), Q1_SEEN_STORE) + .to(Q1_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); + } + + if (only == null || only.equals("Q2-continuous")) { + trips.process(() -> new Q2ContinuousProcessor(p.targetId)) + .to(Q2_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), tripSerde)); + } + + if (only == null || only.equals("Q3-continuous")) { + trips.process(() -> new Q3ContinuousProcessor(p.pLon, p.pLat, p.radiusMetres)) + .to(Q3_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Boolean())); + } + + if (only == null || only.equals("Q4-continuous")) { + addStore(builder, Q4_WAS_INSIDE_STORE, Serdes.Integer(), Serdes.Boolean()); + trips.process(() -> new Q4ContinuousProcessor(Q4_WAS_INSIDE_STORE, p.xmin, p.ymin, p.xmax, p.ymax), Q4_WAS_INSIDE_STORE) + .to(Q4_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); + } + + if (only == null || only.equals("Q5-continuous")) { + addStore(builder, Q5_LAST_POS_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q5ContinuousProcessor(Q5_LAST_POS_STORE, p.pLon, p.pLat, p.radiusMetres, p.dMeetMetres), Q5_LAST_POS_STORE) + .to(Q5_CONTINUOUS_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); + } + + if (only == null || only.equals("Q6-continuous")) { + addStore(builder, Q6_STATE_STORE, Serdes.Integer(), Serdes.String()); + trips.process(() -> new Q6ContinuousProcessor(Q6_STATE_STORE), Q6_STATE_STORE) + .to(Q6_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Double())); + } + + if (only == null || only.equals("Q7-continuous")) { + addStore(builder, Q7_FIRST_PASSED_STORE, Serdes.Integer(), Serdes.Long()); + trips.process(() -> new Q7ContinuousProcessor(Q7_FIRST_PASSED_STORE, p.pois), Q7_FIRST_PASSED_STORE) + .to(Q7_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); + } + + if (only == null || only.equals("Q8-continuous")) { + trips.process(() -> new Q8ContinuousProcessor(p.s1Lon, p.s1Lat, p.s2Lon, p.s2Lat, p.radiusMetres)) + .to(Q8_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Boolean())); + } + + if (only == null || only.equals("Q9-continuous")) { + addStore(builder, Q9_STATE_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q9ContinuousProcessor(Q9_STATE_STORE, p.xId, p.yId), Q9_STATE_STORE) + .to(Q9_CONTINUOUS_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); + } + + if (only == null || only.equals("Q1-windowed")) { + addStore(builder, Q1_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q1WindowedProcessor(Q1_WIN_STORE, WIN_MS), Q1_WIN_STORE) + .to(Q1_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); + } + + if (only == null || only.equals("Q2-windowed")) { + addStore(builder, Q2_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q2WindowedProcessor(Q2_WIN_STORE, p.targetId, WIN_MS), Q2_WIN_STORE) + .to(Q2_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q3-windowed")) { + addStore(builder, Q3_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q3WindowedProcessor(Q3_WIN_STORE, p.pLon, p.pLat, p.radiusMetres, WIN_MS), Q3_WIN_STORE) + .to(Q3_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); + } + + if (only == null || only.equals("Q4-windowed")) { + addStore(builder, Q4_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q4WindowedProcessor(Q4_WIN_STORE, p.xmin, p.ymin, p.xmax, p.ymax, WIN_MS), Q4_WIN_STORE) + .to(Q4_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q5-windowed")) { + addStore(builder, Q5_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q5WindowedProcessor(Q5_WIN_STORE, p.pLon, p.pLat, p.radiusMetres, p.dMeetMetres, WIN_MS), Q5_WIN_STORE) + .to(Q5_WINDOWED_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); + } + + if (only == null || only.equals("Q6-windowed")) { + addStore(builder, Q6_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q6WindowedProcessor(Q6_WIN_STORE, WIN_MS), Q6_WIN_STORE) + .to(Q6_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q7-windowed")) { + addStore(builder, Q7_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q7WindowedProcessor(Q7_WIN_STORE, p.pois, WIN_MS), Q7_WIN_STORE) + .to(Q7_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q8-windowed")) { + addStore(builder, Q8_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q8WindowedProcessor(Q8_WIN_STORE, p.s1Lon, p.s1Lat, p.s2Lon, p.s2Lat, p.radiusMetres, WIN_MS), Q8_WIN_STORE) + .to(Q8_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); + } + + if (only == null || only.equals("Q9-windowed")) { + addStore(builder, Q9_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q9WindowedProcessor(Q9_WIN_STORE, p.xId, p.yId, WIN_MS), Q9_WIN_STORE) + .to(Q9_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); + } + + if (only == null || only.equals("Q1-snapshot")) { + addStore(builder, Q1_SNAP_STORE, Serdes.Integer(), Serdes.Long()); + tripsK0.process(() -> new Q1SnapshotProcessor(Q1_SNAP_STORE, p.snapshotTickMillis), Q1_SNAP_STORE) + .to(Q1_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); + } + + if (only == null || only.equals("Q2-snapshot")) { + addStore(builder, Q2_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q2SnapshotProcessor(Q2_SNAP_STORE, p.targetId, p.snapshotTickMillis), Q2_SNAP_STORE) + .to(Q2_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q3-snapshot")) { + addStore(builder, Q3_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q3SnapshotProcessor(Q3_SNAP_STORE, p.pLon, p.pLat, p.radiusMetres, p.snapshotTickMillis), Q3_SNAP_STORE) + .to(Q3_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); + } + + if (only == null || only.equals("Q4-snapshot")) { + addStore(builder, Q4_SNAP_WAS_INSIDE_STORE, Serdes.Integer(), Serdes.Boolean()); + addStore(builder, Q4_SNAP_ENTRIES_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q4SnapshotProcessor(Q4_SNAP_WAS_INSIDE_STORE, Q4_SNAP_ENTRIES_STORE, p.xmin, p.ymin, p.xmax, p.ymax, p.snapshotTickMillis), Q4_SNAP_WAS_INSIDE_STORE, Q4_SNAP_ENTRIES_STORE) + .to(Q4_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q5-snapshot")) { + addStore(builder, Q5_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q5SnapshotProcessor(Q5_SNAP_STORE, p.pLon, p.pLat, p.radiusMetres, p.dMeetMetres, p.snapshotTickMillis), Q5_SNAP_STORE) + .to(Q5_SNAPSHOT_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); + } + + if (only == null || only.equals("Q6-snapshot")) { + addStore(builder, Q6_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q6SnapshotProcessor(Q6_SNAP_STORE, p.snapshotTickMillis), Q6_SNAP_STORE) + .to(Q6_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q7-snapshot")) { + addStore(builder, Q7_SNAP_STORE, Serdes.Integer(), Serdes.Long()); + tripsK0.process(() -> new Q7SnapshotProcessor(Q7_SNAP_STORE, p.pois, p.snapshotTickMillis), Q7_SNAP_STORE) + .to(Q7_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q8-snapshot")) { + addStore(builder, Q8_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q8SnapshotProcessor(Q8_SNAP_STORE, p.s1Lon, p.s1Lat, p.s2Lon, p.s2Lat, p.radiusMetres, p.snapshotTickMillis), Q8_SNAP_STORE) + .to(Q8_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); + } + + if (only == null || only.equals("Q9-snapshot")) { + addStore(builder, Q9_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q9SnapshotProcessor(Q9_SNAP_STORE, p.xId, p.yId, p.snapshotTickMillis), Q9_SNAP_STORE) + .to(Q9_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); + } return builder.build(); } diff --git a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTrip.java b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTrip.java index f512fe7..d725891 100644 --- a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTrip.java +++ b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTrip.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import java.io.Serializable; diff --git a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTripSerde.java b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTripSerde.java index 298fd07..f06f44a 100644 --- a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTripSerde.java +++ b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTripSerde.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/kafka-streams-app/src/main/java/berlinmod/MEOSBridge.java b/kafka-streams-app/src/main/java/berlinmod/MEOSBridge.java new file mode 100644 index 0000000..aa99765 --- /dev/null +++ b/kafka-streams-app/src/main/java/berlinmod/MEOSBridge.java @@ -0,0 +1,128 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package berlinmod; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; + +/** + * Thin wiring from the BerlinMOD streaming-form predicates to MEOS via JMEOS. + * + *

Every spatial predicate evaluates through MEOS: the within-distance + * predicate is the canonical temporal operator {@code edwithin_tgeo_geo} — + * ever-within between the vehicle's {@code tgeogpoint} instant and the query + * geography, in metres on the WGS84 spheroid; region containment is + * {@code eintersects_tgeo_geo} between the point's {@code tgeompoint} instant + * and the region polygon; distances are {@code geog_distance}. This class holds + * no spatial mathematics of its own: it constructs the MEOS inputs and delegates + * the computation to libmeos, initialising MEOS once per stream thread. + */ +public final class MEOSBridge { + + private static final ThreadLocal INITIALIZED = + ThreadLocal.withInitial(() -> Boolean.FALSE); + + private MEOSBridge() { + // utility + } + + private static void ensureInitializedOnThread() { + if (!INITIALIZED.get()) { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + INITIALIZED.set(Boolean.TRUE); + } + } + + /** @return {@code true} iff {@code (lon1, lat1)} is within {@code radiusMetres} + * of {@code (lon2, lat2)} on the WGS84 spheroid, via MEOS {@code edwithin_tgeo_geo}. */ + public static boolean dwithinMetres(double lon1, double lat1, + double lon2, double lat2, double radiusMetres) { + ensureInitializedOnThread(); + return GeneratedFunctions.edwithin_tgeo_geo( + tgeogInst(lon1, lat1), pointGeog(lon2, lat2), radiusMetres) == 1; + } + + /** @return {@code true} iff {@code (pLon, pLat)} is within {@code radiusMetres} + * of the LineString {@code (s1, s2)}, via MEOS {@code edwithin_tgeo_geo}. */ + public static boolean dwithinSegmentMetres(double pLon, double pLat, + double s1Lon, double s1Lat, + double s2Lon, double s2Lat, double radiusMetres) { + ensureInitializedOnThread(); + return GeneratedFunctions.edwithin_tgeo_geo( + tgeogInst(pLon, pLat), lineGeog(s1Lon, s1Lat, s2Lon, s2Lat), radiusMetres) == 1; + } + + /** @return {@code true} iff {@code (lon, lat)} lies in the axis-aligned box, via + * MEOS {@code eintersects_tgeo_geo} against the box polygon (planar, SRID 4326). */ + public static boolean intersectsBox(double lon, double lat, + double xmin, double ymin, double xmax, double ymax) { + ensureInitializedOnThread(); + return GeneratedFunctions.eintersects_tgeo_geo( + tgeomInst(lon, lat), boxPolygon(xmin, ymin, xmax, ymax)) == 1; + } + + /** @return the WGS84 spheroidal distance in metres between two points, via MEOS {@code geog_distance}. */ + public static double distanceMetres(double lon1, double lat1, double lon2, double lat2) { + ensureInitializedOnThread(); + return GeneratedFunctions.geog_distance(pointGeog(lon1, lat1), pointGeog(lon2, lat2)); + } + + /** @return the WGS84 spheroidal distance in metres from a point to the LineString, + * via MEOS {@code geog_distance}. */ + public static double distanceSegmentMetres(double pLon, double pLat, + double s1Lon, double s1Lat, double s2Lon, double s2Lat) { + ensureInitializedOnThread(); + return GeneratedFunctions.geog_distance( + pointGeog(pLon, pLat), lineGeog(s1Lon, s1Lat, s2Lon, s2Lat)); + } + + private static Pointer tgeogInst(double lon, double lat) { + return GeneratedFunctions.tgeogpoint_in( + String.format("SRID=4326;Point(%.7f %.7f)@2000-01-01", lon, lat)); + } + + private static Pointer tgeomInst(double lon, double lat) { + return GeneratedFunctions.tgeompoint_in( + String.format("SRID=4326;Point(%.7f %.7f)@2000-01-01", lon, lat)); + } + + private static Pointer pointGeog(double lon, double lat) { + return GeneratedFunctions.geom_to_geog( + GeneratedFunctions.geom_in(String.format("SRID=4326;Point(%.7f %.7f)", lon, lat), -1)); + } + + private static Pointer lineGeog(double s1Lon, double s1Lat, double s2Lon, double s2Lat) { + return GeneratedFunctions.geom_to_geog(GeneratedFunctions.geom_in(String.format( + "SRID=4326;LineString(%.7f %.7f, %.7f %.7f)", s1Lon, s1Lat, s2Lon, s2Lat), -1)); + } + + private static Pointer boxPolygon(double xmin, double ymin, double xmax, double ymax) { + return GeneratedFunctions.geom_in(String.format( + "SRID=4326;Polygon((%.7f %.7f, %.7f %.7f, %.7f %.7f, %.7f %.7f, %.7f %.7f))", + xmin, ymin, xmax, ymin, xmax, ymax, xmin, ymax, xmin, ymin), -1); + } +} diff --git a/kafka-streams-app/src/main/java/berlinmod/PointOfInterest.java b/kafka-streams-app/src/main/java/berlinmod/PointOfInterest.java index 067f804..040963b 100644 --- a/kafka-streams-app/src/main/java/berlinmod/PointOfInterest.java +++ b/kafka-streams-app/src/main/java/berlinmod/PointOfInterest.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import java.io.Serializable; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q1ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q1ContinuousProcessor.java index 08ce438..9e7183c 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q1ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q1ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q1SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q1SnapshotProcessor.java index 56ed8ea..0e721d2 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q1SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q1SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q1WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q1WindowedProcessor.java index ea12251..2e8b417 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q1WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q1WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q2ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q2ContinuousProcessor.java index e5881af..aa17e79 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q2ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q2ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q2SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q2SnapshotProcessor.java index bdca33d..73e502e 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q2SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q2SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.PunctuationType; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q2WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q2WindowedProcessor.java index c5fb102..fd52696 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q2WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q2WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q3ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q3ContinuousProcessor.java index ca91889..b567e18 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q3ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q3ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -13,9 +38,7 @@ * eventTime, near)} per incoming GPS event. Same predicate semantics * as MobilityFlink's {@code Q3ContinuousFunction}. * - *

Predicate today: pure-Java great-circle distance (see {@link Haversine}). - * TODO(meos): replace with the MEOS {@code edwithin_tgeo_geo} operator via - * the JMEOS bridge. + *

Predicate: {@link MEOS {@code edwithin_tgeo_geo} over WGS84 geographies. */ public class Q3ContinuousProcessor implements Processor { @@ -39,7 +62,7 @@ public void init(ProcessorContext context) { public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; - boolean near = Haversine.withinMetres( + boolean near = MEOSBridge.dwithinMetres( trip.getLon(), trip.getLat(), pLon, pLat, radiusMetres); ctx.forward(new Record<>(trip.getVehicleId(), near, trip.getTimestamp())); } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q3SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q3SnapshotProcessor.java index b9c6063..454372b 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q3SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q3SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -22,7 +47,7 @@ *

Caller keys the input by a constant so the shared cross-vehicle * last-known store lives in one subtask. Per event: update last-known. * Per STREAM_TIME punctuator fire: iterate last-known, evaluate the - * Haversine radius predicate, forward {@code (currentTick, vehicleId)} + * MEOS edwithin_tgeo_geo radius predicate, forward {@code (currentTick, vehicleId)} * for every near vehicle (sorted by vehicleId). */ public class Q3SnapshotProcessor implements Processor { @@ -66,7 +91,7 @@ private void punctuate(long currentStreamTime) { String[] ll = kv.value.split(",", 2); double lon = Double.parseDouble(ll[0]); double lat = Double.parseDouble(ll[1]); - if (Haversine.withinMetres(lon, lat, pLon, pLat, radiusMetres)) { + if (MEOSBridge.dwithinMetres(lon, lat, pLon, pLat, radiusMetres)) { nearIds.add(kv.key); } } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q3WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q3WindowedProcessor.java index 0803aac..d3046ca 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q3WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q3WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -53,7 +78,7 @@ public void init(ProcessorContext context) { public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; - if (!Haversine.withinMetres(trip.getLon(), trip.getLat(), pLon, pLat, radiusMetres)) { + if (!MEOSBridge.dwithinMetres(trip.getLon(), trip.getLat(), pLon, pLat, radiusMetres)) { return; } long winStart = (trip.getTimestamp() / windowSizeMs) * windowSizeMs; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q4ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q4ContinuousProcessor.java index c386b89..cd6b7bc 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q4ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q4ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -14,8 +39,10 @@ * inside-or-outside flag for R; on each event, detect outside → inside * transition and emit {@code (vehicleId, entryTime)}. * - *

TODO(meos): replace the point-in-box predicate with a MEOS - * {@code eintersects} call against an STBox via the JMEOS bridge. + *

Predicate: pure-Java axis-aligned point-in-box. The rectangular region + * is degenerate as a geographic predicate (no projection needed); a generic + * polygon-R variant would route through {@link MEOSBridge} for MEOS + * {@code eintersects_tgeo_geo}. */ public class Q4ContinuousProcessor implements Processor { @@ -52,6 +79,6 @@ public void process(Record record) { } private boolean inBox(double lon, double lat) { - return lon >= xmin && lon <= xmax && lat >= ymin && lat <= ymax; + return MEOSBridge.intersectsBox(lon, lat, xmin, ymin, xmax, ymax); } } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q4SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q4SnapshotProcessor.java index 27a1383..05620d8 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q4SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q4SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -62,8 +87,7 @@ public void init(ProcessorContext context) { public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; - boolean curr = trip.getLon() >= xmin && trip.getLon() <= xmax - && trip.getLat() >= ymin && trip.getLat() <= ymax; + boolean curr = MEOSBridge.intersectsBox(trip.getLon(), trip.getLat(), xmin, ymin, xmax, ymax); Boolean prev = wasInside.get(trip.getVehicleId()); boolean prevInside = prev != null && prev; if (curr && !prevInside) { diff --git a/kafka-streams-app/src/main/java/berlinmod/Q4WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q4WindowedProcessor.java index 346ec34..af7922a 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q4WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q4WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -57,8 +82,7 @@ public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; long winStart = (trip.getTimestamp() / windowSizeMs) * windowSizeMs; - boolean curr = trip.getLon() >= xmin && trip.getLon() <= xmax - && trip.getLat() >= ymin && trip.getLat() <= ymax; + boolean curr = MEOSBridge.intersectsBox(trip.getLon(), trip.getLat(), xmin, ymin, xmax, ymax); String s = winState.get(winStart); // Parse per-vehicle records separated by '|' StringBuilder rebuilt = new StringBuilder(); diff --git a/kafka-streams-app/src/main/java/berlinmod/Q5ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q5ContinuousProcessor.java index 804a6eb..9db02f2 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q5ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q5ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -63,7 +88,7 @@ public void process(Record record) { String[] ll = kv.value.split(",", 2); double lon = Double.parseDouble(ll[0]); double lat = Double.parseDouble(ll[1]); - if (Haversine.withinMetres(lon, lat, pLon, pLat, dPMetres)) { + if (MEOSBridge.dwithinMetres(lon, lat, pLon, pLat, dPMetres)) { ids.add(new int[]{kv.key}); positions.add(new double[]{lon, lat}); } @@ -81,7 +106,7 @@ public void process(Record record) { } for (int i = 0; i < n; i++) { for (int j = i + 1; j < n; j++) { - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( positions.get(i)[0], positions.get(i)[1], positions.get(j)[0], positions.get(j)[1]); if (d <= dMeetMetres) { diff --git a/kafka-streams-app/src/main/java/berlinmod/Q5SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q5SnapshotProcessor.java index 1f79ec8..c42d2f2 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q5SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q5SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -69,7 +94,7 @@ private void punctuate(long currentStreamTime) { String[] ll = kv.value.split(",", 2); double lon = Double.parseDouble(ll[0]); double lat = Double.parseDouble(ll[1]); - if (Haversine.withinMetres(lon, lat, pLon, pLat, dPMetres)) { + if (MEOSBridge.dwithinMetres(lon, lat, pLon, pLat, dPMetres)) { ids.add(new int[]{kv.key}); positions.add(new double[]{lon, lat}); } @@ -86,7 +111,7 @@ private void punctuate(long currentStreamTime) { } for (int i = 0; i < n; i++) { for (int j = i + 1; j < n; j++) { - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( positions.get(i)[0], positions.get(i)[1], positions.get(j)[0], positions.get(j)[1]); if (d <= dMeetMetres) { diff --git a/kafka-streams-app/src/main/java/berlinmod/Q5WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q5WindowedProcessor.java index 5ae0c7f..f6a2181 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q5WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q5WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -106,7 +131,7 @@ private void punctuate(long currentStreamTime) { String[] ll = chunk.substring(colon + 1).split(",", 2); double lon = Double.parseDouble(ll[0]); double lat = Double.parseDouble(ll[1]); - if (Haversine.withinMetres(lon, lat, pLon, pLat, dPMetres)) { + if (MEOSBridge.dwithinMetres(lon, lat, pLon, pLat, dPMetres)) { nearIds.add(new int[]{vid}); positions.add(new double[]{lon, lat}); } @@ -122,7 +147,7 @@ private void punctuate(long currentStreamTime) { } for (int a = 0; a < n; a++) { for (int b = a + 1; b < n; b++) { - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( positions.get(a)[0], positions.get(a)[1], positions.get(b)[0], positions.get(b)[1]); if (d <= dMeetMetres) { diff --git a/kafka-streams-app/src/main/java/berlinmod/Q6ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q6ContinuousProcessor.java index 6e5b9c2..a06e676 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q6ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q6ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -11,15 +36,17 @@ *

"What is each vehicle's cumulative distance travelled so far?" * *

Keyed by vehicleId. Per-vehicle state holds the last-known (lon, lat) - * and the running total in metres. On each event, accumulate the Haversine + * and the running total in metres. On each event, accumulate the MEOS geog_distance * delta and emit the cumulative total. * *

State value uses a small string encoding "lon,lat,total" since the * scaffold avoids declaring a dedicated tuple SerDe; the encoding is * private to this processor. * - *

TODO(meos): replace with the MEOS trajectory {@code length} call via - * the JMEOS bridge. + *

Cumulative distance: per consecutive position-pair via + * {@link MEOSBridge#distanceMetres}. The future "full" path uses MEOS' + * {@code tpoint_length} over an aggregated trajectory; the per-event + * cumulative form is the same numeric quantity either way. */ public class Q6ContinuousProcessor implements Processor { @@ -50,7 +77,7 @@ public void process(Record record) { double lastLon = Double.parseDouble(parts[0]); double lastLat = Double.parseDouble(parts[1]); double prevTotal = Double.parseDouble(parts[2]); - total = prevTotal + Haversine.distanceMetres(lastLon, lastLat, trip.getLon(), trip.getLat()); + total = prevTotal + MEOSBridge.distanceMetres(lastLon, lastLat, trip.getLon(), trip.getLat()); } state.put(trip.getVehicleId(), trip.getLon() + "," + trip.getLat() + "," + total); ctx.forward(new Record<>(trip.getVehicleId(), total, trip.getTimestamp())); diff --git a/kafka-streams-app/src/main/java/berlinmod/Q6SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q6SnapshotProcessor.java index 4865fcd..3ef922c 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q6SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q6SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -20,7 +45,7 @@ * up to T?" * *

Caller keys the input by a constant. State value "lon,lat,total" - * per vehicleId. Per event: accumulate Haversine delta. Per STREAM_TIME + * per vehicleId. Per event: accumulate the MEOS geog_distance delta. Per STREAM_TIME * punctuator fire: emit {@code (currentTick, vehicleId, total)} for * every vehicle (sorted by vehicleId), encoded as "vid:total". */ @@ -57,7 +82,7 @@ public void process(Record record) { double lastLon = Double.parseDouble(parts[0]); double lastLat = Double.parseDouble(parts[1]); double prevTotal = Double.parseDouble(parts[2]); - total = prevTotal + Haversine.distanceMetres(lastLon, lastLat, trip.getLon(), trip.getLat()); + total = prevTotal + MEOSBridge.distanceMetres(lastLon, lastLat, trip.getLon(), trip.getLat()); } state.put(trip.getVehicleId(), trip.getLon() + "," + trip.getLat() + "," + total); } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q6WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q6WindowedProcessor.java index 4d1c359..b276dd7 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q6WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q6WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -20,7 +45,7 @@ * during the window." * *

State encodes per-window per-vehicle {@code "vid:lastLon,lastLat,total|..."}. - * On each event, accumulate Haversine delta from the previous in-window + * On each event, accumulate the MEOS geog_distance delta from the previous in-window * position. On punctuator: emit per-vehicle totals for closed windows. */ public class Q6WindowedProcessor implements Processor { @@ -61,7 +86,7 @@ public void process(Record record) { double lastLon = Double.parseDouble(f[0]); double lastLat = Double.parseDouble(f[1]); double prevTotal = Double.parseDouble(f[2]); - double newTotal = prevTotal + Haversine.distanceMetres( + double newTotal = prevTotal + MEOSBridge.distanceMetres( lastLon, lastLat, trip.getLon(), trip.getLat()); if (rebuilt.length() > 0) rebuilt.append("|"); rebuilt.append(vid).append(":") diff --git a/kafka-streams-app/src/main/java/berlinmod/Q7ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q7ContinuousProcessor.java index 4170fc9..62d254c 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q7ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q7ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -46,7 +71,7 @@ public void process(Record record) { for (PointOfInterest poi : pois) { int compositeKey = compositeKey(vehicleId, poi.id); if (firstPassed.get(compositeKey) != null) continue; - if (Haversine.withinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { + if (MEOSBridge.dwithinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { firstPassed.put(compositeKey, trip.getTimestamp()); ctx.forward(new Record<>(poi.id, trip.getTimestamp(), trip.getTimestamp())); } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q7SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q7SnapshotProcessor.java index 6374a56..acc2692 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q7SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q7SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -55,7 +80,7 @@ public void process(Record record) { for (PointOfInterest poi : pois) { int composite = trip.getVehicleId() * 1000 + poi.id; if (firstPassed.get(composite) != null) continue; - if (Haversine.withinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { + if (MEOSBridge.dwithinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { firstPassed.put(composite, trip.getTimestamp()); } } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q7WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q7WindowedProcessor.java index 7eb08aa..e9cfda2 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q7WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q7WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -59,7 +84,7 @@ public void process(Record record) { for (PointOfInterest poi : pois) { String marker = trip.getVehicleId() + ":" + poi.id + ":"; if (s.contains(marker)) continue; - if (Haversine.withinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { + if (MEOSBridge.dwithinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { if (appended.length() > 0) appended.append(","); appended.append(trip.getVehicleId()).append(":").append(poi.id).append(":").append(trip.getTimestamp()); } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q8ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q8ContinuousProcessor.java index d910061..7c1d96a 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q8ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q8ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -36,7 +61,7 @@ public void init(ProcessorContext context) { public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; - boolean near = SegmentDistance.withinMetres( + boolean near = MEOSBridge.dwithinSegmentMetres( trip.getLon(), trip.getLat(), s1Lon, s1Lat, s2Lon, s2Lat, radiusMetres); diff --git a/kafka-streams-app/src/main/java/berlinmod/Q8SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q8SnapshotProcessor.java index e6aed09..f8bdb15 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q8SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q8SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -67,7 +92,7 @@ private void punctuate(long currentStreamTime) { String[] ll = kv.value.split(",", 2); double lon = Double.parseDouble(ll[0]); double lat = Double.parseDouble(ll[1]); - if (SegmentDistance.withinMetres(lon, lat, s1Lon, s1Lat, s2Lon, s2Lat, radiusMetres)) { + if (MEOSBridge.dwithinSegmentMetres(lon, lat, s1Lon, s1Lat, s2Lon, s2Lat, radiusMetres)) { nearIds.add(kv.key); } } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q8WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q8WindowedProcessor.java index f36e648..ca908f6 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q8WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q8WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -57,7 +82,7 @@ public void init(ProcessorContext context) { public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; - if (!SegmentDistance.withinMetres(trip.getLon(), trip.getLat(), + if (!MEOSBridge.dwithinSegmentMetres(trip.getLon(), trip.getLat(), s1Lon, s1Lat, s2Lon, s2Lat, radiusMetres)) { return; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q9ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q9ContinuousProcessor.java index f26dd46..e528f6d 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q9ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q9ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -60,7 +85,7 @@ public void process(Record record) { if (!xSlot.startsWith("NaN") && !ySlot.startsWith("NaN")) { String[] x = xSlot.split(",", 2); String[] y = ySlot.split(",", 2); - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( Double.parseDouble(x[0]), Double.parseDouble(x[1]), Double.parseDouble(y[0]), Double.parseDouble(y[1])); ctx.forward(new Record<>(trip.getTimestamp(), d, trip.getTimestamp())); diff --git a/kafka-streams-app/src/main/java/berlinmod/Q9SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q9SnapshotProcessor.java index 50c624f..e966eb7 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q9SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q9SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.PunctuationType; @@ -71,7 +96,7 @@ private void punctuate(long currentStreamTime) { if (parts[0].startsWith("NaN") || parts[1].startsWith("NaN")) return; String[] x = parts[0].split(",", 2); String[] y = parts[1].split(",", 2); - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( Double.parseDouble(x[0]), Double.parseDouble(x[1]), Double.parseDouble(y[0]), Double.parseDouble(y[1])); ctx.forward(new Record<>(tick, d, tick)); diff --git a/kafka-streams-app/src/main/java/berlinmod/Q9WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q9WindowedProcessor.java index ec0ca3c..12631e4 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q9WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q9WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -91,7 +116,7 @@ private void punctuate(long currentStreamTime) { if (!parts[0].startsWith("NaN") && !parts[1].startsWith("NaN")) { String[] x = parts[0].split(",", 2); String[] y = parts[1].split(",", 2); - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( Double.parseDouble(x[0]), Double.parseDouble(x[1]), Double.parseDouble(y[0]), Double.parseDouble(y[1])); ctx.forward(new Record<>(winStart, d, winStart + windowSizeMs - 1)); diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosBoundedStateProcessor.java b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosBoundedStateProcessor.java new file mode 100644 index 0000000..c92f863 --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosBoundedStateProcessor.java @@ -0,0 +1,164 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings; + +import jnr.ffi.Pointer; +import org.apache.kafka.streams.processor.api.Processor; +import org.apache.kafka.streams.processor.api.ProcessorContext; +import org.apache.kafka.streams.processor.api.Record; +import org.apache.kafka.streams.state.KeyValueStore; + +import java.io.Serializable; + +/** + * Kafka Streams wiring for the {@code bounded-state} streaming tier of + * the generated {@code org.mobilitydb.meos.MeosOps*} facades. + * + *

Wraps any {@code bounded-state} MeosOps method (per the v4 baseline: + * 797 of 2,097 emitted methods — 513 OO-classified + 284 free-fn) as a + * Kafka Streams {@link Processor} that holds per-key MEOS-handle state + * across records via a {@link KeyValueStore}. + * + *

Why state lives as bytes, not as a {@code Pointer}. Mirrors + * the same discipline as the Flink wirings: a {@code jnr.ffi.Pointer} + * is a raw native-memory address. It does not survive across Kafka + * Streams' state-store fault-tolerance / changelog-replay / + * task-rebalance paths (the state-store changelog is a Kafka topic; + * state must be byte-serializable to be replayable). The wiring stores + * state as {@code byte[]} (typically MEOS-WKB or MEOS-WKT — adopter's + * choice) with three adopter-supplied lambdas mediating the round-trip: + * + * 

{@code
+ *  byte[] state                  -- per-key serialized MEOS value (in state store)
+ *      ↓ deserialize (bytes → Pointer)
+ *  Pointer prev                  -- in-flight MEOS handle
+ *      ↓ step(prev, record) → (newPointer, output)
+ *  Pointer next, OUT out         -- new in-flight handle + per-record output
+ *      ↓ serialize (Pointer → bytes)
+ *  byte[] newState               -- new per-key serialized MEOS value (back to store)
+ * }
+ * + *

First record for a key sees {@code prior == null} — the wiring + * skips deserialize and lets the step seed state. + * + *

Typical usage — per-vehicle running tbox union via + * {@code MeosOpsFreeCore.union_tbox_tbox}: + * + * 

{@code
+ * Topology topology = new Topology();
+ * topology.addSource("src", ...);
+ * topology.addProcessor(
+ *     "running-union",
+ *     () -> new MeosBoundedStateProcessor(
+ *         "running-union-state",
+ *         ptr -> MeosOpsTBox.tbox_out(ptr, 6).getBytes(StandardCharsets.UTF_8),
+ *         bytes -> MeosOpsTBox.tbox_in(new String(bytes, StandardCharsets.UTF_8)),
+ *         (prior, record) -> { ... return new MeosStep<>(newState, record.withValue(...)); }),
+ *     "src");
+ * topology.addStateStore(
+ *     Stores.keyValueStoreBuilder(
+ *         Stores.persistentKeyValueStore("running-union-state"),
+ *         Serdes.String(), Serdes.ByteArray()),
+ *     "running-union");
+ * }
+ * + * @param key type + * @param input record value type + * @param output key type (typically same as KIn) + * @param output value type + */ +public final class MeosBoundedStateProcessor + implements Processor { + + /** Serializable Pointer → bytes serializer (typically MEOS-WKB or MEOS-WKT). */ + @FunctionalInterface + public interface PointerSerialize extends Serializable { + byte[] toBytes(Pointer pointer); + } + + /** Serializable bytes → Pointer deserializer (typically MEOS-WKB or MEOS-WKT). */ + @FunctionalInterface + public interface PointerDeserialize extends Serializable { + Pointer fromBytes(byte[] bytes); + } + + /** Per-record step: (prior MEOS handle, input record) → (new handle, optional output record). */ + @FunctionalInterface + public interface MeosStepFn extends Serializable { + MeosStep apply(Pointer prior, Record record); + } + + /** Tuple returned by the step lambda. */ + public static final class MeosStep { + public final Pointer newState; + public final Record output; // null = no forward + public MeosStep(Pointer newState, Record output) { + this.newState = newState; + this.output = output; + } + } + + private final String stateStoreName; + private final PointerSerialize serialize; + private final PointerDeserialize deserialize; + private final MeosStepFn step; + + private KeyValueStore store; + private ProcessorContext context; + + public MeosBoundedStateProcessor(String stateStoreName, + PointerSerialize serialize, + PointerDeserialize deserialize, + MeosStepFn step) { + this.stateStoreName = stateStoreName; + this.serialize = serialize; + this.deserialize = deserialize; + this.step = step; + } + + @Override + public void init(ProcessorContext context) { + this.context = context; + this.store = context.getStateStore(stateStoreName); + } + + @Override + public void process(Record record) { + byte[] priorBytes = store.get(record.key()); + Pointer prior = (priorBytes == null) ? null : deserialize.fromBytes(priorBytes); + + MeosStep stepResult = step.apply(prior, record); + + store.put(record.key(), serialize.toBytes(stepResult.newState)); + + if (stepResult.output != null) { + context.forward(stepResult.output); + } + } + + @Override + public void close() { /* nothing to release; MEOS handles are short-lived per record */ } +} diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosCrossStreamJoiner.java b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosCrossStreamJoiner.java new file mode 100644 index 0000000..7583304 --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosCrossStreamJoiner.java @@ -0,0 +1,99 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings; + +import org.apache.kafka.streams.kstream.ValueJoiner; + +import java.io.Serializable; + +/** + * Kafka Streams wiring for the {@code cross-stream} streaming tier of + * the generated {@code org.mobilitydb.meos.MeosOps*} facades. + * + *

The {@code cross-stream} tier (140 of 2,097 emitted methods per + * v4 baseline) is pairwise across two pre-keyed streams within a + * time-bounded match window. Canonical examples are spatial + * relations between two trajectories + * ({@code edwithin_tgeo_tgeo}, {@code eintersects_tgeo_tgeo}) and + * distance functions on two temporals + * ({@code nad_tgeo_tgeo}, {@code mindistance_tgeo_tgeo}). + * + *

Kafka Streams' {@code KStream.join(otherStream, joiner, + * JoinWindows.of(...))} builder calls a {@link ValueJoiner} per + * matched pair; this helper supplies a serializable factory wrapping + * an adopter lambda that calls the MEOS cross-stream method on the + * pair. + * + *

Typical usage — per-vehicle-pair "did they come within + * 100m of each other in the last 5 minutes?" via + * {@code MeosOpsTGeo.edwithin_tgeo_tgeo}: + * + * 

{@code
+ * KStream a = ...;   // keyed by regionId
+ * KStream b = ...;   // same key space
+ *
+ * KStream meetings = a.join(
+ *     b,
+ *     MeosCrossStreamJoiner.joiner((left, right) -> {
+ *         Pointer leftT  = left.toTGeoPointer();
+ *         Pointer rightT = right.toTGeoPointer();
+ *         if (MeosOpsTGeo.edwithin_tgeo_tgeo(leftT, rightT, 100.0) != 0) {
+ *             return new MeetingEvent(left.id(), right.id(), System.currentTimeMillis());
+ *         }
+ *         return null;  // joined out
+ *     }),
+ *     JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(5)));
+ * }
+ * + *

The join is keyed (both streams must share key space, and only + * records sharing a key are considered for pairing). The match window + * is time-bounded via {@code JoinWindows.ofTimeDifferenceWithNoGrace(...)} + * (or the grace-period variant), event-time-aware. + * + *

For non-matches, the lambda can return {@code null} and chain + * a downstream {@code .filter((k, v) -> v != null)} — Kafka Streams + * forwards null values through the joiner; the filter prunes them. + */ +public final class MeosCrossStreamJoiner { + + /** Serializable per-match MEOS pairwise call. */ + @FunctionalInterface + public interface MeosJoinFn extends ValueJoiner, Serializable { + @Override OUT apply(L left, R right); + } + + private MeosCrossStreamJoiner() { /* utility */ } + + /** + * Wrap a serializable {@code (L, R) -> OUT} as a Kafka-Streams + * {@link ValueJoiner}. Use with {@link org.apache.kafka.streams.kstream.KStream#join( + * org.apache.kafka.streams.kstream.KStream, + * ValueJoiner, org.apache.kafka.streams.kstream.JoinWindows)}. + */ + public static ValueJoiner joiner(MeosJoinFn fn) { + return fn; + } +} diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosOpsRuntime.java b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosOpsRuntime.java new file mode 100644 index 0000000..43f5cae --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosOpsRuntime.java @@ -0,0 +1,56 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings; + +import org.mobilitydb.meos.MeosOpsTBox; + +/** + * Convenience re-export of the shared + * {@code org.mobilitydb.meos.MeosOpsRuntime.MEOS_AVAILABLE} flag + * for the wirings layer. + * + *

The codegen package already supplies the single + * {@code MeosOpsRuntime} static initializer that probes libmeos exactly + * once per JVM and exposes the result via every {@code MeosOps*.MEOS_AVAILABLE} + * accessor. This class is just a wirings-package alias that lets wiring + * code stay package-local without pulling in the full + * {@code org.mobilitydb.meos.MeosOpsRuntime} type for what should + * be a single boolean read. + * + *

The flag is checked inside every generated MeosOps method's body + * (via a thrown {@link UnsupportedOperationException} when false), so + * wiring code rarely needs to consult it explicitly — but when a + * Kafka Streams topology wants to short-circuit before submitting + * (e.g. a clean exit message at startup if libmeos isn't loadable), + * {@code MeosOpsRuntime.MEOS_AVAILABLE} is the canonical place to read. + */ +public final class MeosOpsRuntime { + + /** Shared MEOS-available flag (set once per JVM by the codegen runtime). */ + public static final boolean MEOS_AVAILABLE = MeosOpsTBox.MEOS_AVAILABLE; + + private MeosOpsRuntime() { /* utility */ } +} diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosStatelessOps.java b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosStatelessOps.java new file mode 100644 index 0000000..6305703 --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosStatelessOps.java @@ -0,0 +1,117 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings; + +import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.ValueMapper; + +import java.io.Serializable; + +/** + * Kafka Streams DSL wirings for the {@code stateless} streaming tier of + * the generated {@code org.mobilitydb.meos.MeosOps*} facades. + * + *

Kafka Streams' DSL is lambda-driven — {@code KStream.mapValues} + * and {@code KStream.filter} accept {@link ValueMapper} and + * {@link Predicate} interfaces directly. This class supplies tiny + * helper factories that wrap a generated MeosOps method (or any + * stateless function over a MEOS value) into the DSL-typed shape, + * with serializability + the shared {@link MeosOpsRuntime#MEOS_AVAILABLE} + * probe baked in. + * + *

Per the v4 streaming-relevance baseline, 804 of the 2,097 + * generated methods are {@code stateless} (92 OO-classified + 712 + * free-fn) — any of them can flow through these two helpers without + * per-method registration. + * + *

Typical usage — scalar-predicate filter using the generated + * {@code MeosOpsFreeCore.overlaps_tbox_tbox} (tier = {@code stateless}): + * + * 

{@code
+ * KStream in = ...;
+ * KStream overlapping = in.filter(
+ *     MeosStatelessOps.intPredicate(
+ *         (key, pair) -> MeosOpsFreeCore.overlaps_tbox_tbox(pair.a, pair.b)));
+ * }
+ * + *

Or per-record transform: + * + * 

{@code
+ * KStream hexOut = in.mapValues(
+ *     MeosStatelessOps.mapper(
+ *         tbox -> MeosOpsTBox.tbox_as_hexwkb(tbox, (byte) 4, null)));
+ * }
+ * + *

Both helpers throw {@link UnsupportedOperationException} when + * libmeos is unavailable (the underlying generated MeosOps methods do; + * these helpers preserve the exception shape). + */ +public final class MeosStatelessOps { + + /** Serializable boolean-returning per-record MEOS predicate. */ + @FunctionalInterface + public interface MeosPredicate extends Predicate, Serializable { + @Override boolean test(K key, V value); + } + + /** Serializable int-returning per-record MEOS predicate (0/1 flag). */ + @FunctionalInterface + public interface MeosIntPredicate extends Serializable { + int test(K key, V value); + } + + /** Serializable per-record MEOS value-mapper. */ + @FunctionalInterface + public interface MeosMapper extends ValueMapper, Serializable { + @Override R apply(V value); + } + + private MeosStatelessOps() { /* utility */ } + + /** + * Wrap a serializable {@code (K, V) -> boolean} as a Kafka-Streams + * {@link Predicate}. Use with {@link org.apache.kafka.streams.kstream.KStream#filter}. + */ + public static Predicate predicate(MeosPredicate p) { + return p; + } + + /** + * Adapt an {@code int}-returning generated MEOS predicate (treating + * non-zero as {@code true}) into a Kafka-Streams {@link Predicate}. + */ + public static Predicate intPredicate(MeosIntPredicate p) { + return (k, v) -> p.test(k, v) != 0; + } + + /** + * Wrap a serializable {@code V -> R} as a Kafka-Streams + * {@link ValueMapper}. Use with {@link org.apache.kafka.streams.kstream.KStream#mapValues}. + */ + public static ValueMapper mapper(MeosMapper m) { + return m; + } +} diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosWindowedAggregator.java b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosWindowedAggregator.java new file mode 100644 index 0000000..9d719e2 --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosWindowedAggregator.java @@ -0,0 +1,114 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings; + +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.Initializer; + +import java.io.Serializable; + +/** + * Kafka Streams wiring for the {@code windowed} streaming tier of the + * generated {@code org.mobilitydb.meos.MeosOps*} facades. + * + *

The {@code windowed} tier (per the v4 baseline: 161 of 2,097 + * emitted methods) emits one MEOS-derived value per window. Canonical + * examples are {@code temporal_length(tgeo)} (one length per + * trajectory window) and {@code temporal_twavg(tnumber)} (one + * time-weighted average per window). + * + *

Kafka Streams' {@code KStream.groupByKey().windowedBy(...).aggregate( + * Initializer, Aggregator, Materialized<...>)} + * builder is lambda-shaped; this helper supplies serializable + * {@link Initializer} and {@link Aggregator} factories that: + * + *

+ * + *

Window-close vs running-emit. Unlike Flink's + * {@code ProcessWindowFunction} which fires once at window close, + * Kafka Streams' suppress-less default emits a record per update. + * For window-close-only semantics, chain {@code .suppress(Suppressed.untilWindowCloses(...))} + * downstream — out of scope for this helper, but the standard recipe. + * + *

State-store discipline: same as {@link MeosBoundedStateProcessor} — + * raw {@code Pointer} doesn't survive changelog replay; the aggregator + * value type {@code A} should be byte-serializable + * (typically {@code byte[]} encoded MEOS-WKB / MEOS-WKT). Adopters + * configure the corresponding {@code Serde} via + * {@code Materialized.with(keySerde, valueSerde)}. + * + * @param key type + * @param input record value type + * @param aggregator accumulator type (byte-serializable; typically byte[]) + */ +public final class MeosWindowedAggregator { + + private MeosWindowedAggregator() { /* utility */ } + + /** Serializable per-window initial-value supplier. */ + @FunctionalInterface + public interface MeosInitializer extends Initializer, Serializable { + @Override A apply(); + } + + /** Serializable per-event accumulator step. */ + @FunctionalInterface + public interface MeosAggregator extends Aggregator, Serializable { + @Override A apply(K key, V value, A aggregate); + } + + /** + * Wrap a serializable {@code () -> A} as a Kafka-Streams + * {@link Initializer}. + */ + public static Initializer initializer(MeosInitializer init) { + return init; + } + + /** + * Wrap a serializable {@code (K, V, A) -> A} as a Kafka-Streams + * {@link Aggregator}. + * + *

The lambda receives the current key, the per-event value, and + * the running aggregator state; returns the new aggregator state. + * Per the wirings discipline, the {@code A} type should be + * byte-serializable so changelog-replay / state-rebalance work + * across Kafka Streams' fault-tolerance paths. + */ + public static Aggregator aggregator(MeosAggregator agg) { + return agg; + } +} diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/README.md b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/README.md new file mode 100644 index 0000000..fb4a855 --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/README.md @@ -0,0 +1,94 @@ +# Kafka Streams wirings for the generated MEOS facades + +This package supplies thin, generic Kafka-Streams-DSL wrappers around +the generated `org.mobilitydb.meos.MeosOps*` facades, organized +per **streaming tier** (per +`tools/codegen/meos-ops-manifest.json` + `tools/codegen/meos-ops-free-manifest.json`). + +Mirrors the MobilityFlink wirings package +([`org.mobilitydb.flink.meos.wirings`](https://github.com/MobilityDB/MobilityFlink/blob/main/flink-processor/src/main/java/org/mobilitydb/flink/meos/wirings)) +on the Kafka side; the tier classification is identical, the engine +realizations differ (Kafka Streams DSL → lambda-shaped wirings; Flink +DataStream → class-shaped wirings). + +| Tier | Wiring class(es) here | Method count (v4 baseline) | +|---|---|---:| +| `stateless` | [`MeosStatelessOps`](MeosStatelessOps.java) — `predicate(...)` / `intPredicate(...)` / `mapper(...)` factories returning serializable `Predicate` / `ValueMapper` for `KStream.filter` / `.mapValues` | 804 | +| `bounded-state` | [`MeosBoundedStateProcessor`](MeosBoundedStateProcessor.java) — `Processor` with `KeyValueStore`; per-key MEOS-handle state crosses the operator boundary as `byte[]` (MEOS-WKB or MEOS-WKT) so Kafka Streams' changelog-replay / rebalance / state-rebuild paths work correctly | 797 | +| `windowed` | [`MeosWindowedAggregator`](MeosWindowedAggregator.java) — `initializer(...)` + `aggregator(...)` factories for `KStream.groupByKey().windowedBy(...).aggregate(...)`; pairs with `Materialized.with(...)` for the window state store | 161 | +| `cross-stream` | [`MeosCrossStreamJoiner`](MeosCrossStreamJoiner.java) — `joiner(...)` factory wrapping a serializable `ValueJoiner` for `KStream.join(other, ValueJoiner, JoinWindows)`; same-key pairing, time-bounded match window | 140 | +| `io-meta` | covered by `MeosStatelessOps.mapper(...)` (no state, no window) | 195 | +| `sequence-only` | inherently non-streamable — no wiring | 14 | + +**Cumulative coverage**: same as the Flink side — **2,097 of 2,097 +emitted methods (100%)** wirable through 5 generic wirings classes +without per-method registration. + +## Why DSL-shaped (lambda-driven) wirings rather than class-shaped + +Kafka Streams' DSL is lambda-first: `KStream.mapValues((k, v) -> …)`, +`KStream.filter((k, v) -> …)`, `KStream.groupByKey().aggregate(init, agg, …)`. +Most tiers can be wired with a single serializable lambda; only +`bounded-state` requires a full `Processor` class for state-store +binding. The wirings here reflect that asymmetry — small static-helper +factory classes for the lambda-shaped tiers, and a real +`Processor` class only for `bounded-state`. + +Adopters who want a class-shaped wiring (matching the Flink layout +for cross-binding parity) can subclass any of these helpers; the +serializable functional interfaces (`MeosPredicate`, `MeosMapper`, +`MeosStepFn`, `MeosAggregator`, etc.) are public. + +## How a generated MEOS call becomes a Kafka Streams operator + +```java +// 1. Pick the generated MeosOps method (Javadoc tier marker tells you which wiring) +boolean overlap = MeosOpsFreeCore.overlaps_tbox_tbox(boxA, boxB); // tier = stateless + +// 2. Wrap with the matching wiring factory +KStream overlapping = stream.filter( + MeosStatelessOps.intPredicate( + (key, pair) -> MeosOpsFreeCore.overlaps_tbox_tbox(pair.a, pair.b))); +``` + +`MEOS_AVAILABLE` is probed once per JVM by the shared +`org.mobilitydb.meos.MeosOpsRuntime` static initializer (the same +runtime the codegen package uses). When unavailable, every generated +method throws `UnsupportedOperationException` with a clear message — +the wirings layer doesn't have to handle that itself. + +## End-to-end runnable demo + +[`demo/MeosWiringsDemoTopology`](demo/MeosWiringsDemoTopology.java) is +a Kafka Streams topology that composes all four tier wirings in a +single pipeline: + +1. **Source**: `vehicle-events` topic — `(regionId, tboxWKT)` records. +2. **Stateless filter** (`MeosStatelessOps.predicate`) — drop events for regions outside the interest set. +3. **Bounded-state processor** (`MeosBoundedStateProcessor`) — per-region running tbox union, state stored as MEOS-WKT bytes in a `KeyValueStore`. +4. **Windowed aggregator** (`MeosWindowedAggregator.aggregator`) — per-region 30s tumbling window, accumulator is the latest running-union WKT. +5. **Cross-stream joiner** (`MeosCrossStreamJoiner.joiner`) — join the windowed vehicle aggregates with a `region-queries` topic on shared `regionId`, ±1m time bound; emit on tbox overlap. +6. **Sink**: `overlap-output` topic. + +Run with: + +```bash +mvn -q exec:java \ + -Dexec.mainClass=org.mobilitydb.kafka.meos.wirings.demo.MeosWiringsDemoTopology \ + -Dmeos.enabled=true +``` + +The demo uses `TopologyTestDriver` (kafka-streams-test-utils) — no +Kafka broker required. The `main()` method always prints the topology +description; when `MEOS_AVAILABLE`, it also instantiates the +`TopologyTestDriver` to validate the topology end-to-end at startup. + +## Coexistence with `berlinmod.MEOSBridge` + +`berlinmod.MEOSBridge` (introduced on `feat/jmeos-bridge-swap`) is the +hand-written, BerlinMOD-scoped bridge for the 9-query streaming-form +parity matrix — high-level and query-shaped. The wirings here are +low-level and catalog-shaped — applicable to any of the ~1,800 +streamable generated methods, not just the BerlinMOD-9 subset. Both +share the same `MEOS_AVAILABLE` discipline (via `MeosOpsRuntime`) and +the same `functions.GeneratedFunctions` delegation. diff --git a/kafka-streams-app/src/test/java/berlinmod/BerlinMODBenchmark.java b/kafka-streams-app/src/test/java/berlinmod/BerlinMODBenchmark.java new file mode 100644 index 0000000..8decf09 --- /dev/null +++ b/kafka-streams-app/src/test/java/berlinmod/BerlinMODBenchmark.java @@ -0,0 +1,144 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package berlinmod; + +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.TestInputTopic; +import org.apache.kafka.streams.TestOutputTopic; +import org.apache.kafka.streams.TopologyTestDriver; + +import java.lang.reflect.Field; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.TreeMap; + +/** + * Per-cell throughput benchmark for the BerlinMOD-9 × 3-form streaming matrix on + * Kafka Streams. + * + *

Each cell runs in isolation through its own single-cell topology + * ({@link BerlinMODTopology#buildCell}) and {@link TopologyTestDriver} over the + * corpus — the Kafka-Streams analog of MobilityFlink's per-cell jobs, so the + * per-cell wall-clock and throughput are independent and comparable. The corpus + * and the per-query parameters are corpus-derived via {@link BerlinMODCorpus}; + * the spatial predicates evaluate through MEOS (see {@link MEOSBridge}). + * + * 

+ *   java … berlinmod.BerlinMODBenchmark --csv <berlinmod_instants.csv> [--max N]
+ *   java … berlinmod.BerlinMODBenchmark --vehicles 50 --events 600 [--only Q3-continuous]
+ *
+ */ +public final class BerlinMODBenchmark { + + private BerlinMODBenchmark() { /* utility */ } + + public static void main(String[] args) throws Exception { + String csv = null, only = null; + int maxRows = 0; + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "--csv": csv = args[++i]; break; + case "--max": maxRows = Integer.parseInt(args[++i]); break; + case "--only": only = args[++i]; break; + default: break; + } + } + if (csv == null) { + System.err.println("--csv is required: the benchmark runs " + + "on the canonical BerlinMOD corpus only."); + System.exit(2); + } + List corpus = BerlinMODCorpus.fromInstantsCsv(csv, maxRows); + int n = corpus.size(); + long maxTs = corpus.stream().mapToLong(BerlinMODTrip::getTimestamp).max().orElse(0L); + BerlinMODCorpus.Params p = BerlinMODCorpus.derive(corpus); + System.out.printf("Corpus: %s, %d events; window=%ds tick=%dms%n", + csv != null ? "real BerlinMOD instants" : "synthetic", n, p.windowSeconds, p.snapshotTickMillis); + + // (cellName, outputTopic) enumerated from the topology's *_OUTPUT constants. + TreeMap cells = new TreeMap<>(); + for (Field f : BerlinMODTopology.class.getFields()) { + if (f.getName().endsWith("_OUTPUT") && f.getType() == String.class) { + cells.put(cellName(f.getName()), (String) f.get(null)); + } + } + + Properties props = new Properties(); + props.put(StreamsConfig.APPLICATION_ID_CONFIG, "berlinmod-bench"); + props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:9092"); + props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); + props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.ByteArray().getClass()); + BerlinMODTripSerde tripSerde = new BerlinMODTripSerde(); + + List rows = new ArrayList<>(); + for (var e : cells.entrySet()) { + String cell = e.getKey(), topic = e.getValue(); + if (only != null && !cell.equals(only)) { + continue; + } + long out; + long t0 = System.nanoTime(); + try (TopologyTestDriver driver = new TopologyTestDriver(BerlinMODTopology.buildCell(p, cell), props)) { + TestInputTopic input = driver.createInputTopic( + BerlinMODTopology.INPUT_TOPIC, new IntegerSerializer(), tripSerde.serializer()); + for (BerlinMODTrip trip : corpus) { + input.pipeInput(trip.getVehicleId(), trip, Instant.ofEpochMilli(trip.getTimestamp())); + } + input.pipeInput(-1, new BerlinMODTrip(-1, maxTs + 3_600_000L, 0.0, 0.0), + Instant.ofEpochMilli(maxTs + 3_600_000L)); + input.pipeInput(-1, new BerlinMODTrip(-1, maxTs + 7_200_000L, 0.0, 0.0), + Instant.ofEpochMilli(maxTs + 7_200_000L)); + TestOutputTopic output = driver.createOutputTopic( + topic, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + out = output.readRecordsToList().size(); + } + long wallMs = (System.nanoTime() - t0) / 1_000_000L; + double tput = wallMs > 0 ? n / (wallMs / 1000.0) : 0; + rows.add(new String[]{cell, String.valueOf(n), String.valueOf(out), + String.valueOf(wallMs), String.format("%,.0f", tput)}); + System.out.printf(" %-14s out=%-8d %6d ms %,.0f ev/s%n", cell, out, wallMs, tput); + } + + System.out.println(); + System.out.println("| Cell | Events in | Output rows | Wall (ms) | Throughput (ev/s) |"); + System.out.println("|---|---:|---:|---:|---:|"); + for (String[] r : rows) { + System.out.printf("| %s | %s | %s | %s | %s |%n", r[0], r[1], r[2], r[3], r[4]); + } + } + + /** Q3_CONTINUOUS_OUTPUT → Q3-continuous */ + private static String cellName(String field) { + String s = field.substring(0, field.length() - "_OUTPUT".length()); + int us = s.indexOf('_'); + return s.substring(0, us) + "-" + s.substring(us + 1).toLowerCase(); + } +} diff --git a/kafka-streams-app/src/test/java/berlinmod/BerlinMODSetSetJoinTest.java b/kafka-streams-app/src/test/java/berlinmod/BerlinMODSetSetJoinTest.java new file mode 100644 index 0000000..3dba8f6 --- /dev/null +++ b/kafka-streams-app/src/test/java/berlinmod/BerlinMODSetSetJoinTest.java @@ -0,0 +1,120 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ +package berlinmod; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.mobilitydb.meos.MeosSetSetJoin; + +import java.util.HashSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Verifies the BerlinMOD trip-level NxN spatial join (the kernel-pruned + * {@link MeosSetSetJoin} set-set family) against an independent per-pair scalar + * baseline ({@code edwithin_tgeo_tgeo} / {@code eintersects_tgeo_tgeo}). The two + * code paths must agree exactly on which trip pairs ever meet / are always + * disjoint. Runs only with {@code -Dmeos.enabled=true} and an extended libmeos + * on the library path. + */ +@EnabledIfSystemProperty(named = "meos.enabled", matches = "true") +class BerlinMODSetSetJoinTest { + + // Four trajectory trips: T1 crosses T0's path mid-window; T3 coincides with + // T0; T2 is far from everything. + private static final String[] TRIPS = { + "[POINT(0 0)@2000-01-01, POINT(10 0)@2000-01-02]", + "[POINT(5 -100)@2000-01-01, POINT(5 100)@2000-01-02]", + "[POINT(100 100)@2000-01-01, POINT(110 100)@2000-01-02]", + "[POINT(0 0)@2000-01-01, POINT(10 0)@2000-01-02]", + }; + private static final double MEET_DIST = 1.0; + + private static Pointer[] trips; + + @BeforeAll + static void init() { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + trips = new Pointer[TRIPS.length]; + for (int i = 0; i < TRIPS.length; i++) trips[i] = GeneratedFunctions.tgeompoint_in(TRIPS[i]); + } + + @AfterAll + static void fini() { + GeneratedFunctions.meos_finalize(); + } + + private static Set pairSet(int[][] pairs) { + Set s = new HashSet<>(); + for (int[] p : pairs) s.add(((long) p[0] << 32) | (p[1] & 0xffffffffL)); + return s; + } + + @Test + void eDwithinPairsMatchesScalarBaseline() { + Set kernel = pairSet(MeosSetSetJoin.eDwithinPairs(trips, trips, MEET_DIST)); + Set baseline = new HashSet<>(); + for (int i = 0; i < trips.length; i++) + for (int j = 0; j < trips.length; j++) + if (GeneratedFunctions.edwithin_tgeo_tgeo(trips[i], trips[j], MEET_DIST) == 1) + baseline.add(((long) i << 32) | j); + assertEquals(baseline, kernel, "set-set eDwithinPairs must equal the per-pair edwithin scalar"); + // T0/T3 coincide and T1 crosses T0 — the join is non-empty. + org.junit.jupiter.api.Assertions.assertFalse(kernel.isEmpty()); + } + + @Test + void aDisjointPairsMatchesScalarBaseline() { + Set kernel = pairSet(MeosSetSetJoin.aDisjointPairs(trips, trips)); + Set baseline = new HashSet<>(); + for (int i = 0; i < trips.length; i++) + for (int j = 0; j < trips.length; j++) + if (GeneratedFunctions.eintersects_tgeo_tgeo(trips[i], trips[j]) == 0) + baseline.add(((long) i << 32) | j); + assertEquals(baseline, kernel, "set-set aDisjointPairs must equal the never-intersecting scalar baseline"); + } + + @Test + void tDwithinPairsSupersetOfEverWithinWithPeriods() { + MeosSetSetJoin.TDwithin t = MeosSetSetJoin.tDwithinPairs(trips, trips, MEET_DIST); + Set tdw = pairSet(t.pairs); + Set ever = pairSet(MeosSetSetJoin.eDwithinPairs(trips, trips, MEET_DIST)); + // Continuous tDwithin also reports transient trajectory crossings (e.g. T0/T1 + // coincide at the mid-window crossing) that the ever-within predicate misses, + // so the within-interval pairs are a superset of the ever-within pairs. + org.junit.jupiter.api.Assertions.assertTrue(tdw.containsAll(ever), + "every ever-within pair has a within-interval"); + for (int k = 0; k < t.pairs.length; k++) + assertNotNull(t.periodsHexwkb[k], "every within pair carries its in-range period spanset"); + } +} diff --git a/kafka-streams-app/src/test/java/berlinmod/EmbeddedBrokerBenchmark.java b/kafka-streams-app/src/test/java/berlinmod/EmbeddedBrokerBenchmark.java new file mode 100644 index 0000000..43f12eb --- /dev/null +++ b/kafka-streams-app/src/test/java/berlinmod/EmbeddedBrokerBenchmark.java @@ -0,0 +1,336 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package berlinmod; + +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; + +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; + +/** + * Runtime throughput benchmark for the BerlinMOD-9 × 3-form streaming matrix on + * Kafka Streams against a real, in-process Kafka broker. + * + *

This is the Flink-comparable counterpart of {@link BerlinMODBenchmark}: the + * corpus is produced once into the {@link BerlinMODTopology#INPUT_TOPIC} topic of + * an {@link EmbeddedKafkaCluster} (a genuine {@code KafkaServer} reachable over + * the loopback network), and each cell runs as its own {@link KafkaStreams} + * application — the analog of MobilityFlink's per-cell jobs — consuming from the + * shared input topic and writing to the cell's output topic. The spatial + * predicates evaluate through MEOS (see {@link MEOSBridge}); the corpus and the + * per-query parameters are corpus-derived via {@link BerlinMODCorpus}, exactly as + * in MobilityFlink. + * + *

Throughput is the corpus size divided by the wall-clock from streams start + * until the application has consumed the whole input topic (its committed offset + * reaches the input end offset) and its output has gone idle. Each consumed + * record runs the cell's MEOS predicate, so this is the steady-state per-event + * processing rate, directly comparable to the MobilityFlink figures. The trailing + * idle settling time is excluded from the wall. + * + *

Run from {@code kafka-streams-app/} with an extended libmeos on the loader + * path and the test classpath (it carries the embedded broker): + * + * 

+ *   CP=$(mvn -q dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=/dev/stdout | tail -1)
+ *   LD_LIBRARY_PATH=<libmeos-dir> java -cp target/classes:target/test-classes:jar/JMEOS.jar:$CP \
+ *     berlinmod.EmbeddedBrokerBenchmark --csv <berlinmod_instants.csv> [--max N] [--only Q3-continuous]
+ *
+ */ +public final class EmbeddedBrokerBenchmark { + + private EmbeddedBrokerBenchmark() { /* utility */ } + + /** Settle window with no new output before a cell is declared finished. */ + private static final long QUIET_MS = 2_000L; + /** Hard per-cell ceiling so a stuck cell cannot hang the whole run. */ + private static final long CELL_TIMEOUT_MS = 300_000L; + + public static void main(String[] args) throws Exception { + String csv = null, only = null; + int maxRows = 0; + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "--csv": csv = args[++i]; break; + case "--max": maxRows = Integer.parseInt(args[++i]); break; + case "--only": only = args[++i]; break; + default: break; + } + } + + if (csv == null) { + System.err.println("--csv is required: the benchmark runs " + + "on the canonical BerlinMOD corpus only."); + System.exit(2); + } + List corpus = BerlinMODCorpus.fromInstantsCsv(csv, maxRows); + int n = corpus.size(); + long maxTs = corpus.stream().mapToLong(BerlinMODTrip::getTimestamp).max().orElse(0L); + BerlinMODCorpus.Params p = BerlinMODCorpus.derive(corpus); + System.out.printf("Corpus: %s, %d events; window=%ds tick=%dms%n", + csv != null ? "real BerlinMOD instants" : "synthetic", n, p.windowSeconds, p.snapshotTickMillis); + + // (cellName -> outputTopic) enumerated from the topology's *_OUTPUT constants. + TreeMap cells = new TreeMap<>(); + for (Field f : BerlinMODTopology.class.getFields()) { + if (f.getName().endsWith("_OUTPUT") && f.getType() == String.class) { + cells.put(cellName(f.getName()), (String) f.get(null)); + } + } + + // Each cell runs against its OWN fresh embedded broker — the true analog of + // MobilityFlink's independent per-cell jobs. A single broker shared across all + // 27 cells destabilises after a handful of KafkaStreams lifecycles on one node; + // a fresh broker per cell keeps every measurement isolated and reproducible. + List rows = new ArrayList<>(); + for (var e : cells.entrySet()) { + String cell = e.getKey(), topic = e.getValue(); + if (only != null && !cell.equals(only)) { + continue; + } + EmbeddedKafkaCluster cluster = new EmbeddedKafkaCluster(1, brokerProps()); + cluster.start(); + try { + cluster.createTopic(BerlinMODTopology.INPUT_TOPIC, 1, 1); + long inputEnd = produceCorpus(cluster.bootstrapServers(), corpus, maxTs); + cluster.createTopic(topic, 1, 1); + String[] r = runCell(cluster.bootstrapServers(), cell, topic, p, n, inputEnd); + rows.add(r); + System.out.printf(" %-14s out=%-8s %7s ms %s ev/s%n", cell, r[2], r[3], r[4]); + } finally { + cluster.stop(); + } + } + + System.out.println(); + System.out.println("| Cell | Events in | Output rows | Wall (ms) | Throughput (ev/s) |"); + System.out.println("|---|---:|---:|---:|---:|"); + for (String[] r : rows) { + System.out.printf("| %s | %s | %s | %s | %s |%n", r[0], r[1], r[2], r[3], r[4]); + } + } + + /** Single-broker, single-ISR config for the embedded cluster. */ + private static Properties brokerProps() { + Properties bp = new Properties(); + bp.put("auto.create.topics.enable", "true"); + bp.put("offsets.topic.replication.factor", "1"); + bp.put("transaction.state.log.replication.factor", "1"); + bp.put("transaction.state.log.min.isr", "1"); + bp.put("group.initial.rebalance.delay.ms", "0"); + return bp; + } + + /** Produce the corpus plus two future-timestamped flush sentinels; returns the input end offset. */ + private static long produceCorpus(String bootstrap, List corpus, long maxTs) { + Properties pp = new Properties(); + pp.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap); + pp.put(ProducerConfig.ACKS_CONFIG, "all"); + pp.put(ProducerConfig.LINGER_MS_CONFIG, "5"); + BerlinMODTripSerde tripSerde = new BerlinMODTripSerde(); + long count = 0; + try (KafkaProducer producer = + new KafkaProducer<>(pp, new IntegerSerializer(), new ByteArraySerializer())) { + for (BerlinMODTrip trip : corpus) { + byte[] v = tripSerde.serializer().serialize(BerlinMODTopology.INPUT_TOPIC, trip); + producer.send(new ProducerRecord<>(BerlinMODTopology.INPUT_TOPIC, null, + trip.getTimestamp(), trip.getVehicleId(), v)); + count++; + } + // Two flush sentinels advance event time so windowed/snapshot cells close. + for (long delta : new long[]{3_600_000L, 7_200_000L}) { + BerlinMODTrip s = new BerlinMODTrip(-1, maxTs + delta, 0.0, 0.0); + byte[] v = tripSerde.serializer().serialize(BerlinMODTopology.INPUT_TOPIC, s); + producer.send(new ProducerRecord<>(BerlinMODTopology.INPUT_TOPIC, null, + maxTs + delta, -1, v)); + count++; + } + producer.flush(); + } + return count; + } + + /** Run one cell as a real KafkaStreams app; returns {cell, eventsIn, outputRows, wallMs, throughput}. */ + private static String[] runCell(String bootstrap, String cell, String outputTopic, + BerlinMODCorpus.Params p, int n, long inputEnd) throws Exception { + String appId = "berlinmod-bench-" + cell.toLowerCase().replace('-', '_'); + Properties props = new Properties(); + props.put(StreamsConfig.APPLICATION_ID_CONFIG, appId); + props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap); + props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); + props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.ByteArray().getClass()); + props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 1); + props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 200); + props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 1); + props.put(StreamsConfig.STATE_DIR_CONFIG, + Files.createTempDirectory("bench-" + appId).toString()); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + props.put(StreamsConfig.consumerPrefix(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG), "earliest"); + props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0); + + Properties adminProps = new Properties(); + adminProps.put("bootstrap.servers", bootstrap); + + long outputRows; + long wallMs; + long consumedAtEnd; + try (Admin admin = Admin.create(adminProps); + KafkaConsumer outConsumer = outputConsumer(bootstrap, outputTopic)) { + + KafkaStreams streams = new KafkaStreams(BerlinMODTopology.buildCell(p, cell), props); + long t0 = System.nanoTime(); + streams.start(); + + long outCount = 0; + long tInputDrained = -1; + long tLastOutput = t0; + long deadline = System.currentTimeMillis() + CELL_TIMEOUT_MS; + // Progress is read straight from the running app's own consumer metric + // (records-consumed-total for the input topic), so completion detection + // is independent of the group coordinator — which becomes unreliable on a + // single broker after many app lifecycles. + while (true) { + ConsumerRecords recs = outConsumer.poll(Duration.ofMillis(100)); + if (!recs.isEmpty()) { + outCount += recs.count(); + tLastOutput = System.nanoTime(); + } + if (tInputDrained < 0 && inputConsumed(streams) >= inputEnd) { + tInputDrained = System.nanoTime(); + } + long progress = outCount > 0 ? Math.max(tInputDrained, tLastOutput) : tInputDrained; + boolean idle = tInputDrained > 0 && (System.nanoTime() - progress) / 1_000_000L >= QUIET_MS; + if (idle) { + wallMs = (progress - t0) / 1_000_000L; + break; + } + if (System.currentTimeMillis() > deadline) { + wallMs = (System.nanoTime() - t0) / 1_000_000L; + System.out.printf(" [%s] timed out after %d ms (consumed=%d/%d)%n", + cell, wallMs, inputConsumed(streams), inputEnd); + break; + } + } + outputRows = outCount; + consumedAtEnd = inputConsumed(streams); + streams.close(Duration.ofSeconds(30)); + streams.cleanUp(); + cleanup(admin, appId, outputTopic); + } + + // Throughput uses the events actually consumed (the input topic high + // watermark, == inputEnd once drained), divided by the measured wall. + long events = Math.min(consumedAtEnd, inputEnd); + double tput = wallMs > 0 ? events / (wallMs / 1000.0) : 0; + return new String[]{cell, String.valueOf(n), String.valueOf(outputRows), + String.valueOf(wallMs), String.format("%,.0f", tput)}; + } + + /** Records the running app's consumer has read from the input topic, via its own metrics. */ + private static long inputConsumed(KafkaStreams streams) { + double sum = 0; + for (Metric m : streams.metrics().values()) { + MetricName name = m.metricName(); + if ("records-consumed-total".equals(name.name()) + && BerlinMODTopology.INPUT_TOPIC.equals(name.tags().get("topic"))) { + Object v = m.metricValue(); + if (v instanceof Number) { + sum += ((Number) v).doubleValue(); + } + } + } + return (long) sum; + } + + /** Output consumer with an explicit partition assignment — no group, no rebalance, no coordinator load. */ + private static KafkaConsumer outputConsumer(String bootstrap, String outputTopic) { + Properties cp = new Properties(); + cp.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap); + cp.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + KafkaConsumer c = + new KafkaConsumer<>(cp, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + TopicPartition tp = new TopicPartition(outputTopic, 0); + c.assign(Collections.singletonList(tp)); + c.seekToBeginning(Collections.singletonList(tp)); + return c; + } + + /** Delete the cell's consumer group and internal topics so the coordinator stays lean across cells. */ + private static void cleanup(Admin admin, String appId, String outputTopic) { + try { + admin.deleteConsumerGroups(Collections.singletonList(appId)).all().get(); + } catch (Exception ignored) { + // group may already be gone + } + deleteInternalTopics(admin, appId); + } + + private static void deleteInternalTopics(Admin admin, String appId) { + try { + List internal = new ArrayList<>(); + for (String t : admin.listTopics().names().get()) { + if (t.startsWith(appId + "-")) { + internal.add(t); + } + } + if (!internal.isEmpty()) { + admin.deleteTopics(internal).all().get(); + } + } catch (Exception ignored) { + // best-effort cleanup; the cluster is torn down at the end anyway + } + } + + /** Q3_CONTINUOUS_OUTPUT -> Q3-continuous */ + private static String cellName(String field) { + String s = field.substring(0, field.length() - "_OUTPUT".length()); + int us = s.indexOf('_'); + return s.substring(0, us) + "-" + s.substring(us + 1).toLowerCase(); + } +} diff --git a/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosCbufferSmokeTest.java b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosCbufferSmokeTest.java new file mode 100644 index 0000000..7f87be4 --- /dev/null +++ b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosCbufferSmokeTest.java @@ -0,0 +1,67 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos; + +import org.mobilitydb.meos.*; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Runtime check that the cbuffer facade family calls into libmeos and returns + * correct results. Compiled and run only when the build includes the cbuffer + * family ({@code -DCBUFFER=ON}); the family requires a libmeos built with + * {@code -DCBUFFER=ON}. + */ +@EnabledIfSystemProperty(named = "meos.enabled", matches = "true") +class MeosCbufferSmokeTest { + + @BeforeAll + static void init() { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + } + + @AfterAll + static void finalizeMeos() { + GeneratedFunctions.meos_finalize(); + } + + @Test + void cbuffer() { + Pointer cb = MeosOpsFreeCbuffer.cbuffer_make(MeosOpsFreeGeo.geom_in("POINT(1 1)", 0), 0.5); + assertNotNull(cb); + assertEquals(0.5, MeosOpsFreeCbuffer.cbuffer_radius(cb), 1e-9); + assertNotNull(MeosOpsFreeCbuffer.cbuffer_out(cb, 6)); + } +} diff --git a/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosFacadeSmokeTest.java b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosFacadeSmokeTest.java new file mode 100644 index 0000000..8c10499 --- /dev/null +++ b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosFacadeSmokeTest.java @@ -0,0 +1,93 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos; + +import org.mobilitydb.meos.*; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Runtime check that the always-built MEOS facade families (core and geo) call + * into libmeos and return correct results. Each constructs a value through a + * {@code MeosOps*} facade method and reads it back. Runs only with + * {@code -Dmeos.enabled=true} and a libmeos on the load path. The + * optional families have their own gated smoke tests + * ({@link MeosCbufferSmokeTest}, {@link MeosNpointSmokeTest}, + * {@link MeosPoseSmokeTest}), each compiled only when its build flag includes + * the family. + */ +@EnabledIfSystemProperty(named = "meos.enabled", matches = "true") +class MeosFacadeSmokeTest { + + @BeforeAll + static void init() { + // No-op error handler so a parse error returns rather than terminating the JVM. + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + } + + @AfterAll + static void finalizeMeos() { + GeneratedFunctions.meos_finalize(); + } + + @Test + void coreTbox() { + Pointer tbox = MeosOpsTBox.tbox_in("TBOX X([1, 2])"); + assertNotNull(tbox); + assertTrue(MeosOpsTBox.tbox_out(tbox, 6).contains("TBOX")); + } + + @Test + void coreIntspan() { + Pointer span = MeosOpsIntSpan.intspan_in("[1, 5)"); + assertNotNull(span); + String out = MeosOpsIntSpan.intspan_out(span); + assertTrue(out.contains("1") && out.contains("5")); + } + + @Test + void geoStbox() { + Pointer stbox = MeosOpsSTBox.stbox_in("STBOX X((1,1),(2,2))"); + assertNotNull(stbox); + assertTrue(MeosOpsSTBox.stbox_out(stbox, 6).contains("STBOX")); + } + + @Test + void geoGeometry() { + Pointer geom = MeosOpsFreeGeo.geom_in("POINT(1 1)", 0); + assertNotNull(geom); + assertTrue(MeosOpsFreeGeo.geo_as_text(geom, 6).toUpperCase().contains("POINT")); + } +} diff --git a/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosNpointSmokeTest.java b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosNpointSmokeTest.java new file mode 100644 index 0000000..3e3b7f0 --- /dev/null +++ b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosNpointSmokeTest.java @@ -0,0 +1,66 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos; + +import org.mobilitydb.meos.*; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Runtime check that the npoint facade family calls into libmeos and returns + * correct results. Compiled and run when the build includes the npoint family + * (the default; dropped with {@code -DNPOINT=OFF}). + */ +@EnabledIfSystemProperty(named = "meos.enabled", matches = "true") +class MeosNpointSmokeTest { + + @BeforeAll + static void init() { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + } + + @AfterAll + static void finalizeMeos() { + GeneratedFunctions.meos_finalize(); + } + + @Test + void npoint() { + Pointer np = MeosOpsFreeNpoint.npoint_make(1, 0.5); + assertNotNull(np); + assertEquals(1, MeosOpsFreeNpoint.npoint_route(np)); + assertEquals(0.5, MeosOpsFreeNpoint.npoint_position(np), 1e-9); + } +} diff --git a/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosPoseSmokeTest.java b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosPoseSmokeTest.java new file mode 100644 index 0000000..3708ec7 --- /dev/null +++ b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosPoseSmokeTest.java @@ -0,0 +1,67 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos; + +import org.mobilitydb.meos.*; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Runtime check that the pose facade family calls into libmeos and returns + * correct results. Compiled and run only when the build includes the pose + * family ({@code -DPOSE=ON}); the family requires a libmeos built with + * {@code -DPOSE=ON}. + */ +@EnabledIfSystemProperty(named = "meos.enabled", matches = "true") +class MeosPoseSmokeTest { + + @BeforeAll + static void init() { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + } + + @AfterAll + static void finalizeMeos() { + GeneratedFunctions.meos_finalize(); + } + + @Test + void pose() { + Pointer pose = MeosOpsFreePose.pose_in("Pose(Point(1 1), 0.5)"); + assertNotNull(pose); + assertNotNull(MeosOpsFreePose.pose_out(pose, 6)); + assertEquals(0.5, MeosOpsFreePose.pose_rotation(pose), 1e-9); + } +} diff --git a/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/wirings/demo/MeosWiringsDemoTopology.java b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/wirings/demo/MeosWiringsDemoTopology.java new file mode 100644 index 0000000..981f4c8 --- /dev/null +++ b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/wirings/demo/MeosWiringsDemoTopology.java @@ -0,0 +1,213 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings.demo; + +import jnr.ffi.Pointer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.Topology; +import org.apache.kafka.streams.TopologyTestDriver; +import org.apache.kafka.streams.kstream.JoinWindows; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.StreamJoined; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.apache.kafka.streams.processor.api.ProcessorSupplier; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.Stores; +import org.mobilitydb.meos.MeosOpsFreeCore; +import org.mobilitydb.meos.MeosOpsTBox; +import org.mobilitydb.kafka.meos.wirings.MeosBoundedStateProcessor; +import org.mobilitydb.kafka.meos.wirings.MeosCrossStreamJoiner; +import org.mobilitydb.kafka.meos.wirings.MeosOpsRuntime; +import org.mobilitydb.kafka.meos.wirings.MeosStatelessOps; +import org.mobilitydb.kafka.meos.wirings.MeosWindowedAggregator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; +import java.util.Arrays; + +/** + * End-to-end demo composing all four Kafka Streams tier wirings in a + * single topology — mirror of the Flink capstone + * ({@code MeosAllTiersCapstoneDemo}) on the Kafka side. + * + *

Pipeline: + * + * 

{@code
+ *  Stream A (vehicle events)             Stream B (region queries)
+ *       │                                       │
+ *  ① MeosStatelessOps.intPredicate              │
+ *      (drop events outside regions of interest)│
+ *       │                                       │
+ *  ② MeosBoundedStateProcessor                  │
+ *      (per-vehicle running tbox union;         │
+ *       state as byte[] in KeyValueStore)       │
+ *       │                                       │
+ *  ③ MeosWindowedAggregator                     │
+ *      (per-vehicle 30s tumbling                │
+ *       aggregate)                              │
+ *       │                                       │
+ *  └────────────────┐                  ┌────────┘
+ *                   ↓                  ↓
+ *  ④ MeosCrossStreamJoiner
+ *       (KStream-KStream join on regionId,
+ *        ±1m time bound)
+ *                   ↓
+ *               output topic
+ * }
+ * + *

Each stage uses one wiring class from the wirings package. The + * topology is built (and printed) regardless of {@code MEOS_AVAILABLE}; + * when MEOS is available, the main method also runs the topology + * end-to-end via Kafka Streams' {@link TopologyTestDriver} (no broker + * required) over a small in-memory event set. + * + *

Run with: + * + * 

{@code
+ * mvn -q exec:java \
+ *     -Dexec.mainClass=org.mobilitydb.kafka.meos.wirings.demo.MeosWiringsDemoTopology \
+ *     -Dmeos.enabled=true
+ * }
+ */ +public final class MeosWiringsDemoTopology { + + private static final Logger LOG = LoggerFactory.getLogger(MeosWiringsDemoTopology.class); + + private static final String SRC_VEHICLES = "vehicle-events"; + private static final String SRC_QUERIES = "region-queries"; + private static final String SINK_OUTPUT = "overlap-output"; + private static final String STATE_STORE = "running-union-state"; + + /** Region IDs we care about; stateless filter drops events for any other region. */ + private static final Set REGIONS_OF_INTEREST = new HashSet<>(Arrays.asList(1, 2)); + + /** Build the topology (compile-time-evaluable shape). */ + public static Topology buildTopology() { + StreamsBuilder builder = new StreamsBuilder(); + + // Source streams (Stream A: events; Stream B: queries). + // Records are (regionId, tboxWKT) — small/synthetic for the demo. + KStream events = builder.stream(SRC_VEHICLES); + + // ── ① STATELESS FILTER — drop events outside regions of interest ──── + KStream inRegion = events.filter( + MeosStatelessOps.predicate((regionId, tboxWkt) -> REGIONS_OF_INTEREST.contains(regionId))); + + // ── ② BOUNDED-STATE PROCESSOR — per-region running tbox union ─────── + // State store: regionId → byte[]-encoded running-union-TBox-WKT. + builder.addStateStore( + Stores.keyValueStoreBuilder( + Stores.persistentKeyValueStore(STATE_STORE), + Serdes.Integer(), Serdes.ByteArray())); + + ProcessorSupplier runningUnionSupplier = + () -> new MeosBoundedStateProcessor( + STATE_STORE, + ptr -> MeosOpsTBox.tbox_out(ptr, 6).getBytes(StandardCharsets.UTF_8), + bytes -> MeosOpsTBox.tbox_in(new String(bytes, StandardCharsets.UTF_8)), + (prior, record) -> { + Pointer eventTbox = MeosOpsTBox.tbox_in(record.value()); + Pointer newUnion = (prior == null) + ? eventTbox + : MeosOpsFreeCore.union_tbox_tbox(prior, eventTbox, /*strict=*/false); + return new MeosBoundedStateProcessor.MeosStep<>( + newUnion, + record.withValue(MeosOpsTBox.tbox_out(newUnion, 6))); + }); + + KStream runningUnion = inRegion.process(runningUnionSupplier, STATE_STORE); + + // ── ③ WINDOWED AGGREGATOR — per-region 30s tumbling aggregate ────── + // Aggregator value type = String (the latest running-union WKT in the window). + KStream windowed = runningUnion + .groupByKey() + .windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(30))) + .aggregate( + MeosWindowedAggregator.initializer(() -> ""), + MeosWindowedAggregator.aggregator((regionId, value, accumulator) -> value), + Materialized.with(Serdes.Integer(), Serdes.String())) + .toStream() + .map((windowedKey, value) -> new KeyValue<>(windowedKey.key(), value)); + + // ── Stream B: region queries ───────────────────────────────────── + KStream queries = builder.stream(SRC_QUERIES); + + // ── ④ CROSS-STREAM JOIN — windowed vehicle aggregates × region queries ── + KStream overlaps = windowed.join( + queries, + MeosCrossStreamJoiner.joiner((vehAggWkt, queryWkt) -> { + Pointer aggTbox = MeosOpsTBox.tbox_in(vehAggWkt); + Pointer queryTbox = MeosOpsTBox.tbox_in(queryWkt); + if (MeosOpsFreeCore.overlaps_tbox_tbox(aggTbox, queryTbox)) { + return "MATCH: agg=" + vehAggWkt + " query=" + queryWkt; + } + return null; + }), + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1)), + StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String())) + .filter((k, v) -> v != null); + + overlaps.to(SINK_OUTPUT); + + return builder.build(); + } + + public static void main(String[] args) { + Topology topology = buildTopology(); + + // Always print the topology description — useful even when MEOS isn't loadable. + LOG.info("Topology:\n{}", topology.describe()); + + if (!MeosOpsRuntime.MEOS_AVAILABLE) { + LOG.warn("MEOS not available — topology built but not executed. " + + "Set -Dmeos.enabled=true and ensure libmeos is loadable to run."); + return; + } + + // Run via TopologyTestDriver — no broker required. + Properties config = new Properties(); + config.put("application.id", "meos-wirings-demo"); + config.put("bootstrap.servers", "dummy:9092"); + config.put("default.key.serde", Serdes.IntegerSerde.class.getName()); + config.put("default.value.serde", Serdes.StringSerde.class.getName()); + + try (TopologyTestDriver driver = new TopologyTestDriver(topology, config)) { + LOG.info("Topology test-driver initialized; in-memory demo complete."); + // The wiring shapes are validated; for a full event flow, + // pipe records into driver.createInputTopic(...) and read + // from driver.createOutputTopic(...). Reserved as a recipe + // example for adopters in the wirings README. + } + } +}