From 813643a433068d6cb78474df8e0115c8c0eb7d2e Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Fri, 19 Jun 2026 12:29:18 +0200 Subject: [PATCH 1/2] feat(berlinmod): route the streaming parity matrix through MEOS via JMEOS 1.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route the BerlinMOD-9 × 3-form processors through MEOS (libmeos via the JMEOS 1.4 bridge) — edwithin_tgeo_geo, eintersects_tgeo_geo, geog_distance and the set-set join family — and add the MEOS DSL-tier wirings, the per-family facade smoke tests and the canonical BerlinMOD corpus loader. JMEOS and the native libmeos.so are built from source rather than committed to the repository. build-jmeos.sh clones MobilityDB and JMEOS at the deliverable PRs' immutable head commit SHAs, builds libmeos.so, builds the jar and installs it into the local Maven repository, so the pom resolves JMEOS as an ordinary dependency. It pins MobilityDB #1148 (set-set join, stacked on #1162 clean/geo) and JMEOS #25 (facade) as overridable env vars; both are switched to upstream master/main after those PRs are integrated. --- .gitignore | 7 + README.md | 18 + build-jmeos.sh | 167 ++++++++ kafka-streams-app/pom.xml | 27 ++ .../main/java/berlinmod/BerlinMODCorpus.java | 167 ++++++++ .../java/berlinmod/BerlinMODTopology.java | 357 ++++++++++-------- .../main/java/berlinmod/BerlinMODTrip.java | 25 ++ .../java/berlinmod/BerlinMODTripSerde.java | 25 ++ .../src/main/java/berlinmod/MEOSBridge.java | 128 +++++++ .../main/java/berlinmod/PointOfInterest.java | 25 ++ .../java/berlinmod/Q1ContinuousProcessor.java | 25 ++ .../java/berlinmod/Q1SnapshotProcessor.java | 25 ++ .../java/berlinmod/Q1WindowedProcessor.java | 25 ++ .../java/berlinmod/Q2ContinuousProcessor.java | 25 ++ .../java/berlinmod/Q2SnapshotProcessor.java | 25 ++ .../java/berlinmod/Q2WindowedProcessor.java | 25 ++ .../java/berlinmod/Q3ContinuousProcessor.java | 31 +- .../java/berlinmod/Q3SnapshotProcessor.java | 29 +- .../java/berlinmod/Q3WindowedProcessor.java | 27 +- .../java/berlinmod/Q4ContinuousProcessor.java | 33 +- .../java/berlinmod/Q4SnapshotProcessor.java | 28 +- .../java/berlinmod/Q4WindowedProcessor.java | 28 +- .../java/berlinmod/Q5ContinuousProcessor.java | 29 +- .../java/berlinmod/Q5SnapshotProcessor.java | 29 +- .../java/berlinmod/Q5WindowedProcessor.java | 29 +- .../java/berlinmod/Q6ContinuousProcessor.java | 35 +- .../java/berlinmod/Q6SnapshotProcessor.java | 29 +- .../java/berlinmod/Q6WindowedProcessor.java | 29 +- .../java/berlinmod/Q7ContinuousProcessor.java | 27 +- .../java/berlinmod/Q7SnapshotProcessor.java | 27 +- .../java/berlinmod/Q7WindowedProcessor.java | 27 +- .../java/berlinmod/Q8ContinuousProcessor.java | 27 +- .../java/berlinmod/Q8SnapshotProcessor.java | 27 +- .../java/berlinmod/Q8WindowedProcessor.java | 27 +- .../java/berlinmod/Q9ContinuousProcessor.java | 27 +- .../java/berlinmod/Q9SnapshotProcessor.java | 27 +- .../java/berlinmod/Q9WindowedProcessor.java | 27 +- .../wirings/MeosBoundedStateProcessor.java | 164 ++++++++ .../meos/wirings/MeosCrossStreamJoiner.java | 99 +++++ .../kafka/meos/wirings/MeosOpsRuntime.java | 56 +++ .../kafka/meos/wirings/MeosStatelessOps.java | 117 ++++++ .../meos/wirings/MeosWindowedAggregator.java | 114 ++++++ .../mobilitydb/kafka/meos/wirings/README.md | 94 +++++ .../berlinmod/BerlinMODSetSetJoinTest.java | 120 ++++++ .../kafka/meos/MeosCbufferSmokeTest.java | 67 ++++ .../kafka/meos/MeosFacadeSmokeTest.java | 93 +++++ .../kafka/meos/MeosNpointSmokeTest.java | 66 ++++ .../kafka/meos/MeosPoseSmokeTest.java | 67 ++++ .../wirings/demo/MeosWiringsDemoTopology.java | 213 +++++++++++ 49 files changed, 2771 insertions(+), 194 deletions(-) create mode 100755 build-jmeos.sh create mode 100644 kafka-streams-app/src/main/java/berlinmod/BerlinMODCorpus.java create mode 100644 kafka-streams-app/src/main/java/berlinmod/MEOSBridge.java create mode 100644 kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosBoundedStateProcessor.java create mode 100644 kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosCrossStreamJoiner.java create mode 100644 kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosOpsRuntime.java create mode 100644 kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosStatelessOps.java create mode 100644 kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosWindowedAggregator.java create mode 100644 kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/README.md create mode 100644 kafka-streams-app/src/test/java/berlinmod/BerlinMODSetSetJoinTest.java create mode 100644 kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosCbufferSmokeTest.java create mode 100644 kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosFacadeSmokeTest.java create mode 100644 kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosNpointSmokeTest.java create mode 100644 kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosPoseSmokeTest.java create mode 100644 kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/wirings/demo/MeosWiringsDemoTopology.java diff --git a/.gitignore b/.gitignore index 4dd9b3c..6164529 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,13 @@ # Maven build artefacts target/ +# JMEOS native dependency — generated by build-jmeos.sh, never committed +.build-jmeos/ +kafka-streams-app/jar/ +kafka-streams-app/lib/ +*.jar +*.so + # IDE .idea/ .vscode/ diff --git a/README.md b/README.md index d4cedeb..9238699 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,24 @@ Spatial predicates today use pure-Java great-circle (`Haversine`) and planar seg ## Build and run +### MEOS native dependency + +The spatial predicates route through MEOS via the [JMEOS](https://github.com/MobilityDB/JMEOS) +bridge, so the build needs the JMEOS jar and the native `libmeos.so`. Neither is +committed to this repository — generate them from source with the helper script, +which clones MobilityDB and JMEOS at pinned, immutable refs, builds `libmeos.so`, +builds the jar, installs the jar into the local Maven repository, and stages +`libmeos.so` for the runtime: + +``` +./build-jmeos.sh +``` + +Run it once (re-run it only to bump the pinned MobilityDB/JMEOS refs at the top of +the script). After it succeeds, JMEOS resolves as an ordinary Maven dependency. + +### Build the app + ``` cd kafka-streams-app mvn -q clean package -DskipTests diff --git a/build-jmeos.sh b/build-jmeos.sh new file mode 100755 index 0000000..5d792d5 --- /dev/null +++ b/build-jmeos.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# +# build-jmeos.sh — build the JMEOS jar and the native libmeos.so from source and +# install them locally, so the repository never has to carry the binaries. +# +# This is the downstream generation chain for the JVM streaming tools: +# +# MobilityDB/MEOS (deliverable PRs) -> JMEOS (FFI facade + jar) -> MobilityKafka +# +# (The MEOS-API meos-idl.json step is pre-materialized in the JMEOS branch's +# committed codegen/input/meos-idl.json, so this script only has to build the +# two endpoints.) +# +# What it does: +# 1. Clones MobilityDB at the pinned ref and builds libmeos.so (cmake -DMEOS=ON). +# 2. Clones JMEOS at the pinned ref, drops libmeos.so in, and builds JMEOS.jar. +# 3. Registers the jar in the local Maven repository via +# `mvn install:install-file` under the coordinates the kafka-streams-app +# pom depends on (com.mobilitydb:jmeos:1.4.0 by default). +# 4. Copies libmeos.so into kafka-streams-app/lib/ for the test/runtime +# LD_LIBRARY_PATH. +# +# After running this once, `cd kafka-streams-app && mvn test` resolves JMEOS as +# an ordinary dependency — no committed jar/so required. +# +# The refs below point at the DELIVERABLE pull requests that provide the surface +# this project consumes — NOT at an ecosystem-pin tag. A pin is the ecosystem's +# benchmark / evidence vehicle, not a source of truth, so a binding must never +# depend on one. The dependency is expressed as the PRs' immutable head commit +# SHAs (overridable env vars): +# * MobilityDB PR #1148 — the *_tgeoarr_tgeoarr set-set spatial-join MEOS symbols. +# * JMEOS PR #25 — the org.mobilitydb.meos facade incl. MeosSetSetJoin. +# A head SHA is immutable: a later rebase/force-push of either PR creates a NEW +# SHA and leaves the pinned one unchanged. Once both PRs merge upstream, repoint +# MOBILITYDB_REF -> MobilityDB/MobilityDB master and JMEOS_REF -> MobilityDB/JMEOS +# main (or a release tag) — that is the only change needed. +# +set -euo pipefail + +# --------------------------------------------------------------------------- +# Pinned sources (override any of these via the environment). +# --------------------------------------------------------------------------- +# MobilityDB PR #1148 (set-set spatial join, rebased onto #1162 clean/geo so the +# set-set join inherits the geodetic-disjoint surface) — immutable head SHA. +MOBILITYDB_REPO="${MOBILITYDB_REPO:-https://github.com/estebanzimanyi/MobilityDB.git}" +MOBILITYDB_REF="${MOBILITYDB_REF:-fab7025b24f9b2b45db26d6bd0a3118058d5b55c}" # PR #1148 head + +# JMEOS PR #25 (org.mobilitydb.meos facade + MeosSetSetJoin) — immutable head SHA. +JMEOS_REPO="${JMEOS_REPO:-https://github.com/estebanzimanyi/JMEOS.git}" +JMEOS_REF="${JMEOS_REF:-f921d8608f18574a5a824b837af1a6b82c985fc2}" # PR #25 head + +# Maven coordinates the jar is installed under (must match kafka-streams-app/pom.xml). +JMEOS_GROUP_ID="${JMEOS_GROUP_ID:-com.mobilitydb}" +JMEOS_ARTIFACT_ID="${JMEOS_ARTIFACT_ID:-jmeos}" +JMEOS_VERSION="${JMEOS_VERSION:-1.4.0}" + +# --------------------------------------------------------------------------- +# Layout. +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +APP_DIR="${SCRIPT_DIR}/kafka-streams-app" +WORK_DIR="${WORK_DIR:-${SCRIPT_DIR}/.build-jmeos}" +JOBS="${JOBS:-$(nproc 2>/dev/null || echo 4)}" + +log() { printf '\n\033[1;34m==>\033[0m %s\n' "$*"; } + +# --------------------------------------------------------------------------- +# Preconditions. +# --------------------------------------------------------------------------- +for tool in git cmake make mvn; do + command -v "$tool" >/dev/null 2>&1 || { echo "error: '$tool' is required but not on PATH" >&2; exit 1; } +done + +mkdir -p "${WORK_DIR}" + +# clone_at +# Clones (or reuses) and checks out the exact . may be a tag, +# branch or commit SHA; PR-head SHAs are fetched from the pull ref namespace if +# they are not reachable from the default branches. +clone_at() { + local repo="$1" ref="$2" dest="$3" + if [ ! -d "${dest}/.git" ]; then + log "Cloning ${repo}" + git clone "${repo}" "${dest}" + fi + git -C "${dest}" fetch --quiet --tags origin + if ! git -C "${dest}" cat-file -e "${ref}^{commit}" 2>/dev/null; then + # SHA only reachable through an open PR — fetch every PR head, then retry. + git -C "${dest}" fetch --quiet origin '+refs/pull/*/head:refs/remotes/origin/pr/*' || true + fi + log "Checking out ${ref}" + git -C "${dest}" -c advice.detachedHead=false checkout --quiet "${ref}" +} + +# --------------------------------------------------------------------------- +# 1. Build libmeos.so from MobilityDB. +# --------------------------------------------------------------------------- +MDB_DIR="${WORK_DIR}/MobilityDB" +clone_at "${MOBILITYDB_REPO}" "${MOBILITYDB_REF}" "${MDB_DIR}" + +# Enable the MEOS type families the streaming app exercises: circular buffers, +# network points (default ON) and geoposes (which auto-enables rigid geometries). +# The facade smoke tests link these symbols, so they must be in libmeos.so. +# H3 and POINTCLOUD are left OFF — the app does not use them and they require +# extra system libraries (libh3, libpointcloud); enable them via MEOS_CMAKE_ARGS +# if a downstream consumer ever needs them. +MEOS_CMAKE_ARGS="${MEOS_CMAKE_ARGS:--DCBUFFER=ON -DNPOINT=ON -DPOSE=ON}" + +log "Building libmeos.so (MEOS=ON ${MEOS_CMAKE_ARGS})" +rm -rf "${MDB_DIR}/build" +cmake -S "${MDB_DIR}" -B "${MDB_DIR}/build" -DMEOS=ON ${MEOS_CMAKE_ARGS} >/dev/null +cmake --build "${MDB_DIR}/build" --target meos -j "${JOBS}" + +LIBMEOS_SO="$(find "${MDB_DIR}/build" -name 'libmeos.so' -print -quit)" +[ -n "${LIBMEOS_SO}" ] || { echo "error: libmeos.so not produced by the MEOS build" >&2; exit 1; } +log "Built ${LIBMEOS_SO}" + +# --------------------------------------------------------------------------- +# 2. Build JMEOS.jar against that libmeos.so. +# --------------------------------------------------------------------------- +JMEOS_DIR="${WORK_DIR}/JMEOS" +clone_at "${JMEOS_REPO}" "${JMEOS_REF}" "${JMEOS_DIR}" + +# JMEOS' build bundles src/libmeos.so into the jar and JarLibraryLoader extracts it. +cp -f "${LIBMEOS_SO}" "${JMEOS_DIR}/jmeos-core/src/libmeos.so" + +log "Building JMEOS.jar" +# FunctionsGenerator lives in the codegen module, which jmeos-core does not +# declare as a Maven dependency — so '-am' will not build it. Compile it first +# so jmeos-core's build-time facade generation can find it. Use +# 'maven.test.skip' (not 'skipTests'): the jmeos-core pom hardcodes +# false, which overrides -DskipTests but not this. +mvn -f "${JMEOS_DIR}/pom.xml" -q -pl codegen compile +mvn -f "${JMEOS_DIR}/pom.xml" -q -pl jmeos-core -am -Dmaven.test.skip=true package + +JMEOS_JAR="${JMEOS_DIR}/jar/JMEOS.jar" +[ -f "${JMEOS_JAR}" ] || { echo "error: ${JMEOS_JAR} was not produced" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# 3. Install the jar into the local Maven repository. +# --------------------------------------------------------------------------- +log "Installing ${JMEOS_GROUP_ID}:${JMEOS_ARTIFACT_ID}:${JMEOS_VERSION} into the local Maven repo" +mvn -q install:install-file \ + -Dfile="${JMEOS_JAR}" \ + -DgroupId="${JMEOS_GROUP_ID}" \ + -DartifactId="${JMEOS_ARTIFACT_ID}" \ + -Dversion="${JMEOS_VERSION}" \ + -Dpackaging=jar + +# --------------------------------------------------------------------------- +# 4. Stage libmeos.so for the kafka-streams-app runtime (LD_LIBRARY_PATH). +# --------------------------------------------------------------------------- +mkdir -p "${APP_DIR}/lib" +cp -f "${LIBMEOS_SO}" "${APP_DIR}/lib/libmeos.so" + +log "Done." +cat <2.18.2 2.0.16 5.11.4 + true + ${project.basedir}/lib + + + com.mobilitydb + jmeos + 1.4.0 + + + com.github.jnr + jnr-ffi + 2.1.10 + org.apache.kafka kafka-streams @@ -76,6 +93,16 @@ maven-surefire-plugin 3.2.5 + + false + + ${meos.enabled} + + + ${meos.lib.dir} + + --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + maven-shade-plugin diff --git a/kafka-streams-app/src/main/java/berlinmod/BerlinMODCorpus.java b/kafka-streams-app/src/main/java/berlinmod/BerlinMODCorpus.java new file mode 100644 index 0000000..cc74be5 --- /dev/null +++ b/kafka-streams-app/src/main/java/berlinmod/BerlinMODCorpus.java @@ -0,0 +1,167 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package berlinmod; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.TreeSet; +import java.util.stream.Stream; + +/** + * Corpus loader and query-parameter derivation for the BerlinMOD streaming + * benchmark. + * + *

Supplies either a deterministic synthetic corpus or the real BerlinMOD + * instants corpus read from the {@code berlinmod_instants.csv} produced by the + * BerlinMOD generator. Real instants are stored in EPSG:3857; they are + * reprojected to EPSG:4326 through MEOS {@code geo_transform} at load — the + * loader holds no projection mathematics of its own. + * + *

{@link Params} fixes the per-query parameters from the corpus itself (its + * centroid, bounding box, vehicle ids, and time span) so every spatial cell is + * selective and the windowing granularity yields a comparable number of windows + * regardless of the corpus time span. + */ +public final class BerlinMODCorpus { + + private static final DateTimeFormatter TS = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss") + .optionalStart().appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true).optionalEnd() + .appendOffset("+HH", "Z") + .toFormatter(); + + private BerlinMODCorpus() { /* utility */ } + + /** Query parameters derived from a corpus. */ + public static final class Params { + public final double pLon, pLat, radiusMetres, dMeetMetres; + public final double xmin, ymin, xmax, ymax; + public final double s1Lon, s1Lat, s2Lon, s2Lat; + public final List pois; + public final int targetId, xId, yId; + public final long windowSeconds, snapshotTickMillis; + + Params(double pLon, double pLat, double radiusMetres, double dMeetMetres, + double xmin, double ymin, double xmax, double ymax, + double s1Lon, double s1Lat, double s2Lon, double s2Lat, + List pois, int targetId, int xId, int yId, + long windowSeconds, long snapshotTickMillis) { + this.pLon = pLon; this.pLat = pLat; this.radiusMetres = radiusMetres; this.dMeetMetres = dMeetMetres; + this.xmin = xmin; this.ymin = ymin; this.xmax = xmax; this.ymax = ymax; + this.s1Lon = s1Lon; this.s1Lat = s1Lat; this.s2Lon = s2Lon; this.s2Lat = s2Lat; + this.pois = pois; this.targetId = targetId; this.xId = xId; this.yId = yId; + this.windowSeconds = windowSeconds; this.snapshotTickMillis = snapshotTickMillis; + } + } + + /** Real BerlinMOD instants from {@code berlinmod_instants.csv} + * (columns {@code tripid,vehid,day,seqno,geom,t}), reprojected 3857→4326 + * through MEOS, sorted by timestamp. {@code maxRows <= 0} loads all rows. */ + public static List fromInstantsCsv(String path, int maxRows) throws Exception { + ensureMeos(); + List events = new ArrayList<>(); + try (Stream lines = Files.lines(Paths.get(path))) { + java.util.Iterator it = lines.iterator(); + if (it.hasNext()) { + it.next(); // header + } + while (it.hasNext() && (maxRows <= 0 || events.size() < maxRows)) { + String[] f = it.next().split(","); + int vid = Integer.parseInt(f[1].trim()); + long ms = OffsetDateTime.parse(f[5].trim(), TS).toInstant().toEpochMilli(); + Pointer g4326 = GeneratedFunctions.geo_transform( + GeneratedFunctions.geom_in(f[4].trim(), -1), 4326); + String txt = GeneratedFunctions.geo_as_text(g4326, 7); // POINT(lon lat) + String[] xy = txt.substring(txt.indexOf('(') + 1, txt.indexOf(')')).trim().split("\\s+"); + events.add(make(vid, ms, Double.parseDouble(xy[0]), Double.parseDouble(xy[1]))); + } + } + events.sort((a, b) -> Long.compare(a.getTimestamp(), b.getTimestamp())); + return events; + } + + /** Derive selective per-query parameters and a window/tick granularity that + * yields ~200 windows over the corpus time span. */ + public static Params derive(List corpus) { + double sumLon = 0, sumLat = 0, minLon = Double.MAX_VALUE, minLat = Double.MAX_VALUE, + maxLon = -Double.MAX_VALUE, maxLat = -Double.MAX_VALUE, minT = Double.MAX_VALUE, maxT = -Double.MAX_VALUE; + TreeSet ids = new TreeSet<>(); + for (BerlinMODTrip t : corpus) { + sumLon += t.getLon(); sumLat += t.getLat(); + minLon = Math.min(minLon, t.getLon()); maxLon = Math.max(maxLon, t.getLon()); + minLat = Math.min(minLat, t.getLat()); maxLat = Math.max(maxLat, t.getLat()); + minT = Math.min(minT, t.getTimestamp()); maxT = Math.max(maxT, t.getTimestamp()); + ids.add(t.getVehicleId()); + } + int n = corpus.size(); + double cLon = sumLon / n, cLat = sumLat / n; + double exLon = maxLon - minLon, exLat = maxLat - minLat; + List idList = new ArrayList<>(ids); + int targetId = idList.get(idList.size() / 2); + int xId = idList.get(0); + int yId = idList.get(Math.min(idList.size() - 1, idList.size() / 2)); + long span = (long) (maxT - minT); + long windowSeconds = Math.max(1L, span / 1000 / 200); + long tickMillis = Math.max(1000L, windowSeconds * 1000L / 2); + List pois = Arrays.asList( + new PointOfInterest(1, cLon, cLat, 2_000.0), + new PointOfInterest(2, cLon + 0.1 * exLon, cLat + 0.1 * exLat, 1_000.0), + new PointOfInterest(3, cLon - 0.1 * exLon, cLat - 0.1 * exLat, 2_000.0)); + return new Params(cLon, cLat, 5_000.0, 5_000.0, + cLon - 0.25 * exLon, cLat - 0.25 * exLat, cLon + 0.25 * exLon, cLat + 0.25 * exLat, + minLon + 0.25 * exLon, cLat, maxLon - 0.25 * exLon, cLat, + pois, targetId, xId, yId, windowSeconds, tickMillis); + } + + private static BerlinMODTrip make(int vid, long t, double lon, double lat) { + BerlinMODTrip trip = new BerlinMODTrip(); + trip.setVehicleId(vid); + trip.setTimestamp(t); + trip.setLon(lon); + trip.setLat(lat); + return trip; + } + + private static final ThreadLocal MEOS_INIT = ThreadLocal.withInitial(() -> Boolean.FALSE); + private static void ensureMeos() { + if (!MEOS_INIT.get()) { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + MEOS_INIT.set(Boolean.TRUE); + } + } + +} diff --git a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTopology.java b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTopology.java index 3f0a88d..a0babb1 100644 --- a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTopology.java +++ b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTopology.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.common.serialization.Serdes; @@ -58,10 +83,9 @@ public final class BerlinMODTopology { // ---------- Q3 ---------- public static final String Q3_CONTINUOUS_OUTPUT = "berlinmod-q3-continuous"; - // Query params below are anchored to the canonical BerlinMOD sample's - // per-vehicle geometry (vehicles 1-5, centroids 7-18 km apart); radii use - // km-scale margins so the dwithin partition is identical under the - // pure-Java Haversine and the MEOS geodetic engines. + // Query params anchored to the canonical BerlinMOD sample's per-vehicle + // geometry (vehicles 1-5, 7-18 km apart); km-scale radii keep the dwithin + // partition identical under pure-Java Haversine and the MEOS geodetic engine. public static final double Q3_P_LON = 4.4322; // near vehicle 1 public static final double Q3_P_LAT = 50.7670; public static final double Q3_RADIUS_METRES = 5_000.0; @@ -120,165 +144,184 @@ public final class BerlinMODTopology { private BerlinMODTopology() {} - public static Topology build() { + public static Topology build() { return build(defaultParams()); } + public static Topology build(BerlinMODCorpus.Params p) { return buildInternal(p, null); } + public static Topology buildCell(BerlinMODCorpus.Params p, String cell) { return buildInternal(p, cell); } + + private static BerlinMODCorpus.Params defaultParams() { + return new BerlinMODCorpus.Params(Q3_P_LON, Q3_P_LAT, Q3_RADIUS_METRES, Q5_D_MEET_METRES, + Q4_XMIN, Q4_YMIN, Q4_XMAX, Q4_YMAX, Q8_S1_LON, Q8_S1_LAT, Q8_S2_LON, Q8_S2_LAT, + Q7_POIS, Q2_TARGET_VEHICLE_ID, Q9_X_VEHICLE_ID, Q9_Y_VEHICLE_ID, + WINDOW_SIZE_MILLIS / 1000L, SNAPSHOT_TICK_MILLIS); + } + + private static Topology buildInternal(BerlinMODCorpus.Params p, String only) { StreamsBuilder builder = new StreamsBuilder(); + final long WIN_MS = p.windowSeconds * 1000L; BerlinMODTripSerde tripSerde = new BerlinMODTripSerde(); - - // ---- continuous-form state stores ---- - addStore(builder, Q1_SEEN_STORE, Serdes.Integer(), Serdes.Boolean()); - addStore(builder, Q4_WAS_INSIDE_STORE, Serdes.Integer(), Serdes.Boolean()); - addStore(builder, Q5_LAST_POS_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q6_STATE_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q7_FIRST_PASSED_STORE, Serdes.Integer(), Serdes.Long()); - addStore(builder, Q9_STATE_STORE, Serdes.Integer(), Serdes.String()); - - // ---- windowed-form state stores ---- - addStore(builder, Q1_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q2_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q3_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q4_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q5_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q6_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q7_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q8_WIN_STORE, Serdes.Long(), Serdes.String()); - addStore(builder, Q9_WIN_STORE, Serdes.Long(), Serdes.String()); - - // ---- snapshot-form state stores (separate to avoid co-write conflicts with continuous) ---- - addStore(builder, Q1_SNAP_STORE, Serdes.Integer(), Serdes.Long()); - addStore(builder, Q2_SNAP_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q3_SNAP_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q4_SNAP_WAS_INSIDE_STORE, Serdes.Integer(), Serdes.Boolean()); - addStore(builder, Q4_SNAP_ENTRIES_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q5_SNAP_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q6_SNAP_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q7_SNAP_STORE, Serdes.Integer(), Serdes.Long()); - addStore(builder, Q8_SNAP_STORE, Serdes.Integer(), Serdes.String()); - addStore(builder, Q9_SNAP_STORE, Serdes.Integer(), Serdes.String()); - - // ---- streams ---- KStream trips = builder.stream(INPUT_TOPIC, Consumed.with(Serdes.Integer(), tripSerde)); - - // Re-keyed by constant for the shared-state snapshot/multi-vehicle processors KStream tripsK0 = trips.selectKey((k, v) -> 0); - // ====== continuous form ====== - trips.process(() -> new Q1ContinuousProcessor(Q1_SEEN_STORE), Q1_SEEN_STORE) - .to(Q1_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); - - trips.process(() -> new Q2ContinuousProcessor(Q2_TARGET_VEHICLE_ID)) - .to(Q2_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), tripSerde)); - - trips.process(() -> new Q3ContinuousProcessor(Q3_P_LON, Q3_P_LAT, Q3_RADIUS_METRES)) - .to(Q3_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Boolean())); - - trips.process(() -> new Q4ContinuousProcessor(Q4_WAS_INSIDE_STORE, Q4_XMIN, Q4_YMIN, Q4_XMAX, Q4_YMAX), - Q4_WAS_INSIDE_STORE) - .to(Q4_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); - - tripsK0.process(() -> new Q5ContinuousProcessor(Q5_LAST_POS_STORE, - Q5_P_LON, Q5_P_LAT, Q5_D_P_METRES, Q5_D_MEET_METRES), - Q5_LAST_POS_STORE) - .to(Q5_CONTINUOUS_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); - - trips.process(() -> new Q6ContinuousProcessor(Q6_STATE_STORE), Q6_STATE_STORE) - .to(Q6_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Double())); - - trips.process(() -> new Q7ContinuousProcessor(Q7_FIRST_PASSED_STORE, Q7_POIS), - Q7_FIRST_PASSED_STORE) - .to(Q7_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); - - trips.process(() -> new Q8ContinuousProcessor(Q8_S1_LON, Q8_S1_LAT, Q8_S2_LON, Q8_S2_LAT, Q8_RADIUS_METRES)) - .to(Q8_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Boolean())); - - tripsK0.process(() -> new Q9ContinuousProcessor(Q9_STATE_STORE, Q9_X_VEHICLE_ID, Q9_Y_VEHICLE_ID), - Q9_STATE_STORE) - .to(Q9_CONTINUOUS_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); - - // ====== windowed form (distinct-count per tumbling window for Q1/Q3/Q8) ====== - tripsK0.process(() -> new Q1WindowedProcessor(Q1_WIN_STORE, WINDOW_SIZE_MILLIS), Q1_WIN_STORE) - .to(Q1_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); - - tripsK0.process(() -> new Q3WindowedProcessor(Q3_WIN_STORE, - Q3_P_LON, Q3_P_LAT, Q3_RADIUS_METRES, - WINDOW_SIZE_MILLIS), - Q3_WIN_STORE) - .to(Q3_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); - - tripsK0.process(() -> new Q8WindowedProcessor(Q8_WIN_STORE, - Q8_S1_LON, Q8_S1_LAT, Q8_S2_LON, Q8_S2_LAT, - Q8_RADIUS_METRES, WINDOW_SIZE_MILLIS), - Q8_WIN_STORE) - .to(Q8_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); - - tripsK0.process(() -> new Q2WindowedProcessor(Q2_WIN_STORE, Q2_TARGET_VEHICLE_ID, WINDOW_SIZE_MILLIS), - Q2_WIN_STORE) - .to(Q2_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q4WindowedProcessor(Q4_WIN_STORE, - Q4_XMIN, Q4_YMIN, Q4_XMAX, Q4_YMAX, - WINDOW_SIZE_MILLIS), - Q4_WIN_STORE) - .to(Q4_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q5WindowedProcessor(Q5_WIN_STORE, - Q5_P_LON, Q5_P_LAT, Q5_D_P_METRES, Q5_D_MEET_METRES, - WINDOW_SIZE_MILLIS), - Q5_WIN_STORE) - .to(Q5_WINDOWED_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); - - tripsK0.process(() -> new Q6WindowedProcessor(Q6_WIN_STORE, WINDOW_SIZE_MILLIS), Q6_WIN_STORE) - .to(Q6_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q7WindowedProcessor(Q7_WIN_STORE, Q7_POIS, WINDOW_SIZE_MILLIS), Q7_WIN_STORE) - .to(Q7_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q9WindowedProcessor(Q9_WIN_STORE, - Q9_X_VEHICLE_ID, Q9_Y_VEHICLE_ID, - WINDOW_SIZE_MILLIS), - Q9_WIN_STORE) - .to(Q9_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); - - // ====== snapshot form (all via constant key, with STREAM_TIME punctuators) ====== - tripsK0.process(() -> new Q1SnapshotProcessor(Q1_SNAP_STORE, SNAPSHOT_TICK_MILLIS), Q1_SNAP_STORE) - .to(Q1_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); - - tripsK0.process(() -> new Q2SnapshotProcessor(Q2_SNAP_STORE, Q2_TARGET_VEHICLE_ID, SNAPSHOT_TICK_MILLIS), - Q2_SNAP_STORE) - .to(Q2_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q3SnapshotProcessor(Q3_SNAP_STORE, - Q3_P_LON, Q3_P_LAT, Q3_RADIUS_METRES, SNAPSHOT_TICK_MILLIS), - Q3_SNAP_STORE) - .to(Q3_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); - - tripsK0.process(() -> new Q4SnapshotProcessor(Q4_SNAP_WAS_INSIDE_STORE, Q4_SNAP_ENTRIES_STORE, - Q4_XMIN, Q4_YMIN, Q4_XMAX, Q4_YMAX, SNAPSHOT_TICK_MILLIS), - Q4_SNAP_WAS_INSIDE_STORE, Q4_SNAP_ENTRIES_STORE) - .to(Q4_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q5SnapshotProcessor(Q5_SNAP_STORE, - Q5_P_LON, Q5_P_LAT, Q5_D_P_METRES, Q5_D_MEET_METRES, - SNAPSHOT_TICK_MILLIS), - Q5_SNAP_STORE) - .to(Q5_SNAPSHOT_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); - - tripsK0.process(() -> new Q6SnapshotProcessor(Q6_SNAP_STORE, SNAPSHOT_TICK_MILLIS), Q6_SNAP_STORE) - .to(Q6_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q7SnapshotProcessor(Q7_SNAP_STORE, Q7_POIS, SNAPSHOT_TICK_MILLIS), Q7_SNAP_STORE) - .to(Q7_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); - - tripsK0.process(() -> new Q8SnapshotProcessor(Q8_SNAP_STORE, - Q8_S1_LON, Q8_S1_LAT, Q8_S2_LON, Q8_S2_LAT, - Q8_RADIUS_METRES, SNAPSHOT_TICK_MILLIS), - Q8_SNAP_STORE) - .to(Q8_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); - - tripsK0.process(() -> new Q9SnapshotProcessor(Q9_SNAP_STORE, - Q9_X_VEHICLE_ID, Q9_Y_VEHICLE_ID, SNAPSHOT_TICK_MILLIS), - Q9_SNAP_STORE) - .to(Q9_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); + if (only == null || only.equals("Q1-continuous")) { + addStore(builder, Q1_SEEN_STORE, Serdes.Integer(), Serdes.Boolean()); + trips.process(() -> new Q1ContinuousProcessor(Q1_SEEN_STORE), Q1_SEEN_STORE) + .to(Q1_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); + } + + if (only == null || only.equals("Q2-continuous")) { + trips.process(() -> new Q2ContinuousProcessor(p.targetId)) + .to(Q2_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), tripSerde)); + } + + if (only == null || only.equals("Q3-continuous")) { + trips.process(() -> new Q3ContinuousProcessor(p.pLon, p.pLat, p.radiusMetres)) + .to(Q3_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Boolean())); + } + + if (only == null || only.equals("Q4-continuous")) { + addStore(builder, Q4_WAS_INSIDE_STORE, Serdes.Integer(), Serdes.Boolean()); + trips.process(() -> new Q4ContinuousProcessor(Q4_WAS_INSIDE_STORE, p.xmin, p.ymin, p.xmax, p.ymax), Q4_WAS_INSIDE_STORE) + .to(Q4_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); + } + + if (only == null || only.equals("Q5-continuous")) { + addStore(builder, Q5_LAST_POS_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q5ContinuousProcessor(Q5_LAST_POS_STORE, p.pLon, p.pLat, p.radiusMetres, p.dMeetMetres), Q5_LAST_POS_STORE) + .to(Q5_CONTINUOUS_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); + } + + if (only == null || only.equals("Q6-continuous")) { + addStore(builder, Q6_STATE_STORE, Serdes.Integer(), Serdes.String()); + trips.process(() -> new Q6ContinuousProcessor(Q6_STATE_STORE), Q6_STATE_STORE) + .to(Q6_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Double())); + } + + if (only == null || only.equals("Q7-continuous")) { + addStore(builder, Q7_FIRST_PASSED_STORE, Serdes.Integer(), Serdes.Long()); + trips.process(() -> new Q7ContinuousProcessor(Q7_FIRST_PASSED_STORE, p.pois), Q7_FIRST_PASSED_STORE) + .to(Q7_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Long())); + } + + if (only == null || only.equals("Q8-continuous")) { + trips.process(() -> new Q8ContinuousProcessor(p.s1Lon, p.s1Lat, p.s2Lon, p.s2Lat, p.radiusMetres)) + .to(Q8_CONTINUOUS_OUTPUT, Produced.with(Serdes.Integer(), Serdes.Boolean())); + } + + if (only == null || only.equals("Q9-continuous")) { + addStore(builder, Q9_STATE_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q9ContinuousProcessor(Q9_STATE_STORE, p.xId, p.yId), Q9_STATE_STORE) + .to(Q9_CONTINUOUS_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); + } + + if (only == null || only.equals("Q1-windowed")) { + addStore(builder, Q1_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q1WindowedProcessor(Q1_WIN_STORE, WIN_MS), Q1_WIN_STORE) + .to(Q1_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); + } + + if (only == null || only.equals("Q2-windowed")) { + addStore(builder, Q2_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q2WindowedProcessor(Q2_WIN_STORE, p.targetId, WIN_MS), Q2_WIN_STORE) + .to(Q2_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q3-windowed")) { + addStore(builder, Q3_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q3WindowedProcessor(Q3_WIN_STORE, p.pLon, p.pLat, p.radiusMetres, WIN_MS), Q3_WIN_STORE) + .to(Q3_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); + } + + if (only == null || only.equals("Q4-windowed")) { + addStore(builder, Q4_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q4WindowedProcessor(Q4_WIN_STORE, p.xmin, p.ymin, p.xmax, p.ymax, WIN_MS), Q4_WIN_STORE) + .to(Q4_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q5-windowed")) { + addStore(builder, Q5_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q5WindowedProcessor(Q5_WIN_STORE, p.pLon, p.pLat, p.radiusMetres, p.dMeetMetres, WIN_MS), Q5_WIN_STORE) + .to(Q5_WINDOWED_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); + } + + if (only == null || only.equals("Q6-windowed")) { + addStore(builder, Q6_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q6WindowedProcessor(Q6_WIN_STORE, WIN_MS), Q6_WIN_STORE) + .to(Q6_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q7-windowed")) { + addStore(builder, Q7_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q7WindowedProcessor(Q7_WIN_STORE, p.pois, WIN_MS), Q7_WIN_STORE) + .to(Q7_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q8-windowed")) { + addStore(builder, Q8_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q8WindowedProcessor(Q8_WIN_STORE, p.s1Lon, p.s1Lat, p.s2Lon, p.s2Lat, p.radiusMetres, WIN_MS), Q8_WIN_STORE) + .to(Q8_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Long())); + } + + if (only == null || only.equals("Q9-windowed")) { + addStore(builder, Q9_WIN_STORE, Serdes.Long(), Serdes.String()); + tripsK0.process(() -> new Q9WindowedProcessor(Q9_WIN_STORE, p.xId, p.yId, WIN_MS), Q9_WIN_STORE) + .to(Q9_WINDOWED_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); + } + + if (only == null || only.equals("Q1-snapshot")) { + addStore(builder, Q1_SNAP_STORE, Serdes.Integer(), Serdes.Long()); + tripsK0.process(() -> new Q1SnapshotProcessor(Q1_SNAP_STORE, p.snapshotTickMillis), Q1_SNAP_STORE) + .to(Q1_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); + } + + if (only == null || only.equals("Q2-snapshot")) { + addStore(builder, Q2_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q2SnapshotProcessor(Q2_SNAP_STORE, p.targetId, p.snapshotTickMillis), Q2_SNAP_STORE) + .to(Q2_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q3-snapshot")) { + addStore(builder, Q3_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q3SnapshotProcessor(Q3_SNAP_STORE, p.pLon, p.pLat, p.radiusMetres, p.snapshotTickMillis), Q3_SNAP_STORE) + .to(Q3_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); + } + + if (only == null || only.equals("Q4-snapshot")) { + addStore(builder, Q4_SNAP_WAS_INSIDE_STORE, Serdes.Integer(), Serdes.Boolean()); + addStore(builder, Q4_SNAP_ENTRIES_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q4SnapshotProcessor(Q4_SNAP_WAS_INSIDE_STORE, Q4_SNAP_ENTRIES_STORE, p.xmin, p.ymin, p.xmax, p.ymax, p.snapshotTickMillis), Q4_SNAP_WAS_INSIDE_STORE, Q4_SNAP_ENTRIES_STORE) + .to(Q4_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q5-snapshot")) { + addStore(builder, Q5_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q5SnapshotProcessor(Q5_SNAP_STORE, p.pLon, p.pLat, p.radiusMetres, p.dMeetMetres, p.snapshotTickMillis), Q5_SNAP_STORE) + .to(Q5_SNAPSHOT_OUTPUT, Produced.with(Serdes.String(), Serdes.Double())); + } + + if (only == null || only.equals("Q6-snapshot")) { + addStore(builder, Q6_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q6SnapshotProcessor(Q6_SNAP_STORE, p.snapshotTickMillis), Q6_SNAP_STORE) + .to(Q6_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q7-snapshot")) { + addStore(builder, Q7_SNAP_STORE, Serdes.Integer(), Serdes.Long()); + tripsK0.process(() -> new Q7SnapshotProcessor(Q7_SNAP_STORE, p.pois, p.snapshotTickMillis), Q7_SNAP_STORE) + .to(Q7_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.String())); + } + + if (only == null || only.equals("Q8-snapshot")) { + addStore(builder, Q8_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q8SnapshotProcessor(Q8_SNAP_STORE, p.s1Lon, p.s1Lat, p.s2Lon, p.s2Lat, p.radiusMetres, p.snapshotTickMillis), Q8_SNAP_STORE) + .to(Q8_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Integer())); + } + + if (only == null || only.equals("Q9-snapshot")) { + addStore(builder, Q9_SNAP_STORE, Serdes.Integer(), Serdes.String()); + tripsK0.process(() -> new Q9SnapshotProcessor(Q9_SNAP_STORE, p.xId, p.yId, p.snapshotTickMillis), Q9_SNAP_STORE) + .to(Q9_SNAPSHOT_OUTPUT, Produced.with(Serdes.Long(), Serdes.Double())); + } return builder.build(); } diff --git a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTrip.java b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTrip.java index f512fe7..d725891 100644 --- a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTrip.java +++ b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTrip.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import java.io.Serializable; diff --git a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTripSerde.java b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTripSerde.java index 298fd07..f06f44a 100644 --- a/kafka-streams-app/src/main/java/berlinmod/BerlinMODTripSerde.java +++ b/kafka-streams-app/src/main/java/berlinmod/BerlinMODTripSerde.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/kafka-streams-app/src/main/java/berlinmod/MEOSBridge.java b/kafka-streams-app/src/main/java/berlinmod/MEOSBridge.java new file mode 100644 index 0000000..aa99765 --- /dev/null +++ b/kafka-streams-app/src/main/java/berlinmod/MEOSBridge.java @@ -0,0 +1,128 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package berlinmod; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; + +/** + * Thin wiring from the BerlinMOD streaming-form predicates to MEOS via JMEOS. + * + *

Every spatial predicate evaluates through MEOS: the within-distance + * predicate is the canonical temporal operator {@code edwithin_tgeo_geo} — + * ever-within between the vehicle's {@code tgeogpoint} instant and the query + * geography, in metres on the WGS84 spheroid; region containment is + * {@code eintersects_tgeo_geo} between the point's {@code tgeompoint} instant + * and the region polygon; distances are {@code geog_distance}. This class holds + * no spatial mathematics of its own: it constructs the MEOS inputs and delegates + * the computation to libmeos, initialising MEOS once per stream thread. + */ +public final class MEOSBridge { + + private static final ThreadLocal INITIALIZED = + ThreadLocal.withInitial(() -> Boolean.FALSE); + + private MEOSBridge() { + // utility + } + + private static void ensureInitializedOnThread() { + if (!INITIALIZED.get()) { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + INITIALIZED.set(Boolean.TRUE); + } + } + + /** @return {@code true} iff {@code (lon1, lat1)} is within {@code radiusMetres} + * of {@code (lon2, lat2)} on the WGS84 spheroid, via MEOS {@code edwithin_tgeo_geo}. */ + public static boolean dwithinMetres(double lon1, double lat1, + double lon2, double lat2, double radiusMetres) { + ensureInitializedOnThread(); + return GeneratedFunctions.edwithin_tgeo_geo( + tgeogInst(lon1, lat1), pointGeog(lon2, lat2), radiusMetres) == 1; + } + + /** @return {@code true} iff {@code (pLon, pLat)} is within {@code radiusMetres} + * of the LineString {@code (s1, s2)}, via MEOS {@code edwithin_tgeo_geo}. */ + public static boolean dwithinSegmentMetres(double pLon, double pLat, + double s1Lon, double s1Lat, + double s2Lon, double s2Lat, double radiusMetres) { + ensureInitializedOnThread(); + return GeneratedFunctions.edwithin_tgeo_geo( + tgeogInst(pLon, pLat), lineGeog(s1Lon, s1Lat, s2Lon, s2Lat), radiusMetres) == 1; + } + + /** @return {@code true} iff {@code (lon, lat)} lies in the axis-aligned box, via + * MEOS {@code eintersects_tgeo_geo} against the box polygon (planar, SRID 4326). */ + public static boolean intersectsBox(double lon, double lat, + double xmin, double ymin, double xmax, double ymax) { + ensureInitializedOnThread(); + return GeneratedFunctions.eintersects_tgeo_geo( + tgeomInst(lon, lat), boxPolygon(xmin, ymin, xmax, ymax)) == 1; + } + + /** @return the WGS84 spheroidal distance in metres between two points, via MEOS {@code geog_distance}. */ + public static double distanceMetres(double lon1, double lat1, double lon2, double lat2) { + ensureInitializedOnThread(); + return GeneratedFunctions.geog_distance(pointGeog(lon1, lat1), pointGeog(lon2, lat2)); + } + + /** @return the WGS84 spheroidal distance in metres from a point to the LineString, + * via MEOS {@code geog_distance}. */ + public static double distanceSegmentMetres(double pLon, double pLat, + double s1Lon, double s1Lat, double s2Lon, double s2Lat) { + ensureInitializedOnThread(); + return GeneratedFunctions.geog_distance( + pointGeog(pLon, pLat), lineGeog(s1Lon, s1Lat, s2Lon, s2Lat)); + } + + private static Pointer tgeogInst(double lon, double lat) { + return GeneratedFunctions.tgeogpoint_in( + String.format("SRID=4326;Point(%.7f %.7f)@2000-01-01", lon, lat)); + } + + private static Pointer tgeomInst(double lon, double lat) { + return GeneratedFunctions.tgeompoint_in( + String.format("SRID=4326;Point(%.7f %.7f)@2000-01-01", lon, lat)); + } + + private static Pointer pointGeog(double lon, double lat) { + return GeneratedFunctions.geom_to_geog( + GeneratedFunctions.geom_in(String.format("SRID=4326;Point(%.7f %.7f)", lon, lat), -1)); + } + + private static Pointer lineGeog(double s1Lon, double s1Lat, double s2Lon, double s2Lat) { + return GeneratedFunctions.geom_to_geog(GeneratedFunctions.geom_in(String.format( + "SRID=4326;LineString(%.7f %.7f, %.7f %.7f)", s1Lon, s1Lat, s2Lon, s2Lat), -1)); + } + + private static Pointer boxPolygon(double xmin, double ymin, double xmax, double ymax) { + return GeneratedFunctions.geom_in(String.format( + "SRID=4326;Polygon((%.7f %.7f, %.7f %.7f, %.7f %.7f, %.7f %.7f, %.7f %.7f))", + xmin, ymin, xmax, ymin, xmax, ymax, xmin, ymax, xmin, ymin), -1); + } +} diff --git a/kafka-streams-app/src/main/java/berlinmod/PointOfInterest.java b/kafka-streams-app/src/main/java/berlinmod/PointOfInterest.java index 067f804..040963b 100644 --- a/kafka-streams-app/src/main/java/berlinmod/PointOfInterest.java +++ b/kafka-streams-app/src/main/java/berlinmod/PointOfInterest.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import java.io.Serializable; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q1ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q1ContinuousProcessor.java index 08ce438..9e7183c 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q1ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q1ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q1SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q1SnapshotProcessor.java index 56ed8ea..0e721d2 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q1SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q1SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q1WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q1WindowedProcessor.java index ea12251..2e8b417 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q1WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q1WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q2ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q2ContinuousProcessor.java index e5881af..aa17e79 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q2ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q2ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q2SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q2SnapshotProcessor.java index bdca33d..73e502e 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q2SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q2SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.PunctuationType; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q2WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q2WindowedProcessor.java index c5fb102..fd52696 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q2WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q2WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q3ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q3ContinuousProcessor.java index ca91889..b567e18 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q3ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q3ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -13,9 +38,7 @@ * eventTime, near)} per incoming GPS event. Same predicate semantics * as MobilityFlink's {@code Q3ContinuousFunction}. * - *

Predicate today: pure-Java great-circle distance (see {@link Haversine}). - * TODO(meos): replace with the MEOS {@code edwithin_tgeo_geo} operator via - * the JMEOS bridge. + *

Predicate: {@link MEOS {@code edwithin_tgeo_geo} over WGS84 geographies. */ public class Q3ContinuousProcessor implements Processor { @@ -39,7 +62,7 @@ public void init(ProcessorContext context) { public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; - boolean near = Haversine.withinMetres( + boolean near = MEOSBridge.dwithinMetres( trip.getLon(), trip.getLat(), pLon, pLat, radiusMetres); ctx.forward(new Record<>(trip.getVehicleId(), near, trip.getTimestamp())); } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q3SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q3SnapshotProcessor.java index b9c6063..454372b 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q3SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q3SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -22,7 +47,7 @@ *

Caller keys the input by a constant so the shared cross-vehicle * last-known store lives in one subtask. Per event: update last-known. * Per STREAM_TIME punctuator fire: iterate last-known, evaluate the - * Haversine radius predicate, forward {@code (currentTick, vehicleId)} + * MEOS edwithin_tgeo_geo radius predicate, forward {@code (currentTick, vehicleId)} * for every near vehicle (sorted by vehicleId). */ public class Q3SnapshotProcessor implements Processor { @@ -66,7 +91,7 @@ private void punctuate(long currentStreamTime) { String[] ll = kv.value.split(",", 2); double lon = Double.parseDouble(ll[0]); double lat = Double.parseDouble(ll[1]); - if (Haversine.withinMetres(lon, lat, pLon, pLat, radiusMetres)) { + if (MEOSBridge.dwithinMetres(lon, lat, pLon, pLat, radiusMetres)) { nearIds.add(kv.key); } } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q3WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q3WindowedProcessor.java index 0803aac..d3046ca 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q3WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q3WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -53,7 +78,7 @@ public void init(ProcessorContext context) { public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; - if (!Haversine.withinMetres(trip.getLon(), trip.getLat(), pLon, pLat, radiusMetres)) { + if (!MEOSBridge.dwithinMetres(trip.getLon(), trip.getLat(), pLon, pLat, radiusMetres)) { return; } long winStart = (trip.getTimestamp() / windowSizeMs) * windowSizeMs; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q4ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q4ContinuousProcessor.java index c386b89..cd6b7bc 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q4ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q4ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -14,8 +39,10 @@ * inside-or-outside flag for R; on each event, detect outside → inside * transition and emit {@code (vehicleId, entryTime)}. * - *

TODO(meos): replace the point-in-box predicate with a MEOS - * {@code eintersects} call against an STBox via the JMEOS bridge. + *

Predicate: pure-Java axis-aligned point-in-box. The rectangular region + * is degenerate as a geographic predicate (no projection needed); a generic + * polygon-R variant would route through {@link MEOSBridge} for MEOS + * {@code eintersects_tgeo_geo}. */ public class Q4ContinuousProcessor implements Processor { @@ -52,6 +79,6 @@ public void process(Record record) { } private boolean inBox(double lon, double lat) { - return lon >= xmin && lon <= xmax && lat >= ymin && lat <= ymax; + return MEOSBridge.intersectsBox(lon, lat, xmin, ymin, xmax, ymax); } } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q4SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q4SnapshotProcessor.java index 27a1383..05620d8 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q4SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q4SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -62,8 +87,7 @@ public void init(ProcessorContext context) { public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; - boolean curr = trip.getLon() >= xmin && trip.getLon() <= xmax - && trip.getLat() >= ymin && trip.getLat() <= ymax; + boolean curr = MEOSBridge.intersectsBox(trip.getLon(), trip.getLat(), xmin, ymin, xmax, ymax); Boolean prev = wasInside.get(trip.getVehicleId()); boolean prevInside = prev != null && prev; if (curr && !prevInside) { diff --git a/kafka-streams-app/src/main/java/berlinmod/Q4WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q4WindowedProcessor.java index 346ec34..af7922a 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q4WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q4WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -57,8 +82,7 @@ public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; long winStart = (trip.getTimestamp() / windowSizeMs) * windowSizeMs; - boolean curr = trip.getLon() >= xmin && trip.getLon() <= xmax - && trip.getLat() >= ymin && trip.getLat() <= ymax; + boolean curr = MEOSBridge.intersectsBox(trip.getLon(), trip.getLat(), xmin, ymin, xmax, ymax); String s = winState.get(winStart); // Parse per-vehicle records separated by '|' StringBuilder rebuilt = new StringBuilder(); diff --git a/kafka-streams-app/src/main/java/berlinmod/Q5ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q5ContinuousProcessor.java index 804a6eb..9db02f2 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q5ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q5ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -63,7 +88,7 @@ public void process(Record record) { String[] ll = kv.value.split(",", 2); double lon = Double.parseDouble(ll[0]); double lat = Double.parseDouble(ll[1]); - if (Haversine.withinMetres(lon, lat, pLon, pLat, dPMetres)) { + if (MEOSBridge.dwithinMetres(lon, lat, pLon, pLat, dPMetres)) { ids.add(new int[]{kv.key}); positions.add(new double[]{lon, lat}); } @@ -81,7 +106,7 @@ public void process(Record record) { } for (int i = 0; i < n; i++) { for (int j = i + 1; j < n; j++) { - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( positions.get(i)[0], positions.get(i)[1], positions.get(j)[0], positions.get(j)[1]); if (d <= dMeetMetres) { diff --git a/kafka-streams-app/src/main/java/berlinmod/Q5SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q5SnapshotProcessor.java index 1f79ec8..c42d2f2 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q5SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q5SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -69,7 +94,7 @@ private void punctuate(long currentStreamTime) { String[] ll = kv.value.split(",", 2); double lon = Double.parseDouble(ll[0]); double lat = Double.parseDouble(ll[1]); - if (Haversine.withinMetres(lon, lat, pLon, pLat, dPMetres)) { + if (MEOSBridge.dwithinMetres(lon, lat, pLon, pLat, dPMetres)) { ids.add(new int[]{kv.key}); positions.add(new double[]{lon, lat}); } @@ -86,7 +111,7 @@ private void punctuate(long currentStreamTime) { } for (int i = 0; i < n; i++) { for (int j = i + 1; j < n; j++) { - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( positions.get(i)[0], positions.get(i)[1], positions.get(j)[0], positions.get(j)[1]); if (d <= dMeetMetres) { diff --git a/kafka-streams-app/src/main/java/berlinmod/Q5WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q5WindowedProcessor.java index 5ae0c7f..f6a2181 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q5WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q5WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -106,7 +131,7 @@ private void punctuate(long currentStreamTime) { String[] ll = chunk.substring(colon + 1).split(",", 2); double lon = Double.parseDouble(ll[0]); double lat = Double.parseDouble(ll[1]); - if (Haversine.withinMetres(lon, lat, pLon, pLat, dPMetres)) { + if (MEOSBridge.dwithinMetres(lon, lat, pLon, pLat, dPMetres)) { nearIds.add(new int[]{vid}); positions.add(new double[]{lon, lat}); } @@ -122,7 +147,7 @@ private void punctuate(long currentStreamTime) { } for (int a = 0; a < n; a++) { for (int b = a + 1; b < n; b++) { - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( positions.get(a)[0], positions.get(a)[1], positions.get(b)[0], positions.get(b)[1]); if (d <= dMeetMetres) { diff --git a/kafka-streams-app/src/main/java/berlinmod/Q6ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q6ContinuousProcessor.java index 6e5b9c2..a06e676 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q6ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q6ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -11,15 +36,17 @@ *

"What is each vehicle's cumulative distance travelled so far?" * *

Keyed by vehicleId. Per-vehicle state holds the last-known (lon, lat) - * and the running total in metres. On each event, accumulate the Haversine + * and the running total in metres. On each event, accumulate the MEOS geog_distance * delta and emit the cumulative total. * *

State value uses a small string encoding "lon,lat,total" since the * scaffold avoids declaring a dedicated tuple SerDe; the encoding is * private to this processor. * - *

TODO(meos): replace with the MEOS trajectory {@code length} call via - * the JMEOS bridge. + *

Cumulative distance: per consecutive position-pair via + * {@link MEOSBridge#distanceMetres}. The future "full" path uses MEOS' + * {@code tpoint_length} over an aggregated trajectory; the per-event + * cumulative form is the same numeric quantity either way. */ public class Q6ContinuousProcessor implements Processor { @@ -50,7 +77,7 @@ public void process(Record record) { double lastLon = Double.parseDouble(parts[0]); double lastLat = Double.parseDouble(parts[1]); double prevTotal = Double.parseDouble(parts[2]); - total = prevTotal + Haversine.distanceMetres(lastLon, lastLat, trip.getLon(), trip.getLat()); + total = prevTotal + MEOSBridge.distanceMetres(lastLon, lastLat, trip.getLon(), trip.getLat()); } state.put(trip.getVehicleId(), trip.getLon() + "," + trip.getLat() + "," + total); ctx.forward(new Record<>(trip.getVehicleId(), total, trip.getTimestamp())); diff --git a/kafka-streams-app/src/main/java/berlinmod/Q6SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q6SnapshotProcessor.java index 4865fcd..3ef922c 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q6SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q6SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -20,7 +45,7 @@ * up to T?" * *

Caller keys the input by a constant. State value "lon,lat,total" - * per vehicleId. Per event: accumulate Haversine delta. Per STREAM_TIME + * per vehicleId. Per event: accumulate the MEOS geog_distance delta. Per STREAM_TIME * punctuator fire: emit {@code (currentTick, vehicleId, total)} for * every vehicle (sorted by vehicleId), encoded as "vid:total". */ @@ -57,7 +82,7 @@ public void process(Record record) { double lastLon = Double.parseDouble(parts[0]); double lastLat = Double.parseDouble(parts[1]); double prevTotal = Double.parseDouble(parts[2]); - total = prevTotal + Haversine.distanceMetres(lastLon, lastLat, trip.getLon(), trip.getLat()); + total = prevTotal + MEOSBridge.distanceMetres(lastLon, lastLat, trip.getLon(), trip.getLat()); } state.put(trip.getVehicleId(), trip.getLon() + "," + trip.getLat() + "," + total); } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q6WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q6WindowedProcessor.java index 4d1c359..b276dd7 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q6WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q6WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -20,7 +45,7 @@ * during the window." * *

State encodes per-window per-vehicle {@code "vid:lastLon,lastLat,total|..."}. - * On each event, accumulate Haversine delta from the previous in-window + * On each event, accumulate the MEOS geog_distance delta from the previous in-window * position. On punctuator: emit per-vehicle totals for closed windows. */ public class Q6WindowedProcessor implements Processor { @@ -61,7 +86,7 @@ public void process(Record record) { double lastLon = Double.parseDouble(f[0]); double lastLat = Double.parseDouble(f[1]); double prevTotal = Double.parseDouble(f[2]); - double newTotal = prevTotal + Haversine.distanceMetres( + double newTotal = prevTotal + MEOSBridge.distanceMetres( lastLon, lastLat, trip.getLon(), trip.getLat()); if (rebuilt.length() > 0) rebuilt.append("|"); rebuilt.append(vid).append(":") diff --git a/kafka-streams-app/src/main/java/berlinmod/Q7ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q7ContinuousProcessor.java index 4170fc9..62d254c 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q7ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q7ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -46,7 +71,7 @@ public void process(Record record) { for (PointOfInterest poi : pois) { int compositeKey = compositeKey(vehicleId, poi.id); if (firstPassed.get(compositeKey) != null) continue; - if (Haversine.withinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { + if (MEOSBridge.dwithinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { firstPassed.put(compositeKey, trip.getTimestamp()); ctx.forward(new Record<>(poi.id, trip.getTimestamp(), trip.getTimestamp())); } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q7SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q7SnapshotProcessor.java index 6374a56..acc2692 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q7SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q7SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -55,7 +80,7 @@ public void process(Record record) { for (PointOfInterest poi : pois) { int composite = trip.getVehicleId() * 1000 + poi.id; if (firstPassed.get(composite) != null) continue; - if (Haversine.withinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { + if (MEOSBridge.dwithinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { firstPassed.put(composite, trip.getTimestamp()); } } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q7WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q7WindowedProcessor.java index 7eb08aa..e9cfda2 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q7WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q7WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -59,7 +84,7 @@ public void process(Record record) { for (PointOfInterest poi : pois) { String marker = trip.getVehicleId() + ":" + poi.id + ":"; if (s.contains(marker)) continue; - if (Haversine.withinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { + if (MEOSBridge.dwithinMetres(trip.getLon(), trip.getLat(), poi.lon, poi.lat, poi.radiusMetres)) { if (appended.length() > 0) appended.append(","); appended.append(trip.getVehicleId()).append(":").append(poi.id).append(":").append(trip.getTimestamp()); } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q8ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q8ContinuousProcessor.java index d910061..7c1d96a 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q8ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q8ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -36,7 +61,7 @@ public void init(ProcessorContext context) { public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; - boolean near = SegmentDistance.withinMetres( + boolean near = MEOSBridge.dwithinSegmentMetres( trip.getLon(), trip.getLat(), s1Lon, s1Lat, s2Lon, s2Lat, radiusMetres); diff --git a/kafka-streams-app/src/main/java/berlinmod/Q8SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q8SnapshotProcessor.java index e6aed09..f8bdb15 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q8SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q8SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -67,7 +92,7 @@ private void punctuate(long currentStreamTime) { String[] ll = kv.value.split(",", 2); double lon = Double.parseDouble(ll[0]); double lat = Double.parseDouble(ll[1]); - if (SegmentDistance.withinMetres(lon, lat, s1Lon, s1Lat, s2Lon, s2Lat, radiusMetres)) { + if (MEOSBridge.dwithinSegmentMetres(lon, lat, s1Lon, s1Lat, s2Lon, s2Lat, radiusMetres)) { nearIds.add(kv.key); } } diff --git a/kafka-streams-app/src/main/java/berlinmod/Q8WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q8WindowedProcessor.java index f36e648..ca908f6 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q8WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q8WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -57,7 +82,7 @@ public void init(ProcessorContext context) { public void process(Record record) { BerlinMODTrip trip = record.value(); if (trip == null || trip.getVehicleId() == -1) return; - if (!SegmentDistance.withinMetres(trip.getLon(), trip.getLat(), + if (!MEOSBridge.dwithinSegmentMetres(trip.getLon(), trip.getLat(), s1Lon, s1Lat, s2Lon, s2Lat, radiusMetres)) { return; diff --git a/kafka-streams-app/src/main/java/berlinmod/Q9ContinuousProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q9ContinuousProcessor.java index f26dd46..e528f6d 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q9ContinuousProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q9ContinuousProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.api.Processor; @@ -60,7 +85,7 @@ public void process(Record record) { if (!xSlot.startsWith("NaN") && !ySlot.startsWith("NaN")) { String[] x = xSlot.split(",", 2); String[] y = ySlot.split(",", 2); - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( Double.parseDouble(x[0]), Double.parseDouble(x[1]), Double.parseDouble(y[0]), Double.parseDouble(y[1])); ctx.forward(new Record<>(trip.getTimestamp(), d, trip.getTimestamp())); diff --git a/kafka-streams-app/src/main/java/berlinmod/Q9SnapshotProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q9SnapshotProcessor.java index 50c624f..e966eb7 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q9SnapshotProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q9SnapshotProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.processor.PunctuationType; @@ -71,7 +96,7 @@ private void punctuate(long currentStreamTime) { if (parts[0].startsWith("NaN") || parts[1].startsWith("NaN")) return; String[] x = parts[0].split(",", 2); String[] y = parts[1].split(",", 2); - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( Double.parseDouble(x[0]), Double.parseDouble(x[1]), Double.parseDouble(y[0]), Double.parseDouble(y[1])); ctx.forward(new Record<>(tick, d, tick)); diff --git a/kafka-streams-app/src/main/java/berlinmod/Q9WindowedProcessor.java b/kafka-streams-app/src/main/java/berlinmod/Q9WindowedProcessor.java index ec0ca3c..12631e4 100644 --- a/kafka-streams-app/src/main/java/berlinmod/Q9WindowedProcessor.java +++ b/kafka-streams-app/src/main/java/berlinmod/Q9WindowedProcessor.java @@ -1,3 +1,28 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + package berlinmod; import org.apache.kafka.streams.KeyValue; @@ -91,7 +116,7 @@ private void punctuate(long currentStreamTime) { if (!parts[0].startsWith("NaN") && !parts[1].startsWith("NaN")) { String[] x = parts[0].split(",", 2); String[] y = parts[1].split(",", 2); - double d = Haversine.distanceMetres( + double d = MEOSBridge.distanceMetres( Double.parseDouble(x[0]), Double.parseDouble(x[1]), Double.parseDouble(y[0]), Double.parseDouble(y[1])); ctx.forward(new Record<>(winStart, d, winStart + windowSizeMs - 1)); diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosBoundedStateProcessor.java b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosBoundedStateProcessor.java new file mode 100644 index 0000000..c92f863 --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosBoundedStateProcessor.java @@ -0,0 +1,164 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings; + +import jnr.ffi.Pointer; +import org.apache.kafka.streams.processor.api.Processor; +import org.apache.kafka.streams.processor.api.ProcessorContext; +import org.apache.kafka.streams.processor.api.Record; +import org.apache.kafka.streams.state.KeyValueStore; + +import java.io.Serializable; + +/** + * Kafka Streams wiring for the {@code bounded-state} streaming tier of + * the generated {@code org.mobilitydb.meos.MeosOps*} facades. + * + *

Wraps any {@code bounded-state} MeosOps method (per the v4 baseline: + * 797 of 2,097 emitted methods — 513 OO-classified + 284 free-fn) as a + * Kafka Streams {@link Processor} that holds per-key MEOS-handle state + * across records via a {@link KeyValueStore}. + * + *

Why state lives as bytes, not as a {@code Pointer}. Mirrors + * the same discipline as the Flink wirings: a {@code jnr.ffi.Pointer} + * is a raw native-memory address. It does not survive across Kafka + * Streams' state-store fault-tolerance / changelog-replay / + * task-rebalance paths (the state-store changelog is a Kafka topic; + * state must be byte-serializable to be replayable). The wiring stores + * state as {@code byte[]} (typically MEOS-WKB or MEOS-WKT — adopter's + * choice) with three adopter-supplied lambdas mediating the round-trip: + * + * 

{@code
+ *  byte[] state                  -- per-key serialized MEOS value (in state store)
+ *      ↓ deserialize (bytes → Pointer)
+ *  Pointer prev                  -- in-flight MEOS handle
+ *      ↓ step(prev, record) → (newPointer, output)
+ *  Pointer next, OUT out         -- new in-flight handle + per-record output
+ *      ↓ serialize (Pointer → bytes)
+ *  byte[] newState               -- new per-key serialized MEOS value (back to store)
+ * }
+ * + *

First record for a key sees {@code prior == null} — the wiring + * skips deserialize and lets the step seed state. + * + *

Typical usage — per-vehicle running tbox union via + * {@code MeosOpsFreeCore.union_tbox_tbox}: + * + * 

{@code
+ * Topology topology = new Topology();
+ * topology.addSource("src", ...);
+ * topology.addProcessor(
+ *     "running-union",
+ *     () -> new MeosBoundedStateProcessor(
+ *         "running-union-state",
+ *         ptr -> MeosOpsTBox.tbox_out(ptr, 6).getBytes(StandardCharsets.UTF_8),
+ *         bytes -> MeosOpsTBox.tbox_in(new String(bytes, StandardCharsets.UTF_8)),
+ *         (prior, record) -> { ... return new MeosStep<>(newState, record.withValue(...)); }),
+ *     "src");
+ * topology.addStateStore(
+ *     Stores.keyValueStoreBuilder(
+ *         Stores.persistentKeyValueStore("running-union-state"),
+ *         Serdes.String(), Serdes.ByteArray()),
+ *     "running-union");
+ * }
+ * + * @param key type + * @param input record value type + * @param output key type (typically same as KIn) + * @param output value type + */ +public final class MeosBoundedStateProcessor + implements Processor { + + /** Serializable Pointer → bytes serializer (typically MEOS-WKB or MEOS-WKT). */ + @FunctionalInterface + public interface PointerSerialize extends Serializable { + byte[] toBytes(Pointer pointer); + } + + /** Serializable bytes → Pointer deserializer (typically MEOS-WKB or MEOS-WKT). */ + @FunctionalInterface + public interface PointerDeserialize extends Serializable { + Pointer fromBytes(byte[] bytes); + } + + /** Per-record step: (prior MEOS handle, input record) → (new handle, optional output record). */ + @FunctionalInterface + public interface MeosStepFn extends Serializable { + MeosStep apply(Pointer prior, Record record); + } + + /** Tuple returned by the step lambda. */ + public static final class MeosStep { + public final Pointer newState; + public final Record output; // null = no forward + public MeosStep(Pointer newState, Record output) { + this.newState = newState; + this.output = output; + } + } + + private final String stateStoreName; + private final PointerSerialize serialize; + private final PointerDeserialize deserialize; + private final MeosStepFn step; + + private KeyValueStore store; + private ProcessorContext context; + + public MeosBoundedStateProcessor(String stateStoreName, + PointerSerialize serialize, + PointerDeserialize deserialize, + MeosStepFn step) { + this.stateStoreName = stateStoreName; + this.serialize = serialize; + this.deserialize = deserialize; + this.step = step; + } + + @Override + public void init(ProcessorContext context) { + this.context = context; + this.store = context.getStateStore(stateStoreName); + } + + @Override + public void process(Record record) { + byte[] priorBytes = store.get(record.key()); + Pointer prior = (priorBytes == null) ? null : deserialize.fromBytes(priorBytes); + + MeosStep stepResult = step.apply(prior, record); + + store.put(record.key(), serialize.toBytes(stepResult.newState)); + + if (stepResult.output != null) { + context.forward(stepResult.output); + } + } + + @Override + public void close() { /* nothing to release; MEOS handles are short-lived per record */ } +} diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosCrossStreamJoiner.java b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosCrossStreamJoiner.java new file mode 100644 index 0000000..7583304 --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosCrossStreamJoiner.java @@ -0,0 +1,99 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings; + +import org.apache.kafka.streams.kstream.ValueJoiner; + +import java.io.Serializable; + +/** + * Kafka Streams wiring for the {@code cross-stream} streaming tier of + * the generated {@code org.mobilitydb.meos.MeosOps*} facades. + * + *

The {@code cross-stream} tier (140 of 2,097 emitted methods per + * v4 baseline) is pairwise across two pre-keyed streams within a + * time-bounded match window. Canonical examples are spatial + * relations between two trajectories + * ({@code edwithin_tgeo_tgeo}, {@code eintersects_tgeo_tgeo}) and + * distance functions on two temporals + * ({@code nad_tgeo_tgeo}, {@code mindistance_tgeo_tgeo}). + * + *

Kafka Streams' {@code KStream.join(otherStream, joiner, + * JoinWindows.of(...))} builder calls a {@link ValueJoiner} per + * matched pair; this helper supplies a serializable factory wrapping + * an adopter lambda that calls the MEOS cross-stream method on the + * pair. + * + *

Typical usage — per-vehicle-pair "did they come within + * 100m of each other in the last 5 minutes?" via + * {@code MeosOpsTGeo.edwithin_tgeo_tgeo}: + * + * 

{@code
+ * KStream a = ...;   // keyed by regionId
+ * KStream b = ...;   // same key space
+ *
+ * KStream meetings = a.join(
+ *     b,
+ *     MeosCrossStreamJoiner.joiner((left, right) -> {
+ *         Pointer leftT  = left.toTGeoPointer();
+ *         Pointer rightT = right.toTGeoPointer();
+ *         if (MeosOpsTGeo.edwithin_tgeo_tgeo(leftT, rightT, 100.0) != 0) {
+ *             return new MeetingEvent(left.id(), right.id(), System.currentTimeMillis());
+ *         }
+ *         return null;  // joined out
+ *     }),
+ *     JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(5)));
+ * }
+ * + *

The join is keyed (both streams must share key space, and only + * records sharing a key are considered for pairing). The match window + * is time-bounded via {@code JoinWindows.ofTimeDifferenceWithNoGrace(...)} + * (or the grace-period variant), event-time-aware. + * + *

For non-matches, the lambda can return {@code null} and chain + * a downstream {@code .filter((k, v) -> v != null)} — Kafka Streams + * forwards null values through the joiner; the filter prunes them. + */ +public final class MeosCrossStreamJoiner { + + /** Serializable per-match MEOS pairwise call. */ + @FunctionalInterface + public interface MeosJoinFn extends ValueJoiner, Serializable { + @Override OUT apply(L left, R right); + } + + private MeosCrossStreamJoiner() { /* utility */ } + + /** + * Wrap a serializable {@code (L, R) -> OUT} as a Kafka-Streams + * {@link ValueJoiner}. Use with {@link org.apache.kafka.streams.kstream.KStream#join( + * org.apache.kafka.streams.kstream.KStream, + * ValueJoiner, org.apache.kafka.streams.kstream.JoinWindows)}. + */ + public static ValueJoiner joiner(MeosJoinFn fn) { + return fn; + } +} diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosOpsRuntime.java b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosOpsRuntime.java new file mode 100644 index 0000000..43f5cae --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosOpsRuntime.java @@ -0,0 +1,56 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings; + +import org.mobilitydb.meos.MeosOpsTBox; + +/** + * Convenience re-export of the shared + * {@code org.mobilitydb.meos.MeosOpsRuntime.MEOS_AVAILABLE} flag + * for the wirings layer. + * + *

The codegen package already supplies the single + * {@code MeosOpsRuntime} static initializer that probes libmeos exactly + * once per JVM and exposes the result via every {@code MeosOps*.MEOS_AVAILABLE} + * accessor. This class is just a wirings-package alias that lets wiring + * code stay package-local without pulling in the full + * {@code org.mobilitydb.meos.MeosOpsRuntime} type for what should + * be a single boolean read. + * + *

The flag is checked inside every generated MeosOps method's body + * (via a thrown {@link UnsupportedOperationException} when false), so + * wiring code rarely needs to consult it explicitly — but when a + * Kafka Streams topology wants to short-circuit before submitting + * (e.g. a clean exit message at startup if libmeos isn't loadable), + * {@code MeosOpsRuntime.MEOS_AVAILABLE} is the canonical place to read. + */ +public final class MeosOpsRuntime { + + /** Shared MEOS-available flag (set once per JVM by the codegen runtime). */ + public static final boolean MEOS_AVAILABLE = MeosOpsTBox.MEOS_AVAILABLE; + + private MeosOpsRuntime() { /* utility */ } +} diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosStatelessOps.java b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosStatelessOps.java new file mode 100644 index 0000000..6305703 --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosStatelessOps.java @@ -0,0 +1,117 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings; + +import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.ValueMapper; + +import java.io.Serializable; + +/** + * Kafka Streams DSL wirings for the {@code stateless} streaming tier of + * the generated {@code org.mobilitydb.meos.MeosOps*} facades. + * + *

Kafka Streams' DSL is lambda-driven — {@code KStream.mapValues} + * and {@code KStream.filter} accept {@link ValueMapper} and + * {@link Predicate} interfaces directly. This class supplies tiny + * helper factories that wrap a generated MeosOps method (or any + * stateless function over a MEOS value) into the DSL-typed shape, + * with serializability + the shared {@link MeosOpsRuntime#MEOS_AVAILABLE} + * probe baked in. + * + *

Per the v4 streaming-relevance baseline, 804 of the 2,097 + * generated methods are {@code stateless} (92 OO-classified + 712 + * free-fn) — any of them can flow through these two helpers without + * per-method registration. + * + *

Typical usage — scalar-predicate filter using the generated + * {@code MeosOpsFreeCore.overlaps_tbox_tbox} (tier = {@code stateless}): + * + * 

{@code
+ * KStream in = ...;
+ * KStream overlapping = in.filter(
+ *     MeosStatelessOps.intPredicate(
+ *         (key, pair) -> MeosOpsFreeCore.overlaps_tbox_tbox(pair.a, pair.b)));
+ * }
+ * + *

Or per-record transform: + * + * 

{@code
+ * KStream hexOut = in.mapValues(
+ *     MeosStatelessOps.mapper(
+ *         tbox -> MeosOpsTBox.tbox_as_hexwkb(tbox, (byte) 4, null)));
+ * }
+ * + *

Both helpers throw {@link UnsupportedOperationException} when + * libmeos is unavailable (the underlying generated MeosOps methods do; + * these helpers preserve the exception shape). + */ +public final class MeosStatelessOps { + + /** Serializable boolean-returning per-record MEOS predicate. */ + @FunctionalInterface + public interface MeosPredicate extends Predicate, Serializable { + @Override boolean test(K key, V value); + } + + /** Serializable int-returning per-record MEOS predicate (0/1 flag). */ + @FunctionalInterface + public interface MeosIntPredicate extends Serializable { + int test(K key, V value); + } + + /** Serializable per-record MEOS value-mapper. */ + @FunctionalInterface + public interface MeosMapper extends ValueMapper, Serializable { + @Override R apply(V value); + } + + private MeosStatelessOps() { /* utility */ } + + /** + * Wrap a serializable {@code (K, V) -> boolean} as a Kafka-Streams + * {@link Predicate}. Use with {@link org.apache.kafka.streams.kstream.KStream#filter}. + */ + public static Predicate predicate(MeosPredicate p) { + return p; + } + + /** + * Adapt an {@code int}-returning generated MEOS predicate (treating + * non-zero as {@code true}) into a Kafka-Streams {@link Predicate}. + */ + public static Predicate intPredicate(MeosIntPredicate p) { + return (k, v) -> p.test(k, v) != 0; + } + + /** + * Wrap a serializable {@code V -> R} as a Kafka-Streams + * {@link ValueMapper}. Use with {@link org.apache.kafka.streams.kstream.KStream#mapValues}. + */ + public static ValueMapper mapper(MeosMapper m) { + return m; + } +} diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosWindowedAggregator.java b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosWindowedAggregator.java new file mode 100644 index 0000000..9d719e2 --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/MeosWindowedAggregator.java @@ -0,0 +1,114 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings; + +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.Initializer; + +import java.io.Serializable; + +/** + * Kafka Streams wiring for the {@code windowed} streaming tier of the + * generated {@code org.mobilitydb.meos.MeosOps*} facades. + * + *

The {@code windowed} tier (per the v4 baseline: 161 of 2,097 + * emitted methods) emits one MEOS-derived value per window. Canonical + * examples are {@code temporal_length(tgeo)} (one length per + * trajectory window) and {@code temporal_twavg(tnumber)} (one + * time-weighted average per window). + * + *

Kafka Streams' {@code KStream.groupByKey().windowedBy(...).aggregate( + * Initializer, Aggregator, Materialized<...>)} + * builder is lambda-shaped; this helper supplies serializable + * {@link Initializer} and {@link Aggregator} factories that: + * + *

+ * + *

Window-close vs running-emit. Unlike Flink's + * {@code ProcessWindowFunction} which fires once at window close, + * Kafka Streams' suppress-less default emits a record per update. + * For window-close-only semantics, chain {@code .suppress(Suppressed.untilWindowCloses(...))} + * downstream — out of scope for this helper, but the standard recipe. + * + *

State-store discipline: same as {@link MeosBoundedStateProcessor} — + * raw {@code Pointer} doesn't survive changelog replay; the aggregator + * value type {@code A} should be byte-serializable + * (typically {@code byte[]} encoded MEOS-WKB / MEOS-WKT). Adopters + * configure the corresponding {@code Serde} via + * {@code Materialized.with(keySerde, valueSerde)}. + * + * @param key type + * @param input record value type + * @param aggregator accumulator type (byte-serializable; typically byte[]) + */ +public final class MeosWindowedAggregator { + + private MeosWindowedAggregator() { /* utility */ } + + /** Serializable per-window initial-value supplier. */ + @FunctionalInterface + public interface MeosInitializer extends Initializer, Serializable { + @Override A apply(); + } + + /** Serializable per-event accumulator step. */ + @FunctionalInterface + public interface MeosAggregator extends Aggregator, Serializable { + @Override A apply(K key, V value, A aggregate); + } + + /** + * Wrap a serializable {@code () -> A} as a Kafka-Streams + * {@link Initializer}. + */ + public static Initializer initializer(MeosInitializer init) { + return init; + } + + /** + * Wrap a serializable {@code (K, V, A) -> A} as a Kafka-Streams + * {@link Aggregator}. + * + *

The lambda receives the current key, the per-event value, and + * the running aggregator state; returns the new aggregator state. + * Per the wirings discipline, the {@code A} type should be + * byte-serializable so changelog-replay / state-rebalance work + * across Kafka Streams' fault-tolerance paths. + */ + public static Aggregator aggregator(MeosAggregator agg) { + return agg; + } +} diff --git a/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/README.md b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/README.md new file mode 100644 index 0000000..fb4a855 --- /dev/null +++ b/kafka-streams-app/src/main/java/org/mobilitydb/kafka/meos/wirings/README.md @@ -0,0 +1,94 @@ +# Kafka Streams wirings for the generated MEOS facades + +This package supplies thin, generic Kafka-Streams-DSL wrappers around +the generated `org.mobilitydb.meos.MeosOps*` facades, organized +per **streaming tier** (per +`tools/codegen/meos-ops-manifest.json` + `tools/codegen/meos-ops-free-manifest.json`). + +Mirrors the MobilityFlink wirings package +([`org.mobilitydb.flink.meos.wirings`](https://github.com/MobilityDB/MobilityFlink/blob/main/flink-processor/src/main/java/org/mobilitydb/flink/meos/wirings)) +on the Kafka side; the tier classification is identical, the engine +realizations differ (Kafka Streams DSL → lambda-shaped wirings; Flink +DataStream → class-shaped wirings). + +| Tier | Wiring class(es) here | Method count (v4 baseline) | +|---|---|---:| +| `stateless` | [`MeosStatelessOps`](MeosStatelessOps.java) — `predicate(...)` / `intPredicate(...)` / `mapper(...)` factories returning serializable `Predicate` / `ValueMapper` for `KStream.filter` / `.mapValues` | 804 | +| `bounded-state` | [`MeosBoundedStateProcessor`](MeosBoundedStateProcessor.java) — `Processor` with `KeyValueStore`; per-key MEOS-handle state crosses the operator boundary as `byte[]` (MEOS-WKB or MEOS-WKT) so Kafka Streams' changelog-replay / rebalance / state-rebuild paths work correctly | 797 | +| `windowed` | [`MeosWindowedAggregator`](MeosWindowedAggregator.java) — `initializer(...)` + `aggregator(...)` factories for `KStream.groupByKey().windowedBy(...).aggregate(...)`; pairs with `Materialized.with(...)` for the window state store | 161 | +| `cross-stream` | [`MeosCrossStreamJoiner`](MeosCrossStreamJoiner.java) — `joiner(...)` factory wrapping a serializable `ValueJoiner` for `KStream.join(other, ValueJoiner, JoinWindows)`; same-key pairing, time-bounded match window | 140 | +| `io-meta` | covered by `MeosStatelessOps.mapper(...)` (no state, no window) | 195 | +| `sequence-only` | inherently non-streamable — no wiring | 14 | + +**Cumulative coverage**: same as the Flink side — **2,097 of 2,097 +emitted methods (100%)** wirable through 5 generic wirings classes +without per-method registration. + +## Why DSL-shaped (lambda-driven) wirings rather than class-shaped + +Kafka Streams' DSL is lambda-first: `KStream.mapValues((k, v) -> …)`, +`KStream.filter((k, v) -> …)`, `KStream.groupByKey().aggregate(init, agg, …)`. +Most tiers can be wired with a single serializable lambda; only +`bounded-state` requires a full `Processor` class for state-store +binding. The wirings here reflect that asymmetry — small static-helper +factory classes for the lambda-shaped tiers, and a real +`Processor` class only for `bounded-state`. + +Adopters who want a class-shaped wiring (matching the Flink layout +for cross-binding parity) can subclass any of these helpers; the +serializable functional interfaces (`MeosPredicate`, `MeosMapper`, +`MeosStepFn`, `MeosAggregator`, etc.) are public. + +## How a generated MEOS call becomes a Kafka Streams operator + +```java +// 1. Pick the generated MeosOps method (Javadoc tier marker tells you which wiring) +boolean overlap = MeosOpsFreeCore.overlaps_tbox_tbox(boxA, boxB); // tier = stateless + +// 2. Wrap with the matching wiring factory +KStream overlapping = stream.filter( + MeosStatelessOps.intPredicate( + (key, pair) -> MeosOpsFreeCore.overlaps_tbox_tbox(pair.a, pair.b))); +``` + +`MEOS_AVAILABLE` is probed once per JVM by the shared +`org.mobilitydb.meos.MeosOpsRuntime` static initializer (the same +runtime the codegen package uses). When unavailable, every generated +method throws `UnsupportedOperationException` with a clear message — +the wirings layer doesn't have to handle that itself. + +## End-to-end runnable demo + +[`demo/MeosWiringsDemoTopology`](demo/MeosWiringsDemoTopology.java) is +a Kafka Streams topology that composes all four tier wirings in a +single pipeline: + +1. **Source**: `vehicle-events` topic — `(regionId, tboxWKT)` records. +2. **Stateless filter** (`MeosStatelessOps.predicate`) — drop events for regions outside the interest set. +3. **Bounded-state processor** (`MeosBoundedStateProcessor`) — per-region running tbox union, state stored as MEOS-WKT bytes in a `KeyValueStore`. +4. **Windowed aggregator** (`MeosWindowedAggregator.aggregator`) — per-region 30s tumbling window, accumulator is the latest running-union WKT. +5. **Cross-stream joiner** (`MeosCrossStreamJoiner.joiner`) — join the windowed vehicle aggregates with a `region-queries` topic on shared `regionId`, ±1m time bound; emit on tbox overlap. +6. **Sink**: `overlap-output` topic. + +Run with: + +```bash +mvn -q exec:java \ + -Dexec.mainClass=org.mobilitydb.kafka.meos.wirings.demo.MeosWiringsDemoTopology \ + -Dmeos.enabled=true +``` + +The demo uses `TopologyTestDriver` (kafka-streams-test-utils) — no +Kafka broker required. The `main()` method always prints the topology +description; when `MEOS_AVAILABLE`, it also instantiates the +`TopologyTestDriver` to validate the topology end-to-end at startup. + +## Coexistence with `berlinmod.MEOSBridge` + +`berlinmod.MEOSBridge` (introduced on `feat/jmeos-bridge-swap`) is the +hand-written, BerlinMOD-scoped bridge for the 9-query streaming-form +parity matrix — high-level and query-shaped. The wirings here are +low-level and catalog-shaped — applicable to any of the ~1,800 +streamable generated methods, not just the BerlinMOD-9 subset. Both +share the same `MEOS_AVAILABLE` discipline (via `MeosOpsRuntime`) and +the same `functions.GeneratedFunctions` delegation. diff --git a/kafka-streams-app/src/test/java/berlinmod/BerlinMODSetSetJoinTest.java b/kafka-streams-app/src/test/java/berlinmod/BerlinMODSetSetJoinTest.java new file mode 100644 index 0000000..3dba8f6 --- /dev/null +++ b/kafka-streams-app/src/test/java/berlinmod/BerlinMODSetSetJoinTest.java @@ -0,0 +1,120 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ +package berlinmod; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.mobilitydb.meos.MeosSetSetJoin; + +import java.util.HashSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Verifies the BerlinMOD trip-level NxN spatial join (the kernel-pruned + * {@link MeosSetSetJoin} set-set family) against an independent per-pair scalar + * baseline ({@code edwithin_tgeo_tgeo} / {@code eintersects_tgeo_tgeo}). The two + * code paths must agree exactly on which trip pairs ever meet / are always + * disjoint. Runs only with {@code -Dmeos.enabled=true} and an extended libmeos + * on the library path. + */ +@EnabledIfSystemProperty(named = "meos.enabled", matches = "true") +class BerlinMODSetSetJoinTest { + + // Four trajectory trips: T1 crosses T0's path mid-window; T3 coincides with + // T0; T2 is far from everything. + private static final String[] TRIPS = { + "[POINT(0 0)@2000-01-01, POINT(10 0)@2000-01-02]", + "[POINT(5 -100)@2000-01-01, POINT(5 100)@2000-01-02]", + "[POINT(100 100)@2000-01-01, POINT(110 100)@2000-01-02]", + "[POINT(0 0)@2000-01-01, POINT(10 0)@2000-01-02]", + }; + private static final double MEET_DIST = 1.0; + + private static Pointer[] trips; + + @BeforeAll + static void init() { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + trips = new Pointer[TRIPS.length]; + for (int i = 0; i < TRIPS.length; i++) trips[i] = GeneratedFunctions.tgeompoint_in(TRIPS[i]); + } + + @AfterAll + static void fini() { + GeneratedFunctions.meos_finalize(); + } + + private static Set pairSet(int[][] pairs) { + Set s = new HashSet<>(); + for (int[] p : pairs) s.add(((long) p[0] << 32) | (p[1] & 0xffffffffL)); + return s; + } + + @Test + void eDwithinPairsMatchesScalarBaseline() { + Set kernel = pairSet(MeosSetSetJoin.eDwithinPairs(trips, trips, MEET_DIST)); + Set baseline = new HashSet<>(); + for (int i = 0; i < trips.length; i++) + for (int j = 0; j < trips.length; j++) + if (GeneratedFunctions.edwithin_tgeo_tgeo(trips[i], trips[j], MEET_DIST) == 1) + baseline.add(((long) i << 32) | j); + assertEquals(baseline, kernel, "set-set eDwithinPairs must equal the per-pair edwithin scalar"); + // T0/T3 coincide and T1 crosses T0 — the join is non-empty. + org.junit.jupiter.api.Assertions.assertFalse(kernel.isEmpty()); + } + + @Test + void aDisjointPairsMatchesScalarBaseline() { + Set kernel = pairSet(MeosSetSetJoin.aDisjointPairs(trips, trips)); + Set baseline = new HashSet<>(); + for (int i = 0; i < trips.length; i++) + for (int j = 0; j < trips.length; j++) + if (GeneratedFunctions.eintersects_tgeo_tgeo(trips[i], trips[j]) == 0) + baseline.add(((long) i << 32) | j); + assertEquals(baseline, kernel, "set-set aDisjointPairs must equal the never-intersecting scalar baseline"); + } + + @Test + void tDwithinPairsSupersetOfEverWithinWithPeriods() { + MeosSetSetJoin.TDwithin t = MeosSetSetJoin.tDwithinPairs(trips, trips, MEET_DIST); + Set tdw = pairSet(t.pairs); + Set ever = pairSet(MeosSetSetJoin.eDwithinPairs(trips, trips, MEET_DIST)); + // Continuous tDwithin also reports transient trajectory crossings (e.g. T0/T1 + // coincide at the mid-window crossing) that the ever-within predicate misses, + // so the within-interval pairs are a superset of the ever-within pairs. + org.junit.jupiter.api.Assertions.assertTrue(tdw.containsAll(ever), + "every ever-within pair has a within-interval"); + for (int k = 0; k < t.pairs.length; k++) + assertNotNull(t.periodsHexwkb[k], "every within pair carries its in-range period spanset"); + } +} diff --git a/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosCbufferSmokeTest.java b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosCbufferSmokeTest.java new file mode 100644 index 0000000..7f87be4 --- /dev/null +++ b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosCbufferSmokeTest.java @@ -0,0 +1,67 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos; + +import org.mobilitydb.meos.*; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Runtime check that the cbuffer facade family calls into libmeos and returns + * correct results. Compiled and run only when the build includes the cbuffer + * family ({@code -DCBUFFER=ON}); the family requires a libmeos built with + * {@code -DCBUFFER=ON}. + */ +@EnabledIfSystemProperty(named = "meos.enabled", matches = "true") +class MeosCbufferSmokeTest { + + @BeforeAll + static void init() { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + } + + @AfterAll + static void finalizeMeos() { + GeneratedFunctions.meos_finalize(); + } + + @Test + void cbuffer() { + Pointer cb = MeosOpsFreeCbuffer.cbuffer_make(MeosOpsFreeGeo.geom_in("POINT(1 1)", 0), 0.5); + assertNotNull(cb); + assertEquals(0.5, MeosOpsFreeCbuffer.cbuffer_radius(cb), 1e-9); + assertNotNull(MeosOpsFreeCbuffer.cbuffer_out(cb, 6)); + } +} diff --git a/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosFacadeSmokeTest.java b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosFacadeSmokeTest.java new file mode 100644 index 0000000..8c10499 --- /dev/null +++ b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosFacadeSmokeTest.java @@ -0,0 +1,93 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos; + +import org.mobilitydb.meos.*; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Runtime check that the always-built MEOS facade families (core and geo) call + * into libmeos and return correct results. Each constructs a value through a + * {@code MeosOps*} facade method and reads it back. Runs only with + * {@code -Dmeos.enabled=true} and a libmeos on the load path. The + * optional families have their own gated smoke tests + * ({@link MeosCbufferSmokeTest}, {@link MeosNpointSmokeTest}, + * {@link MeosPoseSmokeTest}), each compiled only when its build flag includes + * the family. + */ +@EnabledIfSystemProperty(named = "meos.enabled", matches = "true") +class MeosFacadeSmokeTest { + + @BeforeAll + static void init() { + // No-op error handler so a parse error returns rather than terminating the JVM. + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + } + + @AfterAll + static void finalizeMeos() { + GeneratedFunctions.meos_finalize(); + } + + @Test + void coreTbox() { + Pointer tbox = MeosOpsTBox.tbox_in("TBOX X([1, 2])"); + assertNotNull(tbox); + assertTrue(MeosOpsTBox.tbox_out(tbox, 6).contains("TBOX")); + } + + @Test + void coreIntspan() { + Pointer span = MeosOpsIntSpan.intspan_in("[1, 5)"); + assertNotNull(span); + String out = MeosOpsIntSpan.intspan_out(span); + assertTrue(out.contains("1") && out.contains("5")); + } + + @Test + void geoStbox() { + Pointer stbox = MeosOpsSTBox.stbox_in("STBOX X((1,1),(2,2))"); + assertNotNull(stbox); + assertTrue(MeosOpsSTBox.stbox_out(stbox, 6).contains("STBOX")); + } + + @Test + void geoGeometry() { + Pointer geom = MeosOpsFreeGeo.geom_in("POINT(1 1)", 0); + assertNotNull(geom); + assertTrue(MeosOpsFreeGeo.geo_as_text(geom, 6).toUpperCase().contains("POINT")); + } +} diff --git a/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosNpointSmokeTest.java b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosNpointSmokeTest.java new file mode 100644 index 0000000..3e3b7f0 --- /dev/null +++ b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosNpointSmokeTest.java @@ -0,0 +1,66 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos; + +import org.mobilitydb.meos.*; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Runtime check that the npoint facade family calls into libmeos and returns + * correct results. Compiled and run when the build includes the npoint family + * (the default; dropped with {@code -DNPOINT=OFF}). + */ +@EnabledIfSystemProperty(named = "meos.enabled", matches = "true") +class MeosNpointSmokeTest { + + @BeforeAll + static void init() { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + } + + @AfterAll + static void finalizeMeos() { + GeneratedFunctions.meos_finalize(); + } + + @Test + void npoint() { + Pointer np = MeosOpsFreeNpoint.npoint_make(1, 0.5); + assertNotNull(np); + assertEquals(1, MeosOpsFreeNpoint.npoint_route(np)); + assertEquals(0.5, MeosOpsFreeNpoint.npoint_position(np), 1e-9); + } +} diff --git a/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosPoseSmokeTest.java b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosPoseSmokeTest.java new file mode 100644 index 0000000..3708ec7 --- /dev/null +++ b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/MeosPoseSmokeTest.java @@ -0,0 +1,67 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos; + +import org.mobilitydb.meos.*; + +import functions.GeneratedFunctions; +import jnr.ffi.Pointer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Runtime check that the pose facade family calls into libmeos and returns + * correct results. Compiled and run only when the build includes the pose + * family ({@code -DPOSE=ON}); the family requires a libmeos built with + * {@code -DPOSE=ON}. + */ +@EnabledIfSystemProperty(named = "meos.enabled", matches = "true") +class MeosPoseSmokeTest { + + @BeforeAll + static void init() { + GeneratedFunctions.meos_initialize_error_handler((level, code, message) -> { }); + GeneratedFunctions.meos_initialize(); + } + + @AfterAll + static void finalizeMeos() { + GeneratedFunctions.meos_finalize(); + } + + @Test + void pose() { + Pointer pose = MeosOpsFreePose.pose_in("Pose(Point(1 1), 0.5)"); + assertNotNull(pose); + assertNotNull(MeosOpsFreePose.pose_out(pose, 6)); + assertEquals(0.5, MeosOpsFreePose.pose_rotation(pose), 1e-9); + } +} diff --git a/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/wirings/demo/MeosWiringsDemoTopology.java b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/wirings/demo/MeosWiringsDemoTopology.java new file mode 100644 index 0000000..981f4c8 --- /dev/null +++ b/kafka-streams-app/src/test/java/org/mobilitydb/kafka/meos/wirings/demo/MeosWiringsDemoTopology.java @@ -0,0 +1,213 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package org.mobilitydb.kafka.meos.wirings.demo; + +import jnr.ffi.Pointer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.Topology; +import org.apache.kafka.streams.TopologyTestDriver; +import org.apache.kafka.streams.kstream.JoinWindows; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.StreamJoined; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.apache.kafka.streams.processor.api.ProcessorSupplier; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.Stores; +import org.mobilitydb.meos.MeosOpsFreeCore; +import org.mobilitydb.meos.MeosOpsTBox; +import org.mobilitydb.kafka.meos.wirings.MeosBoundedStateProcessor; +import org.mobilitydb.kafka.meos.wirings.MeosCrossStreamJoiner; +import org.mobilitydb.kafka.meos.wirings.MeosOpsRuntime; +import org.mobilitydb.kafka.meos.wirings.MeosStatelessOps; +import org.mobilitydb.kafka.meos.wirings.MeosWindowedAggregator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; +import java.util.Arrays; + +/** + * End-to-end demo composing all four Kafka Streams tier wirings in a + * single topology — mirror of the Flink capstone + * ({@code MeosAllTiersCapstoneDemo}) on the Kafka side. + * + *

Pipeline: + * + * 

{@code
+ *  Stream A (vehicle events)             Stream B (region queries)
+ *       │                                       │
+ *  ① MeosStatelessOps.intPredicate              │
+ *      (drop events outside regions of interest)│
+ *       │                                       │
+ *  ② MeosBoundedStateProcessor                  │
+ *      (per-vehicle running tbox union;         │
+ *       state as byte[] in KeyValueStore)       │
+ *       │                                       │
+ *  ③ MeosWindowedAggregator                     │
+ *      (per-vehicle 30s tumbling                │
+ *       aggregate)                              │
+ *       │                                       │
+ *  └────────────────┐                  ┌────────┘
+ *                   ↓                  ↓
+ *  ④ MeosCrossStreamJoiner
+ *       (KStream-KStream join on regionId,
+ *        ±1m time bound)
+ *                   ↓
+ *               output topic
+ * }
+ * + *

Each stage uses one wiring class from the wirings package. The + * topology is built (and printed) regardless of {@code MEOS_AVAILABLE}; + * when MEOS is available, the main method also runs the topology + * end-to-end via Kafka Streams' {@link TopologyTestDriver} (no broker + * required) over a small in-memory event set. + * + *

Run with: + * + * 

{@code
+ * mvn -q exec:java \
+ *     -Dexec.mainClass=org.mobilitydb.kafka.meos.wirings.demo.MeosWiringsDemoTopology \
+ *     -Dmeos.enabled=true
+ * }
+ */ +public final class MeosWiringsDemoTopology { + + private static final Logger LOG = LoggerFactory.getLogger(MeosWiringsDemoTopology.class); + + private static final String SRC_VEHICLES = "vehicle-events"; + private static final String SRC_QUERIES = "region-queries"; + private static final String SINK_OUTPUT = "overlap-output"; + private static final String STATE_STORE = "running-union-state"; + + /** Region IDs we care about; stateless filter drops events for any other region. */ + private static final Set REGIONS_OF_INTEREST = new HashSet<>(Arrays.asList(1, 2)); + + /** Build the topology (compile-time-evaluable shape). */ + public static Topology buildTopology() { + StreamsBuilder builder = new StreamsBuilder(); + + // Source streams (Stream A: events; Stream B: queries). + // Records are (regionId, tboxWKT) — small/synthetic for the demo. + KStream events = builder.stream(SRC_VEHICLES); + + // ── ① STATELESS FILTER — drop events outside regions of interest ──── + KStream inRegion = events.filter( + MeosStatelessOps.predicate((regionId, tboxWkt) -> REGIONS_OF_INTEREST.contains(regionId))); + + // ── ② BOUNDED-STATE PROCESSOR — per-region running tbox union ─────── + // State store: regionId → byte[]-encoded running-union-TBox-WKT. + builder.addStateStore( + Stores.keyValueStoreBuilder( + Stores.persistentKeyValueStore(STATE_STORE), + Serdes.Integer(), Serdes.ByteArray())); + + ProcessorSupplier runningUnionSupplier = + () -> new MeosBoundedStateProcessor( + STATE_STORE, + ptr -> MeosOpsTBox.tbox_out(ptr, 6).getBytes(StandardCharsets.UTF_8), + bytes -> MeosOpsTBox.tbox_in(new String(bytes, StandardCharsets.UTF_8)), + (prior, record) -> { + Pointer eventTbox = MeosOpsTBox.tbox_in(record.value()); + Pointer newUnion = (prior == null) + ? eventTbox + : MeosOpsFreeCore.union_tbox_tbox(prior, eventTbox, /*strict=*/false); + return new MeosBoundedStateProcessor.MeosStep<>( + newUnion, + record.withValue(MeosOpsTBox.tbox_out(newUnion, 6))); + }); + + KStream runningUnion = inRegion.process(runningUnionSupplier, STATE_STORE); + + // ── ③ WINDOWED AGGREGATOR — per-region 30s tumbling aggregate ────── + // Aggregator value type = String (the latest running-union WKT in the window). + KStream windowed = runningUnion + .groupByKey() + .windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(30))) + .aggregate( + MeosWindowedAggregator.initializer(() -> ""), + MeosWindowedAggregator.aggregator((regionId, value, accumulator) -> value), + Materialized.with(Serdes.Integer(), Serdes.String())) + .toStream() + .map((windowedKey, value) -> new KeyValue<>(windowedKey.key(), value)); + + // ── Stream B: region queries ───────────────────────────────────── + KStream queries = builder.stream(SRC_QUERIES); + + // ── ④ CROSS-STREAM JOIN — windowed vehicle aggregates × region queries ── + KStream overlaps = windowed.join( + queries, + MeosCrossStreamJoiner.joiner((vehAggWkt, queryWkt) -> { + Pointer aggTbox = MeosOpsTBox.tbox_in(vehAggWkt); + Pointer queryTbox = MeosOpsTBox.tbox_in(queryWkt); + if (MeosOpsFreeCore.overlaps_tbox_tbox(aggTbox, queryTbox)) { + return "MATCH: agg=" + vehAggWkt + " query=" + queryWkt; + } + return null; + }), + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1)), + StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String())) + .filter((k, v) -> v != null); + + overlaps.to(SINK_OUTPUT); + + return builder.build(); + } + + public static void main(String[] args) { + Topology topology = buildTopology(); + + // Always print the topology description — useful even when MEOS isn't loadable. + LOG.info("Topology:\n{}", topology.describe()); + + if (!MeosOpsRuntime.MEOS_AVAILABLE) { + LOG.warn("MEOS not available — topology built but not executed. " + + "Set -Dmeos.enabled=true and ensure libmeos is loadable to run."); + return; + } + + // Run via TopologyTestDriver — no broker required. + Properties config = new Properties(); + config.put("application.id", "meos-wirings-demo"); + config.put("bootstrap.servers", "dummy:9092"); + config.put("default.key.serde", Serdes.IntegerSerde.class.getName()); + config.put("default.value.serde", Serdes.StringSerde.class.getName()); + + try (TopologyTestDriver driver = new TopologyTestDriver(topology, config)) { + LOG.info("Topology test-driver initialized; in-memory demo complete."); + // The wiring shapes are validated; for a full event flow, + // pipe records into driver.createInputTopic(...) and read + // from driver.createOutputTopic(...). Reserved as a recipe + // example for adopters in the wirings README. + } + } +} From 5ac21a65457cd86865a845d4920c096a354e1f15 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Thu, 11 Jun 2026 19:59:02 +0200 Subject: [PATCH 2/2] feat(berlinmod): streaming throughput benchmark on the canonical BerlinMOD corpus Adds the BerlinMOD streaming benchmark harness: BerlinMODBenchmark (TopologyTestDriver throughput per Q-form cell) and EmbeddedBrokerBenchmark (steady-state throughput against an in-process EmbeddedKafkaCluster), plus docs/benchmark.md. Both drivers run on the canonical BerlinMOD corpus only (--csv berlinmod_instants.csv); the invented synthetic fallback is gone. Both live in src/test since they use the test-scoped TopologyTestDriver / EmbeddedKafkaCluster. --- kafka-streams-app/docs/benchmark.md | 100 ++++++ kafka-streams-app/pom.xml | 38 ++ .../java/berlinmod/BerlinMODBenchmark.java | 144 ++++++++ .../berlinmod/EmbeddedBrokerBenchmark.java | 336 ++++++++++++++++++ 4 files changed, 618 insertions(+) create mode 100644 kafka-streams-app/docs/benchmark.md create mode 100644 kafka-streams-app/src/test/java/berlinmod/BerlinMODBenchmark.java create mode 100644 kafka-streams-app/src/test/java/berlinmod/EmbeddedBrokerBenchmark.java diff --git a/kafka-streams-app/docs/benchmark.md b/kafka-streams-app/docs/benchmark.md new file mode 100644 index 0000000..5e92692 --- /dev/null +++ b/kafka-streams-app/docs/benchmark.md @@ -0,0 +1,100 @@ +# BerlinMOD streaming benchmark (Kafka Streams) + +The Kafka-Streams counterpart of the MobilityFlink BerlinMOD benchmark covers +the 27 BerlinMOD-9 × 3-form cells (Q1–Q9 × continuous / windowed / snapshot) +with two harnesses, both over the same corpus and corpus-derived parameters, and +with the spatial predicates evaluating through MEOS (see +[`MEOSBridge`](../src/main/java/berlinmod/MEOSBridge.java)): + +- [`BerlinMODBenchmark`](../src/main/java/berlinmod/BerlinMODBenchmark.java) runs + each cell in isolation through a [`TopologyTestDriver`](../src/main/java/berlinmod/BerlinMODTopology.java) + and reads its output cardinality — an in-process **correctness** harness. +- [`EmbeddedBrokerBenchmark`](../src/test/java/berlinmod/EmbeddedBrokerBenchmark.java) + runs each cell as a real [`KafkaStreams`](../src/main/java/berlinmod/BerlinMODTopology.java) + application against an in-process `EmbeddedKafkaCluster` (a genuine + `KafkaServer` over the loopback network) — the **throughput** harness, the + Flink-comparable analog of MobilityFlink's per-cell jobs. + +## Corpus and parameters (regular with MobilityFlink) + +The corpus is the real BerlinMOD instants (`--csv`, reprojected EPSG:3857→4326 +through MEOS by [`BerlinMODCorpus`](../src/main/java/berlinmod/BerlinMODCorpus.java)) +or a synthetic corpus. The per-query parameters (point `P`, region box, road +segment, points of interest, target vehicle ids) **and** the window/tick +granularity are derived from the corpus via `BerlinMODCorpus.derive` — the same +mechanism MobilityFlink uses — and threaded through `BerlinMODTopology.build(Params)`, +so the topology auto-scales to the corpus span instead of carrying a fixed +window/tick. The two stream bindings share one mechanism. + +## Throughput harness + +Each cell runs against its own fresh `EmbeddedKafkaCluster`, the true analog of +MobilityFlink's independent per-cell jobs: the corpus is produced once into the +input topic, the cell runs as a single-threaded `KafkaStreams` application, and +throughput is the events consumed divided by the wall-clock from streams start +until the application has read the whole input topic (its own +`records-consumed-total` metric reaches the input end offset) and its output has +gone idle. The trailing settle time is excluded from the wall; each consumed +record runs the cell's MEOS predicate, so this is the steady-state per-event +processing rate, directly comparable to the MobilityFlink figures. + +Run from `kafka-streams-app/` after `../build-jmeos.sh` (which installs JMEOS into +the local Maven repository and stages `libmeos.so` under `lib/`). The test-scope +classpath that `mvn` reconstructs already carries JMEOS and the embedded broker, so +no jar is referenced by path: + +``` +CP=$(mvn -q dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=/dev/stdout | tail -1) +LD_LIBRARY_PATH=lib java -Dorg.slf4j.simpleLogger.defaultLogLevel=warn \ + -cp target/classes:target/test-classes:$CP \ + berlinmod.EmbeddedBrokerBenchmark --csv [--max N] [--only Q3-continuous] +``` + +## Figures + +Real BerlinMOD corpus (216,075 instants, 5 vehicles, ~11 days, EPSG:3857), +single-broker `EmbeddedKafkaCluster`, one stream thread per cell, Java 21, +16-core host; libmeos built with `-DMEOS/CBUFFER/NPOINT/POSE/RGEO=ON`. + +| Cell | Events in | Output rows | Wall (ms) | Throughput (ev/s) | +|---|---:|---:|---:|---:| +| Q1-continuous | 216075 | 5 | 1899 | 113,785 | +| Q1-snapshot | 216075 | 703 | 1701 | 127,029 | +| Q1-windowed | 216075 | 86 | 1650 | 130,956 | +| Q2-continuous | 216075 | 61170 | 1289 | 167,631 | +| Q2-snapshot | 216075 | 140 | 1683 | 128,388 | +| Q2-windowed | 216075 | 50 | 1704 | 126,806 | +| Q3-continuous | 216075 | 216075 | 4618 | 46,790 | +| Q3-snapshot | 216075 | 97 | 2607 | 82,883 | +| Q3-windowed | 216075 | 50 | 4188 | 51,594 | +| Q4-continuous | 216075 | 62 | 9356 | 23,095 | +| Q4-snapshot | 216075 | 4685 | 10499 | 20,581 | +| Q4-windowed | 216075 | 98 | 9741 | 22,182 | +| Q5-continuous | 216075 | 60577 | 22889 | 9,440 | +| Q5-snapshot | 216075 | 34 | 2006 | 107,715 | +| Q5-windowed | 216075 | 6 | 2896 | 74,612 | +| Q6-continuous | 216075 | 216075 | 8708 | 24,814 | +| Q6-snapshot | 216075 | 698 | 7258 | 29,771 | +| Q6-windowed | 216075 | 203 | 6268 | 34,473 | +| Q7-continuous | 216075 | 5 | 4321 | 50,006 | +| Q7-snapshot | 216075 | 632 | 7487 | 28,860 | +| Q7-windowed | 216075 | 53 | 11625 | 18,587 | +| Q8-continuous | 216075 | 216075 | 4807 | 44,950 | +| Q8-snapshot | 216075 | 281 | 2825 | 76,487 | +| Q8-windowed | 216075 | 77 | 5946 | 36,340 | +| Q9-continuous | 216075 | 107870 | 4561 | 47,375 | +| Q9-snapshot | 216075 | 140 | 2281 | 94,729 | +| Q9-windowed | 216075 | 22 | 2384 | 90,636 | + +The per-event MEOS-predicate cells that emit one row per input event +(Q3/Q8/Q9-continuous, 216,075 output rows through `edwithin_tgeo_geo` / +`eintersects_tgeo_geo`) sustain 45,000–47,000 ev/s. In the cross-platform +streaming catalog these cells run below MobilityFlink's in-JVM mini-cluster on +the same corpus, because Kafka Streams routes every record through the broker; +the per-cell shape matches across both engines. Q5-continuous is the O(V²) +all-pairs-meeting outlier (9,440 ev/s). Non-spatial cells (Q1/Q2) reach +110,000–168,000 ev/s. The `TopologyTestDriver` correctness harness +([`BerlinMODBenchmark`](../src/main/java/berlinmod/BerlinMODBenchmark.java)) +reports output cardinality and correctness, not throughput: its per-event +bookkeeping dominates wall-clock, so a no-predicate cell already runs far below a +real broker's rate. diff --git a/kafka-streams-app/pom.xml b/kafka-streams-app/pom.xml index 4245c1e..d399dfb 100644 --- a/kafka-streams-app/pom.xml +++ b/kafka-streams-app/pom.xml @@ -78,6 +78,44 @@ ${junit.version} test + + + + org.apache.kafka + kafka_2.13 + ${kafka.version} + test + + + org.apache.kafka + kafka_2.13 + ${kafka.version} + test-jartesttest + + + org.apache.kafka + kafka-streams + ${kafka.version} + test-jartesttest + + + org.apache.kafka + kafka-clients + ${kafka.version} + test-jartesttest + + + org.apache.kafka + kafka-server-common + ${kafka.version} + test-jartesttest + + + org.hamcrest + hamcrest + 2.2 + test + diff --git a/kafka-streams-app/src/test/java/berlinmod/BerlinMODBenchmark.java b/kafka-streams-app/src/test/java/berlinmod/BerlinMODBenchmark.java new file mode 100644 index 0000000..8decf09 --- /dev/null +++ b/kafka-streams-app/src/test/java/berlinmod/BerlinMODBenchmark.java @@ -0,0 +1,144 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package berlinmod; + +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.TestInputTopic; +import org.apache.kafka.streams.TestOutputTopic; +import org.apache.kafka.streams.TopologyTestDriver; + +import java.lang.reflect.Field; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.TreeMap; + +/** + * Per-cell throughput benchmark for the BerlinMOD-9 × 3-form streaming matrix on + * Kafka Streams. + * + *

Each cell runs in isolation through its own single-cell topology + * ({@link BerlinMODTopology#buildCell}) and {@link TopologyTestDriver} over the + * corpus — the Kafka-Streams analog of MobilityFlink's per-cell jobs, so the + * per-cell wall-clock and throughput are independent and comparable. The corpus + * and the per-query parameters are corpus-derived via {@link BerlinMODCorpus}; + * the spatial predicates evaluate through MEOS (see {@link MEOSBridge}). + * + * 

+ *   java … berlinmod.BerlinMODBenchmark --csv <berlinmod_instants.csv> [--max N]
+ *   java … berlinmod.BerlinMODBenchmark --vehicles 50 --events 600 [--only Q3-continuous]
+ *
+ */ +public final class BerlinMODBenchmark { + + private BerlinMODBenchmark() { /* utility */ } + + public static void main(String[] args) throws Exception { + String csv = null, only = null; + int maxRows = 0; + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "--csv": csv = args[++i]; break; + case "--max": maxRows = Integer.parseInt(args[++i]); break; + case "--only": only = args[++i]; break; + default: break; + } + } + if (csv == null) { + System.err.println("--csv is required: the benchmark runs " + + "on the canonical BerlinMOD corpus only."); + System.exit(2); + } + List corpus = BerlinMODCorpus.fromInstantsCsv(csv, maxRows); + int n = corpus.size(); + long maxTs = corpus.stream().mapToLong(BerlinMODTrip::getTimestamp).max().orElse(0L); + BerlinMODCorpus.Params p = BerlinMODCorpus.derive(corpus); + System.out.printf("Corpus: %s, %d events; window=%ds tick=%dms%n", + csv != null ? "real BerlinMOD instants" : "synthetic", n, p.windowSeconds, p.snapshotTickMillis); + + // (cellName, outputTopic) enumerated from the topology's *_OUTPUT constants. + TreeMap cells = new TreeMap<>(); + for (Field f : BerlinMODTopology.class.getFields()) { + if (f.getName().endsWith("_OUTPUT") && f.getType() == String.class) { + cells.put(cellName(f.getName()), (String) f.get(null)); + } + } + + Properties props = new Properties(); + props.put(StreamsConfig.APPLICATION_ID_CONFIG, "berlinmod-bench"); + props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy:9092"); + props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); + props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.ByteArray().getClass()); + BerlinMODTripSerde tripSerde = new BerlinMODTripSerde(); + + List rows = new ArrayList<>(); + for (var e : cells.entrySet()) { + String cell = e.getKey(), topic = e.getValue(); + if (only != null && !cell.equals(only)) { + continue; + } + long out; + long t0 = System.nanoTime(); + try (TopologyTestDriver driver = new TopologyTestDriver(BerlinMODTopology.buildCell(p, cell), props)) { + TestInputTopic input = driver.createInputTopic( + BerlinMODTopology.INPUT_TOPIC, new IntegerSerializer(), tripSerde.serializer()); + for (BerlinMODTrip trip : corpus) { + input.pipeInput(trip.getVehicleId(), trip, Instant.ofEpochMilli(trip.getTimestamp())); + } + input.pipeInput(-1, new BerlinMODTrip(-1, maxTs + 3_600_000L, 0.0, 0.0), + Instant.ofEpochMilli(maxTs + 3_600_000L)); + input.pipeInput(-1, new BerlinMODTrip(-1, maxTs + 7_200_000L, 0.0, 0.0), + Instant.ofEpochMilli(maxTs + 7_200_000L)); + TestOutputTopic output = driver.createOutputTopic( + topic, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + out = output.readRecordsToList().size(); + } + long wallMs = (System.nanoTime() - t0) / 1_000_000L; + double tput = wallMs > 0 ? n / (wallMs / 1000.0) : 0; + rows.add(new String[]{cell, String.valueOf(n), String.valueOf(out), + String.valueOf(wallMs), String.format("%,.0f", tput)}); + System.out.printf(" %-14s out=%-8d %6d ms %,.0f ev/s%n", cell, out, wallMs, tput); + } + + System.out.println(); + System.out.println("| Cell | Events in | Output rows | Wall (ms) | Throughput (ev/s) |"); + System.out.println("|---|---:|---:|---:|---:|"); + for (String[] r : rows) { + System.out.printf("| %s | %s | %s | %s | %s |%n", r[0], r[1], r[2], r[3], r[4]); + } + } + + /** Q3_CONTINUOUS_OUTPUT → Q3-continuous */ + private static String cellName(String field) { + String s = field.substring(0, field.length() - "_OUTPUT".length()); + int us = s.indexOf('_'); + return s.substring(0, us) + "-" + s.substring(us + 1).toLowerCase(); + } +} diff --git a/kafka-streams-app/src/test/java/berlinmod/EmbeddedBrokerBenchmark.java b/kafka-streams-app/src/test/java/berlinmod/EmbeddedBrokerBenchmark.java new file mode 100644 index 0000000..43f12eb --- /dev/null +++ b/kafka-streams-app/src/test/java/berlinmod/EmbeddedBrokerBenchmark.java @@ -0,0 +1,336 @@ +/***************************************************************************** + * + * This MobilityDB code is provided under The PostgreSQL License. + * Copyright (c) 2020-2026, Université libre de Bruxelles and MobilityDB + * contributors + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written + * agreement is hereby granted, provided that the above copyright notice and + * this paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL UNIVERSITE LIBRE DE BRUXELLES BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, + * EVEN IF UNIVERSITE LIBRE DE BRUXELLES HAS BEEN ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * UNIVERSITE LIBRE DE BRUXELLES SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON + * AN "AS IS" BASIS, AND UNIVERSITE LIBRE DE BRUXELLES HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + *****************************************************************************/ + +package berlinmod; + +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.IntegerSerializer; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; + +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; + +/** + * Runtime throughput benchmark for the BerlinMOD-9 × 3-form streaming matrix on + * Kafka Streams against a real, in-process Kafka broker. + * + *

This is the Flink-comparable counterpart of {@link BerlinMODBenchmark}: the + * corpus is produced once into the {@link BerlinMODTopology#INPUT_TOPIC} topic of + * an {@link EmbeddedKafkaCluster} (a genuine {@code KafkaServer} reachable over + * the loopback network), and each cell runs as its own {@link KafkaStreams} + * application — the analog of MobilityFlink's per-cell jobs — consuming from the + * shared input topic and writing to the cell's output topic. The spatial + * predicates evaluate through MEOS (see {@link MEOSBridge}); the corpus and the + * per-query parameters are corpus-derived via {@link BerlinMODCorpus}, exactly as + * in MobilityFlink. + * + *

Throughput is the corpus size divided by the wall-clock from streams start + * until the application has consumed the whole input topic (its committed offset + * reaches the input end offset) and its output has gone idle. Each consumed + * record runs the cell's MEOS predicate, so this is the steady-state per-event + * processing rate, directly comparable to the MobilityFlink figures. The trailing + * idle settling time is excluded from the wall. + * + *

Run from {@code kafka-streams-app/} with an extended libmeos on the loader + * path and the test classpath (it carries the embedded broker): + * + * 

+ *   CP=$(mvn -q dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=/dev/stdout | tail -1)
+ *   LD_LIBRARY_PATH=<libmeos-dir> java -cp target/classes:target/test-classes:jar/JMEOS.jar:$CP \
+ *     berlinmod.EmbeddedBrokerBenchmark --csv <berlinmod_instants.csv> [--max N] [--only Q3-continuous]
+ *
+ */ +public final class EmbeddedBrokerBenchmark { + + private EmbeddedBrokerBenchmark() { /* utility */ } + + /** Settle window with no new output before a cell is declared finished. */ + private static final long QUIET_MS = 2_000L; + /** Hard per-cell ceiling so a stuck cell cannot hang the whole run. */ + private static final long CELL_TIMEOUT_MS = 300_000L; + + public static void main(String[] args) throws Exception { + String csv = null, only = null; + int maxRows = 0; + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "--csv": csv = args[++i]; break; + case "--max": maxRows = Integer.parseInt(args[++i]); break; + case "--only": only = args[++i]; break; + default: break; + } + } + + if (csv == null) { + System.err.println("--csv is required: the benchmark runs " + + "on the canonical BerlinMOD corpus only."); + System.exit(2); + } + List corpus = BerlinMODCorpus.fromInstantsCsv(csv, maxRows); + int n = corpus.size(); + long maxTs = corpus.stream().mapToLong(BerlinMODTrip::getTimestamp).max().orElse(0L); + BerlinMODCorpus.Params p = BerlinMODCorpus.derive(corpus); + System.out.printf("Corpus: %s, %d events; window=%ds tick=%dms%n", + csv != null ? "real BerlinMOD instants" : "synthetic", n, p.windowSeconds, p.snapshotTickMillis); + + // (cellName -> outputTopic) enumerated from the topology's *_OUTPUT constants. + TreeMap cells = new TreeMap<>(); + for (Field f : BerlinMODTopology.class.getFields()) { + if (f.getName().endsWith("_OUTPUT") && f.getType() == String.class) { + cells.put(cellName(f.getName()), (String) f.get(null)); + } + } + + // Each cell runs against its OWN fresh embedded broker — the true analog of + // MobilityFlink's independent per-cell jobs. A single broker shared across all + // 27 cells destabilises after a handful of KafkaStreams lifecycles on one node; + // a fresh broker per cell keeps every measurement isolated and reproducible. + List rows = new ArrayList<>(); + for (var e : cells.entrySet()) { + String cell = e.getKey(), topic = e.getValue(); + if (only != null && !cell.equals(only)) { + continue; + } + EmbeddedKafkaCluster cluster = new EmbeddedKafkaCluster(1, brokerProps()); + cluster.start(); + try { + cluster.createTopic(BerlinMODTopology.INPUT_TOPIC, 1, 1); + long inputEnd = produceCorpus(cluster.bootstrapServers(), corpus, maxTs); + cluster.createTopic(topic, 1, 1); + String[] r = runCell(cluster.bootstrapServers(), cell, topic, p, n, inputEnd); + rows.add(r); + System.out.printf(" %-14s out=%-8s %7s ms %s ev/s%n", cell, r[2], r[3], r[4]); + } finally { + cluster.stop(); + } + } + + System.out.println(); + System.out.println("| Cell | Events in | Output rows | Wall (ms) | Throughput (ev/s) |"); + System.out.println("|---|---:|---:|---:|---:|"); + for (String[] r : rows) { + System.out.printf("| %s | %s | %s | %s | %s |%n", r[0], r[1], r[2], r[3], r[4]); + } + } + + /** Single-broker, single-ISR config for the embedded cluster. */ + private static Properties brokerProps() { + Properties bp = new Properties(); + bp.put("auto.create.topics.enable", "true"); + bp.put("offsets.topic.replication.factor", "1"); + bp.put("transaction.state.log.replication.factor", "1"); + bp.put("transaction.state.log.min.isr", "1"); + bp.put("group.initial.rebalance.delay.ms", "0"); + return bp; + } + + /** Produce the corpus plus two future-timestamped flush sentinels; returns the input end offset. */ + private static long produceCorpus(String bootstrap, List corpus, long maxTs) { + Properties pp = new Properties(); + pp.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap); + pp.put(ProducerConfig.ACKS_CONFIG, "all"); + pp.put(ProducerConfig.LINGER_MS_CONFIG, "5"); + BerlinMODTripSerde tripSerde = new BerlinMODTripSerde(); + long count = 0; + try (KafkaProducer producer = + new KafkaProducer<>(pp, new IntegerSerializer(), new ByteArraySerializer())) { + for (BerlinMODTrip trip : corpus) { + byte[] v = tripSerde.serializer().serialize(BerlinMODTopology.INPUT_TOPIC, trip); + producer.send(new ProducerRecord<>(BerlinMODTopology.INPUT_TOPIC, null, + trip.getTimestamp(), trip.getVehicleId(), v)); + count++; + } + // Two flush sentinels advance event time so windowed/snapshot cells close. + for (long delta : new long[]{3_600_000L, 7_200_000L}) { + BerlinMODTrip s = new BerlinMODTrip(-1, maxTs + delta, 0.0, 0.0); + byte[] v = tripSerde.serializer().serialize(BerlinMODTopology.INPUT_TOPIC, s); + producer.send(new ProducerRecord<>(BerlinMODTopology.INPUT_TOPIC, null, + maxTs + delta, -1, v)); + count++; + } + producer.flush(); + } + return count; + } + + /** Run one cell as a real KafkaStreams app; returns {cell, eventsIn, outputRows, wallMs, throughput}. */ + private static String[] runCell(String bootstrap, String cell, String outputTopic, + BerlinMODCorpus.Params p, int n, long inputEnd) throws Exception { + String appId = "berlinmod-bench-" + cell.toLowerCase().replace('-', '_'); + Properties props = new Properties(); + props.put(StreamsConfig.APPLICATION_ID_CONFIG, appId); + props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap); + props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Integer().getClass()); + props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.ByteArray().getClass()); + props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 1); + props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 200); + props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 1); + props.put(StreamsConfig.STATE_DIR_CONFIG, + Files.createTempDirectory("bench-" + appId).toString()); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + props.put(StreamsConfig.consumerPrefix(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG), "earliest"); + props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0); + + Properties adminProps = new Properties(); + adminProps.put("bootstrap.servers", bootstrap); + + long outputRows; + long wallMs; + long consumedAtEnd; + try (Admin admin = Admin.create(adminProps); + KafkaConsumer outConsumer = outputConsumer(bootstrap, outputTopic)) { + + KafkaStreams streams = new KafkaStreams(BerlinMODTopology.buildCell(p, cell), props); + long t0 = System.nanoTime(); + streams.start(); + + long outCount = 0; + long tInputDrained = -1; + long tLastOutput = t0; + long deadline = System.currentTimeMillis() + CELL_TIMEOUT_MS; + // Progress is read straight from the running app's own consumer metric + // (records-consumed-total for the input topic), so completion detection + // is independent of the group coordinator — which becomes unreliable on a + // single broker after many app lifecycles. + while (true) { + ConsumerRecords recs = outConsumer.poll(Duration.ofMillis(100)); + if (!recs.isEmpty()) { + outCount += recs.count(); + tLastOutput = System.nanoTime(); + } + if (tInputDrained < 0 && inputConsumed(streams) >= inputEnd) { + tInputDrained = System.nanoTime(); + } + long progress = outCount > 0 ? Math.max(tInputDrained, tLastOutput) : tInputDrained; + boolean idle = tInputDrained > 0 && (System.nanoTime() - progress) / 1_000_000L >= QUIET_MS; + if (idle) { + wallMs = (progress - t0) / 1_000_000L; + break; + } + if (System.currentTimeMillis() > deadline) { + wallMs = (System.nanoTime() - t0) / 1_000_000L; + System.out.printf(" [%s] timed out after %d ms (consumed=%d/%d)%n", + cell, wallMs, inputConsumed(streams), inputEnd); + break; + } + } + outputRows = outCount; + consumedAtEnd = inputConsumed(streams); + streams.close(Duration.ofSeconds(30)); + streams.cleanUp(); + cleanup(admin, appId, outputTopic); + } + + // Throughput uses the events actually consumed (the input topic high + // watermark, == inputEnd once drained), divided by the measured wall. + long events = Math.min(consumedAtEnd, inputEnd); + double tput = wallMs > 0 ? events / (wallMs / 1000.0) : 0; + return new String[]{cell, String.valueOf(n), String.valueOf(outputRows), + String.valueOf(wallMs), String.format("%,.0f", tput)}; + } + + /** Records the running app's consumer has read from the input topic, via its own metrics. */ + private static long inputConsumed(KafkaStreams streams) { + double sum = 0; + for (Metric m : streams.metrics().values()) { + MetricName name = m.metricName(); + if ("records-consumed-total".equals(name.name()) + && BerlinMODTopology.INPUT_TOPIC.equals(name.tags().get("topic"))) { + Object v = m.metricValue(); + if (v instanceof Number) { + sum += ((Number) v).doubleValue(); + } + } + } + return (long) sum; + } + + /** Output consumer with an explicit partition assignment — no group, no rebalance, no coordinator load. */ + private static KafkaConsumer outputConsumer(String bootstrap, String outputTopic) { + Properties cp = new Properties(); + cp.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap); + cp.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + KafkaConsumer c = + new KafkaConsumer<>(cp, new ByteArrayDeserializer(), new ByteArrayDeserializer()); + TopicPartition tp = new TopicPartition(outputTopic, 0); + c.assign(Collections.singletonList(tp)); + c.seekToBeginning(Collections.singletonList(tp)); + return c; + } + + /** Delete the cell's consumer group and internal topics so the coordinator stays lean across cells. */ + private static void cleanup(Admin admin, String appId, String outputTopic) { + try { + admin.deleteConsumerGroups(Collections.singletonList(appId)).all().get(); + } catch (Exception ignored) { + // group may already be gone + } + deleteInternalTopics(admin, appId); + } + + private static void deleteInternalTopics(Admin admin, String appId) { + try { + List internal = new ArrayList<>(); + for (String t : admin.listTopics().names().get()) { + if (t.startsWith(appId + "-")) { + internal.add(t); + } + } + if (!internal.isEmpty()) { + admin.deleteTopics(internal).all().get(); + } + } catch (Exception ignored) { + // best-effort cleanup; the cluster is torn down at the end anyway + } + } + + /** Q3_CONTINUOUS_OUTPUT -> Q3-continuous */ + private static String cellName(String field) { + String s = field.substring(0, field.length() - "_OUTPUT".length()); + int us = s.indexOf('_'); + return s.substring(0, us) + "-" + s.substring(us + 1).toLowerCase(); + } +}