Update tracing & metrics

engine/Cargo.toml

@@ -25,7 +25,7 @@ poem = { version = "3.1.6", git = "https://github.com/poem-web/poem", branch = "
     "sse",
     "tempfile",
     "opentelemetry",
-    "requestid"
+    "requestid",
 ] }
 poem-openapi = { version = "5.1.5", git = "https://github.com/poem-web/poem", branch = "master", features = [
     "chrono",
@@ -61,16 +61,32 @@ rust-s3 = { version = "0.36.0-beta.2", default-features = false, features = [
 ] }
 aws-sdk-s3 = "1.64.0"
 aws-config = "1.5.10"
-async_zip = { version = "0.0.17", features = ["bzip2", "lzma", "zstd", "xz", "deflate"] }
+async_zip = { version = "0.0.17", features = [
+    "bzip2",
+    "lzma",
+    "zstd",
+    "xz",
+    "deflate",
+] }
 dashmap = { version = "6.1.0", features = ["serde"] }
 serde_json = "1.0.138"
 moka = { version = "0.12.10", features = ["future"] }
 infer = "0.19.0"
 # tokio = { version = "1.39.0", features = [""], default-features = false}
-tracing-opentelemetry = "0.29.0"
-opentelemetry-otlp = { version = "0.28.0", features = ["trace", "metrics", "grpc-tonic"] }
-opentelemetry_sdk = {version = "0.28.0", features = ["trace", "rt-async-std"]}
-opentelemetry = { version = "0.28.0", features = ["trace"]}
+tracing-opentelemetry = { version = "0.29.0" }
+opentelemetry-otlp = { version = "0.28.0", features = [
+    "trace",
+    "metrics",
+    "grpc-tonic",
+] }
+opentelemetry_sdk = { version = "0.28.0", features = ["trace", "rt-async-std"] }
+opentelemetry = { version = "0.28.0", features = ["trace"] }
 opentelemetry-stdout = "0.28.0"
 opentelemetry-http = "0.28.0"
 opentelemetry-semantic-conventions = "0.28.0"
+opentelemetry-prometheus = "0.28.0"
+prometheus = "0.13.4"
+tracing-log = "0.2.0"
+tracing-futures = "0.2.5"
+opentelemetry-auto-span = "0.4.0"
+delegate = "0.13.2"

engine/compose.yaml

@@ -37,3 +37,70 @@ services:
         echo $0 "$@"
         exec minio $0 "$@"                                                 # start Minio in the foreground
       '
+
+# Monitoring
+
+  # Tempo runs as user 10001, and docker compose creates the volume as root.
+  # As such, we need to chown the volume in order for Tempo to start correctly.
+  init:
+    image: &tempoImage grafana/tempo:latest
+    user: root
+    entrypoint:
+      - "chown"
+      - "10001:10001"
+      - "/var/tempo"
+    volumes:
+      - ./tempo-data:/var/tempo
+
+  memcached:
+    image: memcached:1.6.29
+    container_name: memcached
+    ports:
+      - "11211:11211"
+    environment:
+      - MEMCACHED_MAX_MEMORY=64m  # Set the maximum memory usage
+      - MEMCACHED_THREADS=4       # Number of threads to use
+
+  tempo:
+    image: *tempoImage
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ../shared/tempo.yaml:/etc/tempo.yaml
+      - ./tempo-data:/var/tempo
+    ports:
+      - "14268:14268"  # jaeger ingest
+      - "3200:3200"   # tempo
+      - "9095:9095" # tempo grpc
+      - "4317:4317"  # otlp grpc
+      - "4318:4318"  # otlp http
+      - "9411:9411"   # zipkin
+    depends_on:
+      - init
+      - memcached
+
+  prometheus:
+    image: prom/prometheus:latest
+    command:
+      - --config.file=/etc/prometheus.yaml
+      - --web.enable-remote-write-receiver
+      - --enable-feature=exemplar-storage
+      - --enable-feature=native-histograms
+    volumes:
+      - ../shared/prometheus.yaml:/etc/prometheus.yaml
+    ports:
+      - "9090:9090"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  grafana:
+    image: grafana/grafana:11.2.0
+    volumes:
+      - ../shared/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor metricsSummary
+      - GF_INSTALL_PLUGINS=https://storage.googleapis.com/integration-artifacts/grafana-exploretraces-app/grafana-exploretraces-app-latest.zip;grafana-traces-app
+    ports:
+      - "3001:3000"

engine/src/database.rs

@@ -1,4 +1,7 @@
-use sqlx::{postgres::PgPoolOptions, PgPool};
+use sqlx::{
+    postgres::{PgConnectOptions, PgPoolOptions},
+    ConnectOptions, PgPool,
+};
 use tracing::info;
 
 #[derive(Debug)]
@@ -8,7 +11,23 @@ pub struct Database {
 
 impl Database {
     pub async fn new(url: &str) -> Result<Self, sqlx::Error> {
-        let pool = PgPoolOptions::new().max_connections(5).connect(url).await?;
+        let mut options: PgConnectOptions = url.parse().unwrap();
+
+        options = options.log_statements(tracing_log::log::LevelFilter::Trace);
+
+        let pool = PgPoolOptions::new()
+            .max_connections(5)
+            .before_acquire(|conn, _meta| Box::pin(async move {
+                info!("Before Acquire");
+                Ok(true)
+            }))
+            .after_connect(|conn, _meta| Box::pin(async move {
+                info!("After Connect");
+
+                Ok(())
+            }))
+            .connect_with(options)
+            .await?;
 
         // Initialization code here
         let s = Self { pool };