feat(api): add OpenTelemetry observability

Author: thonatos

.env.example

@@ -61,3 +61,8 @@ MODERATION_SCHEDULE_INTERVAL_MS=3600000
 MODERATION_SCAN_BATCH_SIZE=200
 MODERATION_SCAN_THROTTLE_MS=500
 MODERATION_SCAN_MAX_BATCHES_PER_RUN=100
+
+# OpenTelemetry tracing is disabled by default for local development.
+CNODE_OTEL_ENABLED=0
+CNODE_OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318/v1/traces
+CNODE_OTEL_TRACE_SAMPLE_RATIO=0.1

apps/api/Dockerfile

@@ -42,4 +42,4 @@ COPY --from=build /app/packages/db /app/packages/db
 COPY --from=build /app/packages/shared /app/packages/shared
 COPY --from=build /app/scripts /app/scripts
 EXPOSE 3001
-CMD ["pnpm", "--filter", "@cnode/api", "exec", "tsx", "src/index.ts"]
+CMD ["pnpm", "--filter", "@cnode/api", "exec", "tsx", "src/bootstrap.ts", "api"]

apps/api/package.json

@@ -4,11 +4,11 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "dev": "tsx watch src/index.ts",
+    "dev": "tsx watch src/bootstrap.ts api",
     "build": "tsc",
-    "start": "node dist/index.js",
+    "start": "tsx src/bootstrap.ts api",
     "test": "vp test run",
-    "worker:moderation": "tsx src/worker/moderation-scan.ts",
+    "worker:moderation": "tsx src/bootstrap.ts moderation-worker",
     "typecheck": "tsc --noEmit",
     "gen:openapi": "tsx scripts/gen-openapi.ts"
   },
@@ -18,6 +18,16 @@
     "@hono/node-server": "^1.13.0",
     "@hono/zod-openapi": "^1.5.1",
     "@hono/zod-validator": "^0.4.0",
+    "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/core": "^2.10.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.221.0",
+    "@opentelemetry/instrumentation-http": "^0.221.0",
+    "@opentelemetry/instrumentation-pg": "^0.73.0",
+    "@opentelemetry/instrumentation-undici": "^0.31.0",
+    "@opentelemetry/resources": "^2.10.0",
+    "@opentelemetry/sdk-node": "^0.221.0",
+    "@opentelemetry/sdk-trace-base": "^2.10.0",
+    "@opentelemetry/semantic-conventions": "^1.43.0",
     "@react-email/render": "2.1.0",
     "ali-oss": "^6.21.0",
     "bcryptjs": "catalog:",

apps/api/src/app.ts

@@ -5,10 +5,11 @@ import packageJson from "../package.json";
 import { errorHandler } from "./middleware/error";
 import { authMiddleware, type AuthVars } from "./middleware/auth";
 import { ipBanMiddleware } from "./middleware/ip-ban";
+import { telemetryMiddleware, type TelemetryVariables } from "./middleware/telemetry";
 import { apiRoutes } from "./routes/index";
 
 const app = new Hono<{
-  Variables: AuthVars;
+  Variables: AuthVars & TelemetryVariables;
 }>();
 
 function allowedCorsOrigin(origin: string | undefined) {
@@ -24,6 +25,7 @@ function allowedCorsOrigin(origin: string | undefined) {
   return origin && allowed.has(origin) ? origin : webBaseUrl;
 }
 
+app.use("*", telemetryMiddleware());
 app.use("*", logger());
 app.use(
   "*",

apps/api/src/bootstrap.ts

@@ -0,0 +1,57 @@
+import { pathToFileURL } from "node:url";
+import { configureProxy } from "./load-env";
+import {
+  initializeTelemetry,
+  type TelemetryRuntime,
+} from "./telemetry/index";
+import type { TelemetryRole } from "./telemetry/config";
+
+interface BootstrapDependencies {
+  initialize?: (role: TelemetryRole) => Promise<TelemetryRuntime>;
+  configureRuntimeProxy?: () => Promise<void>;
+  importTarget?: (role: TelemetryRole) => Promise<unknown>;
+  installSignalHandlers?: boolean;
+}
+
+function parseRole(value: string | undefined): TelemetryRole {
+  if (value === "api" || value === "moderation-worker") return value;
+  throw new Error("bootstrap role must be api or moderation-worker");
+}
+
+async function importTarget(role: TelemetryRole) {
+  return role === "api" ? import("./index") : import("./worker/moderation-scan");
+}
+
+function registerShutdown(runtime: TelemetryRuntime) {
+  let stopping = false;
+  const stop = async () => {
+    if (stopping) return;
+    stopping = true;
+    await runtime.shutdown();
+    process.exit(0);
+  };
+  process.once("SIGTERM", stop);
+  process.once("SIGINT", stop);
+}
+
+export async function runBootstrap(role: TelemetryRole, dependencies: BootstrapDependencies = {}) {
+  const initialize = dependencies.initialize ?? initializeTelemetry;
+  const runtime = await initialize(role);
+
+  try {
+    await (dependencies.configureRuntimeProxy ?? configureProxy)();
+    if (dependencies.installSignalHandlers !== false) registerShutdown(runtime);
+    await (dependencies.importTarget ?? importTarget)(role);
+  } catch (error) {
+    await runtime.shutdown();
+    throw error;
+  }
+}
+
+const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (isMain) {
+  runBootstrap(parseRole(process.argv[2])).catch((error) => {
+    console.error("[bootstrap] application startup failed", error);
+    process.exit(1);
+  });
+}

apps/api/src/index.ts

@@ -1,4 +1,3 @@
-import "./load-env";
 import { serve } from "@hono/node-server";
 import app from "./app";
 

apps/api/src/load-env.ts

@@ -1,6 +1,5 @@
 import { existsSync, readFileSync } from "node:fs";
 import { isAbsolute, resolve } from "node:path";
-import { setGlobalDispatcher, ProxyAgent } from "undici";
 
 function findWorkspaceRoot(cwd: string) {
   let current = resolve(cwd);
@@ -69,8 +68,10 @@ function loadRootEnv() {
 
 loadRootEnv();
 
-const proxy = process.env.HTTPS_PROXY || process.env.HTTP_PROXY;
-if (proxy) {
+export async function configureProxy() {
+  const proxy = process.env.HTTPS_PROXY || process.env.HTTP_PROXY;
+  if (!proxy) return;
+  const { setGlobalDispatcher, ProxyAgent } = await import("undici");
   setGlobalDispatcher(new ProxyAgent(proxy));
-  console.log("[proxy] set global dispatcher:", proxy);
+  console.log("[proxy] configured");
 }

apps/api/src/middleware/telemetry.ts

@@ -0,0 +1,50 @@
+import { randomUUID } from "node:crypto";
+import { SpanKind, SpanStatusCode, trace, type Tracer } from "@opentelemetry/api";
+import { createMiddleware } from "hono/factory";
+
+export interface TelemetryVariables {
+  requestId: string;
+}
+
+const allowedErrorTypes = new Set(["Error", "HTTPException"]);
+
+function routeName(method: string, routePath: string) {
+  return `${method.toUpperCase()} ${routePath || "unmatched"}`;
+}
+
+export function telemetryMiddleware(tracer: Tracer = trace.getTracer("cnode-api-hono")) {
+  return createMiddleware<{ Variables: TelemetryVariables }>(async (c, next) => {
+    const requestId = randomUUID();
+    c.set("requestId", requestId);
+    c.header("X-Request-ID", requestId);
+
+    return tracer.startActiveSpan(
+      "HTTP request",
+      {
+        kind: SpanKind.INTERNAL,
+        attributes: {
+          "cnode.request.id": requestId,
+          "http.request.method": c.req.method,
+        },
+      },
+      async (span) => {
+        try {
+          await next();
+          const status = c.res.status;
+          span.setAttribute("http.response.status_code", status);
+          if (status >= 500) span.setStatus({ code: SpanStatusCode.ERROR });
+        } catch (error) {
+          span.setStatus({ code: SpanStatusCode.ERROR });
+          const errorType = error instanceof Error ? error.name : "unknown";
+          if (allowedErrorTypes.has(errorType)) span.setAttribute("error.type", errorType);
+          throw error;
+        } finally {
+          const route = c.req.routePath || "unmatched";
+          span.updateName(routeName(c.req.method, route));
+          span.setAttribute("http.route", route);
+          span.end();
+        }
+      },
+    );
+  });
+}

apps/api/src/telemetry/config.ts

@@ -0,0 +1,72 @@
+import packageJson from "../../package.json";
+
+export type TelemetryRole = "api" | "moderation-worker";
+
+export interface TelemetryConfig {
+  enabled: boolean;
+  endpoint?: string;
+  sampleRatio: number;
+  serviceName: "cnode-api" | "cnode-moderation-worker";
+  serviceVersion: string;
+  commitRevision: string;
+  deploymentEnvironment: string;
+}
+
+export class TelemetryConfigError extends Error {
+  constructor(variable: string, expectation: string) {
+    super(`${variable} ${expectation}`);
+    this.name = "TelemetryConfigError";
+  }
+}
+
+function parseEnabled(value: string | undefined) {
+  if (value === undefined || value === "" || value === "0" || value === "false") return false;
+  if (value === "1" || value === "true") return true;
+  throw new TelemetryConfigError("CNODE_OTEL_ENABLED", "must be a boolean");
+}
+
+function parseSampleRatio(value: string | undefined) {
+  if (value === undefined || value === "") return 0.1;
+  const ratio = Number(value);
+  if (!Number.isFinite(ratio) || ratio < 0 || ratio > 1) {
+    throw new TelemetryConfigError(
+      "CNODE_OTEL_TRACE_SAMPLE_RATIO",
+      "must be a finite number between 0 and 1",
+    );
+  }
+  return ratio;
+}
+
+function parseEndpoint(value: string | undefined) {
+  if (!value) {
+    throw new TelemetryConfigError("CNODE_OTEL_EXPORTER_OTLP_ENDPOINT", "is required");
+  }
+
+  try {
+    const endpoint = new URL(value);
+    if (endpoint.protocol !== "http:" && endpoint.protocol !== "https:") throw new Error();
+    if (endpoint.username || endpoint.password) throw new Error();
+    return endpoint.toString();
+  } catch {
+    throw new TelemetryConfigError(
+      "CNODE_OTEL_EXPORTER_OTLP_ENDPOINT",
+      "must be an HTTP URL without credentials",
+    );
+  }
+}
+
+export function parseTelemetryConfig(
+  role: TelemetryRole,
+  env: NodeJS.ProcessEnv = process.env,
+): TelemetryConfig {
+  const enabled = parseEnabled(env.CNODE_OTEL_ENABLED);
+  return {
+    enabled,
+    endpoint: enabled ? parseEndpoint(env.CNODE_OTEL_EXPORTER_OTLP_ENDPOINT) : undefined,
+    sampleRatio: enabled ? parseSampleRatio(env.CNODE_OTEL_TRACE_SAMPLE_RATIO) : 0,
+    serviceName: role === "api" ? "cnode-api" : "cnode-moderation-worker",
+    serviceVersion: packageJson.version || "unknown",
+    commitRevision: env.CNODE_GIT_SHA || env.GIT_SHA || env.COMMIT_SHA || "unknown",
+    deploymentEnvironment: env.CNODE_ENV || env.NODE_ENV || "unknown",
+  };
+}