Trace Postgres and Redis queries with the contrib instrumentations

Builds the TracerProvider in tavolo.telemetry (kaya-otel's is private) and
injects it into kaya-otel, then instruments asyncpg and redis through
opentelemetry-instrumentation-asyncpg/redis. Redis command values are
redacted and asyncpg spans carry the parameterized SQL only.

server/packages/tavolo-app/README.md

@@ -12,9 +12,11 @@ The deployable tavolo application: the composition root wiring
   `tavolo.app:app` for granian.
 - `config.py` — environment-driven frozen `Settings`.
 - `telemetry.py` — the optional OpenTelemetry wiring beyond kaya-otel:
-  meter/logger providers, the `tavolo.active_users`/`tavolo.ongoing_games`
-  gauges (polled from the stores), process/CPython metrics via
-  `opentelemetry-instrumentation-system-metrics` and OTLP log export.
+  meter/tracer/logger providers, the
+  `tavolo.active_users`/`tavolo.ongoing_games` gauges (polled from the
+  stores), process/CPython metrics via
+  `opentelemetry-instrumentation-system-metrics`, Postgres/Redis query
+  spans via the asyncpg/redis instrumentations, and OTLP log export.
 - `aerich_config.py` — Tortoise ORM configuration consumed by the aerich
   CLI (migrations live in `server/migrations/`; aerich itself is
   configured via the `[tool.aerich]` section of `server/pyproject.toml`).

server/packages/tavolo-app/pyproject.toml

@@ -35,6 +35,8 @@ dev = [
 ]
 otel = [
     "kaya-otel>=0.0.4",
+    "opentelemetry-instrumentation-asyncpg>=0.66b0",
+    "opentelemetry-instrumentation-redis>=0.66b0",
     "opentelemetry-instrumentation-system-metrics>=0.66b0",
 ]
 

server/packages/tavolo-app/src/tavolo/app.py

@@ -80,7 +80,9 @@ def cors_mixin_from_settings(settings: Settings) -> Optional[CorsMixin]:
 
 
 def otel_mixin_from_settings(
-    settings: Settings, meter_provider: Optional[Any] = None
+    settings: Settings,
+    meter_provider: Optional[Any] = None,
+    tracer_provider: Optional[Any] = None,
 ) -> Optional[KayaMixin]:
     """Build a :class:`~kaya.otel.OTelMixin` from the OTEL_* settings.
 
@@ -89,10 +91,11 @@ def otel_mixin_from_settings(
     is imported lazily here: default installs and the test suite never need
     the OpenTelemetry packages.
 
-    ``meter_provider`` is the provider built by
-    :func:`tavolo.telemetry.telemetry_from_settings`; injecting it lets the
-    custom gauges share the same OTLP pipeline (kaya-otel then leaves its
-    shutdown to the telemetry mixin).
+    ``meter_provider``/``tracer_provider`` are the providers built by
+    :func:`tavolo.telemetry.telemetry_from_settings`; injecting them lets
+    the custom instruments and the database instrumentations share the same
+    OTLP pipelines (kaya-otel then leaves their shutdown to the telemetry
+    mixin).
     """
     if not settings.otel_enabled:
         return None
@@ -111,6 +114,8 @@ def otel_mixin_from_settings(
     }
     if meter_provider is not None:
         kwargs["meter_provider"] = meter_provider
+    if tracer_provider is not None:
+        kwargs["tracer_provider"] = tracer_provider
     return OTelMixin(**kwargs)
 
 
@@ -190,7 +195,9 @@ mixins: list[KayaMixin] = [session_mixin, oidc_mixin, tortoise_mixin, openapi_mi
 telemetry = telemetry_from_settings(settings, session_store, game_store)
 if telemetry is not None:
     otel_mixin = otel_mixin_from_settings(
-        settings, meter_provider=telemetry.meter_provider
+        settings,
+        meter_provider=telemetry.meter_provider,
+        tracer_provider=telemetry.tracer_provider,
     )
     assert otel_mixin is not None  # OTEL_ENABLED was checked above
     # kaya-otel first: before hooks run in registration order (after hooks

server/packages/tavolo-app/src/tavolo/telemetry.py

@@ -37,6 +37,7 @@ from .config import Settings
 if TYPE_CHECKING:
     from opentelemetry.sdk._logs import LoggerProvider
     from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.trace import TracerProvider
 
 log = getLogger(__name__)
 
@@ -122,17 +123,19 @@ CountableSessionStore = Union[CountingInMemorySessionStore, CountingRedisSession
 
 
 class TelemetryMixin(KayaMixin):
-    """Application gauges and log export on top of kaya-otel.
+    """Application gauges, DB instrumentation and log export.
 
-    The meter and logger providers are built by
+    The meter, tracer and logger providers are built by
     :func:`telemetry_from_settings` and injected: kaya-otel only shuts
-    down providers it created itself, so this mixin owns their shutdown.
+    down providers it created itself, so this mixin owns their shutdown
+    (and uninstruments the database clients it instrumented).
     """
 
     def __init__(
         self,
         *,
         meter_provider: "MeterProvider",
+        tracer_provider: "TracerProvider",
         logger_provider: Optional["LoggerProvider"],
         log_handler: Optional[logging.Handler],
         session_store: CountableSessionStore,
@@ -140,6 +143,7 @@ class TelemetryMixin(KayaMixin):
         poll_interval_seconds: float,
     ) -> None:
         self._meter_provider = meter_provider
+        self._tracer_provider = tracer_provider
         self._logger_provider = logger_provider
         self._log_handler = log_handler
         self._session_store = session_store
@@ -149,8 +153,11 @@ class TelemetryMixin(KayaMixin):
         self._ongoing_games: Optional[int] = None
         self._poll_task: Optional[asyncio.Task] = None
         self._instrumentor: Optional[Any] = None
+        self._db_instrumentors: list = []
 
     def apply(self, app: KayaApp) -> None:
+        from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor
+        from opentelemetry.instrumentation.redis import RedisInstrumentor
         from opentelemetry.instrumentation.system_metrics import (
             SystemMetricsInstrumentor,
         )
@@ -160,6 +167,16 @@ class TelemetryMixin(KayaMixin):
         self._instrumentor = instrumentor
         instrumentor.instrument(meter_provider=self._meter_provider)
 
+        # Query spans for the Postgres (asyncpg, used by Tortoise) and Redis
+        # clients. Redis command values are redacted by the instrumentation
+        # (``SET ? ?``); asyncpg spans carry the parameterized SQL only.
+        self._db_instrumentors = [
+            AsyncPGInstrumentor(capture_parameters=False),
+            RedisInstrumentor(),
+        ]
+        for db_instrumentor in self._db_instrumentors:
+            db_instrumentor.instrument(tracer_provider=self._tracer_provider)
+
         meter = self._meter_provider.get_meter(__name__)
 
         def active_users(options: CallbackOptions):
@@ -192,12 +209,16 @@ class TelemetryMixin(KayaMixin):
         if self._poll_task is not None:
             self._poll_task.cancel()
             self._poll_task = None
+        for db_instrumentor in self._db_instrumentors:
+            db_instrumentor.uninstrument()
+        self._db_instrumentors = []
         if self._log_handler is not None:
             logging.getLogger().removeHandler(self._log_handler)
             self._log_handler.close()
             self._log_handler = None
         if self._logger_provider is not None:
             self._logger_provider.shutdown()
+        self._tracer_provider.shutdown()
         self._meter_provider.shutdown()
 
     async def _poll(self) -> None:
@@ -222,6 +243,7 @@ class Telemetry:
     """The providers and mixin built from the ``OTEL_*`` settings."""
 
     meter_provider: "MeterProvider"
+    tracer_provider: "TracerProvider"
     logger_provider: Optional["LoggerProvider"]
     log_handler: Optional[logging.Handler]
     mixin: TelemetryMixin
@@ -232,7 +254,7 @@ def telemetry_from_settings(
     session_store: CountableSessionStore,
     game_store: GameStore,
 ) -> Optional[Telemetry]:
-    """Build the metric/log providers, or ``None`` when telemetry is off.
+    """Build the metric/trace/log providers, or ``None`` when telemetry is off.
 
     Only called when ``OTEL_ENABLED`` is set; the OpenTelemetry imports
     are lazy so the default install does not need the ``otel`` extra.
@@ -246,11 +268,16 @@ def telemetry_from_settings(
         from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
             OTLPMetricExporter,
         )
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+            OTLPSpanExporter,
+        )
         from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
         from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
         from opentelemetry.sdk.metrics import MeterProvider
         from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
         from opentelemetry.sdk.resources import Resource
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
     except ImportError as exc:
         raise RuntimeError(
             "OTEL_ENABLED is set but the OpenTelemetry dependencies are not "
@@ -260,6 +287,15 @@ def telemetry_from_settings(
     endpoint = settings.otel_exporter_endpoint
     headers = otel_headers(settings)
     resource = Resource.create({"service.name": settings.otel_service_name})
+    tracer_provider = TracerProvider(resource=resource)
+    tracer_provider.add_span_processor(
+        BatchSpanProcessor(
+            OTLPSpanExporter(
+                endpoint=_signal_endpoint(endpoint, "v1/traces"),
+                headers=headers,
+            )
+        )
+    )
     meter_provider = MeterProvider(
         resource=resource,
         metric_readers=[
@@ -286,6 +322,7 @@ def telemetry_from_settings(
     logging.getLogger().addHandler(handler)
     mixin = TelemetryMixin(
         meter_provider=meter_provider,
+        tracer_provider=tracer_provider,
         logger_provider=logger_provider,
         log_handler=handler,
         session_store=session_store,
@@ -294,6 +331,7 @@ def telemetry_from_settings(
     )
     return Telemetry(
         meter_provider=meter_provider,
+        tracer_provider=tracer_provider,
         logger_provider=logger_provider,
         log_handler=handler,
         mixin=mixin,