Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
08f158ef28
|
||
|
|
834f1f26a9
|
||
|
|
bcd1371019
|
||
|
|
3cf7e2e257
|
||
|
|
2bdf247cb3
|
||
|
|
a3e8ff3206
|
||
|
|
98d4485375
|
@@ -71,13 +71,16 @@ data:
|
|||||||
# CORS_ALLOW_CREDENTIALS: "false"
|
# CORS_ALLOW_CREDENTIALS: "false"
|
||||||
# CORS_EXPOSE_HEADERS: ""
|
# CORS_EXPOSE_HEADERS: ""
|
||||||
# CORS_MAX_AGE: "600"
|
# CORS_MAX_AGE: "600"
|
||||||
# OpenTelemetry (kaya-otel): traces + metrics via OTLP/HTTP, disabled
|
# OpenTelemetry (kaya-otel + contrib instrumentations): traces
|
||||||
# unless OTEL_ENABLED is set. The image bundles the otel extra.
|
# (HTTP/WebSocket, Postgres, Redis), metrics and logs via OTLP/HTTP,
|
||||||
|
# disabled unless OTEL_ENABLED is set. The image bundles the otel extra.
|
||||||
# OTEL_ENABLED: "true"
|
# OTEL_ENABLED: "true"
|
||||||
# OTEL_SERVICE_NAME: "tavolo"
|
# OTEL_SERVICE_NAME: "tavolo"
|
||||||
# OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.observability:4318"
|
# OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.observability:4318"
|
||||||
# OTEL_EXPORTER_OTLP_HEADERS: "Authorization=Bearer ..."
|
# OTEL_EXPORTER_OTLP_HEADERS: "Authorization=Bearer ..."
|
||||||
# OTEL_EXCLUDED_PATHS: "/api/health" # default; paths skipped by tracing
|
# OTEL_EXCLUDED_PATHS: "/api/health" # default; paths skipped by tracing
|
||||||
|
# OTEL_METRIC_EXPORT_INTERVAL_MILLIS: "60000"
|
||||||
|
# OTEL_METRICS_POLL_INTERVAL_SECONDS: "15"
|
||||||
# OIDC (provider lives in another namespace).
|
# OIDC (provider lives in another namespace).
|
||||||
OIDC_CLIENT_ID: tavolo
|
OIDC_CLIENT_ID: tavolo
|
||||||
OIDC_POST_LOGIN_REDIRECT: /
|
OIDC_POST_LOGIN_REDIRECT: /
|
||||||
|
|||||||
+3
-1
@@ -122,7 +122,9 @@ services:
|
|||||||
CORS_ALLOW_CREDENTIALS: ${CORS_ALLOW_CREDENTIALS:-}
|
CORS_ALLOW_CREDENTIALS: ${CORS_ALLOW_CREDENTIALS:-}
|
||||||
CORS_EXPOSE_HEADERS: ${CORS_EXPOSE_HEADERS:-}
|
CORS_EXPOSE_HEADERS: ${CORS_EXPOSE_HEADERS:-}
|
||||||
CORS_MAX_AGE: ${CORS_MAX_AGE:-}
|
CORS_MAX_AGE: ${CORS_MAX_AGE:-}
|
||||||
# OpenTelemetry (kaya-otel): disabled unless OTEL_ENABLED is set.
|
# OpenTelemetry (kaya-otel + contrib instrumentations): traces
|
||||||
|
# (HTTP/WebSocket, Postgres, Redis), metrics and logs via OTLP/HTTP,
|
||||||
|
# disabled unless OTEL_ENABLED is set.
|
||||||
# The image bundles the otel extra (see packages/tavolo-app/pyproject.toml
|
# The image bundles the otel extra (see packages/tavolo-app/pyproject.toml
|
||||||
# and server/requirements.txt).
|
# and server/requirements.txt).
|
||||||
OTEL_ENABLED: ${OTEL_ENABLED:-}
|
OTEL_ENABLED: ${OTEL_ENABLED:-}
|
||||||
|
|||||||
@@ -61,6 +61,23 @@ TURN_TIMEOUT_SECONDS=30
|
|||||||
# Seconds browsers may cache the preflight response (default 600):
|
# Seconds browsers may cache the preflight response (default 600):
|
||||||
#CORS_MAX_AGE=600
|
#CORS_MAX_AGE=600
|
||||||
|
|
||||||
|
# OpenTelemetry (via kaya-otel + the contrib instrumentation packages).
|
||||||
|
# Off unless OTEL_ENABLED is truthy; exports traces (HTTP/WebSocket,
|
||||||
|
# Postgres via asyncpg, Redis), metrics and logs over OTLP/HTTP (requires
|
||||||
|
# the tavolo-app "otel" extra, bundled in the Docker image). The endpoint
|
||||||
|
# defaults to the standard OTEL_EXPORTER_OTLP_ENDPOINT (localhost:4318)
|
||||||
|
# when unset.
|
||||||
|
#OTEL_ENABLED=true
|
||||||
|
#OTEL_SERVICE_NAME=tavolo
|
||||||
|
#OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||||
|
#OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer token,X-Tenant=one
|
||||||
|
# Paths excluded from tracing and metrics (default /api/health).
|
||||||
|
#OTEL_EXCLUDED_PATHS=/api/health
|
||||||
|
# Metric export interval (ms, default 60000) and session/game count poll
|
||||||
|
# interval (seconds, default 15).
|
||||||
|
#OTEL_METRIC_EXPORT_INTERVAL_MILLIS=60000
|
||||||
|
#OTEL_METRICS_POLL_INTERVAL_SECONDS=15
|
||||||
|
|
||||||
# App server
|
# App server
|
||||||
APP_HOST=0.0.0.0
|
APP_HOST=0.0.0.0
|
||||||
APP_PORT=8080
|
APP_PORT=8080
|
||||||
|
|||||||
+35
-1
@@ -81,11 +81,13 @@ All configuration comes from environment variables (see `.env.example`):
|
|||||||
| `CORS_ALLOW_CREDENTIALS` | `false` | `1`/`true`/`yes`/`on` allow cookies/credentials on cross-origin requests |
|
| `CORS_ALLOW_CREDENTIALS` | `false` | `1`/`true`/`yes`/`on` allow cookies/credentials on cross-origin requests |
|
||||||
| `CORS_EXPOSE_HEADERS` | unset | Comma-separated response headers exposed to the browser |
|
| `CORS_EXPOSE_HEADERS` | unset | Comma-separated response headers exposed to the browser |
|
||||||
| `CORS_MAX_AGE` | `600` | Seconds browsers may cache the preflight response |
|
| `CORS_MAX_AGE` | `600` | Seconds browsers may cache the preflight response |
|
||||||
| `OTEL_ENABLED` | `false` | `1`/`true`/`yes`/`on` enable OpenTelemetry traces and metrics (the Docker image bundles the `otel` extra; non-image installs need `pip install tavolo-app[otel]`) |
|
| `OTEL_ENABLED` | `false` | `1`/`true`/`yes`/`on` enable OpenTelemetry traces, metrics and logs (the Docker image bundles the `otel` extra; non-image installs need `pip install tavolo-app[otel]`) |
|
||||||
| `OTEL_SERVICE_NAME` | `tavolo` | `service.name` resource attribute of the exported telemetry |
|
| `OTEL_SERVICE_NAME` | `tavolo` | `service.name` resource attribute of the exported telemetry |
|
||||||
| `OTEL_EXPORTER_OTLP_ENDPOINT` | unset | Base URL of an OTLP/HTTP collector (e.g. `http://localhost:4318`); unset uses the exporter default |
|
| `OTEL_EXPORTER_OTLP_ENDPOINT` | unset | Base URL of an OTLP/HTTP collector (e.g. `http://localhost:4318`); unset uses the exporter default |
|
||||||
| `OTEL_EXPORTER_OTLP_HEADERS` | unset | Comma-separated `key=value` headers sent to the collector (e.g. authentication) |
|
| `OTEL_EXPORTER_OTLP_HEADERS` | unset | Comma-separated `key=value` headers sent to the collector (e.g. authentication) |
|
||||||
| `OTEL_EXCLUDED_PATHS` | `/api/health` | Comma-separated paths excluded from tracing and metrics (exact matches) |
|
| `OTEL_EXCLUDED_PATHS` | `/api/health` | Comma-separated paths excluded from tracing and metrics (exact matches) |
|
||||||
|
| `OTEL_METRIC_EXPORT_INTERVAL_MILLIS` | `60000` | How often the OTLP metric reader exports |
|
||||||
|
| `OTEL_METRICS_POLL_INTERVAL_SECONDS` | `15` | How often the session/game count gauges are refreshed |
|
||||||
| `APP_HOST` / `APP_PORT` | `0.0.0.0` / `8080` | Bind address |
|
| `APP_HOST` / `APP_PORT` | `0.0.0.0` / `8080` | Bind address |
|
||||||
|
|
||||||
## Logging
|
## Logging
|
||||||
@@ -123,6 +125,37 @@ loggers:
|
|||||||
level: INFO
|
level: INFO
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## OpenTelemetry
|
||||||
|
|
||||||
|
With `OTEL_ENABLED` set, the app exports over OTLP/HTTP (kaya-otel plus
|
||||||
|
`tavolo.telemetry`):
|
||||||
|
|
||||||
|
- **Traces**: HTTP server spans, one span per WebSocket connection, plus
|
||||||
|
database spans for the Postgres (asyncpg, through Tortoise) and Redis
|
||||||
|
clients. Redis command values are redacted by the instrumentation
|
||||||
|
(`GET ? ?`) and asyncpg spans carry the parameterized SQL only — no
|
||||||
|
parameter values. Each HTTP/WebSocket span is tagged with the request's
|
||||||
|
session and user, so all traces of a session can be found together:
|
||||||
|
`session.id` is a truncated SHA-256 of the session id (the cookie is a
|
||||||
|
credential, so it is never exported verbatim) and `user.id` is the OIDC
|
||||||
|
`sub`, present once logged in.
|
||||||
|
- **Metrics**: the HTTP server metrics from kaya-otel, plus
|
||||||
|
`tavolo.active_users` (sessions in the session store),
|
||||||
|
`tavolo.ongoing_games` (live matches not finished yet, maintained by the
|
||||||
|
game store), and process/CPython metrics from
|
||||||
|
`opentelemetry-instrumentation-system-metrics`: `process.cpu.time`,
|
||||||
|
`process.cpu.utilization`, `process.memory.usage`,
|
||||||
|
`process.memory.virtual`, `cpython.gc.collections`,
|
||||||
|
`cpython.gc.collected_objects` and `cpython.gc.uncollectable_objects`.
|
||||||
|
- **Logs**: forwarded from the root logger at the configured console level
|
||||||
|
(OpenTelemetry's own records are filtered out to avoid export loops).
|
||||||
|
|
||||||
|
The session/game gauges are polled every
|
||||||
|
`OTEL_METRICS_POLL_INTERVAL_SECONDS` (Redis-backed stores report
|
||||||
|
cluster-wide counts, in-memory stores the process's own). Note that with
|
||||||
|
multiple workers or replicas each process reports the same global counts:
|
||||||
|
aggregate with `max`, not `sum`.
|
||||||
|
|
||||||
## Data model
|
## Data model
|
||||||
|
|
||||||
### Redis (live games)
|
### Redis (live games)
|
||||||
@@ -356,6 +389,7 @@ packages/tavolo-app/ # deployable app: composition root (own pyproject)
|
|||||||
src/tavolo/
|
src/tavolo/
|
||||||
├── app.py # session/OIDC/Tortoise/OpenAPI/Platform/DeadlineScheduler mixins
|
├── app.py # session/OIDC/Tortoise/OpenAPI/Platform/DeadlineScheduler mixins
|
||||||
├── config.py # env -> frozen Settings
|
├── config.py # env -> frozen Settings
|
||||||
|
├── telemetry.py # OTLP metric/log providers, app gauges, count poller
|
||||||
├── aerich_config.py # aerich CLI configuration
|
├── aerich_config.py # aerich CLI configuration
|
||||||
├── logging_config.py # logging setup
|
├── logging_config.py # logging setup
|
||||||
└── static.py # SPA shell hosting
|
└── static.py # SPA shell hosting
|
||||||
|
|||||||
@@ -11,6 +11,12 @@ The deployable tavolo application: the composition root wiring
|
|||||||
`ScoponeEngine` with its configured timeouts, and exposes
|
`ScoponeEngine` with its configured timeouts, and exposes
|
||||||
`tavolo.app:app` for granian.
|
`tavolo.app:app` for granian.
|
||||||
- `config.py` — environment-driven frozen `Settings`.
|
- `config.py` — environment-driven frozen `Settings`.
|
||||||
|
- `telemetry.py` — the optional OpenTelemetry wiring beyond kaya-otel:
|
||||||
|
meter/tracer/logger providers, the
|
||||||
|
`tavolo.active_users`/`tavolo.ongoing_games` gauges (polled from the
|
||||||
|
stores), process/CPython metrics via
|
||||||
|
`opentelemetry-instrumentation-system-metrics`, Postgres/Redis query
|
||||||
|
spans via the asyncpg/redis instrumentations, and OTLP log export.
|
||||||
- `aerich_config.py` — Tortoise ORM configuration consumed by the aerich
|
- `aerich_config.py` — Tortoise ORM configuration consumed by the aerich
|
||||||
CLI (migrations live in `server/migrations/`; aerich itself is
|
CLI (migrations live in `server/migrations/`; aerich itself is
|
||||||
configured via the `[tool.aerich]` section of `server/pyproject.toml`).
|
configured via the `[tool.aerich]` section of `server/pyproject.toml`).
|
||||||
@@ -23,6 +29,7 @@ The deployable tavolo application: the composition root wiring
|
|||||||
.venv/bin/python -m mypy --namespace-packages --explicit-package-bases \
|
.venv/bin/python -m mypy --namespace-packages --explicit-package-bases \
|
||||||
src/tavolo/app.py \
|
src/tavolo/app.py \
|
||||||
src/tavolo/config.py \
|
src/tavolo/config.py \
|
||||||
|
src/tavolo/telemetry.py \
|
||||||
src/tavolo/static.py \
|
src/tavolo/static.py \
|
||||||
src/tavolo/aerich_config.py \
|
src/tavolo/aerich_config.py \
|
||||||
src/tavolo/logging_config.py
|
src/tavolo/logging_config.py
|
||||||
|
|||||||
@@ -35,6 +35,9 @@ dev = [
|
|||||||
]
|
]
|
||||||
otel = [
|
otel = [
|
||||||
"kaya-otel>=0.0.4",
|
"kaya-otel>=0.0.4",
|
||||||
|
"opentelemetry-instrumentation-asyncpg>=0.66b0",
|
||||||
|
"opentelemetry-instrumentation-redis>=0.66b0",
|
||||||
|
"opentelemetry-instrumentation-system-metrics>=0.66b0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[tool.setuptools.packages.find]
|
[tool.setuptools.packages.find]
|
||||||
|
|||||||
@@ -31,14 +31,13 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from importlib.metadata import version as _pkg_version
|
from importlib.metadata import version as _pkg_version
|
||||||
from logging import getLogger
|
from logging import getLogger
|
||||||
from typing import Optional
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
from kaya.core import KayaApp, KayaMixin
|
from kaya.core import KayaApp, KayaMixin
|
||||||
from kaya.cors import CorsMixin
|
from kaya.cors import CorsMixin
|
||||||
from kaya.oidc import OIDCConfig, OIDCMixin
|
from kaya.oidc import OIDCConfig, OIDCMixin
|
||||||
from kaya.openapi import OpenAPIMixin
|
from kaya.openapi import OpenAPIMixin
|
||||||
from kaya.session import InMemorySessionStore, SessionMixin, SessionStore
|
from kaya.session import SessionMixin
|
||||||
from kaya.session.redis import RedisSessionStore
|
|
||||||
from redis.asyncio import Redis
|
from redis.asyncio import Redis
|
||||||
from tavolo.platform import GameRegistry, Platform, PlatformMixin
|
from tavolo.platform import GameRegistry, Platform, PlatformMixin
|
||||||
from tavolo.platform.deadlines import DeadlineScheduler, DeadlineSchedulerMixin
|
from tavolo.platform.deadlines import DeadlineScheduler, DeadlineSchedulerMixin
|
||||||
@@ -48,6 +47,14 @@ from tavolo.scopone import ScoponeEngine
|
|||||||
|
|
||||||
from .config import Settings, settings
|
from .config import Settings, settings
|
||||||
from .logging_config import configure_logging
|
from .logging_config import configure_logging
|
||||||
|
from .telemetry import (
|
||||||
|
CountableSessionStore,
|
||||||
|
CountingInMemorySessionStore,
|
||||||
|
CountingRedisSessionStore,
|
||||||
|
otel_headers,
|
||||||
|
session_tag_hook,
|
||||||
|
telemetry_from_settings,
|
||||||
|
)
|
||||||
|
|
||||||
configure_logging(settings.logging_config)
|
configure_logging(settings.logging_config)
|
||||||
log = getLogger(__name__)
|
log = getLogger(__name__)
|
||||||
@@ -73,13 +80,23 @@ def cors_mixin_from_settings(settings: Settings) -> Optional[CorsMixin]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def otel_mixin_from_settings(settings: Settings) -> Optional[KayaMixin]:
|
def otel_mixin_from_settings(
|
||||||
|
settings: Settings,
|
||||||
|
meter_provider: Optional[Any] = None,
|
||||||
|
tracer_provider: Optional[Any] = None,
|
||||||
|
) -> Optional[KayaMixin]:
|
||||||
"""Build a :class:`~kaya.otel.OTelMixin` from the OTEL_* settings.
|
"""Build a :class:`~kaya.otel.OTelMixin` from the OTEL_* settings.
|
||||||
|
|
||||||
Returns ``None`` — telemetry disabled — unless ``OTEL_ENABLED`` is
|
Returns ``None`` — telemetry disabled — unless ``OTEL_ENABLED`` is
|
||||||
truthy. kaya-otel is an optional dependency (the ``otel`` extra), so it
|
truthy. kaya-otel is an optional dependency (the ``otel`` extra), so it
|
||||||
is imported lazily here: default installs and the test suite never need
|
is imported lazily here: default installs and the test suite never need
|
||||||
the OpenTelemetry packages.
|
the OpenTelemetry packages.
|
||||||
|
|
||||||
|
``meter_provider``/``tracer_provider`` are the providers built by
|
||||||
|
:func:`tavolo.telemetry.telemetry_from_settings`; injecting them lets
|
||||||
|
the custom instruments and the database instrumentations share the same
|
||||||
|
OTLP pipelines (kaya-otel then leaves their shutdown to the telemetry
|
||||||
|
mixin).
|
||||||
"""
|
"""
|
||||||
if not settings.otel_enabled:
|
if not settings.otel_enabled:
|
||||||
return None
|
return None
|
||||||
@@ -90,17 +107,21 @@ def otel_mixin_from_settings(settings: Settings) -> Optional[KayaMixin]:
|
|||||||
"OTEL_ENABLED is set but kaya-otel is not installed; "
|
"OTEL_ENABLED is set but kaya-otel is not installed; "
|
||||||
"install tavolo-app with the 'otel' extra"
|
"install tavolo-app with the 'otel' extra"
|
||||||
) from exc
|
) from exc
|
||||||
headers = dict(
|
kwargs: Dict[str, Any] = {
|
||||||
pair.split("=", 1)
|
"service_name": settings.otel_service_name,
|
||||||
for pair in (settings.otel_exporter_headers or ())
|
"endpoint": settings.otel_exporter_endpoint,
|
||||||
if "=" in pair
|
"headers": otel_headers(settings),
|
||||||
)
|
"excluded_paths": settings.otel_excluded_paths,
|
||||||
return OTelMixin(
|
# Runs after the session mixin's after hook, so the session id has
|
||||||
service_name=settings.otel_service_name,
|
# been persisted/minted and the OIDC claims are on the session.
|
||||||
endpoint=settings.otel_exporter_endpoint,
|
"server_response_hook": session_tag_hook,
|
||||||
headers=headers or None,
|
"websocket_close_hook": session_tag_hook,
|
||||||
excluded_paths=settings.otel_excluded_paths,
|
}
|
||||||
)
|
if meter_provider is not None:
|
||||||
|
kwargs["meter_provider"] = meter_provider
|
||||||
|
if tracer_provider is not None:
|
||||||
|
kwargs["tracer_provider"] = tracer_provider
|
||||||
|
return OTelMixin(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
registry = GameRegistry()
|
registry = GameRegistry()
|
||||||
@@ -117,12 +138,12 @@ log.debug(
|
|||||||
settings.turn_timeout_seconds,
|
settings.turn_timeout_seconds,
|
||||||
)
|
)
|
||||||
|
|
||||||
session_store: SessionStore
|
session_store: CountableSessionStore
|
||||||
game_store: GameStore
|
game_store: GameStore
|
||||||
if settings.redis_url is not None:
|
if settings.redis_url is not None:
|
||||||
# Lazy client: no connection is opened until a session is actually
|
# Lazy client: no connection is opened until a session is actually
|
||||||
# loaded/saved, so importing this module never requires a live Redis.
|
# loaded/saved, so importing this module never requires a live Redis.
|
||||||
session_store = RedisSessionStore(Redis.from_url(settings.redis_url))
|
session_store = CountingRedisSessionStore(Redis.from_url(settings.redis_url))
|
||||||
game_store = RedisGameStore(
|
game_store = RedisGameStore(
|
||||||
Redis.from_url(settings.redis_url, decode_responses=False),
|
Redis.from_url(settings.redis_url, decode_responses=False),
|
||||||
registry,
|
registry,
|
||||||
@@ -130,7 +151,7 @@ if settings.redis_url is not None:
|
|||||||
)
|
)
|
||||||
log.info("using Redis stores (sessions + live games, game TTL %ds)", settings.game_ttl_seconds)
|
log.info("using Redis stores (sessions + live games, game TTL %ds)", settings.game_ttl_seconds)
|
||||||
else:
|
else:
|
||||||
session_store = InMemorySessionStore()
|
session_store = CountingInMemorySessionStore()
|
||||||
game_store = InMemoryGameStore(registry)
|
game_store = InMemoryGameStore(registry)
|
||||||
log.info("REDIS_URL unset: using in-memory stores (sessions + live games)")
|
log.info("REDIS_URL unset: using in-memory stores (sessions + live games)")
|
||||||
|
|
||||||
@@ -176,15 +197,24 @@ platform = Platform(
|
|||||||
|
|
||||||
mixins: list[KayaMixin] = [session_mixin, oidc_mixin, tortoise_mixin, openapi_mixin,
|
mixins: list[KayaMixin] = [session_mixin, oidc_mixin, tortoise_mixin, openapi_mixin,
|
||||||
PlatformMixin(platform), DeadlineSchedulerMixin(scheduler)]
|
PlatformMixin(platform), DeadlineSchedulerMixin(scheduler)]
|
||||||
otel_mixin = otel_mixin_from_settings(settings)
|
telemetry = telemetry_from_settings(settings, session_store, game_store)
|
||||||
if otel_mixin is not None:
|
if telemetry is not None:
|
||||||
# First in the list: before hooks run in registration order (after hooks
|
otel_mixin = otel_mixin_from_settings(
|
||||||
|
settings,
|
||||||
|
meter_provider=telemetry.meter_provider,
|
||||||
|
tracer_provider=telemetry.tracer_provider,
|
||||||
|
)
|
||||||
|
assert otel_mixin is not None # OTEL_ENABLED was checked above
|
||||||
|
# kaya-otel first: before hooks run in registration order (after hooks
|
||||||
# in reverse), so the span covers session loading, OIDC handling and the
|
# in reverse), so the span covers session loading, OIDC handling and the
|
||||||
# handler itself. CORS, when enabled, is still prepended before it so
|
# handler itself. CORS, when enabled, is still prepended before it so
|
||||||
# preflight short-circuits stay untraced.
|
# preflight short-circuits stay untraced. The telemetry mixin carries no
|
||||||
|
# hooks but owns the metric/logger providers' polling and shutdown, so
|
||||||
|
# it must be in the list too.
|
||||||
|
mixins.insert(0, telemetry.mixin)
|
||||||
mixins.insert(0, otel_mixin)
|
mixins.insert(0, otel_mixin)
|
||||||
log.info(
|
log.info(
|
||||||
"OpenTelemetry enabled: service=%s endpoint=%s",
|
"OpenTelemetry enabled: service=%s endpoint=%s (traces, metrics, logs)",
|
||||||
settings.otel_service_name,
|
settings.otel_service_name,
|
||||||
settings.otel_exporter_endpoint or "(OTLP default)",
|
settings.otel_exporter_endpoint or "(OTLP default)",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -125,6 +125,10 @@ class Settings:
|
|||||||
# Paths excluded from tracing and metrics (exact matches). Defaults to
|
# Paths excluded from tracing and metrics (exact matches). Defaults to
|
||||||
# the health endpoint, which k8s probes would otherwise spam.
|
# the health endpoint, which k8s probes would otherwise spam.
|
||||||
otel_excluded_paths: Tuple[str, ...]
|
otel_excluded_paths: Tuple[str, ...]
|
||||||
|
# How often the OTLP metric reader exports (milliseconds) and how often
|
||||||
|
# the telemetry gauges refresh the session/game counts (seconds).
|
||||||
|
otel_metric_export_interval_millis: int
|
||||||
|
otel_metrics_poll_interval_seconds: int
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def from_env() -> "Settings":
|
def from_env() -> "Settings":
|
||||||
@@ -171,14 +175,20 @@ class Settings:
|
|||||||
cors_allow_credentials=_env_bool("CORS_ALLOW_CREDENTIALS"),
|
cors_allow_credentials=_env_bool("CORS_ALLOW_CREDENTIALS"),
|
||||||
cors_expose_headers=_env_list("CORS_EXPOSE_HEADERS"),
|
cors_expose_headers=_env_list("CORS_EXPOSE_HEADERS"),
|
||||||
cors_max_age=int(_env("CORS_MAX_AGE", "600")),
|
cors_max_age=int(_env("CORS_MAX_AGE", "600")),
|
||||||
# OpenTelemetry is opt-in: set OTEL_ENABLED=1 to export traces
|
# OpenTelemetry is opt-in: set OTEL_ENABLED=1 to export traces,
|
||||||
# and metrics via OTLP/HTTP (requires the ``otel`` extra).
|
# metrics and logs via OTLP/HTTP (requires the ``otel`` extra).
|
||||||
otel_enabled=_env_bool("OTEL_ENABLED"),
|
otel_enabled=_env_bool("OTEL_ENABLED"),
|
||||||
otel_service_name=_env("OTEL_SERVICE_NAME", "tavolo"),
|
otel_service_name=_env("OTEL_SERVICE_NAME", "tavolo"),
|
||||||
otel_exporter_endpoint=os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") or None,
|
otel_exporter_endpoint=os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") or None,
|
||||||
# Comma-separated key=value pairs, e.g. "Authorization=Bearer x".
|
# Comma-separated key=value pairs, e.g. "Authorization=Bearer x".
|
||||||
otel_exporter_headers=_env_list("OTEL_EXPORTER_OTLP_HEADERS"),
|
otel_exporter_headers=_env_list("OTEL_EXPORTER_OTLP_HEADERS"),
|
||||||
otel_excluded_paths=_env_list("OTEL_EXCLUDED_PATHS") or ("/api/health",),
|
otel_excluded_paths=_env_list("OTEL_EXCLUDED_PATHS") or ("/api/health",),
|
||||||
|
otel_metric_export_interval_millis=int(
|
||||||
|
_env("OTEL_METRIC_EXPORT_INTERVAL_MILLIS", "60000")
|
||||||
|
),
|
||||||
|
otel_metrics_poll_interval_seconds=int(
|
||||||
|
_env("OTEL_METRICS_POLL_INTERVAL_SECONDS", "15")
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
Configured once at app import time (see :mod:`tavolo.app`). By default a
|
Configured once at app import time (see :mod:`tavolo.app`). By default a
|
||||||
single console handler at DEBUG level is installed, formatting records as
|
single console handler at DEBUG level is installed, formatting records as
|
||||||
``{asctime} [{levelname}] ({processName}/{threadName}) - {name} - {message}``.
|
``{asctime} [{levelname}] ({processName}/{threadName}) - {name} - {message}``;
|
||||||
|
``urllib3.connectionpool`` is capped at INFO so its per-request DEBUG
|
||||||
|
chatter does not drown the log.
|
||||||
Point the ``LOGGING_CONFIG`` environment variable at a YAML file to take
|
Point the ``LOGGING_CONFIG`` environment variable at a YAML file to take
|
||||||
over the configuration entirely; the file follows the
|
over the configuration entirely; the file follows the
|
||||||
:data:`logging.config.dictConfig` schema, e.g.::
|
:data:`logging.config.dictConfig` schema, e.g.::
|
||||||
@@ -55,6 +57,13 @@ DEFAULT_CONFIG = {
|
|||||||
"level": "DEBUG",
|
"level": "DEBUG",
|
||||||
"handlers": ["console"],
|
"handlers": ["console"],
|
||||||
},
|
},
|
||||||
|
"loggers": {
|
||||||
|
# Every pooled HTTP request is a DEBUG record; keep the logger's
|
||||||
|
# INFO+ output, drop the per-connection noise.
|
||||||
|
"urllib3.connectionpool": {
|
||||||
|
"level": "INFO",
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,374 @@
|
|||||||
|
"""OpenTelemetry metrics and log export beyond kaya-otel.
|
||||||
|
|
||||||
|
kaya-otel instruments HTTP requests and websockets with traces and HTTP
|
||||||
|
metrics; this module adds what it does not cover, all behind the optional
|
||||||
|
``otel`` extra (OpenTelemetry imports are lazy, so default installs and
|
||||||
|
the test suite never need them):
|
||||||
|
|
||||||
|
* application gauges: ``tavolo.active_users`` (sessions held in the
|
||||||
|
session store) and ``tavolo.ongoing_games`` (live matches that have not
|
||||||
|
finished yet);
|
||||||
|
* process metrics through ``opentelemetry-instrumentation-system-metrics``:
|
||||||
|
CPU time/utilization, memory usage and CPython garbage-collection
|
||||||
|
counters;
|
||||||
|
* OTLP log export through a stdlib :class:`logging.Handler` attached to
|
||||||
|
the root logger, mirroring what the console sees;
|
||||||
|
* session/user tagging of request and websocket spans through
|
||||||
|
:func:`session_tag_hook` (kaya-otel's response/close hooks), so traces
|
||||||
|
can be correlated with a session (hashed id) and an OIDC user.
|
||||||
|
|
||||||
|
The store counts are asynchronous while OpenTelemetry instrument
|
||||||
|
callbacks are synchronous, so :class:`TelemetryMixin` polls the stores on
|
||||||
|
a background task and its callbacks report the cached values.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from logging import getLogger
|
||||||
|
from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Union
|
||||||
|
|
||||||
|
from kaya.core import KayaApp, KayaMixin
|
||||||
|
from kaya.session import InMemorySessionStore, SessionStore
|
||||||
|
from kaya.session.redis import RedisSessionStore
|
||||||
|
from redis.asyncio import Redis
|
||||||
|
from tavolo.platform.store import GameStore
|
||||||
|
|
||||||
|
from .config import Settings
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from opentelemetry.sdk._logs import LoggerProvider
|
||||||
|
from opentelemetry.sdk.metrics import MeterProvider
|
||||||
|
from opentelemetry.sdk.trace import TracerProvider
|
||||||
|
|
||||||
|
log = getLogger(__name__)
|
||||||
|
|
||||||
|
# Session keys live under kaya's default prefix. Keeping it explicit here
|
||||||
|
# makes the Redis count scan exact and preserves existing sessions.
|
||||||
|
SESSION_KEY_PREFIX = "kaya:session:"
|
||||||
|
|
||||||
|
# The interpreter metrics to export. The instrumentation package can also
|
||||||
|
# emit host-level ``system.*`` metrics; those are deliberately left off:
|
||||||
|
# the application cares about its own process, and per-CPU host metrics
|
||||||
|
# add cardinality without context.
|
||||||
|
#
|
||||||
|
# ``process.cpu.utilization`` ignores its configuration value upstream
|
||||||
|
# (the callback always emits one aggregate observation), so its ``None``
|
||||||
|
# is only for readability.
|
||||||
|
PROCESS_METRICS_CONFIG: Dict[str, Optional[list]] = {
|
||||||
|
"process.cpu.time": ["user", "system"],
|
||||||
|
"process.cpu.utilization": None,
|
||||||
|
"process.memory.usage": None,
|
||||||
|
"process.memory.virtual": None,
|
||||||
|
"cpython.gc.collections": None,
|
||||||
|
"cpython.gc.collected_objects": None,
|
||||||
|
"cpython.gc.uncollectable_objects": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def otel_headers(settings: Settings) -> Optional[Dict[str, str]]:
|
||||||
|
"""Parse ``OTEL_EXPORTER_OTLP_HEADERS`` into a mapping (or ``None``)."""
|
||||||
|
headers = dict(
|
||||||
|
pair.split("=", 1)
|
||||||
|
for pair in (settings.otel_exporter_headers or ())
|
||||||
|
if "=" in pair
|
||||||
|
)
|
||||||
|
return headers or None
|
||||||
|
|
||||||
|
|
||||||
|
def _signal_endpoint(base: Optional[str], signal: str) -> Optional[str]:
|
||||||
|
"""``{base}/v1/<signal>``, or ``None`` for the exporter's own default."""
|
||||||
|
return f"{base}/{signal}" if base else None
|
||||||
|
|
||||||
|
|
||||||
|
def session_tag_hook(span: Any, carrier: Any) -> None:
|
||||||
|
"""Tag an HTTP request/websocket span with the request's session/user.
|
||||||
|
|
||||||
|
Wired into kaya-otel as its ``server_response_hook`` and
|
||||||
|
``websocket_close_hook``: those run after the session mixin's after
|
||||||
|
hook has persisted (and, at login, minted) the session, so the final
|
||||||
|
session id is visible here.
|
||||||
|
|
||||||
|
The session id is the session cookie, i.e. a credential, so only a
|
||||||
|
truncated SHA-256 of it is exported; session ids are
|
||||||
|
``secrets.token_urlsafe(32)`` (256 bits), which makes the digest
|
||||||
|
irreversible in practice. The OIDC ``sub`` is a pseudonymous
|
||||||
|
identifier already persisted in Postgres and served by the API, so it
|
||||||
|
is exported verbatim as ``user.id`` (OpenTelemetry's current end-user
|
||||||
|
attribute; ``enduser.id`` is PII-flagged and being superseded).
|
||||||
|
"""
|
||||||
|
session = getattr(carrier, "session", None)
|
||||||
|
if session is None:
|
||||||
|
return
|
||||||
|
session_id = getattr(session, "id", None)
|
||||||
|
if session_id:
|
||||||
|
span.set_attribute(
|
||||||
|
"session.id",
|
||||||
|
hashlib.sha256(session_id.encode()).hexdigest()[:16],
|
||||||
|
)
|
||||||
|
user = session.get("oidc_user")
|
||||||
|
if isinstance(user, Mapping):
|
||||||
|
sub = user.get("sub")
|
||||||
|
if sub:
|
||||||
|
span.set_attribute("user.id", str(sub))
|
||||||
|
|
||||||
|
|
||||||
|
class _NoOtelLogs(logging.Filter):
|
||||||
|
"""Drop OpenTelemetry's own log records.
|
||||||
|
|
||||||
|
Export failures are logged by the SDK; forwarding those through the
|
||||||
|
same exporter would feed an endless retry loop.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def filter(self, record: logging.LogRecord) -> bool:
|
||||||
|
return not record.name.startswith("opentelemetry")
|
||||||
|
|
||||||
|
|
||||||
|
class CountingInMemorySessionStore(InMemorySessionStore):
|
||||||
|
"""In-memory session store that can report its live session count."""
|
||||||
|
|
||||||
|
async def count(self) -> int:
|
||||||
|
now = self._clock()
|
||||||
|
return sum(
|
||||||
|
1
|
||||||
|
for session_id in self._data
|
||||||
|
if session_id not in self._expires
|
||||||
|
or self._expires[session_id] > now
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CountingRedisSessionStore(RedisSessionStore):
|
||||||
|
"""Redis session store that can report its live session count."""
|
||||||
|
|
||||||
|
def __init__(self, client: Redis, prefix: str = SESSION_KEY_PREFIX) -> None:
|
||||||
|
super().__init__(client, prefix=prefix)
|
||||||
|
|
||||||
|
async def count(self) -> int:
|
||||||
|
total = 0
|
||||||
|
async for _ in self._client.scan_iter(match=f"{self._prefix}*", count=500):
|
||||||
|
total += 1
|
||||||
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
# The session stores the application builds: both expose ``count()`` on top
|
||||||
|
# of kaya's ``SessionStore`` interface. Game stores need no alias — the
|
||||||
|
# platform's ``GameStore`` declares ``count_ongoing()`` itself.
|
||||||
|
CountableSessionStore = Union[CountingInMemorySessionStore, CountingRedisSessionStore]
|
||||||
|
|
||||||
|
|
||||||
|
class TelemetryMixin(KayaMixin):
|
||||||
|
"""Application gauges, DB instrumentation and log export.
|
||||||
|
|
||||||
|
The meter, tracer and logger providers are built by
|
||||||
|
:func:`telemetry_from_settings` and injected: kaya-otel only shuts
|
||||||
|
down providers it created itself, so this mixin owns their shutdown
|
||||||
|
(and uninstruments the database clients it instrumented).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
meter_provider: "MeterProvider",
|
||||||
|
tracer_provider: "TracerProvider",
|
||||||
|
logger_provider: Optional["LoggerProvider"],
|
||||||
|
log_handler: Optional[logging.Handler],
|
||||||
|
session_store: CountableSessionStore,
|
||||||
|
game_store: GameStore,
|
||||||
|
poll_interval_seconds: float,
|
||||||
|
) -> None:
|
||||||
|
self._meter_provider = meter_provider
|
||||||
|
self._tracer_provider = tracer_provider
|
||||||
|
self._logger_provider = logger_provider
|
||||||
|
self._log_handler = log_handler
|
||||||
|
self._session_store = session_store
|
||||||
|
self._game_store = game_store
|
||||||
|
self._poll_interval = poll_interval_seconds
|
||||||
|
self._active_users: Optional[int] = None
|
||||||
|
self._ongoing_games: Optional[int] = None
|
||||||
|
self._poll_task: Optional[asyncio.Task] = None
|
||||||
|
self._instrumentor: Optional[Any] = None
|
||||||
|
self._db_instrumentors: list = []
|
||||||
|
|
||||||
|
def apply(self, app: KayaApp) -> None:
|
||||||
|
from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor
|
||||||
|
from opentelemetry.instrumentation.redis import RedisInstrumentor
|
||||||
|
from opentelemetry.instrumentation.system_metrics import (
|
||||||
|
SystemMetricsInstrumentor,
|
||||||
|
)
|
||||||
|
from opentelemetry.metrics import CallbackOptions, Observation
|
||||||
|
|
||||||
|
instrumentor = SystemMetricsInstrumentor(config=PROCESS_METRICS_CONFIG)
|
||||||
|
self._instrumentor = instrumentor
|
||||||
|
instrumentor.instrument(meter_provider=self._meter_provider)
|
||||||
|
|
||||||
|
# Query spans for the Postgres (asyncpg, used by Tortoise) and Redis
|
||||||
|
# clients. Redis command values are redacted by the instrumentation
|
||||||
|
# (``SET ? ?``); asyncpg spans carry the parameterized SQL only.
|
||||||
|
self._db_instrumentors = [
|
||||||
|
AsyncPGInstrumentor(capture_parameters=False),
|
||||||
|
RedisInstrumentor(),
|
||||||
|
]
|
||||||
|
for db_instrumentor in self._db_instrumentors:
|
||||||
|
db_instrumentor.instrument(tracer_provider=self._tracer_provider)
|
||||||
|
|
||||||
|
meter = self._meter_provider.get_meter(__name__)
|
||||||
|
|
||||||
|
def active_users(options: CallbackOptions):
|
||||||
|
if self._active_users is not None:
|
||||||
|
yield Observation(self._active_users)
|
||||||
|
|
||||||
|
def ongoing_games(options: CallbackOptions):
|
||||||
|
if self._ongoing_games is not None:
|
||||||
|
yield Observation(self._ongoing_games)
|
||||||
|
|
||||||
|
meter.create_observable_gauge(
|
||||||
|
"tavolo.active_users",
|
||||||
|
callbacks=[active_users],
|
||||||
|
unit="{user}",
|
||||||
|
description="Sessions currently held in the session store.",
|
||||||
|
)
|
||||||
|
meter.create_observable_gauge(
|
||||||
|
"tavolo.ongoing_games",
|
||||||
|
callbacks=[ongoing_games],
|
||||||
|
unit="{game}",
|
||||||
|
description="Live games that have not finished yet.",
|
||||||
|
)
|
||||||
|
|
||||||
|
# -- lifecycle ----------------------------------------------------------
|
||||||
|
|
||||||
|
def setup(self, loop: asyncio.AbstractEventLoop) -> None:
|
||||||
|
self._poll_task = loop.create_task(self._poll())
|
||||||
|
|
||||||
|
def shutdown(self, loop: asyncio.AbstractEventLoop) -> None:
|
||||||
|
if self._poll_task is not None:
|
||||||
|
self._poll_task.cancel()
|
||||||
|
self._poll_task = None
|
||||||
|
for db_instrumentor in self._db_instrumentors:
|
||||||
|
db_instrumentor.uninstrument()
|
||||||
|
self._db_instrumentors = []
|
||||||
|
if self._log_handler is not None:
|
||||||
|
logging.getLogger().removeHandler(self._log_handler)
|
||||||
|
self._log_handler.close()
|
||||||
|
self._log_handler = None
|
||||||
|
if self._logger_provider is not None:
|
||||||
|
self._logger_provider.shutdown()
|
||||||
|
self._tracer_provider.shutdown()
|
||||||
|
self._meter_provider.shutdown()
|
||||||
|
|
||||||
|
async def _poll(self) -> None:
|
||||||
|
while True:
|
||||||
|
await self.refresh_counts()
|
||||||
|
await asyncio.sleep(self._poll_interval)
|
||||||
|
|
||||||
|
async def refresh_counts(self) -> None:
|
||||||
|
"""Refresh the cached store counts (the poll loop calls this)."""
|
||||||
|
try:
|
||||||
|
self._active_users = await self._session_store.count()
|
||||||
|
except Exception:
|
||||||
|
log.warning("active_users refresh failed", exc_info=True)
|
||||||
|
try:
|
||||||
|
self._ongoing_games = await self._game_store.count_ongoing()
|
||||||
|
except Exception:
|
||||||
|
log.warning("ongoing_games refresh failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Telemetry:
|
||||||
|
"""The providers and mixin built from the ``OTEL_*`` settings."""
|
||||||
|
|
||||||
|
meter_provider: "MeterProvider"
|
||||||
|
tracer_provider: "TracerProvider"
|
||||||
|
logger_provider: Optional["LoggerProvider"]
|
||||||
|
log_handler: Optional[logging.Handler]
|
||||||
|
mixin: TelemetryMixin
|
||||||
|
|
||||||
|
|
||||||
|
def telemetry_from_settings(
|
||||||
|
settings: Settings,
|
||||||
|
session_store: CountableSessionStore,
|
||||||
|
game_store: GameStore,
|
||||||
|
) -> Optional[Telemetry]:
|
||||||
|
"""Build the metric/trace/log providers, or ``None`` when telemetry is off.
|
||||||
|
|
||||||
|
Only called when ``OTEL_ENABLED`` is set; the OpenTelemetry imports
|
||||||
|
are lazy so the default install does not need the ``otel`` extra.
|
||||||
|
"""
|
||||||
|
if not settings.otel_enabled:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import (
|
||||||
|
OTLPLogExporter,
|
||||||
|
)
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
|
||||||
|
OTLPMetricExporter,
|
||||||
|
)
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
||||||
|
OTLPSpanExporter,
|
||||||
|
)
|
||||||
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||||
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||||
|
from opentelemetry.sdk.metrics import MeterProvider
|
||||||
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||||
|
from opentelemetry.sdk.resources import Resource
|
||||||
|
from opentelemetry.sdk.trace import TracerProvider
|
||||||
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||||
|
except ImportError as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
"OTEL_ENABLED is set but the OpenTelemetry dependencies are not "
|
||||||
|
"installed; install tavolo-app with the 'otel' extra"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
endpoint = settings.otel_exporter_endpoint
|
||||||
|
headers = otel_headers(settings)
|
||||||
|
resource = Resource.create({"service.name": settings.otel_service_name})
|
||||||
|
tracer_provider = TracerProvider(resource=resource)
|
||||||
|
tracer_provider.add_span_processor(
|
||||||
|
BatchSpanProcessor(
|
||||||
|
OTLPSpanExporter(
|
||||||
|
endpoint=_signal_endpoint(endpoint, "v1/traces"),
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
meter_provider = MeterProvider(
|
||||||
|
resource=resource,
|
||||||
|
metric_readers=[
|
||||||
|
PeriodicExportingMetricReader(
|
||||||
|
OTLPMetricExporter(
|
||||||
|
endpoint=_signal_endpoint(endpoint, "v1/metrics"),
|
||||||
|
headers=headers,
|
||||||
|
),
|
||||||
|
export_interval_millis=settings.otel_metric_export_interval_millis,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
logger_provider = LoggerProvider(resource=resource)
|
||||||
|
logger_provider.add_log_record_processor(
|
||||||
|
BatchLogRecordProcessor(
|
||||||
|
OTLPLogExporter(
|
||||||
|
endpoint=_signal_endpoint(endpoint, "v1/logs"),
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
|
||||||
|
handler.addFilter(_NoOtelLogs())
|
||||||
|
logging.getLogger().addHandler(handler)
|
||||||
|
mixin = TelemetryMixin(
|
||||||
|
meter_provider=meter_provider,
|
||||||
|
tracer_provider=tracer_provider,
|
||||||
|
logger_provider=logger_provider,
|
||||||
|
log_handler=handler,
|
||||||
|
session_store=session_store,
|
||||||
|
game_store=game_store,
|
||||||
|
poll_interval_seconds=settings.otel_metrics_poll_interval_seconds,
|
||||||
|
)
|
||||||
|
return Telemetry(
|
||||||
|
meter_provider=meter_provider,
|
||||||
|
tracer_provider=tracer_provider,
|
||||||
|
logger_provider=logger_provider,
|
||||||
|
log_handler=handler,
|
||||||
|
mixin=mixin,
|
||||||
|
)
|
||||||
@@ -15,12 +15,15 @@ from __future__ import annotations
|
|||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
from tortoise.queryset import QuerySet
|
from tortoise.queryset import QuerySet
|
||||||
|
|
||||||
from .http import extract_query_params
|
from .http import extract_query_params
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from tortoise.models import Model
|
||||||
|
|
||||||
DEFAULT_LIMIT = 20
|
DEFAULT_LIMIT = 20
|
||||||
MAX_LIMIT = 100
|
MAX_LIMIT = 100
|
||||||
MIN_LIMIT = 1
|
MIN_LIMIT = 1
|
||||||
@@ -81,6 +84,28 @@ def parse_cursor_params(query_string: str) -> Cursor:
|
|||||||
Sort = List[Tuple[str, str]]
|
Sort = List[Tuple[str, str]]
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_sort_values(
|
||||||
|
model: "type[Model]", sort: Sort, after: Dict[str, Any]
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Restore the Python type of every decoded cursor value.
|
||||||
|
|
||||||
|
The cursor is JSON, so field values travel as strings (datetimes via
|
||||||
|
``default=str``). asyncpg rejects strings where a ``timestamp``
|
||||||
|
parameter is expected, and sqlite only tolerates them by accident, so
|
||||||
|
each value is run through its model field's ``to_python_value`` before
|
||||||
|
it reaches the keyset filter.
|
||||||
|
"""
|
||||||
|
fields = model._meta.fields_map
|
||||||
|
coerced: Dict[str, Any] = {}
|
||||||
|
for field, _ in sort:
|
||||||
|
value = after.get(field)
|
||||||
|
field_obj = fields.get(field)
|
||||||
|
if value is not None and field_obj is not None:
|
||||||
|
value = field_obj.to_python_value(value)
|
||||||
|
coerced[field] = value
|
||||||
|
return coerced
|
||||||
|
|
||||||
|
|
||||||
def _keyset_where(sort: Sort, after: Dict[str, Any]):
|
def _keyset_where(sort: Sort, after: Dict[str, Any]):
|
||||||
"""Build a Tortoise ``Q`` filter from a cursor."""
|
"""Build a Tortoise ``Q`` filter from a cursor."""
|
||||||
from tortoise.queryset import Q # local to keep import edge narrow
|
from tortoise.queryset import Q # local to keep import edge narrow
|
||||||
@@ -116,7 +141,8 @@ async def paginate(
|
|||||||
order_by.append(field if direction == "ASC" else f"-{field}")
|
order_by.append(field if direction == "ASC" else f"-{field}")
|
||||||
qs = queryset.order_by(*order_by)
|
qs = queryset.order_by(*order_by)
|
||||||
if cursor.after:
|
if cursor.after:
|
||||||
qs = qs.filter(_keyset_where(sort, cursor.after))
|
after = _coerce_sort_values(queryset.model, sort, cursor.after)
|
||||||
|
qs = qs.filter(_keyset_where(sort, after))
|
||||||
rows = await qs.limit(cursor.limit + 1)
|
rows = await qs.limit(cursor.limit + 1)
|
||||||
if len(rows) <= cursor.limit:
|
if len(rows) <= cursor.limit:
|
||||||
return rows, None
|
return rows, None
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ verbatim through the ``result`` JSON column.
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import defaultdict
|
||||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
from kaya.core import HttpContext, KayaApp
|
from kaya.core import HttpContext, KayaApp
|
||||||
from kaya.openapi import operation
|
from kaya.openapi import operation
|
||||||
@@ -48,8 +50,9 @@ def _parse_game_type(
|
|||||||
return game_type, None
|
return game_type, None
|
||||||
|
|
||||||
|
|
||||||
async def _serialize_match(match: Match, viewer: str) -> Dict[str, Any]:
|
def _serialize_match(
|
||||||
participants = await MatchPlayer.filter(match_id=match.id).order_by("seat")
|
match: Match, viewer: str, participants: List[MatchPlayer]
|
||||||
|
) -> Dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"id": str(match.id),
|
"id": str(match.id),
|
||||||
"game_type": match.game_type,
|
"game_type": match.game_type,
|
||||||
@@ -109,7 +112,21 @@ def register(app: KayaApp, platform: "Platform") -> None:
|
|||||||
matches, next_cursor = await paginate(
|
matches, next_cursor = await paginate(
|
||||||
queryset, [("finished_at", "DESC"), ("id", "ASC")], cursor
|
queryset, [("finished_at", "DESC"), ("id", "ASC")], cursor
|
||||||
)
|
)
|
||||||
results = [await _serialize_match(m, user.sub) for m in matches]
|
# One participants query for the whole page instead of one per
|
||||||
|
# match; the grouping keeps each match's players seat-ordered.
|
||||||
|
by_match: Dict[UUID, List[MatchPlayer]] = defaultdict(list)
|
||||||
|
if matches:
|
||||||
|
participants = (
|
||||||
|
await MatchPlayer.filter(match_id__in=[m.id for m in matches])
|
||||||
|
.order_by("match_id", "seat")
|
||||||
|
)
|
||||||
|
for player in participants:
|
||||||
|
# Tortoise sets the FK's raw id on instances but does not
|
||||||
|
# declare it on the model class, hence the getattr.
|
||||||
|
by_match[getattr(player, "match_id")].append(player)
|
||||||
|
results = [
|
||||||
|
_serialize_match(m, user.sub, by_match.get(m.id, [])) for m in matches
|
||||||
|
]
|
||||||
await send_json(ctx, 200, {"results": results, "next_cursor": next_cursor})
|
await send_json(ctx, 200, {"results": results, "next_cursor": next_cursor})
|
||||||
|
|
||||||
@app.GET("/api/leaderboard")
|
@app.GET("/api/leaderboard")
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ from __future__ import annotations
|
|||||||
import asyncio
|
import asyncio
|
||||||
import contextlib
|
import contextlib
|
||||||
import json
|
import json
|
||||||
|
import time
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from logging import getLogger
|
from logging import getLogger
|
||||||
@@ -54,6 +55,10 @@ GAME_KEY_PREFIX = "tavolo:game:"
|
|||||||
CODE_KEY_PREFIX = "tavolo:code:"
|
CODE_KEY_PREFIX = "tavolo:code:"
|
||||||
CHANNEL_PREFIX = "tavolo:game:"
|
CHANNEL_PREFIX = "tavolo:game:"
|
||||||
DEADLINES_KEY = "tavolo:deadlines"
|
DEADLINES_KEY = "tavolo:deadlines"
|
||||||
|
# Sorted set of running games (score = expiry timestamp), maintained by the
|
||||||
|
# stores so the telemetry gauge can count ongoing matches without loading
|
||||||
|
# every session. Finished games are removed on their final save.
|
||||||
|
ONGOING_GAMES_KEY = "tavolo:games:ongoing"
|
||||||
|
|
||||||
# Sentinel pushed into in-memory subscriber queues to signal a change.
|
# Sentinel pushed into in-memory subscriber queues to signal a change.
|
||||||
_BUMP = b"update"
|
_BUMP = b"update"
|
||||||
@@ -120,6 +125,14 @@ class GameStore(ABC):
|
|||||||
async def find_by_code(self, code: str) -> Optional[GameSession]:
|
async def find_by_code(self, code: str) -> Optional[GameSession]:
|
||||||
"""Return the live session for a join ``code`` or ``None``."""
|
"""Return the live session for a join ``code`` or ``None``."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def count_ongoing(self) -> int:
|
||||||
|
"""Return the number of live games that have not finished yet.
|
||||||
|
|
||||||
|
Lobbies and matches in progress count; matches that are over but
|
||||||
|
still retained for the final-board view do not.
|
||||||
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def lock(self, game_id: str) -> AsyncContextManager[None]:
|
def lock(self, game_id: str) -> AsyncContextManager[None]:
|
||||||
"""Async context manager serializing mutations of one game."""
|
"""Async context manager serializing mutations of one game."""
|
||||||
@@ -181,9 +194,15 @@ class RedisGameStore(GameStore):
|
|||||||
|
|
||||||
async def save(self, session: GameSession) -> None:
|
async def save(self, session: GameSession) -> None:
|
||||||
payload = json.dumps(session_to_json(session, self._registry))
|
payload = json.dumps(session_to_json(session, self._registry))
|
||||||
|
engine = self._registry.require(session.game_type)
|
||||||
async with self._redis.pipeline(transaction=True) as pipe:
|
async with self._redis.pipeline(transaction=True) as pipe:
|
||||||
pipe.set(f"{GAME_KEY_PREFIX}{session.id}", payload, ex=self._ttl)
|
pipe.set(f"{GAME_KEY_PREFIX}{session.id}", payload, ex=self._ttl)
|
||||||
pipe.set(f"{CODE_KEY_PREFIX}{session.join_code}", session.id, ex=self._ttl)
|
pipe.set(f"{CODE_KEY_PREFIX}{session.join_code}", session.id, ex=self._ttl)
|
||||||
|
if engine.is_finished(session):
|
||||||
|
pipe.zrem(ONGOING_GAMES_KEY, session.id)
|
||||||
|
else:
|
||||||
|
pipe.zadd(ONGOING_GAMES_KEY, {session.id: time.time() + self._ttl})
|
||||||
|
pipe.expire(ONGOING_GAMES_KEY, self._ttl)
|
||||||
await pipe.execute()
|
await pipe.execute()
|
||||||
log.debug("redis save %s (game %s, ttl %ds)", session.id, session.game_type, self._ttl)
|
log.debug("redis save %s (game %s, ttl %ds)", session.id, session.game_type, self._ttl)
|
||||||
|
|
||||||
@@ -195,6 +214,17 @@ class RedisGameStore(GameStore):
|
|||||||
game_id = game_id.decode("utf-8")
|
game_id = game_id.decode("utf-8")
|
||||||
return await self.load(str(game_id))
|
return await self.load(str(game_id))
|
||||||
|
|
||||||
|
async def count_ongoing(self) -> int:
|
||||||
|
# Prune entries whose game has expired since the last poll, then
|
||||||
|
# count what is left. Both commands run in one transaction so the
|
||||||
|
# count cannot see a half-pruned set.
|
||||||
|
now = time.time()
|
||||||
|
async with self._redis.pipeline(transaction=True) as pipe:
|
||||||
|
pipe.zremrangebyscore(ONGOING_GAMES_KEY, "-inf", now)
|
||||||
|
pipe.zcard(ONGOING_GAMES_KEY)
|
||||||
|
_, count = await pipe.execute()
|
||||||
|
return int(count)
|
||||||
|
|
||||||
@contextlib.asynccontextmanager
|
@contextlib.asynccontextmanager
|
||||||
async def subscribe(self, game_id: str) -> AsyncIterator[AsyncIterator[None]]:
|
async def subscribe(self, game_id: str) -> AsyncIterator[AsyncIterator[None]]:
|
||||||
pubsub = self._redis.pubsub()
|
pubsub = self._redis.pubsub()
|
||||||
@@ -243,6 +273,7 @@ class InMemoryGameStore(GameStore):
|
|||||||
self._registry = registry
|
self._registry = registry
|
||||||
self._games: Dict[str, str] = {}
|
self._games: Dict[str, str] = {}
|
||||||
self._codes: Dict[str, str] = {}
|
self._codes: Dict[str, str] = {}
|
||||||
|
self._ongoing: Set[str] = set()
|
||||||
self._locks: Dict[str, asyncio.Lock] = {}
|
self._locks: Dict[str, asyncio.Lock] = {}
|
||||||
self._subscribers: Dict[str, Set[asyncio.Queue]] = {}
|
self._subscribers: Dict[str, Set[asyncio.Queue]] = {}
|
||||||
self._deadlines: Dict[str, float] = {}
|
self._deadlines: Dict[str, float] = {}
|
||||||
@@ -268,6 +299,11 @@ class InMemoryGameStore(GameStore):
|
|||||||
async def save(self, session: GameSession) -> None:
|
async def save(self, session: GameSession) -> None:
|
||||||
self._games[session.id] = json.dumps(session_to_json(session, self._registry))
|
self._games[session.id] = json.dumps(session_to_json(session, self._registry))
|
||||||
self._codes[session.join_code] = session.id
|
self._codes[session.join_code] = session.id
|
||||||
|
engine = self._registry.require(session.game_type)
|
||||||
|
if engine.is_finished(session):
|
||||||
|
self._ongoing.discard(session.id)
|
||||||
|
else:
|
||||||
|
self._ongoing.add(session.id)
|
||||||
|
|
||||||
async def find_by_code(self, code: str) -> Optional[GameSession]:
|
async def find_by_code(self, code: str) -> Optional[GameSession]:
|
||||||
game_id = self._codes.get(code.upper())
|
game_id = self._codes.get(code.upper())
|
||||||
@@ -275,6 +311,9 @@ class InMemoryGameStore(GameStore):
|
|||||||
return None
|
return None
|
||||||
return await self.load(game_id)
|
return await self.load(game_id)
|
||||||
|
|
||||||
|
async def count_ongoing(self) -> int:
|
||||||
|
return len(self._ongoing)
|
||||||
|
|
||||||
@contextlib.asynccontextmanager
|
@contextlib.asynccontextmanager
|
||||||
async def subscribe(self, game_id: str) -> AsyncIterator[AsyncIterator[None]]:
|
async def subscribe(self, game_id: str) -> AsyncIterator[AsyncIterator[None]]:
|
||||||
queue: asyncio.Queue = asyncio.Queue()
|
queue: asyncio.Queue = asyncio.Queue()
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""Keyset pagination tests: cursor values are restored to Python types.
|
||||||
|
|
||||||
|
Cursors are JSON, so a cursor carrying a datetime reaches the next
|
||||||
|
request as a string. Postgres/asyncpg rejects strings for timestamp
|
||||||
|
parameters (sqlite silently coerces them), so ``paginate`` must run the
|
||||||
|
decoded values through their model fields before building the WHERE
|
||||||
|
clause.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any, List
|
||||||
|
|
||||||
|
from tavolo.platform.models import Match
|
||||||
|
from tavolo.platform.pagination import Cursor, decode_cursor, encode_cursor, paginate
|
||||||
|
|
||||||
|
from helpers import async_test
|
||||||
|
|
||||||
|
SORT = [("finished_at", "DESC"), ("id", "ASC")]
|
||||||
|
|
||||||
|
|
||||||
|
class _CapturingQuerySet:
|
||||||
|
"""Minimal stand-in for a Tortoise QuerySet, recording its filter."""
|
||||||
|
|
||||||
|
model = Match
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.queryset_filter: Any = None
|
||||||
|
|
||||||
|
def order_by(self, *orderings: str) -> "_CapturingQuerySet":
|
||||||
|
return self
|
||||||
|
|
||||||
|
def filter(self, q: Any) -> "_CapturingQuerySet":
|
||||||
|
self.queryset_filter = q
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def limit(self, count: int) -> List[Any]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _flatten_q(q: Any) -> List[Any]:
|
||||||
|
nodes = [q]
|
||||||
|
for child in q.children:
|
||||||
|
nodes.extend(_flatten_q(child))
|
||||||
|
return nodes
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_values(q: Any, key: str) -> List[Any]:
|
||||||
|
return [node.filters[key] for node in _flatten_q(q) if key in node.filters]
|
||||||
|
|
||||||
|
|
||||||
|
class PaginateCursorTypesTest(unittest.TestCase):
|
||||||
|
@async_test
|
||||||
|
async def test_cursor_values_are_restored_before_filtering(self) -> None:
|
||||||
|
match_id = uuid.uuid4()
|
||||||
|
finished_at = datetime(2026, 9, 19, 5, 2, 50, 547904, tzinfo=timezone.utc)
|
||||||
|
# Round-trip through the wire format, exactly as a request does.
|
||||||
|
after = decode_cursor(
|
||||||
|
encode_cursor({"finished_at": finished_at, "id": match_id})
|
||||||
|
)
|
||||||
|
assert after is not None
|
||||||
|
|
||||||
|
qs = _CapturingQuerySet()
|
||||||
|
await paginate(qs, SORT, Cursor(limit=1, after=after))
|
||||||
|
|
||||||
|
assert qs.queryset_filter is not None
|
||||||
|
for key in ("finished_at", "finished_at__lt"):
|
||||||
|
values = _filter_values(qs.queryset_filter, key)
|
||||||
|
self.assertEqual(1, len(values))
|
||||||
|
self.assertIsInstance(values[0], datetime)
|
||||||
|
self.assertEqual(finished_at, values[0])
|
||||||
|
id_values = _filter_values(qs.queryset_filter, "id__gt")
|
||||||
|
self.assertEqual(1, len(id_values))
|
||||||
|
self.assertIsInstance(id_values[0], uuid.UUID)
|
||||||
|
self.assertEqual(match_id, id_values[0])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|||||||
import unittest
|
import unittest
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
from httpx import ASGITransport, AsyncClient
|
from httpx import ASGITransport, AsyncClient
|
||||||
|
|
||||||
@@ -197,6 +198,28 @@ class StatsRouteTest(unittest.TestCase):
|
|||||||
second.json()["results"][0]["id"],
|
second.json()["results"][0]["id"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
async def test_my_matches_fetches_participants_in_one_query(self) -> None:
|
||||||
|
app, platform, tortoise_mixin = make_platform()
|
||||||
|
await _seed_two_matches(tortoise_mixin)
|
||||||
|
transport = ASGITransport(app=app)
|
||||||
|
with mock.patch.object(
|
||||||
|
MatchPlayer, "filter", wraps=MatchPlayer.filter
|
||||||
|
) as spy:
|
||||||
|
async with AsyncClient(
|
||||||
|
transport=transport, base_url="http://127.0.0.1"
|
||||||
|
) as client:
|
||||||
|
with oidc_user(platform.oidc, "alice"):
|
||||||
|
response = await client.get("/api/me/matches")
|
||||||
|
self.assertEqual(200, response.status_code)
|
||||||
|
# One query for all matches, not one per match row.
|
||||||
|
self.assertEqual(1, spy.call_count)
|
||||||
|
results = response.json()["results"]
|
||||||
|
self.assertEqual(2, len(results))
|
||||||
|
for result in results:
|
||||||
|
seats = [player["seat"] for player in result["players"]]
|
||||||
|
self.assertEqual([0, 1], seats)
|
||||||
|
|
||||||
@async_test
|
@async_test
|
||||||
async def test_my_matches_requires_auth(self) -> None:
|
async def test_my_matches_requires_auth(self) -> None:
|
||||||
app, _, _ = make_platform()
|
app, _, _ = make_platform()
|
||||||
|
|||||||
@@ -2,11 +2,16 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import time
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from tavolo.platform import GameSession, Seat
|
from tavolo.platform import GameSession, Seat
|
||||||
from tavolo.platform.registry import GameRegistry
|
from tavolo.platform.registry import GameRegistry
|
||||||
from tavolo.platform.store import InMemoryGameStore
|
from tavolo.platform.store import (
|
||||||
|
ONGOING_GAMES_KEY,
|
||||||
|
InMemoryGameStore,
|
||||||
|
RedisGameStore,
|
||||||
|
)
|
||||||
from helpers import DummyEngine, async_test
|
from helpers import DummyEngine, async_test
|
||||||
|
|
||||||
|
|
||||||
@@ -27,6 +32,81 @@ def _session(game_id: str = "g1", code: str = "CODE01") -> GameSession:
|
|||||||
return session
|
return session
|
||||||
|
|
||||||
|
|
||||||
|
class _FakePipeline:
|
||||||
|
"""Records commands and applies them to the fake Redis on execute."""
|
||||||
|
|
||||||
|
def __init__(self, redis: "_FakeRedis") -> None:
|
||||||
|
self._redis = redis
|
||||||
|
self._ops: list = []
|
||||||
|
|
||||||
|
async def __aenter__(self) -> "_FakePipeline":
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *exc: object) -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def set(self, key: str, value: str, ex: object = None) -> None:
|
||||||
|
self._ops.append(("set", key))
|
||||||
|
|
||||||
|
def zadd(self, key: str, mapping: dict) -> None:
|
||||||
|
self._ops.append(("zadd", key, mapping))
|
||||||
|
|
||||||
|
def zrem(self, key: str, member: str) -> None:
|
||||||
|
self._ops.append(("zrem", key, member))
|
||||||
|
|
||||||
|
def expire(self, key: str, ttl: int) -> None:
|
||||||
|
self._ops.append(("expire", key))
|
||||||
|
|
||||||
|
def zremrangebyscore(self, key: str, minimum: object, maximum: object) -> None:
|
||||||
|
self._ops.append(("zremrangebyscore", key))
|
||||||
|
|
||||||
|
def zcard(self, key: str) -> None:
|
||||||
|
self._ops.append(("zcard", key))
|
||||||
|
|
||||||
|
async def execute(self) -> list:
|
||||||
|
results = [self._redis.apply(op) for op in self._ops]
|
||||||
|
self._ops = []
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeRedis:
|
||||||
|
"""The slice of the Redis client the game store uses."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.keys: dict = {}
|
||||||
|
self.zsets: dict = {}
|
||||||
|
|
||||||
|
def pipeline(self, transaction: bool = True) -> _FakePipeline:
|
||||||
|
return _FakePipeline(self)
|
||||||
|
|
||||||
|
def apply(self, op: tuple) -> object:
|
||||||
|
name, *args = op
|
||||||
|
if name == "set":
|
||||||
|
self.keys[args[0]] = "payload"
|
||||||
|
return True
|
||||||
|
if name == "zadd":
|
||||||
|
self.zsets.setdefault(args[0], {}).update(args[1])
|
||||||
|
return 1
|
||||||
|
if name == "zrem":
|
||||||
|
self.zsets.get(args[0], {}).pop(args[1], None)
|
||||||
|
return 1
|
||||||
|
if name == "expire":
|
||||||
|
return True
|
||||||
|
if name == "zremrangebyscore":
|
||||||
|
zset = self.zsets.get(args[0], {})
|
||||||
|
for member in [
|
||||||
|
member
|
||||||
|
for member, score in zset.items()
|
||||||
|
if score <= time.time()
|
||||||
|
]:
|
||||||
|
del zset[member]
|
||||||
|
return 0
|
||||||
|
if name == "zcard":
|
||||||
|
return len(self.zsets.get(args[0], {}))
|
||||||
|
raise AssertionError(f"unexpected command: {name}")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class InMemoryGameStoreTest(unittest.TestCase):
|
class InMemoryGameStoreTest(unittest.TestCase):
|
||||||
@async_test
|
@async_test
|
||||||
async def test_save_load_roundtrip(self) -> None:
|
async def test_save_load_roundtrip(self) -> None:
|
||||||
@@ -114,6 +194,28 @@ class InMemoryGameStoreTest(unittest.TestCase):
|
|||||||
["holder-enter", "holder-exit", "contender"], order
|
["holder-enter", "holder-exit", "contender"], order
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
async def test_count_ongoing_tracks_unfinished_games(self) -> None:
|
||||||
|
store = InMemoryGameStore(_registry())
|
||||||
|
self.assertEqual(0, await store.count_ongoing())
|
||||||
|
|
||||||
|
session = _session()
|
||||||
|
await store.save(session)
|
||||||
|
self.assertEqual(1, await store.count_ongoing())
|
||||||
|
|
||||||
|
other = _session(game_id="g2", code="CODE02")
|
||||||
|
await store.save(other)
|
||||||
|
self.assertEqual(2, await store.count_ongoing())
|
||||||
|
|
||||||
|
# A finished match is retained for the final board but no longer
|
||||||
|
# counts as ongoing.
|
||||||
|
session.state["finished"] = True
|
||||||
|
await store.save(session)
|
||||||
|
self.assertEqual(1, await store.count_ongoing())
|
||||||
|
other.state["finished"] = True
|
||||||
|
await store.save(other)
|
||||||
|
self.assertEqual(0, await store.count_ongoing())
|
||||||
|
|
||||||
@async_test
|
@async_test
|
||||||
async def test_deadline_queue(self) -> None:
|
async def test_deadline_queue(self) -> None:
|
||||||
store = InMemoryGameStore(_registry())
|
store = InMemoryGameStore(_registry())
|
||||||
@@ -138,5 +240,35 @@ class InMemoryGameStoreTest(unittest.TestCase):
|
|||||||
self.assertEqual(["b"], await store.due_deadlines(now=100.0))
|
self.assertEqual(["b"], await store.due_deadlines(now=100.0))
|
||||||
|
|
||||||
|
|
||||||
|
class RedisGameStoreTest(unittest.TestCase):
|
||||||
|
@async_test
|
||||||
|
async def test_save_marks_games_ongoing_and_finished_ones_stop_counting(self) -> None:
|
||||||
|
redis = _FakeRedis()
|
||||||
|
store = RedisGameStore(redis, _registry(), ttl_seconds=3600) # type: ignore[arg-type]
|
||||||
|
session = _session()
|
||||||
|
|
||||||
|
await store.save(session)
|
||||||
|
self.assertEqual(1, await store.count_ongoing())
|
||||||
|
score = redis.zsets[ONGOING_GAMES_KEY][session.id]
|
||||||
|
self.assertGreater(score, time.time())
|
||||||
|
self.assertLessEqual(score, time.time() + 3600)
|
||||||
|
|
||||||
|
session.state["finished"] = True
|
||||||
|
await store.save(session)
|
||||||
|
self.assertNotIn(session.id, redis.zsets[ONGOING_GAMES_KEY])
|
||||||
|
self.assertEqual(0, await store.count_ongoing())
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
async def test_count_ongoing_prunes_expired_entries(self) -> None:
|
||||||
|
redis = _FakeRedis()
|
||||||
|
redis.zsets[ONGOING_GAMES_KEY] = {
|
||||||
|
"stale": time.time() - 10,
|
||||||
|
"live": time.time() + 100,
|
||||||
|
}
|
||||||
|
store = RedisGameStore(redis, _registry()) # type: ignore[arg-type]
|
||||||
|
self.assertEqual(1, await store.count_ongoing())
|
||||||
|
self.assertNotIn("stale", redis.zsets[ONGOING_GAMES_KEY])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+29
-1
@@ -93,6 +93,10 @@ opentelemetry-api==1.45.0
|
|||||||
# via
|
# via
|
||||||
# opentelemetry-exporter-http-transport
|
# opentelemetry-exporter-http-transport
|
||||||
# opentelemetry-exporter-otlp-proto-http
|
# opentelemetry-exporter-otlp-proto-http
|
||||||
|
# opentelemetry-instrumentation
|
||||||
|
# opentelemetry-instrumentation-asyncpg
|
||||||
|
# opentelemetry-instrumentation-redis
|
||||||
|
# opentelemetry-instrumentation-system-metrics
|
||||||
# opentelemetry-sdk
|
# opentelemetry-sdk
|
||||||
# opentelemetry-semantic-conventions
|
# opentelemetry-semantic-conventions
|
||||||
opentelemetry-exporter-http-transport[urllib3]==0.66b0
|
opentelemetry-exporter-http-transport[urllib3]==0.66b0
|
||||||
@@ -103,6 +107,17 @@ opentelemetry-exporter-otlp-proto-common==1.45.0
|
|||||||
# via opentelemetry-exporter-otlp-proto-http
|
# via opentelemetry-exporter-otlp-proto-http
|
||||||
opentelemetry-exporter-otlp-proto-http==1.45.0
|
opentelemetry-exporter-otlp-proto-http==1.45.0
|
||||||
# via kaya-otel
|
# via kaya-otel
|
||||||
|
opentelemetry-instrumentation==0.66b0
|
||||||
|
# via
|
||||||
|
# opentelemetry-instrumentation-asyncpg
|
||||||
|
# opentelemetry-instrumentation-redis
|
||||||
|
# opentelemetry-instrumentation-system-metrics
|
||||||
|
opentelemetry-instrumentation-asyncpg==0.66b0
|
||||||
|
# via tavolo-app (pyproject.toml)
|
||||||
|
opentelemetry-instrumentation-redis==0.66b0
|
||||||
|
# via tavolo-app (pyproject.toml)
|
||||||
|
opentelemetry-instrumentation-system-metrics==0.66b0
|
||||||
|
# via tavolo-app (pyproject.toml)
|
||||||
opentelemetry-proto==1.45.0
|
opentelemetry-proto==1.45.0
|
||||||
# via
|
# via
|
||||||
# opentelemetry-exporter-otlp-proto-common
|
# opentelemetry-exporter-otlp-proto-common
|
||||||
@@ -113,11 +128,20 @@ opentelemetry-sdk==1.45.0
|
|||||||
# opentelemetry-exporter-otlp-common
|
# opentelemetry-exporter-otlp-common
|
||||||
# opentelemetry-exporter-otlp-proto-http
|
# opentelemetry-exporter-otlp-proto-http
|
||||||
opentelemetry-semantic-conventions==0.66b0
|
opentelemetry-semantic-conventions==0.66b0
|
||||||
# via opentelemetry-sdk
|
# via
|
||||||
|
# opentelemetry-instrumentation
|
||||||
|
# opentelemetry-instrumentation-asyncpg
|
||||||
|
# opentelemetry-instrumentation-redis
|
||||||
|
# opentelemetry-instrumentation-system-metrics
|
||||||
|
# opentelemetry-sdk
|
||||||
|
packaging==26.3
|
||||||
|
# via opentelemetry-instrumentation
|
||||||
protobuf==7.36.2
|
protobuf==7.36.2
|
||||||
# via
|
# via
|
||||||
# googleapis-common-protos
|
# googleapis-common-protos
|
||||||
# opentelemetry-proto
|
# opentelemetry-proto
|
||||||
|
psutil==7.2.2
|
||||||
|
# via opentelemetry-instrumentation-system-metrics
|
||||||
pwo==0.1.2
|
pwo==0.1.2
|
||||||
# via
|
# via
|
||||||
# kaya-core
|
# kaya-core
|
||||||
@@ -151,3 +175,7 @@ typing-extensions==4.16.0
|
|||||||
# pwo
|
# pwo
|
||||||
urllib3==2.8.0
|
urllib3==2.8.0
|
||||||
# via opentelemetry-exporter-http-transport
|
# via opentelemetry-exporter-http-transport
|
||||||
|
wrapt==2.5.0
|
||||||
|
# via
|
||||||
|
# opentelemetry-instrumentation
|
||||||
|
# opentelemetry-instrumentation-redis
|
||||||
|
|||||||
@@ -164,6 +164,19 @@ class OTelSettingsTests(unittest.TestCase):
|
|||||||
settings = _settings({"OTEL_EXCLUDED_PATHS": "/api/health, /metrics"})
|
settings = _settings({"OTEL_EXCLUDED_PATHS": "/api/health, /metrics"})
|
||||||
self.assertEqual(("/api/health", "/metrics"), settings.otel_excluded_paths)
|
self.assertEqual(("/api/health", "/metrics"), settings.otel_excluded_paths)
|
||||||
|
|
||||||
|
def test_otel_interval_defaults(self):
|
||||||
|
settings = _settings({})
|
||||||
|
self.assertEqual(60000, settings.otel_metric_export_interval_millis)
|
||||||
|
self.assertEqual(15, settings.otel_metrics_poll_interval_seconds)
|
||||||
|
|
||||||
|
def test_otel_intervals_parse(self):
|
||||||
|
settings = _settings({
|
||||||
|
"OTEL_METRIC_EXPORT_INTERVAL_MILLIS": "5000",
|
||||||
|
"OTEL_METRICS_POLL_INTERVAL_SECONDS": "3",
|
||||||
|
})
|
||||||
|
self.assertEqual(5000, settings.otel_metric_export_interval_millis)
|
||||||
|
self.assertEqual(3, settings.otel_metrics_poll_interval_seconds)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -18,12 +18,15 @@ class LoggingConfigTest(unittest.TestCase):
|
|||||||
self._root_level = root.level
|
self._root_level = root.level
|
||||||
tavolo = logging.getLogger("tavolo")
|
tavolo = logging.getLogger("tavolo")
|
||||||
self._tavolo_level = tavolo.level
|
self._tavolo_level = tavolo.level
|
||||||
|
pool = logging.getLogger("urllib3.connectionpool")
|
||||||
|
self._pool_level = pool.level
|
||||||
|
|
||||||
def tearDown(self) -> None:
|
def tearDown(self) -> None:
|
||||||
root = logging.getLogger()
|
root = logging.getLogger()
|
||||||
root.handlers = self._root_handlers
|
root.handlers = self._root_handlers
|
||||||
root.level = self._root_level
|
root.level = self._root_level
|
||||||
logging.getLogger("tavolo").level = self._tavolo_level
|
logging.getLogger("tavolo").level = self._tavolo_level
|
||||||
|
logging.getLogger("urllib3.connectionpool").level = self._pool_level
|
||||||
|
|
||||||
def test_default_config_when_unset(self) -> None:
|
def test_default_config_when_unset(self) -> None:
|
||||||
configure_logging(None)
|
configure_logging(None)
|
||||||
@@ -33,6 +36,12 @@ class LoggingConfigTest(unittest.TestCase):
|
|||||||
any(isinstance(h, logging.StreamHandler) for h in root.handlers),
|
any(isinstance(h, logging.StreamHandler) for h in root.handlers),
|
||||||
"default config installs a console stream handler",
|
"default config installs a console stream handler",
|
||||||
)
|
)
|
||||||
|
# The noisy per-request HTTP logger is capped at INFO while the
|
||||||
|
# root stays at DEBUG.
|
||||||
|
self.assertEqual(
|
||||||
|
logging.INFO,
|
||||||
|
logging.getLogger("urllib3.connectionpool").getEffectiveLevel(),
|
||||||
|
)
|
||||||
|
|
||||||
def test_yaml_config_is_applied(self) -> None:
|
def test_yaml_config_is_applied(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from unittest.mock import patch
|
|||||||
|
|
||||||
from tavolo.app import otel_mixin_from_settings
|
from tavolo.app import otel_mixin_from_settings
|
||||||
from tavolo.config import Settings
|
from tavolo.config import Settings
|
||||||
|
from tavolo.telemetry import session_tag_hook
|
||||||
|
|
||||||
|
|
||||||
def _settings(env: dict) -> Settings:
|
def _settings(env: dict) -> Settings:
|
||||||
@@ -58,8 +59,17 @@ class OTelMixinFromSettingsTests(unittest.TestCase):
|
|||||||
"endpoint": "http://collector:4318",
|
"endpoint": "http://collector:4318",
|
||||||
"headers": {"Authorization": "Bearer t", "X-Tenant": "one"},
|
"headers": {"Authorization": "Bearer t", "X-Tenant": "one"},
|
||||||
"excluded_paths": ("/api/health",),
|
"excluded_paths": ("/api/health",),
|
||||||
|
"server_response_hook": session_tag_hook,
|
||||||
|
"websocket_close_hook": session_tag_hook,
|
||||||
}, mixin.kwargs)
|
}, mixin.kwargs)
|
||||||
|
|
||||||
|
def test_session_tag_hooks_are_wired(self):
|
||||||
|
with _stub_kaya_otel():
|
||||||
|
mixin = otel_mixin_from_settings(_settings({"OTEL_ENABLED": "1"}))
|
||||||
|
assert isinstance(mixin, _StubOTelMixin)
|
||||||
|
self.assertIs(session_tag_hook, mixin.kwargs["server_response_hook"])
|
||||||
|
self.assertIs(session_tag_hook, mixin.kwargs["websocket_close_hook"])
|
||||||
|
|
||||||
def test_defaults_when_only_enabled(self):
|
def test_defaults_when_only_enabled(self):
|
||||||
with _stub_kaya_otel():
|
with _stub_kaya_otel():
|
||||||
mixin = otel_mixin_from_settings(_settings({"OTEL_ENABLED": "on"}))
|
mixin = otel_mixin_from_settings(_settings({"OTEL_ENABLED": "on"}))
|
||||||
@@ -69,6 +79,24 @@ class OTelMixinFromSettingsTests(unittest.TestCase):
|
|||||||
self.assertIsNone(mixin.kwargs["headers"])
|
self.assertIsNone(mixin.kwargs["headers"])
|
||||||
self.assertEqual(("/api/health",), mixin.kwargs["excluded_paths"])
|
self.assertEqual(("/api/health",), mixin.kwargs["excluded_paths"])
|
||||||
|
|
||||||
|
def test_meter_provider_is_passed_through(self):
|
||||||
|
with _stub_kaya_otel():
|
||||||
|
provider = object()
|
||||||
|
mixin = otel_mixin_from_settings(
|
||||||
|
_settings({"OTEL_ENABLED": "1"}), meter_provider=provider
|
||||||
|
)
|
||||||
|
assert isinstance(mixin, _StubOTelMixin)
|
||||||
|
self.assertIs(provider, mixin.kwargs["meter_provider"])
|
||||||
|
|
||||||
|
def test_tracer_provider_is_passed_through(self):
|
||||||
|
with _stub_kaya_otel():
|
||||||
|
provider = object()
|
||||||
|
mixin = otel_mixin_from_settings(
|
||||||
|
_settings({"OTEL_ENABLED": "1"}), tracer_provider=provider
|
||||||
|
)
|
||||||
|
assert isinstance(mixin, _StubOTelMixin)
|
||||||
|
self.assertIs(provider, mixin.kwargs["tracer_provider"])
|
||||||
|
|
||||||
def test_excluded_paths_are_passed_through(self):
|
def test_excluded_paths_are_passed_through(self):
|
||||||
with _stub_kaya_otel():
|
with _stub_kaya_otel():
|
||||||
mixin = otel_mixin_from_settings(_settings({
|
mixin = otel_mixin_from_settings(_settings({
|
||||||
|
|||||||
@@ -0,0 +1,410 @@
|
|||||||
|
"""Telemetry unit tests: app gauges, process metrics, log export and counts.
|
||||||
|
|
||||||
|
The OpenTelemetry packages are an optional extra (``tavolo-app[otel]``),
|
||||||
|
so the provider-dependent tests skip when they are not installed.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import Optional
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from kaya.session import Session
|
||||||
|
from tavolo.telemetry import (
|
||||||
|
CountingInMemorySessionStore,
|
||||||
|
CountingRedisSessionStore,
|
||||||
|
TelemetryMixin,
|
||||||
|
_NoOtelLogs,
|
||||||
|
session_tag_hook,
|
||||||
|
telemetry_from_settings,
|
||||||
|
)
|
||||||
|
from tavolo.config import Settings
|
||||||
|
|
||||||
|
from tests.helpers import async_test
|
||||||
|
|
||||||
|
try:
|
||||||
|
from opentelemetry.sdk._logs import LoggingHandler, LoggerProvider
|
||||||
|
from opentelemetry.sdk._logs.export import (
|
||||||
|
InMemoryLogRecordExporter,
|
||||||
|
SimpleLogRecordProcessor,
|
||||||
|
)
|
||||||
|
from opentelemetry.sdk.metrics import MeterProvider
|
||||||
|
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
|
||||||
|
from opentelemetry.sdk.resources import Resource
|
||||||
|
from opentelemetry.sdk.trace import TracerProvider
|
||||||
|
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||||
|
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
|
||||||
|
InMemorySpanExporter,
|
||||||
|
)
|
||||||
|
|
||||||
|
HAVE_OTEL = True
|
||||||
|
except ImportError: # pragma: no cover - installs without the otel extra
|
||||||
|
HAVE_OTEL = False
|
||||||
|
|
||||||
|
|
||||||
|
def _settings(env: dict) -> Settings:
|
||||||
|
with patch.dict(os.environ, env, clear=True):
|
||||||
|
return Settings.from_env()
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeSessions:
|
||||||
|
def __init__(self, count: int = 0) -> None:
|
||||||
|
self.value: Optional[int] = count
|
||||||
|
self.error: Optional[Exception] = None
|
||||||
|
|
||||||
|
async def count(self) -> int:
|
||||||
|
if self.error is not None:
|
||||||
|
raise self.error
|
||||||
|
assert self.value is not None
|
||||||
|
return self.value
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeGames:
|
||||||
|
def __init__(self, count: int = 0) -> None:
|
||||||
|
self.value: Optional[int] = count
|
||||||
|
|
||||||
|
async def count_ongoing(self) -> int:
|
||||||
|
assert self.value is not None
|
||||||
|
return self.value
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeRedis:
|
||||||
|
"""Minimal async Redis double covering the session-count scan."""
|
||||||
|
|
||||||
|
def __init__(self, keys: list) -> None:
|
||||||
|
self.keys = keys
|
||||||
|
|
||||||
|
async def scan_iter(self, match: str = "", count: int = 0):
|
||||||
|
prefix = match.rstrip("*")
|
||||||
|
for key in self.keys:
|
||||||
|
if key.startswith(prefix):
|
||||||
|
yield key
|
||||||
|
|
||||||
|
|
||||||
|
class TelemetrySetupTest(unittest.TestCase):
|
||||||
|
def test_disabled_returns_none(self) -> None:
|
||||||
|
self.assertIsNone(
|
||||||
|
telemetry_from_settings(
|
||||||
|
_settings({}), _FakeSessions(), _FakeGames()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_missing_otel_dependencies_raise(self) -> None:
|
||||||
|
settings = _settings({"OTEL_ENABLED": "true"})
|
||||||
|
missing = "opentelemetry.exporter.otlp.proto.http._log_exporter"
|
||||||
|
with patch.dict(sys.modules, {missing: None}):
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
telemetry_from_settings(settings, _FakeSessions(), _FakeGames())
|
||||||
|
|
||||||
|
|
||||||
|
class SessionCountTest(unittest.TestCase):
|
||||||
|
@async_test
|
||||||
|
async def test_in_memory_counts_live_sessions(self) -> None:
|
||||||
|
now = [1000.0]
|
||||||
|
store = CountingInMemorySessionStore(clock=lambda: now[0])
|
||||||
|
await store.save("a", Session("a", {"user": "alice"}), max_age=10)
|
||||||
|
await store.save("b", Session("b", {"user": "bob"}), max_age=10)
|
||||||
|
await store.save("c", Session("c", {"user": "carol"})) # no expiry
|
||||||
|
self.assertEqual(3, await store.count())
|
||||||
|
|
||||||
|
now[0] += 11 # a and b have expired
|
||||||
|
self.assertEqual(1, await store.count())
|
||||||
|
await store.delete("c")
|
||||||
|
self.assertEqual(0, await store.count())
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
async def test_redis_counts_only_session_keys(self) -> None:
|
||||||
|
store = CountingRedisSessionStore(
|
||||||
|
_FakeRedis(["kaya:session:a", "kaya:session:b", "tavolo:code:X"])
|
||||||
|
)
|
||||||
|
self.assertEqual(2, await store.count())
|
||||||
|
|
||||||
|
|
||||||
|
class NoOtelLogsFilterTest(unittest.TestCase):
|
||||||
|
def test_filters_opentelemetry_records_only(self) -> None:
|
||||||
|
filter_ = _NoOtelLogs()
|
||||||
|
self.assertFalse(
|
||||||
|
filter_.filter(logging.makeLogRecord({"name": "opentelemetry.sdk"}))
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
filter_.filter(logging.makeLogRecord({"name": "tavolo.app"}))
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
filter_.filter(logging.makeLogRecord({"name": "granian.server"}))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _RecordedSpan:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.attributes: dict = {}
|
||||||
|
|
||||||
|
def set_attribute(self, key: str, value) -> None:
|
||||||
|
self.attributes[key] = value
|
||||||
|
|
||||||
|
|
||||||
|
class SessionTagHookTest(unittest.TestCase):
|
||||||
|
def test_tags_authenticated_session(self) -> None:
|
||||||
|
span = _RecordedSpan()
|
||||||
|
carrier = SimpleNamespace(
|
||||||
|
session=Session("sid", {"oidc_user": {"sub": "alice"}})
|
||||||
|
)
|
||||||
|
session_tag_hook(span, carrier)
|
||||||
|
self.assertEqual(
|
||||||
|
hashlib.sha256(b"sid").hexdigest()[:16],
|
||||||
|
span.attributes["session.id"],
|
||||||
|
)
|
||||||
|
self.assertEqual("alice", span.attributes["user.id"])
|
||||||
|
|
||||||
|
def test_tags_session_without_user(self) -> None:
|
||||||
|
span = _RecordedSpan()
|
||||||
|
session_tag_hook(span, SimpleNamespace(session=Session("sid", {})))
|
||||||
|
self.assertIn("session.id", span.attributes)
|
||||||
|
self.assertNotIn("user.id", span.attributes)
|
||||||
|
|
||||||
|
def test_tags_user_when_session_id_not_yet_minted(self) -> None:
|
||||||
|
span = _RecordedSpan()
|
||||||
|
carrier = SimpleNamespace(
|
||||||
|
session=Session(None, {"oidc_user": {"sub": "alice"}})
|
||||||
|
)
|
||||||
|
session_tag_hook(span, carrier)
|
||||||
|
self.assertNotIn("session.id", span.attributes)
|
||||||
|
self.assertEqual("alice", span.attributes["user.id"])
|
||||||
|
|
||||||
|
def test_missing_session_is_a_noop(self) -> None:
|
||||||
|
span = _RecordedSpan()
|
||||||
|
session_tag_hook(span, SimpleNamespace())
|
||||||
|
self.assertEqual({}, span.attributes)
|
||||||
|
|
||||||
|
def test_malformed_user_is_ignored(self) -> None:
|
||||||
|
span = _RecordedSpan()
|
||||||
|
carrier = SimpleNamespace(session=Session("sid", {"oidc_user": "alice"}))
|
||||||
|
session_tag_hook(span, carrier)
|
||||||
|
self.assertIn("session.id", span.attributes)
|
||||||
|
self.assertNotIn("user.id", span.attributes)
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(HAVE_OTEL, "tavolo-app[otel] not installed")
|
||||||
|
class SessionTagIntegrationTest(unittest.TestCase):
|
||||||
|
"""The hook is wired as kaya-otel's response hook, so it sees the
|
||||||
|
session after the session mixin persisted and minted it."""
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
async def test_spans_carry_hashed_session_and_oidc_sub(self) -> None:
|
||||||
|
from httpx import ASGITransport, AsyncClient
|
||||||
|
from kaya.core import HttpContext, KayaApp
|
||||||
|
from kaya.otel import OTelMixin
|
||||||
|
from kaya.session import InMemorySessionStore, SessionMixin
|
||||||
|
|
||||||
|
span_exporter = InMemorySpanExporter()
|
||||||
|
tracer_provider = TracerProvider(
|
||||||
|
resource=Resource.create({"service.name": "test"})
|
||||||
|
)
|
||||||
|
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
|
||||||
|
app = KayaApp(
|
||||||
|
mixins=[
|
||||||
|
OTelMixin(
|
||||||
|
service_name="test",
|
||||||
|
tracer_provider=tracer_provider,
|
||||||
|
meter_provider=MeterProvider(
|
||||||
|
metric_readers=[InMemoryMetricReader()]
|
||||||
|
),
|
||||||
|
server_response_hook=session_tag_hook,
|
||||||
|
websocket_close_hook=session_tag_hook,
|
||||||
|
),
|
||||||
|
SessionMixin(InMemorySessionStore()),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
@app.GET("/login")
|
||||||
|
async def login(ctx: HttpContext) -> None:
|
||||||
|
session = ctx.session
|
||||||
|
assert session is not None
|
||||||
|
session["oidc_user"] = {"sub": "alice"}
|
||||||
|
await ctx.send_str(200, "ok")
|
||||||
|
|
||||||
|
@app.GET("/me")
|
||||||
|
async def me(ctx: HttpContext) -> None:
|
||||||
|
await ctx.send_str(200, "ok")
|
||||||
|
|
||||||
|
transport = ASGITransport(app=app) # type: ignore[arg-type]
|
||||||
|
async with AsyncClient(
|
||||||
|
transport=transport, base_url="http://127.0.0.1"
|
||||||
|
) as client:
|
||||||
|
first = await client.get("/login")
|
||||||
|
second = await client.get("/me")
|
||||||
|
self.assertEqual(200, first.status_code)
|
||||||
|
self.assertEqual(200, second.status_code)
|
||||||
|
# The cookie value is the session id; only its digest is exported.
|
||||||
|
cookie = first.cookies["session_id"]
|
||||||
|
digest = hashlib.sha256(cookie.encode()).hexdigest()[:16]
|
||||||
|
|
||||||
|
spans = {span.name: span for span in span_exporter.get_finished_spans()}
|
||||||
|
self.assertEqual({"GET /login", "GET /me"}, set(spans))
|
||||||
|
for name, span in spans.items():
|
||||||
|
attributes = span.attributes
|
||||||
|
assert attributes is not None
|
||||||
|
self.assertEqual(digest, attributes["session.id"], name)
|
||||||
|
self.assertEqual("alice", attributes["user.id"], name)
|
||||||
|
|
||||||
|
|
||||||
|
def _metric_points(reader) -> dict:
|
||||||
|
"""Map metric name -> list of (attributes, value) data points."""
|
||||||
|
data = reader.get_metrics_data()
|
||||||
|
assert data is not None
|
||||||
|
points = {}
|
||||||
|
for resource_metrics in data.resource_metrics:
|
||||||
|
for scope_metrics in resource_metrics.scope_metrics:
|
||||||
|
for metric in scope_metrics.metrics:
|
||||||
|
points[metric.name] = [
|
||||||
|
(dict(point.attributes), point.value)
|
||||||
|
for point in metric.data.data_points
|
||||||
|
]
|
||||||
|
return points
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(HAVE_OTEL, "tavolo-app[otel] not installed")
|
||||||
|
class MetricExportTest(unittest.TestCase):
|
||||||
|
# The instrumentation packages are process-wide singletons whose
|
||||||
|
# callbacks cannot be re-registered, so the providers and mixin are
|
||||||
|
# built once for the whole class (as the application does).
|
||||||
|
sessions = _FakeSessions(0)
|
||||||
|
games = _FakeGames(0)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
resource = Resource.create({"service.name": "test"})
|
||||||
|
cls.reader = InMemoryMetricReader()
|
||||||
|
cls.provider = MeterProvider(resource=resource, metric_readers=[cls.reader])
|
||||||
|
cls.span_exporter = InMemorySpanExporter()
|
||||||
|
cls.tracer_provider = TracerProvider(resource=resource)
|
||||||
|
cls.tracer_provider.add_span_processor(
|
||||||
|
SimpleSpanProcessor(cls.span_exporter)
|
||||||
|
)
|
||||||
|
cls.mixin = TelemetryMixin(
|
||||||
|
meter_provider=cls.provider,
|
||||||
|
tracer_provider=cls.tracer_provider,
|
||||||
|
logger_provider=None,
|
||||||
|
log_handler=None,
|
||||||
|
session_store=cls.sessions,
|
||||||
|
game_store=cls.games,
|
||||||
|
poll_interval_seconds=1,
|
||||||
|
)
|
||||||
|
cls.mixin.apply(None) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cls.mixin.shutdown(None) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
async def test_app_gauges_report_polled_counts(self) -> None:
|
||||||
|
self.sessions.value = 3
|
||||||
|
self.sessions.error = None
|
||||||
|
self.games.value = 2
|
||||||
|
await self.mixin.refresh_counts()
|
||||||
|
points = _metric_points(self.reader)
|
||||||
|
self.assertEqual([({}, 3)], points["tavolo.active_users"])
|
||||||
|
self.assertEqual([({}, 2)], points["tavolo.ongoing_games"])
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
async def test_failed_refresh_keeps_last_value(self) -> None:
|
||||||
|
self.sessions.value = 5
|
||||||
|
self.sessions.error = None
|
||||||
|
await self.mixin.refresh_counts()
|
||||||
|
self.sessions.error = RuntimeError("redis down")
|
||||||
|
await self.mixin.refresh_counts()
|
||||||
|
points = _metric_points(self.reader)
|
||||||
|
self.assertEqual([({}, 5)], points["tavolo.active_users"])
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
async def test_process_and_gc_metrics_are_exported(self) -> None:
|
||||||
|
await self.mixin.refresh_counts()
|
||||||
|
points = _metric_points(self.reader)
|
||||||
|
for name in (
|
||||||
|
"process.cpu.time",
|
||||||
|
"process.cpu.utilization",
|
||||||
|
"process.memory.usage",
|
||||||
|
"process.memory.virtual",
|
||||||
|
"cpython.gc.collections",
|
||||||
|
"cpython.gc.collected_objects",
|
||||||
|
"cpython.gc.uncollectable_objects",
|
||||||
|
):
|
||||||
|
self.assertIn(name, points)
|
||||||
|
self.assertGreater(points["process.memory.usage"][0][1], 0)
|
||||||
|
modes = {attrs["type"] for attrs, _ in points["process.cpu.time"]}
|
||||||
|
self.assertEqual({"user", "system"}, modes)
|
||||||
|
generations = {
|
||||||
|
attrs["cpython.gc.generation"]
|
||||||
|
for attrs, _ in points["cpython.gc.collections"]
|
||||||
|
}
|
||||||
|
self.assertEqual({0, 1, 2}, generations)
|
||||||
|
# Host-level metrics are deliberately not configured.
|
||||||
|
self.assertNotIn("system.cpu.time", points)
|
||||||
|
|
||||||
|
@async_test
|
||||||
|
async def test_database_clients_are_instrumented(self) -> None:
|
||||||
|
import asyncpg.connection
|
||||||
|
import redis.asyncio
|
||||||
|
|
||||||
|
self.assertTrue(hasattr(redis.asyncio.Redis.execute_command, "__wrapped__"))
|
||||||
|
self.assertTrue(hasattr(asyncpg.connection.Connection.fetch, "__wrapped__"))
|
||||||
|
|
||||||
|
# Exercise each wrapper through stubs: the downstream call fails (no
|
||||||
|
# server), but the span is recorded before it does.
|
||||||
|
client = object.__new__(redis.asyncio.Redis)
|
||||||
|
client.connection_pool = SimpleNamespace(
|
||||||
|
connection_kwargs={"host": "127.0.0.1", "port": 6379, "db": 0}
|
||||||
|
)
|
||||||
|
with self.assertRaises(AttributeError):
|
||||||
|
await client.execute_command("GET", "key")
|
||||||
|
|
||||||
|
connection = object.__new__(asyncpg.connection.Connection)
|
||||||
|
connection._aborted = True # keep __del__ quiet on the stub
|
||||||
|
with self.assertRaises(Exception):
|
||||||
|
await connection.fetch("SELECT 1")
|
||||||
|
|
||||||
|
spans = {
|
||||||
|
span.name: span for span in self.span_exporter.get_finished_spans()
|
||||||
|
}
|
||||||
|
self.assertEqual("GET ?", spans["GET"].attributes["db.statement"])
|
||||||
|
self.assertEqual("redis", spans["GET"].attributes["db.system"])
|
||||||
|
self.assertEqual("SELECT 1", spans["SELECT"].attributes["db.statement"])
|
||||||
|
self.assertEqual("postgresql", spans["SELECT"].attributes["db.system"])
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(HAVE_OTEL, "tavolo-app[otel] not installed")
|
||||||
|
class LogExportTest(unittest.TestCase):
|
||||||
|
def test_logs_are_exported_and_otel_records_filtered(self) -> None:
|
||||||
|
exporter = InMemoryLogRecordExporter()
|
||||||
|
resource = Resource.create({"service.name": "test"})
|
||||||
|
provider = LoggerProvider(resource=resource)
|
||||||
|
provider.add_log_record_processor(SimpleLogRecordProcessor(exporter))
|
||||||
|
handler = LoggingHandler(level=logging.NOTSET, logger_provider=provider)
|
||||||
|
handler.addFilter(_NoOtelLogs())
|
||||||
|
root = logging.getLogger()
|
||||||
|
previous_level = root.level
|
||||||
|
root.addHandler(handler)
|
||||||
|
root.setLevel(logging.DEBUG)
|
||||||
|
try:
|
||||||
|
logging.getLogger("tavolo.test").info("exported message")
|
||||||
|
logging.getLogger("opentelemetry.test").info("filtered message")
|
||||||
|
bodies = [r.log_record.body for r in exporter.get_finished_logs()]
|
||||||
|
self.assertIn("exported message", bodies)
|
||||||
|
self.assertNotIn("filtered message", bodies)
|
||||||
|
records = exporter.get_finished_logs()
|
||||||
|
self.assertEqual(
|
||||||
|
"test", records[0].resource.attributes["service.name"]
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
root.removeHandler(handler)
|
||||||
|
handler.close()
|
||||||
|
root.setLevel(previous_level)
|
||||||
|
provider.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -291,6 +291,13 @@ pub fn fmt_points(points: f64) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Render a calendar instant as `YYYY-MM-DD HH:MM`. `month` is 0-based
|
||||||
|
/// (the JavaScript `Date` convention); the other components are as
|
||||||
|
/// returned by `Date`'s local-time getters.
|
||||||
|
pub fn fmt_datetime_parts(year: u32, month: u32, day: u32, hour: u32, minute: u32) -> String {
|
||||||
|
format!("{year:04}-{:02}-{day:02} {hour:02}:{minute:02}", month + 1)
|
||||||
|
}
|
||||||
|
|
||||||
/// The caller's Elo rating for one game type (GET /api/me/ratings).
|
/// The caller's Elo rating for one game type (GET /api/me/ratings).
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
@@ -658,6 +665,12 @@ mod tests {
|
|||||||
assert_eq!("2.5", fmt_points(2.5));
|
assert_eq!("2.5", fmt_points(2.5));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fmt_datetime_parts_pads_and_offsets_month() {
|
||||||
|
assert_eq!("2026-09-19 05:02", fmt_datetime_parts(2026, 8, 19, 5, 2));
|
||||||
|
assert_eq!("2026-01-01 00:00", fmt_datetime_parts(2026, 0, 1, 0, 0));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn leaderboard_points_accept_floats() {
|
fn leaderboard_points_accept_floats() {
|
||||||
let e: LeaderboardEntry = serde_json::from_value(json!({
|
let e: LeaderboardEntry = serde_json::from_value(json!({
|
||||||
|
|||||||
@@ -68,6 +68,23 @@ fn game_name(game_types: &[GameTypeInfo], game_type: &str) -> String {
|
|||||||
.unwrap_or_else(|| game_type.to_string())
|
.unwrap_or_else(|| game_type.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The match's finish instant in the browser's local timezone, as
|
||||||
|
/// `YYYY-MM-DD HH:MM`. Falls back to the raw UTC prefix when the browser
|
||||||
|
/// cannot parse the timestamp.
|
||||||
|
fn local_finished(iso: &str) -> String {
|
||||||
|
let date = js_sys::Date::new(&wasm_bindgen::JsValue::from_str(iso));
|
||||||
|
if date.get_time().is_nan() {
|
||||||
|
return iso.replace('T', " ").chars().take(16).collect();
|
||||||
|
}
|
||||||
|
crate::model::fmt_datetime_parts(
|
||||||
|
date.get_full_year(),
|
||||||
|
date.get_month(),
|
||||||
|
date.get_date(),
|
||||||
|
date.get_hours(),
|
||||||
|
date.get_minutes(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
#[component]
|
#[component]
|
||||||
pub fn HistoryPage() -> View {
|
pub fn HistoryPage() -> View {
|
||||||
let page = create_signal(Option::<MatchesPage>::None);
|
let page = create_signal(Option::<MatchesPage>::None);
|
||||||
@@ -131,12 +148,7 @@ pub fn HistoryPage() -> View {
|
|||||||
.get_clone()
|
.get_clone()
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|m| {
|
.map(|m| {
|
||||||
let finished = m
|
let finished = local_finished(&m.finished_at);
|
||||||
.finished_at
|
|
||||||
.replace('T', " ")
|
|
||||||
.chars()
|
|
||||||
.take(16)
|
|
||||||
.collect::<String>();
|
|
||||||
let game = game_name(&names, &m.game_type);
|
let game = game_name(&names, &m.game_type);
|
||||||
let mut cells: Vec<View> = team_columns(&m)
|
let mut cells: Vec<View> = team_columns(&m)
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
Reference in New Issue
Block a user