Restore cursor value types before keyset filtering

Cursors are JSON, so a datetime sort key reached the next request as a
string. Tortoise's DatetimeField.to_db_value does not parse strings, so
asyncpg rejected the string where a timestamptz parameter was expected;
sqlite tolerated it, hiding the bug in tests. Coerce decoded cursor
values through their model field's to_python_value before building the
keyset filter.
This commit is contained in:
2026-09-28 04:26:00 +08:00
parent a3e8ff3206
commit 2bdf247cb3
2 changed files with 109 additions and 2 deletions
@@ -15,12 +15,15 @@ from __future__ import annotations
import base64
import json
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Tuple
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
from tortoise.queryset import QuerySet
from .http import extract_query_params
if TYPE_CHECKING:
from tortoise.models import Model
DEFAULT_LIMIT = 20
MAX_LIMIT = 100
MIN_LIMIT = 1
@@ -81,6 +84,28 @@ def parse_cursor_params(query_string: str) -> Cursor:
Sort = List[Tuple[str, str]]
def _coerce_sort_values(
model: "type[Model]", sort: Sort, after: Dict[str, Any]
) -> Dict[str, Any]:
"""Restore the Python type of every decoded cursor value.
The cursor is JSON, so field values travel as strings (datetimes via
``default=str``). asyncpg rejects strings where a ``timestamp``
parameter is expected, and sqlite only tolerates them by accident, so
each value is run through its model field's ``to_python_value`` before
it reaches the keyset filter.
"""
fields = model._meta.fields_map
coerced: Dict[str, Any] = {}
for field, _ in sort:
value = after.get(field)
field_obj = fields.get(field)
if value is not None and field_obj is not None:
value = field_obj.to_python_value(value)
coerced[field] = value
return coerced
def _keyset_where(sort: Sort, after: Dict[str, Any]):
"""Build a Tortoise ``Q`` filter from a cursor."""
from tortoise.queryset import Q # local to keep import edge narrow
@@ -116,7 +141,8 @@ async def paginate(
order_by.append(field if direction == "ASC" else f"-{field}")
qs = queryset.order_by(*order_by)
if cursor.after:
qs = qs.filter(_keyset_where(sort, cursor.after))
after = _coerce_sort_values(queryset.model, sort, cursor.after)
qs = qs.filter(_keyset_where(sort, after))
rows = await qs.limit(cursor.limit + 1)
if len(rows) <= cursor.limit:
return rows, None
@@ -0,0 +1,81 @@
"""Keyset pagination tests: cursor values are restored to Python types.
Cursors are JSON, so a cursor carrying a datetime reaches the next
request as a string. Postgres/asyncpg rejects strings for timestamp
parameters (sqlite silently coerces them), so ``paginate`` must run the
decoded values through their model fields before building the WHERE
clause.
"""
from __future__ import annotations
import unittest
import uuid
from datetime import datetime, timezone
from typing import Any, List
from tavolo.platform.models import Match
from tavolo.platform.pagination import Cursor, decode_cursor, encode_cursor, paginate
from helpers import async_test
SORT = [("finished_at", "DESC"), ("id", "ASC")]
class _CapturingQuerySet:
"""Minimal stand-in for a Tortoise QuerySet, recording its filter."""
model = Match
def __init__(self) -> None:
self.queryset_filter: Any = None
def order_by(self, *orderings: str) -> "_CapturingQuerySet":
return self
def filter(self, q: Any) -> "_CapturingQuerySet":
self.queryset_filter = q
return self
async def limit(self, count: int) -> List[Any]:
return []
def _flatten_q(q: Any) -> List[Any]:
nodes = [q]
for child in q.children:
nodes.extend(_flatten_q(child))
return nodes
def _filter_values(q: Any, key: str) -> List[Any]:
return [node.filters[key] for node in _flatten_q(q) if key in node.filters]
class PaginateCursorTypesTest(unittest.TestCase):
@async_test
async def test_cursor_values_are_restored_before_filtering(self) -> None:
match_id = uuid.uuid4()
finished_at = datetime(2026, 9, 19, 5, 2, 50, 547904, tzinfo=timezone.utc)
# Round-trip through the wire format, exactly as a request does.
after = decode_cursor(
encode_cursor({"finished_at": finished_at, "id": match_id})
)
assert after is not None
qs = _CapturingQuerySet()
await paginate(qs, SORT, Cursor(limit=1, after=after))
assert qs.queryset_filter is not None
for key in ("finished_at", "finished_at__lt"):
values = _filter_values(qs.queryset_filter, key)
self.assertEqual(1, len(values))
self.assertIsInstance(values[0], datetime)
self.assertEqual(finished_at, values[0])
id_values = _filter_values(qs.queryset_filter, "id__gt")
self.assertEqual(1, len(id_values))
self.assertIsInstance(id_values[0], uuid.UUID)
self.assertEqual(match_id, id_values[0])
if __name__ == "__main__":
unittest.main()