Restore cursor value types before keyset filtering

Cursors are JSON, so a datetime sort key reached the next request as a
string. Tortoise's DatetimeField.to_db_value does not parse strings, so
asyncpg rejected the string where a timestamptz parameter was expected;
sqlite tolerated it, hiding the bug in tests. Coerce decoded cursor
values through their model field's to_python_value before building the
keyset filter.
This commit is contained in:
woggioni committed 2026-09-28 04:26:00 +08:00
1 parent a3e8ff3206
commit 2bdf247cb3
2 files changed
+109 -2

No files matched your search

@@ -15,12 +15,15 @@ from __future__ import annotations
import base64 import base64
import json import json
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Tuple from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
from tortoise.queryset import QuerySet from tortoise.queryset import QuerySet
from .http import extract_query_params from .http import extract_query_params
if TYPE_CHECKING:
from tortoise.models import Model
DEFAULT_LIMIT = 20 DEFAULT_LIMIT = 20
MAX_LIMIT = 100 MAX_LIMIT = 100
MIN_LIMIT = 1 MIN_LIMIT = 1
@@ -81,6 +84,28 @@ def parse_cursor_params(query_string: str) -> Cursor:
Sort = List[Tuple[str, str]] Sort = List[Tuple[str, str]]
def _coerce_sort_values(
model: "type[Model]", sort: Sort, after: Dict[str, Any]
) -> Dict[str, Any]:
"""Restore the Python type of every decoded cursor value.
The cursor is JSON, so field values travel as strings (datetimes via
``default=str``). asyncpg rejects strings where a ``timestamp``
parameter is expected, and sqlite only tolerates them by accident, so
each value is run through its model field's ``to_python_value`` before
it reaches the keyset filter.
"""
fields = model._meta.fields_map
coerced: Dict[str, Any] = {}
for field, _ in sort:
value = after.get(field)
field_obj = fields.get(field)
if value is not None and field_obj is not None:
value = field_obj.to_python_value(value)
coerced[field] = value
return coerced
def _keyset_where(sort: Sort, after: Dict[str, Any]): def _keyset_where(sort: Sort, after: Dict[str, Any]):
"""Build a Tortoise ``Q`` filter from a cursor.""" """Build a Tortoise ``Q`` filter from a cursor."""
from tortoise.queryset import Q # local to keep import edge narrow from tortoise.queryset import Q # local to keep import edge narrow
@@ -116,7 +141,8 @@ async def paginate(
order_by.append(field if direction == "ASC" else f"-{field}") order_by.append(field if direction == "ASC" else f"-{field}")
qs = queryset.order_by(*order_by) qs = queryset.order_by(*order_by)
if cursor.after: if cursor.after:
qs = qs.filter(_keyset_where(sort, cursor.after)) after = _coerce_sort_values(queryset.model, sort, cursor.after)
qs = qs.filter(_keyset_where(sort, after))
rows = await qs.limit(cursor.limit + 1) rows = await qs.limit(cursor.limit + 1)
if len(rows) <= cursor.limit: if len(rows) <= cursor.limit:
return rows, None return rows, None
@@ -0,0 +1,81 @@
"""Keyset pagination tests: cursor values are restored to Python types.
Cursors are JSON, so a cursor carrying a datetime reaches the next
request as a string. Postgres/asyncpg rejects strings for timestamp
parameters (sqlite silently coerces them), so ``paginate`` must run the
decoded values through their model fields before building the WHERE
clause.
"""
from __future__ import annotations
import unittest
import uuid
from datetime import datetime, timezone
from typing import Any, List
from tavolo.platform.models import Match
from tavolo.platform.pagination import Cursor, decode_cursor, encode_cursor, paginate
from helpers import async_test
SORT = [("finished_at", "DESC"), ("id", "ASC")]
class _CapturingQuerySet:
"""Minimal stand-in for a Tortoise QuerySet, recording its filter."""
model = Match
def __init__(self) -> None:
self.queryset_filter: Any = None
def order_by(self, *orderings: str) -> "_CapturingQuerySet":
return self
def filter(self, q: Any) -> "_CapturingQuerySet":
self.queryset_filter = q
return self
async def limit(self, count: int) -> List[Any]:
return []
def _flatten_q(q: Any) -> List[Any]:
nodes = [q]
for child in q.children:
nodes.extend(_flatten_q(child))
return nodes
def _filter_values(q: Any, key: str) -> List[Any]:
return [node.filters[key] for node in _flatten_q(q) if key in node.filters]
class PaginateCursorTypesTest(unittest.TestCase):
@async_test
async def test_cursor_values_are_restored_before_filtering(self) -> None:
match_id = uuid.uuid4()
finished_at = datetime(2026, 9, 19, 5, 2, 50, 547904, tzinfo=timezone.utc)
# Round-trip through the wire format, exactly as a request does.
after = decode_cursor(
encode_cursor({"finished_at": finished_at, "id": match_id})
)
assert after is not None
qs = _CapturingQuerySet()
await paginate(qs, SORT, Cursor(limit=1, after=after))
assert qs.queryset_filter is not None
for key in ("finished_at", "finished_at__lt"):
values = _filter_values(qs.queryset_filter, key)
self.assertEqual(1, len(values))
self.assertIsInstance(values[0], datetime)
self.assertEqual(finished_at, values[0])
id_values = _filter_values(qs.queryset_filter, "id__gt")
self.assertEqual(1, len(id_values))
self.assertIsInstance(id_values[0], uuid.UUID)
self.assertEqual(match_id, id_values[0])
if __name__ == "__main__":
unittest.main()