Restore cursor value types before keyset filtering
Cursors are JSON, so a datetime sort key reached the next request as a string. Tortoise's DatetimeField.to_db_value does not parse strings, so asyncpg rejected the string where a timestamptz parameter was expected; sqlite tolerated it, hiding the bug in tests. Coerce decoded cursor values through their model field's to_python_value before building the keyset filter.
This commit is contained in:
2 files changed
+109
-2
No files matched your search
@@ -15,12 +15,15 @@ from __future__ import annotations
|
|||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
from tortoise.queryset import QuerySet
|
from tortoise.queryset import QuerySet
|
||||||
|
|
||||||
from .http import extract_query_params
|
from .http import extract_query_params
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from tortoise.models import Model
|
||||||
|
|
||||||
DEFAULT_LIMIT = 20
|
DEFAULT_LIMIT = 20
|
||||||
MAX_LIMIT = 100
|
MAX_LIMIT = 100
|
||||||
MIN_LIMIT = 1
|
MIN_LIMIT = 1
|
||||||
@@ -81,6 +84,28 @@ def parse_cursor_params(query_string: str) -> Cursor:
|
|||||||
Sort = List[Tuple[str, str]]
|
Sort = List[Tuple[str, str]]
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_sort_values(
|
||||||
|
model: "type[Model]", sort: Sort, after: Dict[str, Any]
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Restore the Python type of every decoded cursor value.
|
||||||
|
|
||||||
|
The cursor is JSON, so field values travel as strings (datetimes via
|
||||||
|
``default=str``). asyncpg rejects strings where a ``timestamp``
|
||||||
|
parameter is expected, and sqlite only tolerates them by accident, so
|
||||||
|
each value is run through its model field's ``to_python_value`` before
|
||||||
|
it reaches the keyset filter.
|
||||||
|
"""
|
||||||
|
fields = model._meta.fields_map
|
||||||
|
coerced: Dict[str, Any] = {}
|
||||||
|
for field, _ in sort:
|
||||||
|
value = after.get(field)
|
||||||
|
field_obj = fields.get(field)
|
||||||
|
if value is not None and field_obj is not None:
|
||||||
|
value = field_obj.to_python_value(value)
|
||||||
|
coerced[field] = value
|
||||||
|
return coerced
|
||||||
|
|
||||||
|
|
||||||
def _keyset_where(sort: Sort, after: Dict[str, Any]):
|
def _keyset_where(sort: Sort, after: Dict[str, Any]):
|
||||||
"""Build a Tortoise ``Q`` filter from a cursor."""
|
"""Build a Tortoise ``Q`` filter from a cursor."""
|
||||||
from tortoise.queryset import Q # local to keep import edge narrow
|
from tortoise.queryset import Q # local to keep import edge narrow
|
||||||
@@ -116,7 +141,8 @@ async def paginate(
|
|||||||
order_by.append(field if direction == "ASC" else f"-{field}")
|
order_by.append(field if direction == "ASC" else f"-{field}")
|
||||||
qs = queryset.order_by(*order_by)
|
qs = queryset.order_by(*order_by)
|
||||||
if cursor.after:
|
if cursor.after:
|
||||||
qs = qs.filter(_keyset_where(sort, cursor.after))
|
after = _coerce_sort_values(queryset.model, sort, cursor.after)
|
||||||
|
qs = qs.filter(_keyset_where(sort, after))
|
||||||
rows = await qs.limit(cursor.limit + 1)
|
rows = await qs.limit(cursor.limit + 1)
|
||||||
if len(rows) <= cursor.limit:
|
if len(rows) <= cursor.limit:
|
||||||
return rows, None
|
return rows, None
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""Keyset pagination tests: cursor values are restored to Python types.
|
||||||
|
|
||||||
|
Cursors are JSON, so a cursor carrying a datetime reaches the next
|
||||||
|
request as a string. Postgres/asyncpg rejects strings for timestamp
|
||||||
|
parameters (sqlite silently coerces them), so ``paginate`` must run the
|
||||||
|
decoded values through their model fields before building the WHERE
|
||||||
|
clause.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any, List
|
||||||
|
|
||||||
|
from tavolo.platform.models import Match
|
||||||
|
from tavolo.platform.pagination import Cursor, decode_cursor, encode_cursor, paginate
|
||||||
|
|
||||||
|
from helpers import async_test
|
||||||
|
|
||||||
|
SORT = [("finished_at", "DESC"), ("id", "ASC")]
|
||||||
|
|
||||||
|
|
||||||
|
class _CapturingQuerySet:
|
||||||
|
"""Minimal stand-in for a Tortoise QuerySet, recording its filter."""
|
||||||
|
|
||||||
|
model = Match
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.queryset_filter: Any = None
|
||||||
|
|
||||||
|
def order_by(self, *orderings: str) -> "_CapturingQuerySet":
|
||||||
|
return self
|
||||||
|
|
||||||
|
def filter(self, q: Any) -> "_CapturingQuerySet":
|
||||||
|
self.queryset_filter = q
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def limit(self, count: int) -> List[Any]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _flatten_q(q: Any) -> List[Any]:
|
||||||
|
nodes = [q]
|
||||||
|
for child in q.children:
|
||||||
|
nodes.extend(_flatten_q(child))
|
||||||
|
return nodes
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_values(q: Any, key: str) -> List[Any]:
|
||||||
|
return [node.filters[key] for node in _flatten_q(q) if key in node.filters]
|
||||||
|
|
||||||
|
|
||||||
|
class PaginateCursorTypesTest(unittest.TestCase):
|
||||||
|
@async_test
|
||||||
|
async def test_cursor_values_are_restored_before_filtering(self) -> None:
|
||||||
|
match_id = uuid.uuid4()
|
||||||
|
finished_at = datetime(2026, 9, 19, 5, 2, 50, 547904, tzinfo=timezone.utc)
|
||||||
|
# Round-trip through the wire format, exactly as a request does.
|
||||||
|
after = decode_cursor(
|
||||||
|
encode_cursor({"finished_at": finished_at, "id": match_id})
|
||||||
|
)
|
||||||
|
assert after is not None
|
||||||
|
|
||||||
|
qs = _CapturingQuerySet()
|
||||||
|
await paginate(qs, SORT, Cursor(limit=1, after=after))
|
||||||
|
|
||||||
|
assert qs.queryset_filter is not None
|
||||||
|
for key in ("finished_at", "finished_at__lt"):
|
||||||
|
values = _filter_values(qs.queryset_filter, key)
|
||||||
|
self.assertEqual(1, len(values))
|
||||||
|
self.assertIsInstance(values[0], datetime)
|
||||||
|
self.assertEqual(finished_at, values[0])
|
||||||
|
id_values = _filter_values(qs.queryset_filter, "id__gt")
|
||||||
|
self.assertEqual(1, len(id_values))
|
||||||
|
self.assertIsInstance(id_values[0], uuid.UUID)
|
||||||
|
self.assertEqual(match_id, id_values[0])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in new issue
Block a user