Implement kaya-rbcs: simplified RBCS clone on the Kaya framework
CI / Build and push docker image (push) Successful in 2m57s
CI / Build and push docker image (push) Successful in 2m57s
- KayaApp with GET/PUT recursive routes backed by memcached (aiomcache) - env-based configuration, metadata-aware value encoding, key prefix/digest - Granian RSGI launcher (rbcs-server / python -m kaya_rbcs) - pinned requirements via pip-compile, alpine:3.24 Dockerfile with healthcheck - offline tests using an in-process fake memcached - Gitea Actions workflow to build and push the multi-arch Docker image
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
from .app import app, create_app
|
||||
from .config import Config
|
||||
from .store import CacheValueMetadata, MemcacheStore
|
||||
|
||||
__version__ = '0.1.0'
|
||||
|
||||
|
||||
def main() -> None:
|
||||
from kaya_rbcs.__main__ import main as _main
|
||||
|
||||
_main()
|
||||
|
||||
|
||||
__all__ = [
|
||||
'app',
|
||||
'create_app',
|
||||
'Config',
|
||||
'CacheValueMetadata',
|
||||
'MemcacheStore',
|
||||
]
|
||||
@@ -0,0 +1,32 @@
|
||||
import sys
|
||||
from typing import List, NoReturn
|
||||
|
||||
from .config import Config
|
||||
|
||||
|
||||
def main() -> NoReturn:
|
||||
try:
|
||||
from granian.cli import entrypoint
|
||||
except ImportError as exc:
|
||||
raise SystemExit(
|
||||
"Granian is required to run the server, "
|
||||
"install it with 'pip install kaya-rbcs[server]'"
|
||||
) from exc
|
||||
|
||||
config = Config.from_env()
|
||||
args: List[str] = [
|
||||
'--interface',
|
||||
'rsgi',
|
||||
'--host',
|
||||
config.host,
|
||||
'--port',
|
||||
str(config.port),
|
||||
'kaya_rbcs.app:app',
|
||||
]
|
||||
sys.argv = [sys.argv[0], *args]
|
||||
entrypoint() # type: ignore[no-untyped-call]
|
||||
raise SystemExit(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
from logging import getLogger
|
||||
from posixpath import normpath
|
||||
from typing import List, Mapping, Optional, Sequence
|
||||
|
||||
from kaya.core import HttpContext, KayaApp
|
||||
|
||||
from .config import Config
|
||||
from .store import CacheValueMetadata, MemcacheStore
|
||||
|
||||
log = getLogger(__name__)
|
||||
|
||||
_APPLICATION_OCTET_STREAM = 'application/octet-stream'
|
||||
_TEXT_PLAIN = 'text/plain'
|
||||
|
||||
|
||||
def _normalize_prefix(prefix: str) -> str:
|
||||
stripped = prefix.strip('/')
|
||||
return '/' + stripped if stripped else '/'
|
||||
|
||||
|
||||
def _route_pattern(prefix: str) -> str:
|
||||
stripped = prefix.strip('/')
|
||||
return '/*' if not stripped else '/' + stripped + '/*'
|
||||
|
||||
|
||||
def key_from_path(path: str, prefix: str) -> Optional[str]:
|
||||
"""Normalize ``path`` and compute the cache key relative to ``prefix``.
|
||||
|
||||
Returns ``None`` when the path does not fall under the prefix (or resolves
|
||||
to an empty key), mirroring the 400 responses of the original RBCS for
|
||||
unhandled / escaping paths.
|
||||
"""
|
||||
normalized = normpath(path)
|
||||
prefix = _normalize_prefix(prefix)
|
||||
if prefix == '/':
|
||||
key = normalized.lstrip('/')
|
||||
else:
|
||||
if normalized != prefix and not normalized.startswith(prefix + '/'):
|
||||
return None
|
||||
key = normalized[len(prefix):].lstrip('/')
|
||||
return key or None
|
||||
|
||||
|
||||
def _first_header(headers: Mapping[str, Sequence[str]], name: str) -> Optional[str]:
|
||||
values = headers.get(name)
|
||||
if values is None:
|
||||
return None
|
||||
if isinstance(values, str):
|
||||
return values
|
||||
return values[0] if len(values) > 0 else None
|
||||
|
||||
|
||||
def create_app(config: Config) -> KayaApp:
|
||||
store = MemcacheStore(
|
||||
host=config.memcache_host,
|
||||
port=config.memcache_port,
|
||||
key_prefix=config.key_prefix,
|
||||
digest=config.digest,
|
||||
max_age=config.max_age,
|
||||
)
|
||||
prefix = config.path_prefix
|
||||
app = KayaApp()
|
||||
app.store = store # type: ignore[attr-defined]
|
||||
|
||||
@app.GET(_route_pattern(prefix), recursive=True)
|
||||
async def get_handler(ctx: HttpContext, path: List[str]) -> None:
|
||||
key = key_from_path(ctx.path, prefix)
|
||||
if key is None:
|
||||
log.warning('Got GET request for unhandled path %s', ctx.path)
|
||||
await ctx.send_empty(400)
|
||||
return
|
||||
result = await store.get(key)
|
||||
if result is None:
|
||||
log.debug('Cache miss for key %s', key)
|
||||
await ctx.send_empty(404)
|
||||
return
|
||||
metadata, value = result
|
||||
log.debug('Cache hit for key %s', key)
|
||||
headers: dict[str, str] = {
|
||||
'content-type': metadata.content_type or _APPLICATION_OCTET_STREAM,
|
||||
}
|
||||
if metadata.content_disposition is not None:
|
||||
headers['content-disposition'] = metadata.content_disposition
|
||||
await ctx.send_bytes(200, value, headers)
|
||||
|
||||
@app.PUT(_route_pattern(prefix), recursive=True)
|
||||
async def put_handler(ctx: HttpContext, path: List[str]) -> None:
|
||||
key = key_from_path(ctx.path, prefix)
|
||||
if key is None:
|
||||
log.warning('Got PUT request for unhandled path %s', ctx.path)
|
||||
await ctx.send_empty(400)
|
||||
return
|
||||
chunks: List[bytes] = []
|
||||
async for chunk in ctx.request_body:
|
||||
chunks.append(chunk)
|
||||
value = b''.join(chunks)
|
||||
metadata = CacheValueMetadata(
|
||||
content_type=_first_header(ctx.headers, 'content-type'),
|
||||
content_disposition=_first_header(ctx.headers, 'content-disposition'),
|
||||
)
|
||||
await store.put(key, value, metadata)
|
||||
log.debug('Added value for key %s', key)
|
||||
await ctx.send_str(201, key, {'content-type': _TEXT_PLAIN})
|
||||
|
||||
return app
|
||||
|
||||
|
||||
app = create_app(Config.from_env())
|
||||
@@ -0,0 +1,44 @@
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import timedelta
|
||||
from os import getenv
|
||||
|
||||
|
||||
def _parse_max_age(raw: str) -> timedelta:
|
||||
if raw.endswith('s'):
|
||||
return timedelta(seconds=int(raw[:-1]))
|
||||
if raw.endswith('m'):
|
||||
return timedelta(minutes=int(raw[:-1]))
|
||||
if raw.endswith('h'):
|
||||
return timedelta(hours=int(raw[:-1]))
|
||||
if raw.endswith('d'):
|
||||
return timedelta(days=int(raw[:-1]))
|
||||
return timedelta(seconds=int(raw))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
host: str = '127.0.0.1'
|
||||
port: int = 8080
|
||||
path_prefix: str = '/'
|
||||
memcache_host: str = '127.0.0.1'
|
||||
memcache_port: int = 11211
|
||||
key_prefix: str | None = None
|
||||
digest: str | None = None
|
||||
max_age: timedelta = field(default_factory=lambda: timedelta(days=7))
|
||||
|
||||
@staticmethod
|
||||
def from_env() -> 'Config':
|
||||
def get_int(name: str, default: int) -> int:
|
||||
raw = getenv(name)
|
||||
return int(raw) if raw is not None else default
|
||||
|
||||
return Config(
|
||||
host=getenv('RBCS_HOST', '127.0.0.1'),
|
||||
port=get_int('RBCS_PORT', 8080),
|
||||
path_prefix=getenv('RBCS_PATH_PREFIX', '/'),
|
||||
memcache_host=getenv('RBCS_MEMCACHE_HOST', '127.0.0.1'),
|
||||
memcache_port=get_int('RBCS_MEMCACHE_PORT', 11211),
|
||||
key_prefix=getenv('RBCS_KEY_PREFIX'),
|
||||
digest=getenv('RBCS_DIGEST'),
|
||||
max_age=_parse_max_age(getenv('RBCS_MAX_AGE', '7d')),
|
||||
)
|
||||
@@ -0,0 +1,117 @@
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import timedelta
|
||||
from time import time
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
import aiomcache
|
||||
|
||||
_THIRTY_DAYS = 30 * 24 * 60 * 60
|
||||
|
||||
|
||||
def _exptime(max_age: timedelta, clock: Callable[[], float]) -> int:
|
||||
"""Convert a relative TTL to a memcached ``exptime`` value.
|
||||
|
||||
Memcached interprets ``exptime`` values larger than 30 days as absolute
|
||||
Unix timestamps, so large TTLs must be converted explicitly.
|
||||
"""
|
||||
seconds = int(max_age.total_seconds())
|
||||
if seconds > _THIRTY_DAYS:
|
||||
return int(clock()) + seconds
|
||||
return seconds
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CacheValueMetadata:
|
||||
"""Metadata stored alongside a cached value, mirroring RBCS' metadata."""
|
||||
|
||||
content_type: Optional[str] = None
|
||||
content_disposition: Optional[str] = None
|
||||
|
||||
|
||||
def process_cache_key(key: str, key_prefix: Optional[str], digest: Optional[str]) -> bytes:
|
||||
"""Compute the memcached key, mirroring RBCS' ``RBCS.processCacheKey``.
|
||||
|
||||
The key prefix is appended to the key (as in the original implementation)
|
||||
and, if a digest algorithm is configured, the result is hashed and returned
|
||||
as a hexadecimal string so that it stays a valid memcached key.
|
||||
"""
|
||||
prefixed_key = key + key_prefix if key_prefix is not None else key
|
||||
data = prefixed_key.encode('utf-8')
|
||||
if digest is not None:
|
||||
return hashlib.new(digest, data).hexdigest().encode('utf-8')
|
||||
return data
|
||||
|
||||
|
||||
def encode_value(value: bytes, metadata: CacheValueMetadata) -> bytes:
|
||||
"""Encode a cached value as ``[4-byte metadata length][json metadata][value]``."""
|
||||
metadata_bytes = json.dumps(asdict(metadata)).encode('utf-8')
|
||||
return struct.pack('>I', len(metadata_bytes)) + metadata_bytes + value
|
||||
|
||||
|
||||
def decode_value(payload: bytes) -> Tuple[CacheValueMetadata, bytes]:
|
||||
"""Decode a value previously encoded with :func:`encode_value`."""
|
||||
(size,) = struct.unpack('>I', payload[:4])
|
||||
metadata = json.loads(payload[4:4 + size].decode('utf-8'))
|
||||
return CacheValueMetadata(**metadata), payload[4 + size:]
|
||||
|
||||
|
||||
class MemcacheStore:
|
||||
"""Memcached-backed cache store with RBCS-compatible value encoding.
|
||||
|
||||
The underlying ``aiomcache.Client`` is created lazily on first use so that
|
||||
it is always bound to the currently running event loop; call :meth:`start`
|
||||
eagerly inside a running loop if you want to fail fast, and :meth:`close`
|
||||
to release the pool.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str = '127.0.0.1',
|
||||
port: int = 11211,
|
||||
key_prefix: Optional[str] = None,
|
||||
digest: Optional[str] = None,
|
||||
max_age: timedelta = timedelta(days=7),
|
||||
clock: Callable[[], float] = time,
|
||||
) -> None:
|
||||
self._host = host
|
||||
self._port = port
|
||||
self._key_prefix = key_prefix
|
||||
self._digest = digest
|
||||
self._max_age = max_age
|
||||
self._clock = clock
|
||||
self._client: Optional[aiomcache.Client] = None
|
||||
|
||||
def _ensure_client(self) -> aiomcache.Client:
|
||||
if self._client is None:
|
||||
self._client = aiomcache.Client(self._host, self._port)
|
||||
return self._client
|
||||
|
||||
async def start(self) -> None:
|
||||
self._ensure_client()
|
||||
|
||||
def _memcache_key(self, key: str) -> bytes:
|
||||
return process_cache_key(key, self._key_prefix, self._digest)
|
||||
|
||||
async def get(self, key: str) -> Optional[Tuple[CacheValueMetadata, bytes]]:
|
||||
client = self._ensure_client()
|
||||
payload = await client.get(self._memcache_key(key))
|
||||
if payload is None:
|
||||
return None
|
||||
return decode_value(payload)
|
||||
|
||||
async def put(self, key: str, value: bytes, metadata: CacheValueMetadata) -> None:
|
||||
client = self._ensure_client()
|
||||
exptime = _exptime(self._max_age, self._clock)
|
||||
await client.set(
|
||||
self._memcache_key(key),
|
||||
encode_value(value, metadata),
|
||||
exptime=exptime,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._client is not None:
|
||||
await self._client.close()
|
||||
self._client = None
|
||||
Reference in New Issue
Block a user