5 Commits

Author SHA1 Message Date
02737bf9b4 added command line arguments
All checks were successful
CI / Build Pip package (push) Successful in 18s
CI / Build Docker image (push) Successful in 1m2s
2024-10-28 00:10:56 +08:00
29bdad09bf reduced Docker image size
All checks were successful
CI / Build Pip package (push) Successful in 16s
CI / Build Docker image (push) Successful in 3m6s
2024-10-25 07:25:13 +08:00
16dbd3a82a addded nginx confioguration file
All checks were successful
CI / Build Pip package (push) Successful in 16s
CI / Build Docker image (push) Successful in 3m33s
2024-10-24 23:55:21 +08:00
33a3858b02 added cli 2024-10-24 23:26:06 +08:00
e2e4083321 switch from aiohttp to httpx 2024-10-24 23:18:23 +08:00
13 changed files with 245 additions and 178 deletions

View File

@@ -9,10 +9,10 @@ RUN adduser -D luser
USER luser
WORKDIR /home/luser
COPY --chown=luser:users ./requirements-dev.txt ./requirements-dev.txt
COPY --chown=luser:users ./requirements-dev.txt ./requirements-run.txt
COPY --chown=luser:users ./requirements-run.txt ./requirements-run.txt
WORKDIR /home/luser/
RUN python -m venv .venv
RUN --mount=type=cache,target=/home/luser/.cache/pip,uid=1000,gid=1000 .venv/bin/pip wheel -w /home/luser/wheel -r requirements-dev.txt pygraphviz
RUN --mount=type=cache,target=/home/luser/.cache/pip,uid=1000,gid=1000 .venv/bin/pip wheel -w /home/luser/wheel pygraphviz
RUN --mount=type=cache,target=/home/luser/.cache/pip,uid=1000,gid=1000 .venv/bin/pip install -r requirements-dev.txt /home/luser/wheel/*.whl
COPY --chown=luser:users . /home/luser/bugis
WORKDIR /home/luser/bugis
@@ -21,12 +21,12 @@ RUN --mount=type=cache,target=/home/luser/.cache/pip,uid=1000,gid=1000 /home/lus
FROM base AS release
RUN mkdir /srv/http
RUN adduser -D -h /var/bugis -u 1000 bugis
RUN adduser -D -h /var/lib/bugis -u 1000 bugis
USER bugis
WORKDIR /var/bugis
WORKDIR /var/lib/bugis
COPY --chown=bugis:users conf/pip.conf ./.pip/pip.conf
RUN python -m venv .venv
RUN --mount=type=cache,target=/var/bugis/.cache/pip,uid=1000,gid=1000 --mount=type=bind,ro,from=build,source=/home/luser/bugis/requirements-run.txt,target=/requirements-run.txt --mount=type=bind,ro,from=build,source=/home/luser/wheel,target=/wheel .venv/bin/pip install -r /requirements-run.txt /wheel/*.whl
RUN --mount=type=cache,target=/var/bugis/.cache/pip,uid=1000,gid=1000 --mount=type=bind,ro,from=build,source=/home/luser/requirements-run.txt,target=/requirements-run.txt --mount=type=bind,ro,from=build,source=/home/luser/wheel,target=/wheel .venv/bin/pip install -r /requirements-run.txt /wheel/*.whl
RUN --mount=type=cache,target=/var/bugis/.cache/pip,uid=1000,gid=1000 --mount=type=bind,ro,from=build,source=/home/luser/bugis/dist,target=/dist .venv/bin/pip install /dist/*.whl
VOLUME /srv/http
WORKDIR /srv/http
@@ -36,7 +36,7 @@ ENV GRANIAN_PORT=8000
ENV GRANIAN_INTERFACE=asgi
ENV GRANIAN_LOOP=asyncio
ENV GRANIAN_LOG_ENABLED=false
ENTRYPOINT ["/var/bugis/.venv/bin/python", "-m", "granian", "bugis.asgi:application"]
ENV GRANIAN_LOG_ACCESS_ENABLED=true
ENTRYPOINT ["/var/lib/bugis/.venv/bin/python", "-m", "granian", "bugis.asgi:application"]
EXPOSE 8000/tcp

14
conf/nginx-bugis.conf Normal file
View File

@@ -0,0 +1,14 @@
server {
listen 8080;
http2 on;
server_name localhost;
location / {
proxy_pass http://granian:8000;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 60s;
}
}

View File

@@ -29,7 +29,7 @@ dependencies = [
"PyYAML",
"pygraphviz",
"aiofiles",
"aiohttp[speedups]",
"httpx[http2]"
]
[project.optional-dependencies]

View File

@@ -7,30 +7,21 @@
--index-url https://gitea.woggioni.net/api/packages/woggioni/pypi/simple
--extra-index-url https://pypi.org/simple
aiodns==3.2.0
# via aiohttp
aiofiles==24.1.0
# via bugis (pyproject.toml)
aiohappyeyeballs==2.4.3
# via aiohttp
aiohttp[speedups]==3.10.10
# via bugis (pyproject.toml)
aiosignal==1.3.1
# via aiohttp
anyio==4.6.2.post1
# via httpx
asttokens==2.4.1
# via stack-data
attrs==24.2.0
# via aiohttp
brotli==1.1.0
# via aiohttp
build==1.2.2.post1
# via bugis (pyproject.toml)
certifi==2024.8.30
# via requests
cffi==1.17.1
# via
# cryptography
# pycares
# httpcore
# httpx
# requests
cffi==1.17.1
# via cryptography
charset-normalizer==3.4.0
# via requests
click==8.1.7
@@ -45,16 +36,25 @@ docutils==0.21.2
# via readme-renderer
executing==2.1.0
# via stack-data
frozenlist==1.5.0
# via
# aiohttp
# aiosignal
granian==1.6.1
# via bugis (pyproject.toml)
h11==0.14.0
# via httpcore
h2==4.1.0
# via httpx
hpack==4.0.0
# via h2
httpcore==1.0.6
# via httpx
httpx[http2]==0.27.2
# via bugis (pyproject.toml)
hyperframe==6.0.1
# via h2
idna==3.10
# via
# anyio
# httpx
# requests
# yarl
importlib-metadata==8.5.0
# via twine
ipdb==0.13.13
@@ -87,10 +87,6 @@ more-itertools==10.5.0
# via
# jaraco-classes
# jaraco-functools
multidict==6.1.0
# via
# aiohttp
# yarl
mypy==1.13.0
# via bugis (pyproject.toml)
mypy-extensions==1.0.0
@@ -107,16 +103,12 @@ pkginfo==1.10.0
# via twine
prompt-toolkit==3.0.48
# via ipython
propcache==0.2.0
# via yarl
ptyprocess==0.7.0
# via pexpect
pure-eval==0.2.3
# via stack-data
pwo==0.0.4
# via bugis (pyproject.toml)
pycares==4.4.0
# via aiodns
pycparser==2.22
# via cffi
pygments==2.18.0
@@ -147,6 +139,10 @@ secretstorage==3.3.3
# via keyring
six==1.16.0
# via asttokens
sniffio==1.3.1
# via
# anyio
# httpx
stack-data==0.6.3
# via ipython
traitlets==5.14.3
@@ -169,7 +165,5 @@ watchdog==5.0.3
# via bugis (pyproject.toml)
wcwidth==0.2.13
# via prompt-toolkit
yarl==1.16.0
# via aiohttp
zipp==3.20.2
# via importlib-metadata

View File

@@ -7,57 +7,51 @@
--index-url https://gitea.woggioni.net/api/packages/woggioni/pypi/simple
--extra-index-url https://pypi.org/simple
aiodns==3.2.0
# via aiohttp
aiofiles==24.1.0
# via bugis (pyproject.toml)
aiohappyeyeballs==2.4.3
# via aiohttp
aiohttp[speedups]==3.10.10
# via bugis (pyproject.toml)
aiosignal==1.3.1
# via aiohttp
attrs==24.2.0
# via aiohttp
brotli==1.1.0
# via aiohttp
cffi==1.17.1
# via pycares
anyio==4.6.2.post1
# via httpx
certifi==2024.8.30
# via
# httpcore
# httpx
click==8.1.7
# via granian
frozenlist==1.5.0
# via
# aiohttp
# aiosignal
granian==1.6.1
# via bugis (pyproject.toml)
h11==0.14.0
# via httpcore
h2==4.1.0
# via httpx
hpack==4.0.0
# via h2
httpcore==1.0.6
# via httpx
httpx[http2]==0.27.2
# via bugis (pyproject.toml)
hyperframe==6.0.1
# via h2
idna==3.10
# via yarl
# via
# anyio
# httpx
markdown==3.7
# via bugis (pyproject.toml)
multidict==6.1.0
# via
# aiohttp
# yarl
propcache==0.2.0
# via yarl
pwo==0.0.4
# via bugis (pyproject.toml)
pycares==4.4.0
# via aiodns
pycparser==2.22
# via cffi
pygments==2.18.0
# via bugis (pyproject.toml)
pygraphviz==1.14
# via bugis (pyproject.toml)
pyyaml==6.0.2
# via bugis (pyproject.toml)
sniffio==1.3.1
# via
# anyio
# httpx
typing-extensions==4.12.2
# via pwo
uvloop==0.21.0
# via granian
watchdog==5.0.3
# via bugis (pyproject.toml)
yarl==1.16.0
# via aiohttp

View File

@@ -2,56 +2,50 @@
# This file is autogenerated by pip-compile with Python 3.12
# by the following command:
#
# pip-compile pyproject.toml
# pip-compile --output-file=requirements.txt pyproject.toml
#
--index-url https://gitea.woggioni.net/api/packages/woggioni/pypi/simple
--extra-index-url https://pypi.org/simple
aiodns==3.2.0
# via aiohttp
aiofiles==24.1.0
# via bugis (pyproject.toml)
aiohappyeyeballs==2.4.3
# via aiohttp
aiohttp[speedups]==3.10.10
# via bugis (pyproject.toml)
aiosignal==1.3.1
# via aiohttp
attrs==24.2.0
# via aiohttp
brotli==1.1.0
# via aiohttp
cffi==1.17.1
# via pycares
frozenlist==1.5.0
anyio==4.6.2.post1
# via httpx
certifi==2024.8.30
# via
# aiohttp
# aiosignal
# httpcore
# httpx
h11==0.14.0
# via httpcore
h2==4.1.0
# via httpx
hpack==4.0.0
# via h2
httpcore==1.0.6
# via httpx
httpx[http2]==0.27.2
# via bugis (pyproject.toml)
hyperframe==6.0.1
# via h2
idna==3.10
# via yarl
# via
# anyio
# httpx
markdown==3.7
# via bugis (pyproject.toml)
multidict==6.1.0
# via
# aiohttp
# yarl
propcache==0.2.0
# via yarl
pwo==0.0.4
# via bugis (pyproject.toml)
pycares==4.4.0
# via aiodns
pycparser==2.22
# via cffi
pygments==2.18.0
# via bugis (pyproject.toml)
pygraphviz==1.14
# via bugis (pyproject.toml)
pyyaml==6.0.2
# via bugis (pyproject.toml)
sniffio==1.3.1
# via
# anyio
# httpx
typing-extensions==4.12.2
# via pwo
watchdog==5.0.3
# via bugis (pyproject.toml)
yarl==1.16.0
# via aiohttp

View File

@@ -1,5 +1,4 @@
import argparse
from cli import main
from .cli import main
import sys
main(sys.argv[1:])

View File

@@ -1,27 +1,18 @@
import logging
from logging.config import dictConfig as configure_logging
from yaml import safe_load
from .configuration import Configuration
with open(Configuration.instance.logging_configuration_file, 'r') as input_file:
conf = safe_load(input_file)
configure_logging(conf)
from asyncio import get_running_loop
from typing import Optional, Awaitable, Callable, Any, Mapping
from pwo import Maybe
from .server import Server
from asyncio import get_running_loop
from .asgi_utils import decode_headers
from typing import Optional
log = logging.getLogger('access')
log.propagate = False
_server: Optional[Server] = None
async def application(scope, receive, send):
async def application(scope, receive, send : Callable[[Mapping[str, Any]], Awaitable[None]]):
global _server
if scope['type'] == 'lifespan':
while True:
@@ -33,22 +24,6 @@ async def application(scope, receive, send):
await _server.stop()
await send({'type': 'lifespan.shutdown.complete'})
else:
def maybe_log(evt):
d = {
'response_headers': (Maybe.of_nullable(evt.get('headers'))
.map(decode_headers)
.or_none()),
'status': evt['status']
}
log.info(None, extra=dict(**{k : v for k, v in d.items() if k is not None}, **scope))
def wrapped_send(*args, **kwargs):
result = send(*args, **kwargs)
(Maybe.of(args)
.filter(lambda it: len(it) > 0)
.map(lambda it: it[0])
.filter(lambda it: it.get('type') == 'http.response.start')
.if_present(maybe_log))
return result
pathsend = (Maybe.of_nullable(scope.get('extensions'))
.map(lambda it: it.get("http.response.pathsend"))
.is_present)
@@ -61,7 +36,7 @@ async def application(scope, receive, send):
.map(lambda it: it.decode())
.or_else(None),
Maybe.of_nullable(scope.get('query_string', None)).map(lambda it: it.decode()).or_else(None),
wrapped_send,
send,
pathsend
)

View File

@@ -9,7 +9,7 @@ from granian import Granian
from pwo import Maybe
from .configuration import Configuration
from granian.constants import HTTPModes, Interfaces, ThreadModes, Loops
def main(args: Optional[Sequence[str]] = None):
parser = argparse.ArgumentParser(description="A simple CLI program to render Markdown files")
@@ -27,15 +27,88 @@ def main(args: Optional[Sequence[str]] = None):
default=default_configuration_file,
type=Path,
)
parser.add_argument(
'-a',
'--address',
help='Server bind address',
default='127.0.0.1',
)
parser.add_argument(
'-p',
'--port',
help='Server port',
default='8000',
type=int
)
parser.add_argument(
'--access-log',
help='Enable access log',
action='store_true',
dest='log_access'
)
parser.add_argument(
'--logging-configuration',
help='Logging configuration file',
dest='log_config_file'
)
parser.add_argument(
'-w', '--workers',
help='Number of worker processes',
default='1',
dest='workers',
type = int
)
parser.add_argument(
'-t', '--threads',
help='Number of threads per worker',
default='1',
dest='threads',
type=int
)
parser.add_argument(
'--http',
help='HTTP protocol version',
dest='http',
type=lambda it: HTTPModes(it),
choices=[str(mode) for mode in HTTPModes],
default = 'auto',
)
parser.add_argument(
'--threading-mode',
help='Threading mode',
dest='threading_mode',
type=lambda it: ThreadModes(it),
choices=[str(mode) for mode in ThreadModes],
default=ThreadModes.workers
)
parser.add_argument(
'--loop',
help='Loop',
dest='loop',
type=lambda it: Loops(it),
choices=[str(mode) for mode in Loops]
)
args = parser.parse_args(args)
def parse(configuration: Path):
with open(configuration, 'r') as f:
return Configuration.from_dict(yaml.safe_load(f))
return yaml.safe_load(f)
conf = Maybe.of_nullable(args.configuration).map(parse).or_else(Configuration.instance)
def assign(it: Configuration):
Configuration.instance = it
Maybe.of_nullable(args.configuration).map(parse).if_present(assign)
conf = Configuration.instance
granian_conf = asdict(conf).setdefault('granian', dict())
for k, v in vars(args).items():
if v is not None:
granian_conf[k] = v
if args.log_config_file:
with open(args.log_config_file, 'r') as f:
granian_conf['log_dictconfig'] = yaml.safe_load(f)
granian_conf = Configuration.GranianConfiguration.from_dict(granian_conf)
Granian(
"bugis.asgi:application",
**asdict(conf.granian)
**asdict(granian_conf)
).serve()

View File

@@ -1,6 +1,8 @@
from os import environ
from pathlib import Path
from dataclasses import dataclass, field, asdict
import yaml
from granian.constants import Loops, Interfaces, ThreadModes, HTTPModes, StrEnum
from granian.log import LogLevels
from granian.http import HTTP1Settings, HTTP2Settings
@@ -8,16 +10,27 @@ from typing import Optional, Sequence, Dict, Any
from pwo import classproperty, Maybe
from yaml import add_representer, SafeDumper, SafeLoader
def parse_log_config(conf_file=None) -> Dict[str, Any]:
if conf_file is None:
conf_file = environ.get("LOGGING_CONFIGURATION_FILE",
Path(__file__).parent / 'default-conf' / 'logging.yaml')
with open(conf_file, 'r') as file:
return yaml.safe_load(file)
@dataclass(frozen=True)
class Configuration:
logging_configuration_file: str = environ.get("LOGGING_CONFIGURATION_FILE",
Path(__file__).parent / 'default-conf' / 'logging.yaml')
plant_uml_server_address: str = environ.get('PLANT_UML_SERVER_ADDRESS', None)
_instance = None
@classproperty
def instance(cls) -> 'Configuration':
return Configuration()
if cls._instance is None:
cls._instance = Configuration()
return cls._instance
@instance.setter
def bar(cls, value):
cls._instance = value
@dataclass(frozen=True)
class GranianConfiguration:
@@ -77,7 +90,7 @@ class Configuration:
http2_settings=Maybe.of_nullable(d.get('http2_settings')).map(lambda it: HTTP2Settings(**it)).or_else(None),
log_enabled=d.get('log_enabled', None),
log_level=Maybe.of_nullable(d.get('log_level')).map(lambda it: LogLevels(it)).or_else(None),
# log_dictconfig: Optional[Dict[str, Any]] = None,
log_dictconfig=parse_log_config(d.get('log_config_file')),
log_access=d.get('log_access', None),
log_access_format=d.get('log_access_format', None),
ssl_cert=d.get('ssl_cert', None),

View File

@@ -1,5 +1,5 @@
version: 1
disable_existing_loggers: True
disable_existing_loggers: False
handlers:
console:
class : logging.StreamHandler
@@ -8,28 +8,23 @@ handlers:
stream : ext://sys.stderr
access:
class : logging.StreamHandler
formatter: request
formatter: access
level : INFO
stream : ext://sys.stderr
stream : ext://sys.stdout
formatters:
brief:
format: '%(message)s'
default:
format: '{asctime} [{levelname}] ({processName:s}/{threadName:s}) - {name} - {message}'
style: '{'
datefmt: '%Y-%m-%d %H:%M:%S'
request:
format: '{asctime} {client[0]}:{client[1]} HTTP/{http_version} {method} {path} - {status}'
format: '{asctime}.{msecs:0<3.0f} [{levelname}] ({processName:s}/{threadName:s}) - {name} - {message}'
style: '{'
datefmt: '%Y-%m-%d %H:%M:%S'
access:
format: '%(message)s'
loggers:
root:
handlers: [console]
level: DEBUG
access:
_granian:
level: INFO
propagate: False
granian.access:
handlers: [ access ]
level: INFO
watchdog.observers.inotify_buffer:
level: INFO
MARKDOWN:
level: INFO
propagate: False

View File

@@ -1,17 +1,19 @@
from typing import TYPE_CHECKING
from aiofiles import open as async_open
from aiohttp import ClientSession
from .configuration import Configuration
from yarl import URL
if TYPE_CHECKING:
from _typeshed import StrOrBytesPath
from httpx import AsyncClient, URL
from typing import Callable, Awaitable
from urllib.parse import urljoin
async def render_plant_uml(path: 'StrOrBytesPath') -> bytes:
async with ClientSession() as session:
url = URL(Configuration.instance.plant_uml_server_address) / 'svg'
chunk_size = 0x10000
async def render_plant_uml(client: AsyncClient, path: 'StrOrBytesPath', send : Callable[[bytes], Awaitable[None]]):
url = URL(urljoin(Configuration.instance.plant_uml_server_address, 'svg'))
async with async_open(path, 'rb') as file:
source = await file.read()
async with session.post(url, data=source) as response:
response = await client.post(url, content=source)
response.raise_for_status()
return await response.read()
async for chunk in response.aiter_bytes(chunk_size=chunk_size):
await send(chunk)

View File

@@ -6,7 +6,7 @@ from io import BytesIO
from mimetypes import init as mimeinit, guess_type
from os import getcwd
from os.path import join, normpath, splitext, relpath, basename
from typing import Callable, TYPE_CHECKING, Optional, Awaitable, AsyncGenerator, Any
from typing import Callable, TYPE_CHECKING, Optional, Awaitable, AsyncGenerator, Any, Mapping
import pygraphviz as pgv
from aiofiles import open as async_open
@@ -14,10 +14,12 @@ from aiofiles.base import AiofilesContextManager
from aiofiles.os import listdir
from aiofiles.ospath import exists, isdir, isfile, getmtime
from aiofiles.threadpool.binary import AsyncBufferedReader
from httpx import AsyncClient
from pwo import Maybe
from .asgi_utils import encode_headers
from .async_watchdog import FileWatcher
from .configuration import Configuration
from .md2html import compile_html, load_from_cache, STATIC_RESOURCES, MARDOWN_EXTENSIONS
from .plantuml import render_plant_uml
@@ -56,7 +58,10 @@ logger = logging.getLogger(__name__)
class Server:
root_dir: 'StrOrBytesPath'
prefix: Optional['StrOrBytesPath']
_loop: AbstractEventLoop
_client: AsyncClient
def __init__(self,
root_dir: 'StrOrBytesPath' = getcwd(),
@@ -67,13 +72,16 @@ class Server:
self.file_watcher = FileWatcher(cwd)
self.prefix = prefix and normpath(f'{prefix.decode()}')
self._loop = loop
self._client = AsyncClient()
async def handle_request(self,
method: str,
url_path: str,
etag: Optional[str],
query_string: Optional[str], send,
pathsend: bool = False):
query_string: Optional[str],
send: Callable[[Mapping[str, Any]], Awaitable[None]],
pathsend: bool = False
):
if method != 'GET':
await send({
'type': 'http.response.start',
@@ -172,9 +180,8 @@ class Server:
'type': 'http.response.body',
'body': body
})
elif is_plant_uml(path):
elif Configuration.instance.plant_uml_server_address and is_plant_uml(path):
logger.debug("Starting PlantUML rendering for file '%s'", path)
body = await render_plant_uml(path)
logger.debug("Completed PlantUML rendering for file '%s'", path)
await send({
'type': 'http.response.start',
@@ -185,9 +192,15 @@ class Server:
'Cache-Control': 'no-cache'
})
})
await render_plant_uml(self._client, path, lambda chunk: send({
'type': 'http.response.body',
'body': chunk,
'more_body': True
}))
await send({
'type': 'http.response.body',
'body': body
'body': '',
'more_body': False
})
else:
async def read_file(file_path, buffer_size=0x10000):
@@ -392,3 +405,4 @@ class Server:
async def stop(self):
await self.file_watcher.stop()
await self._client.aclose()