From d0b965e61ea20bf6f6dbbf76bfa18a01a4d4d459 Mon Sep 17 00:00:00 2001 From: Walter Oggioni Date: Sun, 3 Sep 2023 21:45:29 +0800 Subject: [PATCH] initial commit --- .gitignore | 5 + jpacrepo_uploader/config.py | 56 +++++++ jpacrepo_uploader/maybe.py | 72 ++++++++ jpacrepo_uploader/uploader.py | 302 ++++++++++++++++++++++++++++++++++ pyproject.toml | 43 +++++ requirements.txt | 15 ++ 6 files changed, 493 insertions(+) create mode 100644 .gitignore create mode 100644 jpacrepo_uploader/config.py create mode 100644 jpacrepo_uploader/maybe.py create mode 100644 jpacrepo_uploader/uploader.py create mode 100644 pyproject.toml create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff1149f --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +dist +.idea +*.pyc +*.egg-info +env diff --git a/jpacrepo_uploader/config.py b/jpacrepo_uploader/config.py new file mode 100644 index 0000000..faca6fc --- /dev/null +++ b/jpacrepo_uploader/config.py @@ -0,0 +1,56 @@ +from configparser import RawConfigParser +from os import environ +from .maybe import Maybe +from pathlib import Path +from itertools import chain +from dataclasses import dataclass +from typing import Optional + +_config_file_name = 'client.properties' + + +@dataclass +class Config: + server_url: str + auth_server_url: str + repo_folders: tuple[Path, ...] + + +config_file_candidates = [ + (Maybe.of(Path('/etc') / 'jpacrepo' / _config_file_name) + .filter(Path.exists)), + (Maybe.of_nullable(environ.get('XDG_CONFIG_HOME', None)) + .map(Path) + .map(lambda xdg_config_home: xdg_config_home / 'jpacrepo' / _config_file_name) + .filter(Path.exists)), + (Maybe.of_nullable(environ['HOME']) + .map(Path) + .map(lambda home: home / '.config' / _config_file_name) + .filter(Path.exists)) +] + + +def load_configuration() -> Config: + main_section = 'main' + server_url: Optional[str] = None + auth_server_url: Optional[str] = None + repo_folders: list[Path] = [] + for config_file_maybe in config_file_candidates: + def process_configuration(config_file: Path) -> None: + nonlocal server_url + nonlocal auth_server_url + config = RawConfigParser() + with open(config_file, 'r') as lines: + config.read_file(chain((f'[{main_section}]',), lines)) + (Maybe.of_nullable(config.get(main_section, 'RepoFolder', fallback=None)) + .map(lambda v: v.format(**environ)) + .map(Path) + .if_present(repo_folders.append)) + server_url = config.get(main_section, 'ServerURL', fallback=server_url) + auth_server_url = config.get(main_section, 'AuthServerURL', fallback=auth_server_url) + + config_file_maybe.if_present(process_configuration) + return Config(server_url or 'https://woggioni.net/jpacrepo/', + auth_server_url or 'https://woggioni.net/auth/realms/woggioni.net', + tuple(repo_folders) or (Path('/var/cache/pacman/pkg'),) + ) diff --git a/jpacrepo_uploader/maybe.py b/jpacrepo_uploader/maybe.py new file mode 100644 index 0000000..1b3252c --- /dev/null +++ b/jpacrepo_uploader/maybe.py @@ -0,0 +1,72 @@ +from __future__ import annotations +from typing import TypeVar, Generic, Optional, Callable, Any + +T = TypeVar('T') +U = TypeVar('U') + + +class Maybe(Generic[T]): + + def __init__(self, value: Optional[T] = None): + self._value: Optional[T] = value + + @staticmethod + def of(obj: T) -> Maybe[T]: + return Maybe(obj) + + @staticmethod + def of_nullable(obj: Optional[T]) -> Maybe[T]: + return Maybe(obj) + + @staticmethod + def empty() -> Maybe[U]: + return _empty + + @property + def value(self) -> T: + value = self._value + if not value: + raise ValueError('Empty Maybe') + else: + return value + + @property + def is_present(self) -> bool: + return self._value is not None + + @property + def is_empty(self) -> bool: + return not self.is_present + + def map(self, transformer: Callable[[T], U]) -> Maybe[U]: + result: Maybe[U] + if self.is_present: + result = Maybe(transformer(self.value)) + else: + result = Maybe.empty() + return result + + def filter(self, predicate: Callable[[T], bool]) -> Maybe[T]: + return self if self.is_present and predicate(self.value) else Maybe.empty() + + def flat_map(self, transformer: Callable[[T], Maybe[U]]) -> Maybe[U]: + return transformer(self.value) if self.is_present else Maybe.empty() + + def or_else(self, alt: T) -> T: + return self.value if self.is_present else alt + + def or_else_throw(self, supplier: Callable[[], Exception]) -> T: + if self.is_present: + return self.value + else: + raise supplier() + + def or_else_get(self, supplier: Callable[[], T]) -> Maybe[T]: + return self if self.is_present else Maybe.of_nullable(supplier()) + + def if_present(self, callback: Callable[[T], U]) -> None: + if self.is_present: + callback(self.value) + + +_empty: Maybe[Any] = Maybe(None) diff --git a/jpacrepo_uploader/uploader.py b/jpacrepo_uploader/uploader.py new file mode 100644 index 0000000..a4b008a --- /dev/null +++ b/jpacrepo_uploader/uploader.py @@ -0,0 +1,302 @@ +import urllib.parse +import re +import math +import json +import logging +import pycurl +import certifi +from time import time, monotonic +from threading import Thread, Condition +from dataclasses import dataclass, fields + +import oidc_client +from oidc_client.discovery import fetch_provider_config +from oidc_client.config import ProviderConfig, DEFAULT_REDIRECT_URI +from oidc_client.oauth import TokenResponse +from urllib.request import Request +from urllib.parse import urlparse, urlunparse, quote, urlencode, ParseResult +from pathlib import Path +from progress import Progress +from progress.bar import Bar +from typing import Optional, Any +from typing_extensions import Self + +from .config import load_configuration, Config + +logger = logging.getLogger('jpacrepo.uploader') + +package_file_pattern = re.compile('.*\.pkg\.tar\.(xz|zst|gz)$') + +_size_uoms = ('B', 'KiB', 'MiB', 'GiB', 'KiB') + + +def format_filesize(size: int) -> str: + counter = 0 + tmp_size = size + while tmp_size > 0: + tmp_size //= 1024 + counter += 1 + counter -= 1 + return '%.2f ' % (size / math.pow(1024, counter)) + _size_uoms[counter] + + +@dataclass +class UploadPt: + uploaded: int + time: float + + +class PackageUploadProgressBar(Bar): + + def __init__(self, + uploaded_size: int, + packages_total_size: int, + start_ts: float = monotonic(), + *args: Any, + **kwargs: Any): + kwargs.setdefault('suffix', + 'speed: %(total_speed)s, completed: %(percent).2f%% - ETA: %(eta_td)s') + kwargs.setdefault('width', 48) + super().__init__(*args, **kwargs) + self.uploaded_size = uploaded_size + self.packages_total_size = packages_total_size + self.start_ts = start_ts + + @Progress.percent.getter + def percent(self) -> float: + return (self.uploaded_size + self.index) * 100 / self.packages_total_size + + @property + def total_avg(self) -> int: + return int((self.uploaded_size + self.index) / (monotonic() - self.start_ts)) + + @property + def total_speed(self) -> str: + return format_filesize(self.total_avg) + '/s' + + @Progress.eta.getter + def eta(self) -> int: + total_avg = self.total_avg + if total_avg > 0: + return int( + math.ceil( + (self.packages_total_size - self.uploaded_size - self.index) / self.total_avg + ) + ) + else: + return 0 + + @property + def upload_progress(self) -> str: + return f'{format_filesize(self.index)} / {format_filesize(self.max)}' + + +class XferProgress: + + def __init__(self, + packages_to_upload: int, + packages_total_size: int, + uploaded_size: int = 0, + packages_uploaded: int = 0): + self.packages_uploaded = packages_uploaded + self.packages_total_size = packages_total_size + self.uploaded_size = uploaded_size + self.packages_to_upload = packages_to_upload + self.bar: Optional[PackageUploadProgressBar] = None + + def update(self, uploaded: int) -> None: + if self.bar: + self.bar.goto(uploaded) + self.uploaded_size += self.bar.max + else: + raise RuntimeError('Progress bar is None') + + def complete(self) -> None: + self.packages_uploaded += 1 + + +class JpacrepoClient: + + def __init__(self, config: Config): + self.config: Config = config + self.token: Optional[TokenResponse] = None + self.provider_config: ProviderConfig = fetch_provider_config(self.config.auth_server_url) + self.token_expiry: Optional[int] = None + self.cond = Condition() + self.thread: Optional[Thread] = None + + def __enter__(self) -> Self: + return self + + def __exit__(self, exc_type: None, exc_val: None, exc_tb: None) -> None: + cond = self.cond + with cond: + cond.notify() + thread = self.thread + self.thread = None + if thread: + thread.join() + + def authenticate(self) -> None: + token = oidc_client.login( + provider_config=self.provider_config, + client_id='jpacrepo-client', + interactive=True, + redirect_uri=DEFAULT_REDIRECT_URI) + self.token = token + self.token_expiry = (token.created_at or int(time())) + (token.expires_in or 10) + + def thread_callback() -> None: + cond = self.cond + with cond: + while self.thread: + expires_in = (self.token_expiry or 0) - int(time()) + if expires_in < 30: + self.refresh_token() + else: + cond.wait(expires_in - 30) + + thread = Thread(target=thread_callback) + self.thread = thread + thread.start() + + def refresh_token(self) -> None: + token = self.token + if not token: + raise ValueError('token is None') + request = urllib.request.Request( + self.provider_config.token_endpoint, + method='POST', + data=urlencode( + dict( + grant_type='refresh_token', + client_id='jpacrepo-client', + refresh_token=token.refresh_token, + audience=self, + scope=token.scope + ) + ).encode() + ) + with urllib.request.urlopen(request) as response: + if response.code == 200: + token = TokenResponse( + **{ + key: value + for key, value in json.load(response).items() + # Ignore extra keys that are not token response fields + if key in (field.name for field in fields(TokenResponse)) + } + ) + self.token = token + self.token_expiry = (token.created_at or int(time())) + (token.expires_in or 0) + logger.debug(f'refreshed OIDC token') + else: + raise RuntimeError(f'Received HTTP error code: {response.code}') + + def packages_to_upload(self) -> tuple[Path, ...]: + package_files: dict[str, Path] = {file.name: file for package_cache in self.config.repo_folders + for file in package_cache.iterdir() + if file.is_file() and package_file_pattern.match(file.name)} + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + } + token = self.token + if isinstance(token, TokenResponse): + headers['Authorization'] = f'Bearer {token.access_token}' + url = urlparse(self.config.server_url) + new_path = Path(url.path) / 'api/pkg/doYouWantAny' + url = url._replace(path=str(new_path)) + request = Request( + urlunparse(url), + headers=headers, + data=json.dumps([filename for filename in package_files.keys()]).encode(), + method='POST' + ) + with urllib.request.urlopen(request) as response: + if response.code == 200: + return tuple((package_files[filename] for filename in json.load(response))) + else: + raise RuntimeError(f'Received HTTP error code: {response.code}') + + def upload(self, files: tuple[Path, ...]) -> None: + total_size: int = 0 + for package_file in files: + if package_file.exists(): + total_size += package_file.stat().st_size + logger.info(f'A total of {format_filesize(total_size)} are going to be uploaded') + curl: pycurl.Curl = pycurl.Curl() + progress = XferProgress(packages_to_upload=len(files), packages_total_size=total_size) + start_ts = monotonic() + for i, file in enumerate(files): + upload_size = file.stat().st_size + kwargs = dict( + width=64, + max=upload_size, + message=f'({i}/{len(files)}) {file.name}', + start_ts=start_ts + ) + + with PackageUploadProgressBar(progress.uploaded_size, total_size, **kwargs) as bar: + bar.start_ts = start_ts + progress.bar = bar + self._upload_file(curl, file, progress) + progress.uploaded_size += upload_size + progress.packages_uploaded += 1 + curl.close() + + def _upload_file(self, curl: pycurl.Curl, file_path: Path, progress: XferProgress) -> None: + parse_result = urlparse(self.config.server_url) + new_path = Path(parse_result.path) / 'api/pkg/upload' + url: str = (urlunparse(parse_result._replace(path=str(new_path))) + + ';filename=' + quote(file_path.name)) + curl.setopt(pycurl.POST, 1) + curl.setopt(pycurl.URL, url) + headers = [ + 'Content-Type: application/octet-stream', + 'User-Agent: jpacrepo-client' + ] + token = self.token + if isinstance(token, TokenResponse): + headers.append(f'Authorization: Bearer {token.access_token}') + + curl.setopt(pycurl.HTTPHEADER, headers) + + def progress_callback(dltotal: int, + dlnow: int, + ultotal: int, + ulnow: int) -> int: + bar = progress.bar + if bar: + bar.goto(ulnow) + else: + raise RuntimeError('bar is None') + return 0 + + curl.setopt(pycurl.XFERINFOFUNCTION, progress_callback) + curl.setopt(pycurl.NOPROGRESS, False) + curl.setopt(pycurl.VERBOSE, False) + curl.setopt(pycurl.CAINFO, certifi.where()) + + with open(str(file_path), 'rb') as file: + curl.setopt(pycurl.READDATA, file) + curl.perform() + http_status_code = curl.getinfo(pycurl.RESPONSE_CODE) + if http_status_code != 201: + raise RuntimeError(f'Server returned {http_status_code}') + + +def main() -> None: + logging.basicConfig(encoding='utf-8', level=logging.INFO) + with JpacrepoClient(load_configuration()) as client: + client.authenticate() + files = client.packages_to_upload() + if len(files): + logger.debug(f'Files to be uploaded: {files}') + client.upload(files) + else: + logger.info('No packages will be uploaded') + + +if __name__ == '__main__': + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b7bca1a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "jpacrepo-uploader" +version = "0.0.1" +authors = [ + { name="Walter Oggioni", email="oggioni.walter@gmail.com" }, +] +description = "Jpacrepo package uploader" +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dependencies = [ + "oidc-client >= 0.2.6", + 'certifi==2023.7.22', + 'progress==1.6', + 'pycurl==7.45.2', + 'types-pycurl==7.45.2.5', + 'typing_extensions==4.7.1' +] + +[project.urls] +"Homepage" = "https://github.com/woggioni/jpacrepo-uploader" +"Bug Tracker" = "https://github.com/woggioni/jpacrepo-uploader/issues" + +[project.scripts] +jpacrepo-uploader = "jpacrepo_uploader.uploader:main" + +[tool.mypy] +python_version = "3.10" +disallow_untyped_defs = true +show_error_codes = true +no_implicit_optional = true +warn_return_any = true +warn_unused_ignores = true +exclude = ["scripts", "docs", "test"] +strict = true \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9d9ab84 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +argcomplete==3.1.1 +build==0.10.0 +certifi==2023.7.22 +click==8.1.7 +mypy==1.5.1 +mypy-extensions==1.0.0 +oidc-client==0.2.6 +packaging==23.1 +pipx==1.2.0 +progress==1.6 +pycurl==7.45.2 +pyproject_hooks==1.0.0 +types-pycurl==7.45.2.5 +typing_extensions==4.7.1 +userpath==1.9.0