diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml new file mode 100644 index 0000000..7f0c0e1 --- /dev/null +++ b/.gitea/workflows/build.yaml @@ -0,0 +1,29 @@ +name: CI +on: + push: + branches: [ master ] +jobs: + build: + runs-on: woryzen + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + cache: 'pip' + - name: Create virtualenv + run: | + python -m venv .venv + .venv/bin/pip install -r requirements.txt +# - name: Run unit tests +# run: .venv/bin/python -m unittest discover -s tests + - name: Execute build + run: | + .venv/bin/python -m build + - name: Publish artifacts + env: + TWINE_REPOSITORY_URL: ${{ vars.PYPI_REGISTRY_URL }} + TWINE_USERNAME: ${{ vars.PUBLISHER_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PUBLISHER_TOKEN }} + run: | + .venv/bin/python -m twine upload --repository gitea dist/*{.whl,tar.gz} diff --git a/.gitignore b/.gitignore index 1eb16bf..b228f96 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ dist .idea +__pycache__ +.venv *.pyc *.egg-info -venv + diff --git a/jpacrepo_uploader/maybe.py b/jpacrepo_uploader/maybe.py deleted file mode 100644 index 1b3252c..0000000 --- a/jpacrepo_uploader/maybe.py +++ /dev/null @@ -1,72 +0,0 @@ -from __future__ import annotations -from typing import TypeVar, Generic, Optional, Callable, Any - -T = TypeVar('T') -U = TypeVar('U') - - -class Maybe(Generic[T]): - - def __init__(self, value: Optional[T] = None): - self._value: Optional[T] = value - - @staticmethod - def of(obj: T) -> Maybe[T]: - return Maybe(obj) - - @staticmethod - def of_nullable(obj: Optional[T]) -> Maybe[T]: - return Maybe(obj) - - @staticmethod - def empty() -> Maybe[U]: - return _empty - - @property - def value(self) -> T: - value = self._value - if not value: - raise ValueError('Empty Maybe') - else: - return value - - @property - def is_present(self) -> bool: - return self._value is not None - - @property - def is_empty(self) -> bool: - return not self.is_present - - def map(self, transformer: Callable[[T], U]) -> Maybe[U]: - result: Maybe[U] - if self.is_present: - result = Maybe(transformer(self.value)) - else: - result = Maybe.empty() - return result - - def filter(self, predicate: Callable[[T], bool]) -> Maybe[T]: - return self if self.is_present and predicate(self.value) else Maybe.empty() - - def flat_map(self, transformer: Callable[[T], Maybe[U]]) -> Maybe[U]: - return transformer(self.value) if self.is_present else Maybe.empty() - - def or_else(self, alt: T) -> T: - return self.value if self.is_present else alt - - def or_else_throw(self, supplier: Callable[[], Exception]) -> T: - if self.is_present: - return self.value - else: - raise supplier() - - def or_else_get(self, supplier: Callable[[], T]) -> Maybe[T]: - return self if self.is_present else Maybe.of_nullable(supplier()) - - def if_present(self, callback: Callable[[T], U]) -> None: - if self.is_present: - callback(self.value) - - -_empty: Maybe[Any] = Maybe(None) diff --git a/pyproject.toml b/pyproject.toml index b7bca1a..5db05a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "jpacrepo-uploader" -version = "0.0.1" +version = "0.0.2" authors = [ { name="Walter Oggioni", email="oggioni.walter@gmail.com" }, ] @@ -22,7 +22,8 @@ dependencies = [ 'progress==1.6', 'pycurl==7.45.2', 'types-pycurl==7.45.2.5', - 'typing_extensions==4.7.1' + 'typing_extensions==4.7.1', + 'pwo >= 0.0.2' ] [project.urls] diff --git a/requirements.txt b/requirements.txt index 9d9ab84..e3b2ae4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,17 @@ -argcomplete==3.1.1 -build==0.10.0 -certifi==2023.7.22 +argcomplete==3.4.0 +build==1.2.1 +certifi==2024.6.2 click==8.1.7 -mypy==1.5.1 +mypy==1.10.0 mypy-extensions==1.0.0 oidc-client==0.2.6 -packaging==23.1 -pipx==1.2.0 +packaging==24.1 +pipx==1.6.0 +platformdirs==4.2.2 progress==1.6 -pycurl==7.45.2 -pyproject_hooks==1.0.0 -types-pycurl==7.45.2.5 +pwo==0.0.2 +pycurl==7.45.3 +pyproject_hooks==1.1.0 +types-pycurl==7.45.3.20240421 typing_extensions==4.7.1 -userpath==1.9.0 +userpath==1.9.2 diff --git a/src/jpacrepo_uploader/__init__.py b/src/jpacrepo_uploader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jpacrepo_uploader/config.py b/src/jpacrepo_uploader/config.py similarity index 79% rename from jpacrepo_uploader/config.py rename to src/jpacrepo_uploader/config.py index efee6f9..8ed13ec 100644 --- a/jpacrepo_uploader/config.py +++ b/src/jpacrepo_uploader/config.py @@ -1,6 +1,6 @@ from configparser import RawConfigParser from os import environ -from .maybe import Maybe +from pwo.maybe import Maybe from pathlib import Path from itertools import chain from dataclasses import dataclass @@ -55,11 +55,12 @@ def load_configuration() -> Config: client_secret = config.get(main_section, 'ClientSecret', fallback=client_secret) config_file_maybe.if_present(process_configuration) - return Config(server_url=server_url or 'https://woggioni.net/jpacrepo/', - auth_server_url=auth_server_url or 'https://woggioni.net/auth/realms/woggioni.net', - repo_folders=tuple(repo_folders) or (Path('/var/cache/pacman/pkg'),), - client_id=client_id or 'jpacrepo-client', - client_secret=Maybe.of_nullable(client_secret) - .map(lambda v: v.format(**environ)) - .or_else(None) - ) + return Config( + server_url=server_url or 'https://woggioni.net/jpacrepo/', + auth_server_url=auth_server_url or 'https://woggioni.net/auth/realms/woggioni.net', + repo_folders=tuple(repo_folders) or (Path('/var/cache/pacman/pkg'),), + client_id=client_id or 'jpacrepo-client', + client_secret=Maybe.of_nullable(client_secret) + .map(lambda v: v.format(**environ)) + .or_none() + ) diff --git a/jpacrepo_uploader/uploader.py b/src/jpacrepo_uploader/uploader.py similarity index 77% rename from jpacrepo_uploader/uploader.py rename to src/jpacrepo_uploader/uploader.py index 0847966..1270232 100644 --- a/jpacrepo_uploader/uploader.py +++ b/src/jpacrepo_uploader/uploader.py @@ -9,6 +9,7 @@ from time import time, monotonic from typing import Optional, Any from urllib.parse import urlparse, urlunparse, quote, urlencode from urllib.request import Request +from threading import Thread, Condition import certifi import math @@ -19,24 +20,28 @@ from oidc_client.discovery import fetch_provider_config from oidc_client.oauth import TokenResponse from progress import Progress from progress.bar import Bar +from typing_extensions import Self +from pwo import format_filesize, retry, ExceptionHandlerOutcome from .config import load_configuration, Config logger = logging.getLogger('jpacrepo.uploader') -package_file_pattern = re.compile('.*\.pkg\.tar\.(xz|zst|gz)$') +package_file_pattern = re.compile('.*\\.pkg\\.tar\\.(xz|zst|gz)$') -_size_uoms = ('B', 'KiB', 'MiB', 'GiB', 'KiB') _supported_compression_formats = ('xz', 'zst', 'gz') -def format_filesize(size: int) -> str: - counter = 0 - tmp_size = size - while tmp_size > 0: - tmp_size //= 1024 - counter += 1 - counter -= 1 - return '%.2f ' % (size / math.pow(1024, counter)) + _size_uoms[counter] + +class HttpException(Exception): + http_status_code : int + message: Optional[str] + + def __init__(self, http_status_code: int, msg: Optional[str] = None): + self.message = msg + self.http_status_code = http_status_code + + def __repr__(self) -> str: + return f'HTTP status {self.http_status_code}' + f': {self.message}' if self.message else '' @dataclass @@ -116,14 +121,42 @@ class XferProgress: class JpacrepoClient: - def __init__(self, config: Config, **kwargs): + config: Config + token: Optional[TokenResponse] + provider_config: ProviderConfig + token_expiry: Optional[int] + cond: Condition + thread: Optional[Thread] + verbose: bool + http2: bool + http3: bool + + def __init__(self, config: Config, + verbose: bool = False, + http2: bool = False, + http3: bool = False + ): self.config: Config = config self.token: Optional[TokenResponse] = None self.provider_config: ProviderConfig = fetch_provider_config(self.config.auth_server_url) self.token_expiry: Optional[int] = None - self.verbose: bool = kwargs.get('verbose', False) - self.http2: bool = kwargs.get('http2', False) - self.http3: bool = kwargs.get('http3', False) + self.cond = Condition() + self.thread: Optional[Thread] = None + self.verbose: bool = verbose + self.http2: bool = http2 + self.http3: bool = http3 + + def __enter__(self) -> Self: + return self + + def __exit__(self, exc_type: None, exc_val: None, exc_tb: None) -> None: + cond = self.cond + with cond: + cond.notify() + thread = self.thread + self.thread = None + if thread: + thread.join() def authenticate(self) -> None: token = oidc_client.login( @@ -135,6 +168,20 @@ class JpacrepoClient: self.token = token self.token_expiry = (token.created_at or int(time())) + (token.expires_in or 10) + def thread_callback() -> None: + cond = self.cond + with cond: + while self.thread: + expires_in = (self.token_expiry or 0) - int(time()) + if expires_in < 60: + self.refresh_token() + else: + cond.wait(expires_in - 60) + + thread = Thread(target=thread_callback) + self.thread = thread + thread.start() + def refresh_token(self) -> None: token = self.token if not token: @@ -169,7 +216,8 @@ class JpacrepoClient: raise RuntimeError(f'Received HTTP error code: {response.code}') def packages_to_upload(self) -> tuple[Path, ...]: - package_files: dict[str, Path] = {file.name: file for ext in _supported_compression_formats for package_cache in self.config.repo_folders + package_files: dict[str, Path] = {file.name: file for ext in _supported_compression_formats for package_cache in + self.config.repo_folders for file in package_cache.glob(f'**/*.pkg.tar.{ext}') if file.is_file() and package_file_pattern.match(file.name)} headers = { @@ -204,9 +252,6 @@ class JpacrepoClient: progress = XferProgress(packages_to_upload=len(files), packages_total_size=total_size) start_ts = monotonic() for i, file in enumerate(files): - expires_in = (self.token_expiry or 0) - int(time()) - if expires_in < 30: - self.refresh_token() upload_size = file.stat().st_size kwargs = dict( width=64, @@ -223,6 +268,17 @@ class JpacrepoClient: progress.packages_uploaded += 1 curl.close() + _RETRIABLE_HTTP_STATUS_CODES = {401, 403, 409, 429, 504} + + @staticmethod + def error_handler(ex: Exception) -> ExceptionHandlerOutcome: + if (isinstance(ex, HttpException) + and ex.http_status_code in JpacrepoClient._RETRIABLE_HTTP_STATUS_CODES): + return ExceptionHandlerOutcome.CONTINUE + else: + return ExceptionHandlerOutcome.THROW + + @retry(max_attempts=3, initial_delay=0, exception_handler=error_handler) def _upload_file(self, curl: pycurl.Curl, file_path: Path, progress: XferProgress) -> None: parse_result = urlparse(self.config.server_url) new_path = Path(parse_result.path) / 'api/pkg/upload' @@ -265,7 +321,7 @@ class JpacrepoClient: curl.perform() http_status_code = curl.getinfo(pycurl.RESPONSE_CODE) if http_status_code != 201: - raise RuntimeError(f'Server returned {http_status_code}') + raise HttpException(http_status_code) def main() -> None: @@ -287,14 +343,14 @@ def main() -> None: help="Enable HTTP/3 protocol") args = parser.parse_args() logging.basicConfig(encoding='utf-8', level=logging.INFO) - client = JpacrepoClient(load_configuration(), **args.__dict__) - client.authenticate() - files = client.packages_to_upload() - if len(files): - logger.debug(f'Files to be uploaded: {files}') - client.upload(files) - else: - logger.info('No packages will be uploaded') + with JpacrepoClient(load_configuration(), **vars(args)) as client: + client.authenticate() + files = client.packages_to_upload() + if len(files): + logger.debug(f'Files to be uploaded: {files}') + client.upload(files) + else: + logger.info('No packages will be uploaded') if __name__ == '__main__':