initial commit

.gitignore

@@ -0,0 +1,5 @@
+dist
+.idea
+*.pyc
+*.egg-info
+env

jpacrepo_uploader/config.py

@@ -0,0 +1,56 @@
+from configparser import RawConfigParser
+from os import environ
+from .maybe import Maybe
+from pathlib import Path
+from itertools import chain
+from dataclasses import dataclass
+from typing import Optional
+
+_config_file_name = 'client.properties'
+
+
+@dataclass
+class Config:
+    server_url: str
+    auth_server_url: str
+    repo_folders: tuple[Path, ...]
+
+
+config_file_candidates = [
+    (Maybe.of(Path('/etc') / 'jpacrepo' / _config_file_name)
+     .filter(Path.exists)),
+    (Maybe.of_nullable(environ.get('XDG_CONFIG_HOME', None))
+     .map(Path)
+     .map(lambda xdg_config_home: xdg_config_home / 'jpacrepo' / _config_file_name)
+     .filter(Path.exists)),
+    (Maybe.of_nullable(environ['HOME'])
+     .map(Path)
+     .map(lambda home: home / '.config' / _config_file_name)
+     .filter(Path.exists))
+]
+
+
+def load_configuration() -> Config:
+    main_section = 'main'
+    server_url: Optional[str] = None
+    auth_server_url: Optional[str] = None
+    repo_folders: list[Path] = []
+    for config_file_maybe in config_file_candidates:
+        def process_configuration(config_file: Path) -> None:
+            nonlocal server_url
+            nonlocal auth_server_url
+            config = RawConfigParser()
+            with open(config_file, 'r') as lines:
+                config.read_file(chain((f'[{main_section}]',), lines))
+            (Maybe.of_nullable(config.get(main_section, 'RepoFolder', fallback=None))
+             .map(lambda v: v.format(**environ))
+             .map(Path)
+             .if_present(repo_folders.append))
+            server_url = config.get(main_section, 'ServerURL', fallback=server_url)
+            auth_server_url = config.get(main_section, 'AuthServerURL', fallback=auth_server_url)
+
+        config_file_maybe.if_present(process_configuration)
+    return Config(server_url or 'https://woggioni.net/jpacrepo/',
+                  auth_server_url or 'https://woggioni.net/auth/realms/woggioni.net',
+                  tuple(repo_folders) or (Path('/var/cache/pacman/pkg'),)
+                  )

jpacrepo_uploader/maybe.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+from typing import TypeVar, Generic, Optional, Callable, Any
+
+T = TypeVar('T')
+U = TypeVar('U')
+
+
+class Maybe(Generic[T]):
+
+    def __init__(self, value: Optional[T] = None):
+        self._value: Optional[T] = value
+
+    @staticmethod
+    def of(obj: T) -> Maybe[T]:
+        return Maybe(obj)
+
+    @staticmethod
+    def of_nullable(obj: Optional[T]) -> Maybe[T]:
+        return Maybe(obj)
+
+    @staticmethod
+    def empty() -> Maybe[U]:
+        return _empty
+
+    @property
+    def value(self) -> T:
+        value = self._value
+        if not value:
+            raise ValueError('Empty Maybe')
+        else:
+            return value
+
+    @property
+    def is_present(self) -> bool:
+        return self._value is not None
+
+    @property
+    def is_empty(self) -> bool:
+        return not self.is_present
+
+    def map(self, transformer: Callable[[T], U]) -> Maybe[U]:
+        result: Maybe[U]
+        if self.is_present:
+            result = Maybe(transformer(self.value))
+        else:
+            result = Maybe.empty()
+        return result
+
+    def filter(self, predicate: Callable[[T], bool]) -> Maybe[T]:
+        return self if self.is_present and predicate(self.value) else Maybe.empty()
+
+    def flat_map(self, transformer: Callable[[T], Maybe[U]]) -> Maybe[U]:
+        return transformer(self.value) if self.is_present else Maybe.empty()
+
+    def or_else(self, alt: T) -> T:
+        return self.value if self.is_present else alt
+
+    def or_else_throw(self, supplier: Callable[[], Exception]) -> T:
+        if self.is_present:
+            return self.value
+        else:
+            raise supplier()
+
+    def or_else_get(self, supplier: Callable[[], T]) -> Maybe[T]:
+        return self if self.is_present else Maybe.of_nullable(supplier())
+
+    def if_present(self, callback: Callable[[T], U]) -> None:
+        if self.is_present:
+            callback(self.value)
+
+
+_empty: Maybe[Any] = Maybe(None)

jpacrepo_uploader/uploader.py

@@ -0,0 +1,302 @@
+import urllib.parse
+import re
+import math
+import json
+import logging
+import pycurl
+import certifi
+from time import time, monotonic
+from threading import Thread, Condition
+from dataclasses import dataclass, fields
+
+import oidc_client
+from oidc_client.discovery import fetch_provider_config
+from oidc_client.config import ProviderConfig, DEFAULT_REDIRECT_URI
+from oidc_client.oauth import TokenResponse
+from urllib.request import Request
+from urllib.parse import urlparse, urlunparse, quote, urlencode, ParseResult
+from pathlib import Path
+from progress import Progress
+from progress.bar import Bar
+from typing import Optional, Any
+from typing_extensions import Self
+
+from .config import load_configuration, Config
+
+logger = logging.getLogger('jpacrepo.uploader')
+
+package_file_pattern = re.compile('.*\.pkg\.tar\.(xz|zst|gz)$')
+
+_size_uoms = ('B', 'KiB', 'MiB', 'GiB', 'KiB')
+
+
+def format_filesize(size: int) -> str:
+    counter = 0
+    tmp_size = size
+    while tmp_size > 0:
+        tmp_size //= 1024
+        counter += 1
+    counter -= 1
+    return '%.2f ' % (size / math.pow(1024, counter)) + _size_uoms[counter]
+
+
+@dataclass
+class UploadPt:
+    uploaded: int
+    time: float
+
+
+class PackageUploadProgressBar(Bar):
+
+    def __init__(self,
+                 uploaded_size: int,
+                 packages_total_size: int,
+                 start_ts: float = monotonic(),
+                 *args: Any,
+                 **kwargs: Any):
+        kwargs.setdefault('suffix',
+                          'speed: %(total_speed)s, completed: %(percent).2f%% - ETA: %(eta_td)s')
+        kwargs.setdefault('width', 48)
+        super().__init__(*args, **kwargs)
+        self.uploaded_size = uploaded_size
+        self.packages_total_size = packages_total_size
+        self.start_ts = start_ts
+
+    @Progress.percent.getter
+    def percent(self) -> float:
+        return (self.uploaded_size + self.index) * 100 / self.packages_total_size
+
+    @property
+    def total_avg(self) -> int:
+        return int((self.uploaded_size + self.index) / (monotonic() - self.start_ts))
+
+    @property
+    def total_speed(self) -> str:
+        return format_filesize(self.total_avg) + '/s'
+
+    @Progress.eta.getter
+    def eta(self) -> int:
+        total_avg = self.total_avg
+        if total_avg > 0:
+            return int(
+                math.ceil(
+                    (self.packages_total_size - self.uploaded_size - self.index) / self.total_avg
+                )
+            )
+        else:
+            return 0
+
+    @property
+    def upload_progress(self) -> str:
+        return f'{format_filesize(self.index)} / {format_filesize(self.max)}'
+
+
+class XferProgress:
+
+    def __init__(self,
+                 packages_to_upload: int,
+                 packages_total_size: int,
+                 uploaded_size: int = 0,
+                 packages_uploaded: int = 0):
+        self.packages_uploaded = packages_uploaded
+        self.packages_total_size = packages_total_size
+        self.uploaded_size = uploaded_size
+        self.packages_to_upload = packages_to_upload
+        self.bar: Optional[PackageUploadProgressBar] = None
+
+    def update(self, uploaded: int) -> None:
+        if self.bar:
+            self.bar.goto(uploaded)
+            self.uploaded_size += self.bar.max
+        else:
+            raise RuntimeError('Progress bar is None')
+
+    def complete(self) -> None:
+        self.packages_uploaded += 1
+
+
+class JpacrepoClient:
+
+    def __init__(self, config: Config):
+        self.config: Config = config
+        self.token: Optional[TokenResponse] = None
+        self.provider_config: ProviderConfig = fetch_provider_config(self.config.auth_server_url)
+        self.token_expiry: Optional[int] = None
+        self.cond = Condition()
+        self.thread: Optional[Thread] = None
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: None, exc_val: None, exc_tb: None) -> None:
+        cond = self.cond
+        with cond:
+            cond.notify()
+        thread = self.thread
+        self.thread = None
+        if thread:
+            thread.join()
+
+    def authenticate(self) -> None:
+        token = oidc_client.login(
+            provider_config=self.provider_config,
+            client_id='jpacrepo-client',
+            interactive=True,
+            redirect_uri=DEFAULT_REDIRECT_URI)
+        self.token = token
+        self.token_expiry = (token.created_at or int(time())) + (token.expires_in or 10)
+
+        def thread_callback() -> None:
+            cond = self.cond
+            with cond:
+                while self.thread:
+                    expires_in = (self.token_expiry or 0) - int(time())
+                    if expires_in < 30:
+                        self.refresh_token()
+                    else:
+                        cond.wait(expires_in - 30)
+
+        thread = Thread(target=thread_callback)
+        self.thread = thread
+        thread.start()
+
+    def refresh_token(self) -> None:
+        token = self.token
+        if not token:
+            raise ValueError('token is None')
+        request = urllib.request.Request(
+            self.provider_config.token_endpoint,
+            method='POST',
+            data=urlencode(
+                dict(
+                    grant_type='refresh_token',
+                    client_id='jpacrepo-client',
+                    refresh_token=token.refresh_token,
+                    audience=self,
+                    scope=token.scope
+                )
+            ).encode()
+        )
+        with urllib.request.urlopen(request) as response:
+            if response.code == 200:
+                token = TokenResponse(
+                    **{
+                        key: value
+                        for key, value in json.load(response).items()
+                        # Ignore extra keys that are not token response fields
+                        if key in (field.name for field in fields(TokenResponse))
+                    }
+                )
+                self.token = token
+                self.token_expiry = (token.created_at or int(time())) + (token.expires_in or 0)
+                logger.debug(f'refreshed OIDC token')
+            else:
+                raise RuntimeError(f'Received HTTP error code: {response.code}')
+
+    def packages_to_upload(self) -> tuple[Path, ...]:
+        package_files: dict[str, Path] = {file.name: file for package_cache in self.config.repo_folders
+                                          for file in package_cache.iterdir()
+                                          if file.is_file() and package_file_pattern.match(file.name)}
+        headers = {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+        }
+        token = self.token
+        if isinstance(token, TokenResponse):
+            headers['Authorization'] = f'Bearer {token.access_token}'
+        url = urlparse(self.config.server_url)
+        new_path = Path(url.path) / 'api/pkg/doYouWantAny'
+        url = url._replace(path=str(new_path))
+        request = Request(
+            urlunparse(url),
+            headers=headers,
+            data=json.dumps([filename for filename in package_files.keys()]).encode(),
+            method='POST'
+        )
+        with urllib.request.urlopen(request) as response:
+            if response.code == 200:
+                return tuple((package_files[filename] for filename in json.load(response)))
+            else:
+                raise RuntimeError(f'Received HTTP error code: {response.code}')
+
+    def upload(self, files: tuple[Path, ...]) -> None:
+        total_size: int = 0
+        for package_file in files:
+            if package_file.exists():
+                total_size += package_file.stat().st_size
+        logger.info(f'A total of {format_filesize(total_size)} are going to be uploaded')
+        curl: pycurl.Curl = pycurl.Curl()
+        progress = XferProgress(packages_to_upload=len(files), packages_total_size=total_size)
+        start_ts = monotonic()
+        for i, file in enumerate(files):
+            upload_size = file.stat().st_size
+            kwargs = dict(
+                width=64,
+                max=upload_size,
+                message=f'({i}/{len(files)}) {file.name}',
+                start_ts=start_ts
+            )
+
+            with PackageUploadProgressBar(progress.uploaded_size, total_size, **kwargs) as bar:
+                bar.start_ts = start_ts
+                progress.bar = bar
+                self._upload_file(curl, file, progress)
+            progress.uploaded_size += upload_size
+            progress.packages_uploaded += 1
+        curl.close()
+
+    def _upload_file(self, curl: pycurl.Curl, file_path: Path, progress: XferProgress) -> None:
+        parse_result = urlparse(self.config.server_url)
+        new_path = Path(parse_result.path) / 'api/pkg/upload'
+        url: str = (urlunparse(parse_result._replace(path=str(new_path)))
+                    + ';filename=' + quote(file_path.name))
+        curl.setopt(pycurl.POST, 1)
+        curl.setopt(pycurl.URL, url)
+        headers = [
+            'Content-Type: application/octet-stream',
+            'User-Agent: jpacrepo-client'
+        ]
+        token = self.token
+        if isinstance(token, TokenResponse):
+            headers.append(f'Authorization: Bearer {token.access_token}')
+
+        curl.setopt(pycurl.HTTPHEADER, headers)
+
+        def progress_callback(dltotal: int,
+                              dlnow: int,
+                              ultotal: int,
+                              ulnow: int) -> int:
+            bar = progress.bar
+            if bar:
+                bar.goto(ulnow)
+            else:
+                raise RuntimeError('bar is None')
+            return 0
+
+        curl.setopt(pycurl.XFERINFOFUNCTION, progress_callback)
+        curl.setopt(pycurl.NOPROGRESS, False)
+        curl.setopt(pycurl.VERBOSE, False)
+        curl.setopt(pycurl.CAINFO, certifi.where())
+
+        with open(str(file_path), 'rb') as file:
+            curl.setopt(pycurl.READDATA, file)
+            curl.perform()
+            http_status_code = curl.getinfo(pycurl.RESPONSE_CODE)
+            if http_status_code != 201:
+                raise RuntimeError(f'Server returned {http_status_code}')
+
+
+def main() -> None:
+    logging.basicConfig(encoding='utf-8', level=logging.INFO)
+    with JpacrepoClient(load_configuration()) as client:
+        client.authenticate()
+        files = client.packages_to_upload()
+        if len(files):
+            logger.debug(f'Files to be uploaded: {files}')
+            client.upload(files)
+        else:
+            logger.info('No packages will be uploaded')
+
+
+if __name__ == '__main__':
+    main()

pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "jpacrepo-uploader"
+version = "0.0.1"
+authors = [
+  { name="Walter Oggioni", email="oggioni.walter@gmail.com" },
+]
+description = "Jpacrepo package uploader"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "oidc-client >= 0.2.6",
+    'certifi==2023.7.22',
+    'progress==1.6',
+    'pycurl==7.45.2',
+    'types-pycurl==7.45.2.5',
+    'typing_extensions==4.7.1'
+]
+
+[project.urls]
+"Homepage" = "https://github.com/woggioni/jpacrepo-uploader"
+"Bug Tracker" = "https://github.com/woggioni/jpacrepo-uploader/issues"
+
+[project.scripts]
+jpacrepo-uploader = "jpacrepo_uploader.uploader:main"
+
+[tool.mypy]
+python_version = "3.10"
+disallow_untyped_defs = true
+show_error_codes = true
+no_implicit_optional = true
+warn_return_any = true
+warn_unused_ignores = true
+exclude = ["scripts", "docs", "test"]
+strict = true

requirements.txt

@@ -0,0 +1,15 @@
+argcomplete==3.1.1
+build==0.10.0
+certifi==2023.7.22
+click==8.1.7
+mypy==1.5.1
+mypy-extensions==1.0.0
+oidc-client==0.2.6
+packaging==23.1
+pipx==1.2.0
+progress==1.6
+pycurl==7.45.2
+pyproject_hooks==1.0.0
+types-pycurl==7.45.2.5
+typing_extensions==4.7.1
+userpath==1.9.0