Compare commits
51 Commits
47ec94caf2
...
doc
Author | SHA1 | Date | |
---|---|---|---|
c19bc9e91e
|
|||
af79e74b95
|
|||
78ae21caa4
|
|||
6c0eadb9fb
|
|||
5fef1b932e
|
|||
5e173dbf62
|
|||
53b24e3d54
|
|||
7d0f24fa58
|
|||
1b6cf1bd96
|
|||
4180df2352
|
|||
c2e388b931
|
|||
6c62ac85c0
|
|||
89153b60f8
|
|||
a2a40ab60f
|
|||
45458761f3
|
|||
90a5834f5f
|
|||
1823d0b9ca
|
|||
649cbba954
|
|||
eb9ccce3be
|
|||
316f64cf9d
|
|||
24a49779f9
|
|||
423b749db9
|
|||
9ce3e7fa0a
|
|||
1e6ece37a5
|
|||
fc9900d821
|
|||
1a78c8092b
|
|||
3d1847c408
|
|||
702556bfbb
|
|||
06e9e7ca09
|
|||
fa5bb55baa
|
|||
007d0fffd6
|
|||
75ebf2248f
|
|||
241d95fe1c
|
|||
3b7030c302
|
|||
a8670277e7
|
|||
03ee75266d
|
|||
05a265e4b4
|
|||
5af99330f8
|
|||
747168cda3
|
|||
225f156864
|
|||
696cb74740
|
|||
59f267426c
|
|||
608a9d18de
|
|||
d2c00402df
|
|||
d701157b06
|
|||
01d5b1462c
|
|||
d5a2c4a591
|
|||
0fdb37fb54
|
|||
688a196a52
|
|||
13f7ecc88a
|
|||
f28ecca45e
|
64
.gitea/workflows/build.yaml
Normal file
64
.gitea/workflows/build.yaml
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
name: CI
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- '*'
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: hostinger
|
||||||
|
steps:
|
||||||
|
- name: Checkout sources
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
- name: Setup Gradle
|
||||||
|
uses: gradle/actions/setup-gradle@v3
|
||||||
|
- name: Execute Gradle build
|
||||||
|
run: ./gradlew build
|
||||||
|
- name: Prepare Docker image build
|
||||||
|
run: ./gradlew prepareDockerBuild
|
||||||
|
- name: Get project version
|
||||||
|
id: retrieve-version
|
||||||
|
run: ./gradlew -q version >> "$GITHUB_OUTPUT"
|
||||||
|
- name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v3
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
with:
|
||||||
|
driver: docker-container
|
||||||
|
- name: Login to Gitea container registry
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
registry: gitea.woggioni.net
|
||||||
|
username: woggioni
|
||||||
|
password: ${{ secrets.PUBLISHER_TOKEN }}
|
||||||
|
-
|
||||||
|
name: Build rbcs Docker image
|
||||||
|
uses: docker/build-push-action@v5.3.0
|
||||||
|
with:
|
||||||
|
context: "docker/build/docker"
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
push: true
|
||||||
|
pull: true
|
||||||
|
tags: |
|
||||||
|
gitea.woggioni.net/woggioni/rbcs:latest
|
||||||
|
gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}
|
||||||
|
target: release
|
||||||
|
cache-from: type=registry,ref=gitea.woggioni.net/woggioni/rbcs:buildx
|
||||||
|
-
|
||||||
|
name: Build rbcs memcache Docker image
|
||||||
|
uses: docker/build-push-action@v5.3.0
|
||||||
|
with:
|
||||||
|
context: "docker/build/docker"
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
push: true
|
||||||
|
pull: true
|
||||||
|
tags: |
|
||||||
|
gitea.woggioni.net/woggioni/rbcs:memcache
|
||||||
|
gitea.woggioni.net/woggioni/rbcs:memcache-${{ steps.retrieve-version.outputs.VERSION }}
|
||||||
|
target: release-memcache
|
||||||
|
cache-from: type=registry,ref=gitea.woggioni.net/woggioni/rbcs:buildx
|
||||||
|
cache-to: type=registry,mode=max,compression=zstd,image-manifest=true,oci-mediatypes=true,ref=gitea.woggioni.net/woggioni/rbcs:buildx
|
||||||
|
- name: Publish artifacts
|
||||||
|
env:
|
||||||
|
PUBLISHER_TOKEN: ${{ secrets.PUBLISHER_TOKEN }}
|
||||||
|
run: ./gradlew publish
|
||||||
|
|
2
.gitignore
vendored
2
.gitignore
vendored
@@ -3,3 +3,5 @@
|
|||||||
|
|
||||||
# Ignore Gradle build output directory
|
# Ignore Gradle build output directory
|
||||||
build
|
build
|
||||||
|
|
||||||
|
rbcs-cli/native-image/*.json
|
||||||
|
20
LICENSE
Normal file
20
LICENSE
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
The MIT License (MIT)
|
||||||
|
|
||||||
|
Copyright (c) 2017 Y. T. CHUNG <zonyitoo@gmail.com>
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
|
this software and associated documentation files (the "Software"), to deal in
|
||||||
|
the Software without restriction, including without limitation the rights to
|
||||||
|
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||||
|
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||||
|
subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||||
|
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||||
|
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||||
|
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||||
|
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
110
README.md
Normal file
110
README.md
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
# Remote Build Cache Server
|
||||||
|
Remote Build Cache Server (shortened to RBCS) allows you to share and reuse unchanged build
|
||||||
|
and test outputs across the team. This speeds up local and CI builds since cycles are not wasted
|
||||||
|
re-building components that are unaffected by new code changes. RBCS supports both Gradle and
|
||||||
|
Maven build tool environments.
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
### Downloading the jar file
|
||||||
|
You can download the latest version from [this link](https://gitea.woggioni.net/woggioni/-/packages/maven/net.woggioni-rbcs-cli/)
|
||||||
|
|
||||||
|
If you want to use memcache as a storage backend you'll also need to download [the memcache plugin](https://gitea.woggioni.net/woggioni/-/packages/maven/net.woggioni-rbcs-server-memcache/)
|
||||||
|
|
||||||
|
### Using the Docker image
|
||||||
|
You can pull the latest Docker image with
|
||||||
|
```bash
|
||||||
|
docker pull gitea.woggioni.net/woggioni/rbcs:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
## Configuration
|
||||||
|
### Using RBCS with Gradle
|
||||||
|
|
||||||
|
```groovy
|
||||||
|
buildCache {
|
||||||
|
remote(HttpBuildCache) {
|
||||||
|
url = 'https://rbcs.example.com/'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Using RBCS with Maven
|
||||||
|
|
||||||
|
Read [here](https://maven.apache.org/extensions/maven-build-cache-extension/remote-cache.html)
|
||||||
|
|
||||||
|
## FAQ
|
||||||
|
### Why should I use a build cache?
|
||||||
|
|
||||||
|
#### Build Caches Improve Build & Test Performance
|
||||||
|
|
||||||
|
Building software consists of a number of steps, like compiling sources, executing tests, and linking binaries. We’ve seen that a binary artifact repository helps when such a step requires an external component by downloading the artifact from the repository rather than building it locally.
|
||||||
|
However, there are many additional steps in this build process which can be optimized to reduce the build time. An obvious strategy is to avoid executing build steps which dominate the total build time when these build steps are not needed.
|
||||||
|
Most build times are dominated by the testing step.
|
||||||
|
|
||||||
|
While binary repositories cannot capture the outcome of a test build step (only the test reports
|
||||||
|
when included in binary artifacts), build caches are designed to eliminate redundant executions
|
||||||
|
for every build step. Moreover, it generalizes the concept of avoiding work associated with any
|
||||||
|
incremental step of the build, including test execution, compilation and resource processing.
|
||||||
|
The mechanism itself is comparable to a pure function. That is, given some inputs such as source
|
||||||
|
files and environment parameters we know that the output is always going to be the same.
|
||||||
|
As a result, we can cache it and retrieve it based on a simple cryptographic hash of the inputs.
|
||||||
|
Build caching is supported natively by some build tools.
|
||||||
|
|
||||||
|
#### Improve CI builds with a remote build cache
|
||||||
|
|
||||||
|
When analyzing the role of a build cache it is important to take into account the granularity
|
||||||
|
of the changes that it caches. Imagine a full build for a project with 40 to 50 modules
|
||||||
|
which fails at the last step (deployment) because the staging environment is temporarily unavailable.
|
||||||
|
Although the vast majority of the build steps (potentially thousands) succeed,
|
||||||
|
the change can not be deployed to the staging environment.
|
||||||
|
Without a build cache one typically relies on a very complex CI configuration to reuse build step outputs
|
||||||
|
or would have to repeat the full build once the environment is available.
|
||||||
|
|
||||||
|
Some build tools don’t support incremental builds properly. For example, outputs of a build started
|
||||||
|
from scratch may vary when compared to subsequent builds that rely on the initial build’s output.
|
||||||
|
As a result, to preserve build integrity, it’s crucial to rebuild from scratch, or ‘cleanly,’ in this
|
||||||
|
scenario.
|
||||||
|
|
||||||
|
With a build cache, only the last step needs to be executed and the build can be re-triggered
|
||||||
|
when the environment is back online. This automatically saves all of the time and
|
||||||
|
resources required across the different build steps which were successfully executed.
|
||||||
|
Instead of executing the intermediate steps, the build tool pulls the outputs from the build cache,
|
||||||
|
avoiding a lot of redundant work
|
||||||
|
|
||||||
|
#### Share outputs with a remote build cache
|
||||||
|
|
||||||
|
One of the most important advantages of a remote build cache is the ability to share build outputs.
|
||||||
|
In most CI configurations, for example, a number of pipelines are created.
|
||||||
|
These may include one for building the sources, one for testing, one for publishing the outcomes
|
||||||
|
to a remote repository, and other pipelines to test on different platforms.
|
||||||
|
There are even situations where CI builds partially build a project (i.e. some modules and not others).
|
||||||
|
|
||||||
|
Most of those pipelines share a lot of intermediate build steps. All builds which perform testing
|
||||||
|
require the binaries to be ready. All publishing builds require all previous steps to be executed.
|
||||||
|
And because modern CI infrastructure means executing everything in containerized (isolated) environments,
|
||||||
|
significant resources are wasted by repeatedly building the same intermediate artifacts.
|
||||||
|
|
||||||
|
A remote build cache greatly reduces this overhead by orders of magnitudes because it provides a way
|
||||||
|
for all those pipelines to share their outputs. After all, there is no point recreating an output that
|
||||||
|
is already available in the cache.
|
||||||
|
|
||||||
|
Because there are inherent dependencies between software components of a build,
|
||||||
|
introducing a build cache dramatically reduces the impact of exploding a component into multiple pieces,
|
||||||
|
allowing for increased modularity without increased overhead.
|
||||||
|
|
||||||
|
#### Make local developers more efficient with remote build caches
|
||||||
|
|
||||||
|
It is common for different teams within a company to work on different modules of a single large
|
||||||
|
application. In this case, most teams don’t care about building the other parts of the software.
|
||||||
|
By introducing a remote cache developers immediately benefit from pre-built artifacts when checking out code.
|
||||||
|
Because it has already been built on CI, they don’t have to do it locally.
|
||||||
|
|
||||||
|
Introducing a remote cache is a huge benefit for those developers. Consider that a typical developer’s
|
||||||
|
day begins by performing a code checkout. Most likely the checked out code has already been built on CI.
|
||||||
|
Therefore, no time is wasted running the first build of the day. The remote cache provides all of the
|
||||||
|
intermediate artifacts needed. And, in the event local changes are made, the remote cache still leverages
|
||||||
|
partial cache hits for projects which are independent. As other developers in the organization request
|
||||||
|
CI builds, the remote cache continues to populate, increasing the likelihood of these remote cache hits
|
||||||
|
across team members.
|
||||||
|
|
116
build.gradle
116
build.gradle
@@ -1,81 +1,123 @@
|
|||||||
plugins {
|
plugins {
|
||||||
alias catalog.plugins.kotlin
|
alias catalog.plugins.kotlin.jvm apply false
|
||||||
alias catalog.plugins.envelope
|
alias catalog.plugins.sambal
|
||||||
id 'maven-publish'
|
alias catalog.plugins.lombok apply false
|
||||||
}
|
}
|
||||||
|
|
||||||
import net.woggioni.gradle.envelope.EnvelopeJarTask
|
|
||||||
|
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
|
||||||
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
|
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
|
||||||
|
|
||||||
|
allprojects { subproject ->
|
||||||
group = 'net.woggioni'
|
group = 'net.woggioni'
|
||||||
|
|
||||||
version = getProperty('gbcs.version')
|
if(project.currentTag.isPresent()) {
|
||||||
|
version = project.currentTag.map { it[0] }.get()
|
||||||
|
} else {
|
||||||
|
version = project.gitRevision.map { gitRevision ->
|
||||||
|
"${getProperty('rbcs.version')}.${gitRevision[0..10]}"
|
||||||
|
}.get()
|
||||||
|
}
|
||||||
|
|
||||||
repositories {
|
repositories {
|
||||||
maven {
|
maven {
|
||||||
url = 'https://woggioni.net/mvn'
|
url = getProperty('gitea.maven.url')
|
||||||
content {
|
content {
|
||||||
includeModule 'net.woggioni', 'jwo'
|
includeModule 'net.woggioni', 'jwo'
|
||||||
|
includeModule 'net.woggioni', 'xmemcached'
|
||||||
includeGroup 'com.lys'
|
includeGroup 'com.lys'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
mavenCentral()
|
mavenCentral()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pluginManager.withPlugin('java-library') {
|
||||||
|
|
||||||
|
ext {
|
||||||
|
jpmsModuleName = subproject.group + '.' + subproject.name.replace('-', '.')
|
||||||
|
}
|
||||||
|
|
||||||
|
java {
|
||||||
|
withSourcesJar()
|
||||||
|
modularity.inferModulePath = true
|
||||||
|
toolchain {
|
||||||
|
languageVersion = JavaLanguageVersion.of(21)
|
||||||
|
vendor = JvmVendorSpec.ORACLE
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
dependencies {
|
dependencies {
|
||||||
implementation catalog.jwo
|
|
||||||
implementation catalog.slf4j.api
|
|
||||||
implementation catalog.netty.codec.http
|
|
||||||
|
|
||||||
runtimeOnly catalog.slf4j.jdk14
|
|
||||||
|
|
||||||
testImplementation catalog.junit.jupiter.api
|
testImplementation catalog.junit.jupiter.api
|
||||||
testImplementation catalog.junit.jupiter.params
|
testImplementation catalog.junit.jupiter.params
|
||||||
testRuntimeOnly catalog.junit.jupiter.engine
|
testRuntimeOnly catalog.junit.jupiter.engine
|
||||||
}
|
}
|
||||||
|
|
||||||
java {
|
test {
|
||||||
withSourcesJar()
|
useJUnitPlatform()
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.withType(JavaCompile) {
|
||||||
|
modularity.inferModulePath = true
|
||||||
|
options.release = 21
|
||||||
}
|
}
|
||||||
|
|
||||||
tasks.named(JavaPlugin.COMPILE_JAVA_TASK_NAME, JavaCompile) {
|
tasks.named(JavaPlugin.COMPILE_JAVA_TASK_NAME, JavaCompile) {
|
||||||
modularity.inferModulePath = true
|
options.compilerArgumentProviders << new CommandLineArgumentProvider() {
|
||||||
options.compilerArgs << '--patch-module' << 'net.woggioni.gbcs=' + project.sourceSets.main.output.asPath
|
@Override
|
||||||
options.release = 17
|
Iterable<String> asArguments() {
|
||||||
|
return ['--patch-module', subproject.jpmsModuleName + '=' + subproject.sourceSets.main.output.asPath]
|
||||||
}
|
}
|
||||||
|
}
|
||||||
tasks.named("compileKotlin", KotlinCompile.class) {
|
options.javaModuleVersion = version
|
||||||
kotlinOptions {
|
|
||||||
jvmTarget = 17
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Provider<EnvelopeJarTask> envelopeJarTaskProvider = tasks.named('envelopeJar', EnvelopeJarTask.class) {
|
pluginManager.withPlugin('jacoco') {
|
||||||
mainModule = 'net.woggioni.gbcs'
|
test {
|
||||||
mainClass = 'net.woggioni.gbcs.GradleBuildCacheServer'
|
finalizedBy jacocoTestReport
|
||||||
systemProperty 'java.util.logging.config.class', 'net.woggioni.gbcs.LoggingConfig'
|
}
|
||||||
systemProperty 'log.config.source', 'logging.properties'
|
jacocoTestReport {
|
||||||
|
dependsOn test
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
wrapper {
|
pluginManager.withPlugin(catalog.plugins.kotlin.jvm.get().pluginId) {
|
||||||
distributionType = Wrapper.DistributionType.BIN
|
tasks.withType(KotlinCompile.class) {
|
||||||
gradleVersion = getProperty('gradle.version')
|
compilerOptions.jvmTarget = JvmTarget.JVM_21
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def envelopeJarArtifact = artifacts.add('archives', envelopeJarTaskProvider.get().archiveFile.get().asFile) {
|
pluginManager.withPlugin(catalog.plugins.lombok.get().pluginId) {
|
||||||
type = 'jar'
|
lombok {
|
||||||
builtBy envelopeJarTaskProvider
|
version = catalog.versions.lombok
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pluginManager.withPlugin('maven-publish') {
|
||||||
|
|
||||||
publishing {
|
publishing {
|
||||||
repositories {
|
repositories {
|
||||||
maven {
|
maven {
|
||||||
url = 'https://mvn.woggioni.net/'
|
name = "Gitea"
|
||||||
|
url = uri(getProperty('gitea.maven.url'))
|
||||||
|
|
||||||
|
credentials(HttpHeaderCredentials) {
|
||||||
|
name = "Authorization"
|
||||||
|
value = "token ${System.getenv()["PUBLISHER_TOKEN"]}"
|
||||||
}
|
}
|
||||||
}
|
|
||||||
publications {
|
authentication {
|
||||||
maven(MavenPublication) {
|
header(HttpHeaderAuthentication)
|
||||||
artifact envelopeJarArtifact
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.register('version') {
|
||||||
|
doLast {
|
||||||
|
println("VERSION=$version")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
16
docker/Dockerfile
Normal file
16
docker/Dockerfile
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
FROM eclipse-temurin:21-jre-alpine AS base-release
|
||||||
|
RUN adduser -D luser
|
||||||
|
USER luser
|
||||||
|
WORKDIR /home/luser
|
||||||
|
|
||||||
|
FROM base-release AS release
|
||||||
|
ADD rbcs-cli-envelope-*.jar rbcs.jar
|
||||||
|
ENTRYPOINT ["java", "-XX:+UseZGC", "-XX:+ZGenerational", "-jar", "/home/luser/rbcs.jar", "server"]
|
||||||
|
|
||||||
|
FROM base-release AS release-memcache
|
||||||
|
ADD --chown=luser:luser rbcs-cli-envelope-*.jar rbcs.jar
|
||||||
|
RUN mkdir plugins
|
||||||
|
WORKDIR /home/luser/plugins
|
||||||
|
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-memcache*.tar
|
||||||
|
WORKDIR /home/luser
|
||||||
|
ENTRYPOINT ["java", "-XX:+UseZGC", "-XX:+ZGenerational", "-jar", "/home/luser/rbcs.jar", "server"]
|
67
docker/build.gradle
Normal file
67
docker/build.gradle
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
plugins {
|
||||||
|
id 'base'
|
||||||
|
alias(catalog.plugins.gradle.docker)
|
||||||
|
}
|
||||||
|
|
||||||
|
import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
|
||||||
|
import com.bmuschko.gradle.docker.tasks.image.DockerPushImage
|
||||||
|
import com.bmuschko.gradle.docker.tasks.image.DockerTagImage
|
||||||
|
|
||||||
|
|
||||||
|
configurations {
|
||||||
|
docker {
|
||||||
|
canBeResolved = true
|
||||||
|
transitive = false
|
||||||
|
visible = false
|
||||||
|
canBeConsumed = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
docker project(path: ':rbcs-cli', configuration: 'release')
|
||||||
|
docker project(path: ':rbcs-server-memcache', configuration: 'release')
|
||||||
|
}
|
||||||
|
|
||||||
|
Provider<Task> cleanTaskProvider = tasks.named(BasePlugin.CLEAN_TASK_NAME) {}
|
||||||
|
|
||||||
|
Provider<Copy> prepareDockerBuild = tasks.register('prepareDockerBuild', Copy) {
|
||||||
|
dependsOn cleanTaskProvider
|
||||||
|
group = 'docker'
|
||||||
|
into project.layout.buildDirectory.file('docker')
|
||||||
|
from(configurations.docker)
|
||||||
|
from(file('Dockerfile'))
|
||||||
|
}
|
||||||
|
|
||||||
|
Provider<DockerBuildImage> dockerBuild = tasks.register('dockerBuildImage', DockerBuildImage) {
|
||||||
|
group = 'docker'
|
||||||
|
dependsOn prepareDockerBuild
|
||||||
|
images.add('gitea.woggioni.net/woggioni/rbcs:latest')
|
||||||
|
images.add("gitea.woggioni.net/woggioni/rbcs:${version}")
|
||||||
|
}
|
||||||
|
|
||||||
|
Provider<DockerTagImage> dockerTag = tasks.register('dockerTagImage', DockerTagImage) {
|
||||||
|
group = 'docker'
|
||||||
|
repository = 'gitea.woggioni.net/woggioni/rbcs'
|
||||||
|
imageId = 'gitea.woggioni.net/woggioni/rbcs:latest'
|
||||||
|
tag = version
|
||||||
|
}
|
||||||
|
|
||||||
|
Provider<DockerTagImage> dockerTagMemcache = tasks.register('dockerTagMemcacheImage', DockerTagImage) {
|
||||||
|
group = 'docker'
|
||||||
|
repository = 'gitea.woggioni.net/woggioni/rbcs'
|
||||||
|
imageId = 'gitea.woggioni.net/woggioni/rbcs:memcache'
|
||||||
|
tag = "${version}-memcache"
|
||||||
|
}
|
||||||
|
|
||||||
|
Provider<DockerPushImage> dockerPush = tasks.register('dockerPushImage', DockerPushImage) {
|
||||||
|
group = 'docker'
|
||||||
|
dependsOn dockerTag, dockerTagMemcache
|
||||||
|
registryCredentials {
|
||||||
|
url = getProperty('docker.registry.url')
|
||||||
|
username = 'woggioni'
|
||||||
|
password = System.getenv().get("PUBLISHER_TOKEN")
|
||||||
|
}
|
||||||
|
images = [dockerTag.flatMap{ it.tag }, dockerTagMemcache.flatMap{ it.tag }]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@@ -1,4 +1,12 @@
|
|||||||
gbcs.version = 0.1-SNAPSHOT
|
org.gradle.configuration-cache=false
|
||||||
|
org.gradle.parallel=true
|
||||||
|
org.gradle.caching=true
|
||||||
|
|
||||||
gradle.version = 7.5.1
|
rbcs.version = 0.1.4
|
||||||
lys.version = 0.1-SNAPSHOT
|
|
||||||
|
lys.version = 2025.02.05
|
||||||
|
|
||||||
|
gitea.maven.url = https://gitea.woggioni.net/api/packages/woggioni/maven
|
||||||
|
docker.registry.url=gitea.woggioni.net
|
||||||
|
|
||||||
|
jpms-check.configurationName = runtimeClasspath
|
||||||
|
BIN
gradle/wrapper/gradle-wrapper.jar
vendored
BIN
gradle/wrapper/gradle-wrapper.jar
vendored
Binary file not shown.
4
gradle/wrapper/gradle-wrapper.properties
vendored
4
gradle/wrapper/gradle-wrapper.properties
vendored
@@ -1,5 +1,7 @@
|
|||||||
distributionBase=GRADLE_USER_HOME
|
distributionBase=GRADLE_USER_HOME
|
||||||
distributionPath=wrapper/dists
|
distributionPath=wrapper/dists
|
||||||
distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
|
distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip
|
||||||
|
networkTimeout=10000
|
||||||
|
validateDistributionUrl=true
|
||||||
zipStoreBase=GRADLE_USER_HOME
|
zipStoreBase=GRADLE_USER_HOME
|
||||||
zipStorePath=wrapper/dists
|
zipStorePath=wrapper/dists
|
||||||
|
38
gradlew
vendored
38
gradlew
vendored
@@ -15,6 +15,8 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
#
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
|
||||||
##############################################################################
|
##############################################################################
|
||||||
#
|
#
|
||||||
@@ -55,7 +57,7 @@
|
|||||||
# Darwin, MinGW, and NonStop.
|
# Darwin, MinGW, and NonStop.
|
||||||
#
|
#
|
||||||
# (3) This script is generated from the Groovy template
|
# (3) This script is generated from the Groovy template
|
||||||
# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||||
# within the Gradle project.
|
# within the Gradle project.
|
||||||
#
|
#
|
||||||
# You can find Gradle at https://github.com/gradle/gradle/.
|
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||||
@@ -80,13 +82,12 @@ do
|
|||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
|
# This is normally unused
|
||||||
|
# shellcheck disable=SC2034
|
||||||
APP_NAME="Gradle"
|
|
||||||
APP_BASE_NAME=${0##*/}
|
APP_BASE_NAME=${0##*/}
|
||||||
|
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
|
||||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
' "$PWD" ) || exit
|
||||||
|
|
||||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||||
MAX_FD=maximum
|
MAX_FD=maximum
|
||||||
@@ -133,22 +134,29 @@ location of your Java installation."
|
|||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
JAVACMD=java
|
JAVACMD=java
|
||||||
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
if ! command -v java >/dev/null 2>&1
|
||||||
|
then
|
||||||
|
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||||
|
|
||||||
Please set the JAVA_HOME variable in your environment to match the
|
Please set the JAVA_HOME variable in your environment to match the
|
||||||
location of your Java installation."
|
location of your Java installation."
|
||||||
fi
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# Increase the maximum file descriptors if we can.
|
# Increase the maximum file descriptors if we can.
|
||||||
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||||
case $MAX_FD in #(
|
case $MAX_FD in #(
|
||||||
max*)
|
max*)
|
||||||
|
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||||
|
# shellcheck disable=SC2039,SC3045
|
||||||
MAX_FD=$( ulimit -H -n ) ||
|
MAX_FD=$( ulimit -H -n ) ||
|
||||||
warn "Could not query maximum file descriptor limit"
|
warn "Could not query maximum file descriptor limit"
|
||||||
esac
|
esac
|
||||||
case $MAX_FD in #(
|
case $MAX_FD in #(
|
||||||
'' | soft) :;; #(
|
'' | soft) :;; #(
|
||||||
*)
|
*)
|
||||||
|
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||||
|
# shellcheck disable=SC2039,SC3045
|
||||||
ulimit -n "$MAX_FD" ||
|
ulimit -n "$MAX_FD" ||
|
||||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||||
esac
|
esac
|
||||||
@@ -193,11 +201,15 @@ if "$cygwin" || "$msys" ; then
|
|||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Collect all arguments for the java command;
|
|
||||||
# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
|
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||||
# shell script including quotes and variable substitutions, so put them in
|
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||||
# double quotes to make sure that they get re-expanded; and
|
|
||||||
# * put everything else in single quotes, so that it's not re-expanded.
|
# Collect all arguments for the java command:
|
||||||
|
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||||
|
# and any embedded shellness will be escaped.
|
||||||
|
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||||
|
# treated as '${Hostname}' itself on the command line.
|
||||||
|
|
||||||
set -- \
|
set -- \
|
||||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||||
|
23
gradlew.bat
vendored
23
gradlew.bat
vendored
@@ -13,6 +13,8 @@
|
|||||||
@rem See the License for the specific language governing permissions and
|
@rem See the License for the specific language governing permissions and
|
||||||
@rem limitations under the License.
|
@rem limitations under the License.
|
||||||
@rem
|
@rem
|
||||||
|
@rem SPDX-License-Identifier: Apache-2.0
|
||||||
|
@rem
|
||||||
|
|
||||||
@if "%DEBUG%"=="" @echo off
|
@if "%DEBUG%"=="" @echo off
|
||||||
@rem ##########################################################################
|
@rem ##########################################################################
|
||||||
@@ -26,6 +28,7 @@ if "%OS%"=="Windows_NT" setlocal
|
|||||||
|
|
||||||
set DIRNAME=%~dp0
|
set DIRNAME=%~dp0
|
||||||
if "%DIRNAME%"=="" set DIRNAME=.
|
if "%DIRNAME%"=="" set DIRNAME=.
|
||||||
|
@rem This is normally unused
|
||||||
set APP_BASE_NAME=%~n0
|
set APP_BASE_NAME=%~n0
|
||||||
set APP_HOME=%DIRNAME%
|
set APP_HOME=%DIRNAME%
|
||||||
|
|
||||||
@@ -42,11 +45,11 @@ set JAVA_EXE=java.exe
|
|||||||
%JAVA_EXE% -version >NUL 2>&1
|
%JAVA_EXE% -version >NUL 2>&1
|
||||||
if %ERRORLEVEL% equ 0 goto execute
|
if %ERRORLEVEL% equ 0 goto execute
|
||||||
|
|
||||||
echo.
|
echo. 1>&2
|
||||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
|
||||||
echo.
|
echo. 1>&2
|
||||||
echo Please set the JAVA_HOME variable in your environment to match the
|
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||||
echo location of your Java installation.
|
echo location of your Java installation. 1>&2
|
||||||
|
|
||||||
goto fail
|
goto fail
|
||||||
|
|
||||||
@@ -56,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
|||||||
|
|
||||||
if exist "%JAVA_EXE%" goto execute
|
if exist "%JAVA_EXE%" goto execute
|
||||||
|
|
||||||
echo.
|
echo. 1>&2
|
||||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
|
||||||
echo.
|
echo. 1>&2
|
||||||
echo Please set the JAVA_HOME variable in your environment to match the
|
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
|
||||||
echo location of your Java installation.
|
echo location of your Java installation. 1>&2
|
||||||
|
|
||||||
goto fail
|
goto fail
|
||||||
|
|
||||||
|
17
rbcs-api/build.gradle
Normal file
17
rbcs-api/build.gradle
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
plugins {
|
||||||
|
id 'java-library'
|
||||||
|
id 'maven-publish'
|
||||||
|
alias catalog.plugins.lombok
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
api catalog.netty.buffer
|
||||||
|
}
|
||||||
|
|
||||||
|
publishing {
|
||||||
|
publications {
|
||||||
|
maven(MavenPublication) {
|
||||||
|
from(components["java"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
7
rbcs-api/src/main/java/module-info.java
Normal file
7
rbcs-api/src/main/java/module-info.java
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
module net.woggioni.rbcs.api {
|
||||||
|
requires static lombok;
|
||||||
|
requires java.xml;
|
||||||
|
requires io.netty.buffer;
|
||||||
|
exports net.woggioni.rbcs.api;
|
||||||
|
exports net.woggioni.rbcs.api.exception;
|
||||||
|
}
|
14
rbcs-api/src/main/java/net/woggioni/rbcs/api/Cache.java
Normal file
14
rbcs-api/src/main/java/net/woggioni/rbcs/api/Cache.java
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
package net.woggioni.rbcs.api;
|
||||||
|
|
||||||
|
import io.netty.buffer.ByteBuf;
|
||||||
|
import net.woggioni.rbcs.api.exception.ContentTooLargeException;
|
||||||
|
|
||||||
|
import java.nio.channels.ReadableByteChannel;
|
||||||
|
import java.util.concurrent.CompletableFuture;
|
||||||
|
|
||||||
|
|
||||||
|
public interface Cache extends AutoCloseable {
|
||||||
|
CompletableFuture<ReadableByteChannel> get(String key);
|
||||||
|
|
||||||
|
CompletableFuture<Void> put(String key, ByteBuf content) throws ContentTooLargeException;
|
||||||
|
}
|
@@ -0,0 +1,17 @@
|
|||||||
|
package net.woggioni.rbcs.api;
|
||||||
|
|
||||||
|
import org.w3c.dom.Document;
|
||||||
|
import org.w3c.dom.Element;
|
||||||
|
|
||||||
|
public interface CacheProvider<T extends Configuration.Cache> {
|
||||||
|
|
||||||
|
String getXmlSchemaLocation();
|
||||||
|
|
||||||
|
String getXmlNamespace();
|
||||||
|
|
||||||
|
String getXmlType();
|
||||||
|
|
||||||
|
T deserialize(Element parent);
|
||||||
|
|
||||||
|
Element serialize(Document doc, T cache);
|
||||||
|
}
|
170
rbcs-api/src/main/java/net/woggioni/rbcs/api/Configuration.java
Normal file
170
rbcs-api/src/main/java/net/woggioni/rbcs/api/Configuration.java
Normal file
@@ -0,0 +1,170 @@
|
|||||||
|
package net.woggioni.rbcs.api;
|
||||||
|
|
||||||
|
|
||||||
|
import lombok.EqualsAndHashCode;
|
||||||
|
import lombok.NonNull;
|
||||||
|
import lombok.Value;
|
||||||
|
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.security.cert.X509Certificate;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public class Configuration {
|
||||||
|
String host;
|
||||||
|
int port;
|
||||||
|
int incomingConnectionsBacklogSize;
|
||||||
|
String serverPath;
|
||||||
|
@NonNull
|
||||||
|
EventExecutor eventExecutor;
|
||||||
|
@NonNull
|
||||||
|
Connection connection;
|
||||||
|
Map<String, User> users;
|
||||||
|
Map<String, Group> groups;
|
||||||
|
Cache cache;
|
||||||
|
Authentication authentication;
|
||||||
|
Tls tls;
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class EventExecutor {
|
||||||
|
boolean useVirtualThreads;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class Connection {
|
||||||
|
Duration readTimeout;
|
||||||
|
Duration writeTimeout;
|
||||||
|
Duration idleTimeout;
|
||||||
|
Duration readIdleTimeout;
|
||||||
|
Duration writeIdleTimeout;
|
||||||
|
int maxRequestSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class Quota {
|
||||||
|
long calls;
|
||||||
|
Duration period;
|
||||||
|
long initialAvailableCalls;
|
||||||
|
long maxAvailableCalls;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class Group {
|
||||||
|
@EqualsAndHashCode.Include
|
||||||
|
String name;
|
||||||
|
Set<Role> roles;
|
||||||
|
Quota groupQuota;
|
||||||
|
Quota userQuota;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class User {
|
||||||
|
@EqualsAndHashCode.Include
|
||||||
|
String name;
|
||||||
|
String password;
|
||||||
|
Set<Group> groups;
|
||||||
|
Quota quota;
|
||||||
|
|
||||||
|
public Set<Role> getRoles() {
|
||||||
|
return groups.stream()
|
||||||
|
.flatMap(group -> group.getRoles().stream())
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface UserExtractor {
|
||||||
|
User extract(X509Certificate cert);
|
||||||
|
}
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface GroupExtractor {
|
||||||
|
Group extract(X509Certificate cert);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class Throttling {
|
||||||
|
KeyStore keyStore;
|
||||||
|
TrustStore trustStore;
|
||||||
|
boolean verifyClients;
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum ClientCertificate {
|
||||||
|
REQUIRED, OPTIONAL
|
||||||
|
}
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class Tls {
|
||||||
|
KeyStore keyStore;
|
||||||
|
TrustStore trustStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class KeyStore {
|
||||||
|
Path file;
|
||||||
|
String password;
|
||||||
|
String keyAlias;
|
||||||
|
String keyPassword;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class TrustStore {
|
||||||
|
Path file;
|
||||||
|
String password;
|
||||||
|
boolean checkCertificateStatus;
|
||||||
|
boolean requireClientCertificate;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class TlsCertificateExtractor {
|
||||||
|
String rdnType;
|
||||||
|
String pattern;
|
||||||
|
}
|
||||||
|
|
||||||
|
public interface Authentication {}
|
||||||
|
|
||||||
|
public static class BasicAuthentication implements Authentication {}
|
||||||
|
|
||||||
|
@Value
|
||||||
|
public static class ClientCertificateAuthentication implements Authentication {
|
||||||
|
TlsCertificateExtractor userExtractor;
|
||||||
|
TlsCertificateExtractor groupExtractor;
|
||||||
|
}
|
||||||
|
|
||||||
|
public interface Cache {
|
||||||
|
net.woggioni.rbcs.api.Cache materialize();
|
||||||
|
String getNamespaceURI();
|
||||||
|
String getTypeName();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Configuration of(
|
||||||
|
String host,
|
||||||
|
int port,
|
||||||
|
int incomingConnectionsBacklogSize,
|
||||||
|
String serverPath,
|
||||||
|
EventExecutor eventExecutor,
|
||||||
|
Connection connection,
|
||||||
|
Map<String, User> users,
|
||||||
|
Map<String, Group> groups,
|
||||||
|
Cache cache,
|
||||||
|
Authentication authentication,
|
||||||
|
Tls tls
|
||||||
|
) {
|
||||||
|
return new Configuration(
|
||||||
|
host,
|
||||||
|
port,
|
||||||
|
incomingConnectionsBacklogSize,
|
||||||
|
serverPath != null && !serverPath.isEmpty() && !serverPath.equals("/") ? serverPath : null,
|
||||||
|
eventExecutor,
|
||||||
|
connection,
|
||||||
|
users,
|
||||||
|
groups,
|
||||||
|
cache,
|
||||||
|
authentication,
|
||||||
|
tls
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
5
rbcs-api/src/main/java/net/woggioni/rbcs/api/Role.java
Normal file
5
rbcs-api/src/main/java/net/woggioni/rbcs/api/Role.java
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
package net.woggioni.rbcs.api;
|
||||||
|
|
||||||
|
public enum Role {
|
||||||
|
Reader, Writer
|
||||||
|
}
|
@@ -0,0 +1,11 @@
|
|||||||
|
package net.woggioni.rbcs.api.exception;
|
||||||
|
|
||||||
|
public class CacheException extends RbcsException {
|
||||||
|
public CacheException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CacheException(String message) {
|
||||||
|
this(message, null);
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,11 @@
|
|||||||
|
package net.woggioni.rbcs.api.exception;
|
||||||
|
|
||||||
|
public class ConfigurationException extends RbcsException {
|
||||||
|
public ConfigurationException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ConfigurationException(String message) {
|
||||||
|
this(message, null);
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,7 @@
|
|||||||
|
package net.woggioni.rbcs.api.exception;
|
||||||
|
|
||||||
|
public class ContentTooLargeException extends RbcsException {
|
||||||
|
public ContentTooLargeException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,7 @@
|
|||||||
|
package net.woggioni.rbcs.api.exception;
|
||||||
|
|
||||||
|
public class RbcsException extends RuntimeException {
|
||||||
|
public RbcsException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
}
|
101
rbcs-cli/build.gradle
Normal file
101
rbcs-cli/build.gradle
Normal file
@@ -0,0 +1,101 @@
|
|||||||
|
plugins {
|
||||||
|
id 'java-library'
|
||||||
|
alias catalog.plugins.kotlin.jvm
|
||||||
|
alias catalog.plugins.envelope
|
||||||
|
alias catalog.plugins.sambal
|
||||||
|
alias catalog.plugins.graalvm.native.image
|
||||||
|
alias catalog.plugins.graalvm.jlink
|
||||||
|
alias catalog.plugins.jpms.check
|
||||||
|
id 'maven-publish'
|
||||||
|
}
|
||||||
|
|
||||||
|
import net.woggioni.gradle.envelope.EnvelopeJarTask
|
||||||
|
import net.woggioni.gradle.graalvm.NativeImageConfigurationTask
|
||||||
|
import net.woggioni.gradle.graalvm.NativeImagePlugin
|
||||||
|
import net.woggioni.gradle.graalvm.NativeImageTask
|
||||||
|
import net.woggioni.gradle.graalvm.JlinkPlugin
|
||||||
|
import net.woggioni.gradle.graalvm.JlinkTask
|
||||||
|
|
||||||
|
Property<String> mainModuleName = objects.property(String.class)
|
||||||
|
mainModuleName.set('net.woggioni.rbcs.cli')
|
||||||
|
Property<String> mainClassName = objects.property(String.class)
|
||||||
|
mainClassName.set('net.woggioni.rbcs.cli.RemoteBuildCacheServerCli')
|
||||||
|
|
||||||
|
tasks.named(JavaPlugin.COMPILE_JAVA_TASK_NAME, JavaCompile) {
|
||||||
|
options.javaModuleMainClass = mainClassName
|
||||||
|
}
|
||||||
|
|
||||||
|
configurations {
|
||||||
|
release {
|
||||||
|
transitive = false
|
||||||
|
canBeConsumed = true
|
||||||
|
canBeResolved = true
|
||||||
|
visible = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
envelopeJar {
|
||||||
|
mainModule = mainModuleName
|
||||||
|
mainClass = mainClassName
|
||||||
|
|
||||||
|
extraClasspath = ["plugins"]
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
implementation catalog.jwo
|
||||||
|
implementation catalog.slf4j.api
|
||||||
|
implementation catalog.netty.codec.http
|
||||||
|
implementation catalog.picocli
|
||||||
|
|
||||||
|
implementation project(':rbcs-client')
|
||||||
|
implementation project(':rbcs-server')
|
||||||
|
|
||||||
|
// runtimeOnly catalog.slf4j.jdk14
|
||||||
|
runtimeOnly catalog.logback.classic
|
||||||
|
// runtimeOnly catalog.slf4j.simple
|
||||||
|
}
|
||||||
|
|
||||||
|
Provider<EnvelopeJarTask> envelopeJarTaskProvider = tasks.named('envelopeJar', EnvelopeJarTask.class) {
|
||||||
|
// systemProperties['java.util.logging.config.class'] = 'net.woggioni.rbcs.LoggingConfig'
|
||||||
|
// systemProperties['log.config.source'] = 'net/woggioni/rbcs/cli/logging.properties'
|
||||||
|
// systemProperties['java.util.logging.config.file'] = 'classpath:net/woggioni/rbcs/cli/logging.properties'
|
||||||
|
systemProperties['logback.configurationFile'] = 'classpath:net/woggioni/rbcs/cli/logback.xml'
|
||||||
|
systemProperties['io.netty.leakDetectionLevel'] = 'DISABLED'
|
||||||
|
|
||||||
|
// systemProperties['org.slf4j.simpleLogger.showDateTime'] = 'true'
|
||||||
|
// systemProperties['org.slf4j.simpleLogger.defaultLogLevel'] = 'debug'
|
||||||
|
// systemProperties['org.slf4j.simpleLogger.log.com.google.code.yanf4j'] = 'warn'
|
||||||
|
// systemProperties['org.slf4j.simpleLogger.log.net.rubyeye.xmemcached'] = 'warn'
|
||||||
|
// systemProperties['org.slf4j.simpleLogger.dateTimeFormat'] = 'yyyy-MM-dd\'T\'HH:mm:ss.SSSZ'
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.named(NativeImagePlugin.CONFIGURE_NATIVE_IMAGE_TASK_NAME, NativeImageConfigurationTask) {
|
||||||
|
mainClass = mainClassName
|
||||||
|
mainModule = mainModuleName
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.named(NativeImagePlugin.NATIVE_IMAGE_TASK_NAME, NativeImageTask) {
|
||||||
|
mainClass = mainClassName
|
||||||
|
mainModule = mainModuleName
|
||||||
|
useMusl = true
|
||||||
|
buildStaticImage = true
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.named(JlinkPlugin.JLINK_TASK_NAME, JlinkTask) {
|
||||||
|
mainClass = mainClassName
|
||||||
|
mainModule = 'net.woggioni.rbcs.cli'
|
||||||
|
}
|
||||||
|
|
||||||
|
artifacts {
|
||||||
|
release(envelopeJarTaskProvider)
|
||||||
|
}
|
||||||
|
|
||||||
|
publishing {
|
||||||
|
publications {
|
||||||
|
maven(MavenPublication) {
|
||||||
|
artifact envelopeJar
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
2
rbcs-cli/native-image/native-image.properties
Normal file
2
rbcs-cli/native-image/native-image.properties
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
Args=-H:Optimize=3 --gc=serial --initialize-at-run-time=io.netty
|
||||||
|
#-H:TraceClassInitialization=io.netty.handler.ssl.BouncyCastleAlpnSslUtils
|
17
rbcs-cli/src/main/java/module-info.java
Normal file
17
rbcs-cli/src/main/java/module-info.java
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
module net.woggioni.rbcs.cli {
|
||||||
|
requires org.slf4j;
|
||||||
|
requires net.woggioni.rbcs.server;
|
||||||
|
requires info.picocli;
|
||||||
|
requires net.woggioni.rbcs.common;
|
||||||
|
requires net.woggioni.rbcs.client;
|
||||||
|
requires kotlin.stdlib;
|
||||||
|
requires net.woggioni.jwo;
|
||||||
|
requires net.woggioni.rbcs.api;
|
||||||
|
|
||||||
|
exports net.woggioni.rbcs.cli.impl.converters to info.picocli;
|
||||||
|
opens net.woggioni.rbcs.cli.impl.commands to info.picocli;
|
||||||
|
opens net.woggioni.rbcs.cli.impl to info.picocli;
|
||||||
|
opens net.woggioni.rbcs.cli to info.picocli, net.woggioni.rbcs.common;
|
||||||
|
|
||||||
|
exports net.woggioni.rbcs.cli;
|
||||||
|
}
|
@@ -0,0 +1,69 @@
|
|||||||
|
package net.woggioni.rbcs.cli
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.cli.impl.AbstractVersionProvider
|
||||||
|
import net.woggioni.rbcs.cli.impl.RbcsCommand
|
||||||
|
import net.woggioni.rbcs.cli.impl.commands.BenchmarkCommand
|
||||||
|
import net.woggioni.rbcs.cli.impl.commands.ClientCommand
|
||||||
|
import net.woggioni.rbcs.cli.impl.commands.GetCommand
|
||||||
|
import net.woggioni.rbcs.cli.impl.commands.HealthCheckCommand
|
||||||
|
import net.woggioni.rbcs.cli.impl.commands.PasswordHashCommand
|
||||||
|
import net.woggioni.rbcs.cli.impl.commands.PutCommand
|
||||||
|
import net.woggioni.rbcs.cli.impl.commands.ServerCommand
|
||||||
|
import net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.jwo.Application
|
||||||
|
import picocli.CommandLine
|
||||||
|
import picocli.CommandLine.Model.CommandSpec
|
||||||
|
|
||||||
|
|
||||||
|
@CommandLine.Command(
|
||||||
|
name = "rbcs", versionProvider = RemoteBuildCacheServerCli.VersionProvider::class
|
||||||
|
)
|
||||||
|
class RemoteBuildCacheServerCli : RbcsCommand() {
|
||||||
|
|
||||||
|
class VersionProvider : AbstractVersionProvider()
|
||||||
|
companion object {
|
||||||
|
@JvmStatic
|
||||||
|
fun main(vararg args: String) {
|
||||||
|
val currentClassLoader = RemoteBuildCacheServerCli::class.java.classLoader
|
||||||
|
Thread.currentThread().contextClassLoader = currentClassLoader
|
||||||
|
if(currentClassLoader.javaClass.name == "net.woggioni.envelope.loader.ModuleClassLoader") {
|
||||||
|
//We're running in an envelope jar and custom URL protocols won't work
|
||||||
|
RbcsUrlStreamHandlerFactory.install()
|
||||||
|
}
|
||||||
|
val log = contextLogger()
|
||||||
|
val app = Application.builder("rbcs")
|
||||||
|
.configurationDirectoryEnvVar("RBCS_CONFIGURATION_DIR")
|
||||||
|
.configurationDirectoryPropertyKey("net.woggioni.rbcs.conf.dir")
|
||||||
|
.build()
|
||||||
|
val rbcsCli = RemoteBuildCacheServerCli()
|
||||||
|
val commandLine = CommandLine(rbcsCli)
|
||||||
|
commandLine.setExecutionExceptionHandler { ex, cl, parseResult ->
|
||||||
|
log.error(ex.message, ex)
|
||||||
|
CommandLine.ExitCode.SOFTWARE
|
||||||
|
}
|
||||||
|
commandLine.addSubcommand(ServerCommand(app))
|
||||||
|
commandLine.addSubcommand(PasswordHashCommand())
|
||||||
|
commandLine.addSubcommand(
|
||||||
|
CommandLine(ClientCommand(app)).apply {
|
||||||
|
addSubcommand(BenchmarkCommand())
|
||||||
|
addSubcommand(PutCommand())
|
||||||
|
addSubcommand(GetCommand())
|
||||||
|
addSubcommand(HealthCheckCommand())
|
||||||
|
})
|
||||||
|
System.exit(commandLine.execute(*args))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@CommandLine.Option(names = ["-V", "--version"], versionHelp = true)
|
||||||
|
var versionHelp = false
|
||||||
|
private set
|
||||||
|
|
||||||
|
@CommandLine.Spec
|
||||||
|
private lateinit var spec: CommandSpec
|
||||||
|
|
||||||
|
|
||||||
|
override fun run() {
|
||||||
|
spec.commandLine().usage(System.out);
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,30 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl
|
||||||
|
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.util.jar.Attributes
|
||||||
|
import java.util.jar.JarFile
|
||||||
|
import java.util.jar.Manifest
|
||||||
|
|
||||||
|
|
||||||
|
abstract class AbstractVersionProvider : CommandLine.IVersionProvider {
|
||||||
|
private val version: String
|
||||||
|
private val vcsHash: String
|
||||||
|
|
||||||
|
init {
|
||||||
|
val mf = Manifest()
|
||||||
|
javaClass.module.getResourceAsStream(JarFile.MANIFEST_NAME).use { `is` ->
|
||||||
|
mf.read(`is`)
|
||||||
|
}
|
||||||
|
val mainAttributes = mf.mainAttributes
|
||||||
|
version = mainAttributes.getValue(Attributes.Name.SPECIFICATION_VERSION) ?: throw RuntimeException("Version information not found in manifest")
|
||||||
|
vcsHash = mainAttributes.getValue(Attributes.Name.IMPLEMENTATION_VERSION) ?: throw RuntimeException("Version information not found in manifest")
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun getVersion(): Array<String?> {
|
||||||
|
return if (version.endsWith("-SNAPSHOT")) {
|
||||||
|
arrayOf(version, vcsHash)
|
||||||
|
} else {
|
||||||
|
arrayOf(version)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,19 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl
|
||||||
|
|
||||||
|
import net.woggioni.jwo.Application
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
|
||||||
|
abstract class RbcsCommand : Runnable {
|
||||||
|
|
||||||
|
@CommandLine.Option(names = ["-h", "--help"], usageHelp = true)
|
||||||
|
var usageHelp = false
|
||||||
|
private set
|
||||||
|
|
||||||
|
protected fun findConfigurationFile(app: Application, fileName : String): Path {
|
||||||
|
val confDir = app.computeConfigurationDirectory()
|
||||||
|
val configurationFile = confDir.resolve(fileName)
|
||||||
|
return configurationFile
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,142 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl.commands
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.cli.impl.RbcsCommand
|
||||||
|
import net.woggioni.rbcs.client.RemoteBuildCacheClient
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.rbcs.common.error
|
||||||
|
import net.woggioni.rbcs.common.info
|
||||||
|
import net.woggioni.jwo.JWO
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.security.SecureRandom
|
||||||
|
import java.time.Duration
|
||||||
|
import java.time.Instant
|
||||||
|
import java.util.concurrent.LinkedBlockingQueue
|
||||||
|
import java.util.concurrent.Semaphore
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
import kotlin.random.Random
|
||||||
|
|
||||||
|
@CommandLine.Command(
|
||||||
|
name = "benchmark",
|
||||||
|
description = ["Run a load test against the server"],
|
||||||
|
showDefaultValues = true
|
||||||
|
)
|
||||||
|
class BenchmarkCommand : RbcsCommand() {
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
@CommandLine.Spec
|
||||||
|
private lateinit var spec: CommandLine.Model.CommandSpec
|
||||||
|
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-e", "--entries"],
|
||||||
|
description = ["Total number of elements to be added to the cache"],
|
||||||
|
paramLabel = "NUMBER_OF_ENTRIES"
|
||||||
|
)
|
||||||
|
private var numberOfEntries = 1000
|
||||||
|
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-s", "--size"],
|
||||||
|
description = ["Size of a cache value in bytes"],
|
||||||
|
paramLabel = "SIZE"
|
||||||
|
)
|
||||||
|
private var size = 0x1000
|
||||||
|
|
||||||
|
override fun run() {
|
||||||
|
val clientCommand = spec.parent().userObject() as ClientCommand
|
||||||
|
val profile = clientCommand.profileName.let { profileName ->
|
||||||
|
clientCommand.configuration.profiles[profileName]
|
||||||
|
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
|
||||||
|
}
|
||||||
|
RemoteBuildCacheClient(profile).use { client ->
|
||||||
|
|
||||||
|
val entryGenerator = sequence {
|
||||||
|
val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong())
|
||||||
|
while (true) {
|
||||||
|
val key = JWO.bytesToHex(random.nextBytes(16))
|
||||||
|
val content = random.nextInt().toByte()
|
||||||
|
val value = ByteArray(size, { _ -> content })
|
||||||
|
yield(key to value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info {
|
||||||
|
"Starting insertion"
|
||||||
|
}
|
||||||
|
val entries = let {
|
||||||
|
val completionCounter = AtomicLong(0)
|
||||||
|
val completionQueue = LinkedBlockingQueue<Pair<String, ByteArray>>(numberOfEntries)
|
||||||
|
val start = Instant.now()
|
||||||
|
val semaphore = Semaphore(profile.maxConnections * 3)
|
||||||
|
val iterator = entryGenerator.take(numberOfEntries).iterator()
|
||||||
|
while (completionCounter.get() < numberOfEntries) {
|
||||||
|
if (iterator.hasNext()) {
|
||||||
|
val entry = iterator.next()
|
||||||
|
semaphore.acquire()
|
||||||
|
val future = client.put(entry.first, entry.second).thenApply { entry }
|
||||||
|
future.whenComplete { result, ex ->
|
||||||
|
if (ex != null) {
|
||||||
|
log.error(ex.message, ex)
|
||||||
|
} else {
|
||||||
|
completionQueue.put(result)
|
||||||
|
}
|
||||||
|
semaphore.release()
|
||||||
|
completionCounter.incrementAndGet()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Thread.sleep(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val inserted = completionQueue.toList()
|
||||||
|
val end = Instant.now()
|
||||||
|
log.info {
|
||||||
|
val elapsed = Duration.between(start, end).toMillis()
|
||||||
|
val opsPerSecond = String.format("%.2f", numberOfEntries.toDouble() / elapsed * 1000)
|
||||||
|
"Insertion rate: $opsPerSecond ops/s"
|
||||||
|
}
|
||||||
|
inserted
|
||||||
|
}
|
||||||
|
log.info {
|
||||||
|
"Inserted ${entries.size} entries"
|
||||||
|
}
|
||||||
|
log.info {
|
||||||
|
"Starting retrieval"
|
||||||
|
}
|
||||||
|
if (entries.isNotEmpty()) {
|
||||||
|
val completionCounter = AtomicLong(0)
|
||||||
|
val semaphore = Semaphore(profile.maxConnections * 3)
|
||||||
|
val start = Instant.now()
|
||||||
|
val it = entries.iterator()
|
||||||
|
while (completionCounter.get() < entries.size) {
|
||||||
|
if (it.hasNext()) {
|
||||||
|
val entry = it.next()
|
||||||
|
val future = client.get(entry.first).thenApply {
|
||||||
|
if (it == null) {
|
||||||
|
log.error {
|
||||||
|
"Missing entry for key '${entry.first}'"
|
||||||
|
}
|
||||||
|
} else if (!entry.second.contentEquals(it)) {
|
||||||
|
log.error {
|
||||||
|
"Retrieved a value different from what was inserted for key '${entry.first}'"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
future.whenComplete { _, _ ->
|
||||||
|
completionCounter.incrementAndGet()
|
||||||
|
semaphore.release()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Thread.sleep(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val end = Instant.now()
|
||||||
|
log.info {
|
||||||
|
val elapsed = Duration.between(start, end).toMillis()
|
||||||
|
val opsPerSecond = String.format("%.2f", entries.size.toDouble() / elapsed * 1000)
|
||||||
|
"Retrieval rate: $opsPerSecond ops/s"
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.error("Skipping retrieval benchmark as it was not possible to insert any entry in the cache")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,41 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl.commands
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.cli.impl.RbcsCommand
|
||||||
|
import net.woggioni.rbcs.client.RemoteBuildCacheClient
|
||||||
|
import net.woggioni.jwo.Application
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
@CommandLine.Command(
|
||||||
|
name = "client",
|
||||||
|
description = ["RBCS client"],
|
||||||
|
showDefaultValues = true
|
||||||
|
)
|
||||||
|
class ClientCommand(app : Application) : RbcsCommand() {
|
||||||
|
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-c", "--configuration"],
|
||||||
|
description = ["Path to the client configuration file"],
|
||||||
|
paramLabel = "CONFIGURATION_FILE"
|
||||||
|
)
|
||||||
|
private var configurationFile : Path = findConfigurationFile(app, "rbcs-client.xml")
|
||||||
|
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-p", "--profile"],
|
||||||
|
description = ["Name of the client profile to be used"],
|
||||||
|
paramLabel = "PROFILE",
|
||||||
|
required = true
|
||||||
|
)
|
||||||
|
var profileName : String? = null
|
||||||
|
|
||||||
|
val configuration : RemoteBuildCacheClient.Configuration by lazy {
|
||||||
|
RemoteBuildCacheClient.Configuration.parse(configurationFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun run() {
|
||||||
|
println("Available profiles:")
|
||||||
|
configuration.profiles.forEach { (profileName, _) ->
|
||||||
|
println(profileName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,51 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl.commands
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.cli.impl.RbcsCommand
|
||||||
|
import net.woggioni.rbcs.client.RemoteBuildCacheClient
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
@CommandLine.Command(
|
||||||
|
name = "get",
|
||||||
|
description = ["Fetch a value from the cache with the specified key"],
|
||||||
|
showDefaultValues = true
|
||||||
|
)
|
||||||
|
class GetCommand : RbcsCommand() {
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
@CommandLine.Spec
|
||||||
|
private lateinit var spec: CommandLine.Model.CommandSpec
|
||||||
|
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-k", "--key"],
|
||||||
|
description = ["The key for the new value"],
|
||||||
|
paramLabel = "KEY"
|
||||||
|
)
|
||||||
|
private var key : String = ""
|
||||||
|
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-v", "--value"],
|
||||||
|
description = ["Path to a file where the retrieved value will be written (defaults to stdout)"],
|
||||||
|
paramLabel = "VALUE_FILE",
|
||||||
|
)
|
||||||
|
private var output : Path? = null
|
||||||
|
|
||||||
|
override fun run() {
|
||||||
|
val clientCommand = spec.parent().userObject() as ClientCommand
|
||||||
|
val profile = clientCommand.profileName.let { profileName ->
|
||||||
|
clientCommand.configuration.profiles[profileName]
|
||||||
|
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
|
||||||
|
}
|
||||||
|
RemoteBuildCacheClient(profile).use { client ->
|
||||||
|
client.get(key).thenApply { value ->
|
||||||
|
value?.let {
|
||||||
|
(output?.let(Files::newOutputStream) ?: System.out).use {
|
||||||
|
it.write(value)
|
||||||
|
}
|
||||||
|
} ?: throw NoSuchElementException("No value found for key $key")
|
||||||
|
}.get()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,45 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl.commands
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.cli.impl.RbcsCommand
|
||||||
|
import net.woggioni.rbcs.client.RemoteBuildCacheClient
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.security.SecureRandom
|
||||||
|
import kotlin.random.Random
|
||||||
|
|
||||||
|
@CommandLine.Command(
|
||||||
|
name = "health",
|
||||||
|
description = ["Check server health"],
|
||||||
|
showDefaultValues = true
|
||||||
|
)
|
||||||
|
class HealthCheckCommand : RbcsCommand() {
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
@CommandLine.Spec
|
||||||
|
private lateinit var spec: CommandLine.Model.CommandSpec
|
||||||
|
|
||||||
|
override fun run() {
|
||||||
|
val clientCommand = spec.parent().userObject() as ClientCommand
|
||||||
|
val profile = clientCommand.profileName.let { profileName ->
|
||||||
|
clientCommand.configuration.profiles[profileName]
|
||||||
|
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
|
||||||
|
}
|
||||||
|
RemoteBuildCacheClient(profile).use { client ->
|
||||||
|
val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong())
|
||||||
|
val nonce = ByteArray(0xa0)
|
||||||
|
random.nextBytes(nonce)
|
||||||
|
client.healthCheck(nonce).thenApply { value ->
|
||||||
|
if(value == null) {
|
||||||
|
throw IllegalStateException("Empty response from server")
|
||||||
|
}
|
||||||
|
for(i in 0 until nonce.size) {
|
||||||
|
for(j in value.size - nonce.size until nonce.size) {
|
||||||
|
if(nonce[i] != value[j]) {
|
||||||
|
throw IllegalStateException("Server nonce does not match")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}.get()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,37 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl.commands
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.cli.impl.RbcsCommand
|
||||||
|
import net.woggioni.rbcs.cli.impl.converters.OutputStreamConverter
|
||||||
|
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
|
||||||
|
import net.woggioni.jwo.UncloseableOutputStream
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.io.OutputStream
|
||||||
|
import java.io.OutputStreamWriter
|
||||||
|
import java.io.PrintWriter
|
||||||
|
|
||||||
|
|
||||||
|
@CommandLine.Command(
|
||||||
|
name = "password",
|
||||||
|
description = ["Generate a password hash to add to RBCS configuration file"],
|
||||||
|
showDefaultValues = true
|
||||||
|
)
|
||||||
|
class PasswordHashCommand : RbcsCommand() {
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-o", "--output-file"],
|
||||||
|
description = ["Write the output to a file instead of stdout"],
|
||||||
|
converter = [OutputStreamConverter::class],
|
||||||
|
showDefaultValue = CommandLine.Help.Visibility.NEVER,
|
||||||
|
paramLabel = "OUTPUT_FILE"
|
||||||
|
)
|
||||||
|
private var outputStream: OutputStream = UncloseableOutputStream(System.out)
|
||||||
|
|
||||||
|
override fun run() {
|
||||||
|
val password1 = String(System.console().readPassword("Type your password:"))
|
||||||
|
val password2 = String(System.console().readPassword("Type your password again for confirmation:"))
|
||||||
|
if(password1 != password2) throw IllegalArgumentException("Passwords do not match")
|
||||||
|
|
||||||
|
PrintWriter(OutputStreamWriter(outputStream, Charsets.UTF_8)).use {
|
||||||
|
it.println(hashPassword(password1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,48 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl.commands
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.cli.impl.RbcsCommand
|
||||||
|
import net.woggioni.rbcs.cli.impl.converters.InputStreamConverter
|
||||||
|
import net.woggioni.rbcs.client.RemoteBuildCacheClient
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.io.InputStream
|
||||||
|
|
||||||
|
@CommandLine.Command(
|
||||||
|
name = "put",
|
||||||
|
description = ["Add or replace a value to the cache with the specified key"],
|
||||||
|
showDefaultValues = true
|
||||||
|
)
|
||||||
|
class PutCommand : RbcsCommand() {
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
@CommandLine.Spec
|
||||||
|
private lateinit var spec: CommandLine.Model.CommandSpec
|
||||||
|
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-k", "--key"],
|
||||||
|
description = ["The key for the new value"],
|
||||||
|
paramLabel = "KEY"
|
||||||
|
)
|
||||||
|
private var key : String = ""
|
||||||
|
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-v", "--value"],
|
||||||
|
description = ["Path to a file containing the value to be added (defaults to stdin)"],
|
||||||
|
paramLabel = "VALUE_FILE",
|
||||||
|
converter = [InputStreamConverter::class]
|
||||||
|
)
|
||||||
|
private var value : InputStream = System.`in`
|
||||||
|
|
||||||
|
override fun run() {
|
||||||
|
val clientCommand = spec.parent().userObject() as ClientCommand
|
||||||
|
val profile = clientCommand.profileName.let { profileName ->
|
||||||
|
clientCommand.configuration.profiles[profileName]
|
||||||
|
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
|
||||||
|
}
|
||||||
|
RemoteBuildCacheClient(profile).use { client ->
|
||||||
|
value.use {
|
||||||
|
client.put(key, it.readAllBytes())
|
||||||
|
}.get()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,76 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl.commands
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.cli.impl.RbcsCommand
|
||||||
|
import net.woggioni.rbcs.cli.impl.converters.DurationConverter
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.rbcs.common.debug
|
||||||
|
import net.woggioni.rbcs.common.info
|
||||||
|
import net.woggioni.rbcs.server.RemoteBuildCacheServer
|
||||||
|
import net.woggioni.rbcs.server.RemoteBuildCacheServer.Companion.DEFAULT_CONFIGURATION_URL
|
||||||
|
import net.woggioni.jwo.Application
|
||||||
|
import net.woggioni.jwo.JWO
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.io.ByteArrayOutputStream
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Path
|
||||||
|
import java.time.Duration
|
||||||
|
|
||||||
|
@CommandLine.Command(
|
||||||
|
name = "server",
|
||||||
|
description = ["RBCS server"],
|
||||||
|
showDefaultValues = true
|
||||||
|
)
|
||||||
|
class ServerCommand(app : Application) : RbcsCommand() {
|
||||||
|
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
private fun createDefaultConfigurationFile(configurationFile: Path) {
|
||||||
|
log.info {
|
||||||
|
"Creating default configuration file at '$configurationFile'"
|
||||||
|
}
|
||||||
|
val defaultConfigurationFileResource = DEFAULT_CONFIGURATION_URL
|
||||||
|
Files.newOutputStream(configurationFile).use { outputStream ->
|
||||||
|
defaultConfigurationFileResource.openStream().use { inputStream ->
|
||||||
|
JWO.copy(inputStream, outputStream)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-t", "--timeout"],
|
||||||
|
description = ["Exit after the specified time"],
|
||||||
|
paramLabel = "TIMEOUT",
|
||||||
|
converter = [DurationConverter::class]
|
||||||
|
)
|
||||||
|
private var timeout: Duration? = null
|
||||||
|
|
||||||
|
@CommandLine.Option(
|
||||||
|
names = ["-c", "--config-file"],
|
||||||
|
description = ["Read the application configuration from this file"],
|
||||||
|
paramLabel = "CONFIG_FILE"
|
||||||
|
)
|
||||||
|
private var configurationFile: Path = findConfigurationFile(app, "rbcs-server.xml")
|
||||||
|
|
||||||
|
override fun run() {
|
||||||
|
if (!Files.exists(configurationFile)) {
|
||||||
|
Files.createDirectories(configurationFile.parent)
|
||||||
|
createDefaultConfigurationFile(configurationFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
val configuration = RemoteBuildCacheServer.loadConfiguration(configurationFile)
|
||||||
|
log.debug {
|
||||||
|
ByteArrayOutputStream().also {
|
||||||
|
RemoteBuildCacheServer.dumpConfiguration(configuration, it)
|
||||||
|
}.let {
|
||||||
|
"Server configuration:\n${String(it.toByteArray())}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val server = RemoteBuildCacheServer(configuration)
|
||||||
|
server.run().use { server ->
|
||||||
|
timeout?.let {
|
||||||
|
Thread.sleep(it)
|
||||||
|
server.shutdown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,11 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl.converters
|
||||||
|
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.time.Duration
|
||||||
|
|
||||||
|
|
||||||
|
class DurationConverter : CommandLine.ITypeConverter<Duration> {
|
||||||
|
override fun convert(value: String): Duration {
|
||||||
|
return Duration.parse(value)
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,13 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl.converters
|
||||||
|
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.io.InputStream
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Paths
|
||||||
|
|
||||||
|
|
||||||
|
class InputStreamConverter : CommandLine.ITypeConverter<InputStream> {
|
||||||
|
override fun convert(value: String): InputStream {
|
||||||
|
return Files.newInputStream(Paths.get(value))
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,13 @@
|
|||||||
|
package net.woggioni.rbcs.cli.impl.converters
|
||||||
|
|
||||||
|
import picocli.CommandLine
|
||||||
|
import java.io.OutputStream
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Paths
|
||||||
|
|
||||||
|
|
||||||
|
class OutputStreamConverter : CommandLine.ITypeConverter<OutputStream> {
|
||||||
|
override fun convert(value: String): OutputStream {
|
||||||
|
return Files.newOutputStream(Paths.get(value))
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,18 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" ?>
|
||||||
|
<!DOCTYPE configuration>
|
||||||
|
|
||||||
|
<configuration>
|
||||||
|
<import class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"/>
|
||||||
|
<import class="ch.qos.logback.core.ConsoleAppender"/>
|
||||||
|
|
||||||
|
<appender name="console" class="ConsoleAppender">
|
||||||
|
<target>System.err</target>
|
||||||
|
<encoder class="PatternLayoutEncoder">
|
||||||
|
<pattern>%d [%highlight(%-5level)] \(%thread\) %logger{36} -%kvp- %msg %n</pattern>
|
||||||
|
</encoder>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<root level="info">
|
||||||
|
<appender-ref ref="console"/>
|
||||||
|
</root>
|
||||||
|
</configuration>
|
@@ -2,8 +2,8 @@
|
|||||||
|
|
||||||
handlers = java.util.logging.ConsoleHandler
|
handlers = java.util.logging.ConsoleHandler
|
||||||
|
|
||||||
java.util.logging.ConsoleHandler.level = FINEST
|
java.util.logging.ConsoleHandler.level = FINER
|
||||||
java.util.logging.ConsoleHandler.filter =
|
java.util.logging.ConsoleHandler.filter =
|
||||||
java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
|
java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
|
||||||
java.util.logging.SimpleFormatter.format = %1$tF %1$tT [%4$s] %2$s %5$s %n
|
java.util.logging.SimpleFormatter.format = %1$tF %1$tT [%4$s] %2$s %5$s %6$s%n
|
||||||
java.util.logging.ConsoleHandler.encoding =
|
java.util.logging.ConsoleHandler.encoding =
|
17
rbcs-client/build.gradle
Normal file
17
rbcs-client/build.gradle
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
plugins {
|
||||||
|
id 'java-library'
|
||||||
|
alias catalog.plugins.kotlin.jvm
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
implementation project(':rbcs-api')
|
||||||
|
implementation project(':rbcs-common')
|
||||||
|
implementation catalog.picocli
|
||||||
|
implementation catalog.slf4j.api
|
||||||
|
implementation catalog.netty.buffer
|
||||||
|
implementation catalog.netty.codec.http
|
||||||
|
|
||||||
|
testRuntimeOnly catalog.logback.classic
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@@ -1,15 +1,17 @@
|
|||||||
module net.woggioni.gbcs {
|
module net.woggioni.rbcs.client {
|
||||||
requires java.xml;
|
|
||||||
requires java.logging;
|
|
||||||
requires kotlin.stdlib;
|
|
||||||
requires io.netty.buffer;
|
|
||||||
requires io.netty.transport;
|
|
||||||
requires io.netty.codec.http;
|
|
||||||
requires io.netty.common;
|
|
||||||
requires io.netty.handler;
|
requires io.netty.handler;
|
||||||
|
requires io.netty.codec.http;
|
||||||
|
requires io.netty.transport;
|
||||||
|
requires kotlin.stdlib;
|
||||||
|
requires io.netty.common;
|
||||||
|
requires io.netty.buffer;
|
||||||
|
requires java.xml;
|
||||||
|
requires net.woggioni.rbcs.common;
|
||||||
|
requires net.woggioni.rbcs.api;
|
||||||
requires io.netty.codec;
|
requires io.netty.codec;
|
||||||
requires org.slf4j;
|
requires org.slf4j;
|
||||||
requires net.woggioni.jwo;
|
|
||||||
|
|
||||||
exports net.woggioni.gbcs;
|
exports net.woggioni.rbcs.client;
|
||||||
|
|
||||||
|
opens net.woggioni.rbcs.client.schema;
|
||||||
}
|
}
|
344
rbcs-client/src/main/kotlin/net/woggioni/rbcs/client/Client.kt
Normal file
344
rbcs-client/src/main/kotlin/net/woggioni/rbcs/client/Client.kt
Normal file
@@ -0,0 +1,344 @@
|
|||||||
|
package net.woggioni.rbcs.client
|
||||||
|
|
||||||
|
import io.netty.bootstrap.Bootstrap
|
||||||
|
import io.netty.buffer.ByteBuf
|
||||||
|
import io.netty.buffer.Unpooled
|
||||||
|
import io.netty.channel.Channel
|
||||||
|
import io.netty.channel.ChannelHandlerContext
|
||||||
|
import io.netty.channel.ChannelOption
|
||||||
|
import io.netty.channel.ChannelPipeline
|
||||||
|
import io.netty.channel.SimpleChannelInboundHandler
|
||||||
|
import io.netty.channel.nio.NioEventLoopGroup
|
||||||
|
import io.netty.channel.pool.AbstractChannelPoolHandler
|
||||||
|
import io.netty.channel.pool.ChannelPool
|
||||||
|
import io.netty.channel.pool.FixedChannelPool
|
||||||
|
import io.netty.channel.socket.nio.NioSocketChannel
|
||||||
|
import io.netty.handler.codec.DecoderException
|
||||||
|
import io.netty.handler.codec.http.DefaultFullHttpRequest
|
||||||
|
import io.netty.handler.codec.http.FullHttpRequest
|
||||||
|
import io.netty.handler.codec.http.FullHttpResponse
|
||||||
|
import io.netty.handler.codec.http.HttpClientCodec
|
||||||
|
import io.netty.handler.codec.http.HttpContentDecompressor
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderNames
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderValues
|
||||||
|
import io.netty.handler.codec.http.HttpMethod
|
||||||
|
import io.netty.handler.codec.http.HttpObjectAggregator
|
||||||
|
import io.netty.handler.codec.http.HttpResponseStatus
|
||||||
|
import io.netty.handler.codec.http.HttpVersion
|
||||||
|
import io.netty.handler.ssl.SslContext
|
||||||
|
import io.netty.handler.ssl.SslContextBuilder
|
||||||
|
import io.netty.handler.stream.ChunkedWriteHandler
|
||||||
|
import io.netty.util.concurrent.Future
|
||||||
|
import io.netty.util.concurrent.GenericFutureListener
|
||||||
|
import net.woggioni.rbcs.client.impl.Parser
|
||||||
|
import net.woggioni.rbcs.common.Xml
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.rbcs.common.debug
|
||||||
|
import net.woggioni.rbcs.common.trace
|
||||||
|
import java.net.InetSocketAddress
|
||||||
|
import java.net.URI
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Path
|
||||||
|
import java.security.PrivateKey
|
||||||
|
import java.security.cert.X509Certificate
|
||||||
|
import java.time.Duration
|
||||||
|
import java.util.Base64
|
||||||
|
import java.util.concurrent.CompletableFuture
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger
|
||||||
|
import io.netty.util.concurrent.Future as NettyFuture
|
||||||
|
|
||||||
|
|
||||||
|
class RemoteBuildCacheClient(private val profile: Configuration.Profile) : AutoCloseable {
|
||||||
|
private val group: NioEventLoopGroup
|
||||||
|
private var sslContext: SslContext
|
||||||
|
private val log = contextLogger()
|
||||||
|
private val pool: ChannelPool
|
||||||
|
|
||||||
|
data class Configuration(
|
||||||
|
val profiles: Map<String, Profile>
|
||||||
|
) {
|
||||||
|
sealed class Authentication {
|
||||||
|
data class TlsClientAuthenticationCredentials(
|
||||||
|
val key: PrivateKey,
|
||||||
|
val certificateChain: Array<X509Certificate>
|
||||||
|
) : Authentication()
|
||||||
|
|
||||||
|
data class BasicAuthenticationCredentials(val username: String, val password: String) : Authentication()
|
||||||
|
}
|
||||||
|
|
||||||
|
class RetryPolicy(
|
||||||
|
val maxAttempts: Int,
|
||||||
|
val initialDelayMillis: Long,
|
||||||
|
val exp: Double
|
||||||
|
)
|
||||||
|
|
||||||
|
data class Profile(
|
||||||
|
val serverURI: URI,
|
||||||
|
val authentication: Authentication?,
|
||||||
|
val connectionTimeout: Duration?,
|
||||||
|
val maxConnections: Int,
|
||||||
|
val retryPolicy: RetryPolicy?,
|
||||||
|
)
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
fun parse(path: Path): Configuration {
|
||||||
|
return Files.newInputStream(path).use {
|
||||||
|
Xml.parseXml(path.toUri().toURL(), it)
|
||||||
|
}.let(Parser::parse)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
init {
|
||||||
|
group = NioEventLoopGroup()
|
||||||
|
sslContext = SslContextBuilder.forClient().also { builder ->
|
||||||
|
(profile.authentication as? Configuration.Authentication.TlsClientAuthenticationCredentials)?.let { tlsClientAuthenticationCredentials ->
|
||||||
|
builder.keyManager(
|
||||||
|
tlsClientAuthenticationCredentials.key,
|
||||||
|
*tlsClientAuthenticationCredentials.certificateChain
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}.build()
|
||||||
|
|
||||||
|
val (scheme, host, port) = profile.serverURI.run {
|
||||||
|
Triple(
|
||||||
|
if (scheme == null) "http" else profile.serverURI.scheme,
|
||||||
|
host,
|
||||||
|
port.takeIf { it > 0 } ?: if ("https" == scheme.lowercase()) 443 else 80
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
val bootstrap = Bootstrap().apply {
|
||||||
|
group(group)
|
||||||
|
channel(NioSocketChannel::class.java)
|
||||||
|
option(ChannelOption.TCP_NODELAY, true)
|
||||||
|
option(ChannelOption.SO_KEEPALIVE, true)
|
||||||
|
remoteAddress(InetSocketAddress(host, port))
|
||||||
|
profile.connectionTimeout?.let {
|
||||||
|
option(ChannelOption.CONNECT_TIMEOUT_MILLIS, it.toMillis().toInt())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val channelPoolHandler = object : AbstractChannelPoolHandler() {
|
||||||
|
|
||||||
|
@Volatile
|
||||||
|
private var connectionCount = AtomicInteger()
|
||||||
|
|
||||||
|
@Volatile
|
||||||
|
private var leaseCount = AtomicInteger()
|
||||||
|
|
||||||
|
override fun channelReleased(ch: Channel) {
|
||||||
|
val activeLeases = leaseCount.decrementAndGet()
|
||||||
|
log.trace {
|
||||||
|
"Released channel ${ch.id().asShortText()}, number of active leases: $activeLeases"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun channelAcquired(ch: Channel) {
|
||||||
|
val activeLeases = leaseCount.getAndIncrement()
|
||||||
|
log.trace {
|
||||||
|
"Acquired channel ${ch.id().asShortText()}, number of active leases: $activeLeases"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun channelCreated(ch: Channel) {
|
||||||
|
val connectionId = connectionCount.getAndIncrement()
|
||||||
|
log.debug {
|
||||||
|
"Created connection $connectionId, total number of active connections: $connectionId"
|
||||||
|
}
|
||||||
|
ch.closeFuture().addListener {
|
||||||
|
val activeConnections = connectionCount.decrementAndGet()
|
||||||
|
log.debug {
|
||||||
|
"Closed connection $connectionId, total number of active connections: $activeConnections"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val pipeline: ChannelPipeline = ch.pipeline()
|
||||||
|
|
||||||
|
// Add SSL handler if needed
|
||||||
|
if ("https".equals(scheme, ignoreCase = true)) {
|
||||||
|
pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port))
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTP handlers
|
||||||
|
pipeline.addLast("codec", HttpClientCodec())
|
||||||
|
pipeline.addLast("decompressor", HttpContentDecompressor())
|
||||||
|
pipeline.addLast("aggregator", HttpObjectAggregator(134217728))
|
||||||
|
pipeline.addLast("chunked", ChunkedWriteHandler())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pool = FixedChannelPool(bootstrap, channelPoolHandler, profile.maxConnections)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun executeWithRetry(operation: () -> CompletableFuture<FullHttpResponse>): CompletableFuture<FullHttpResponse> {
|
||||||
|
val retryPolicy = profile.retryPolicy
|
||||||
|
return if (retryPolicy != null) {
|
||||||
|
val outcomeHandler = OutcomeHandler<FullHttpResponse> { outcome ->
|
||||||
|
when (outcome) {
|
||||||
|
is OperationOutcome.Success -> {
|
||||||
|
val response = outcome.result
|
||||||
|
val status = response.status()
|
||||||
|
when (status) {
|
||||||
|
HttpResponseStatus.TOO_MANY_REQUESTS -> {
|
||||||
|
val retryAfter = response.headers()[HttpHeaderNames.RETRY_AFTER]?.let { headerValue ->
|
||||||
|
try {
|
||||||
|
headerValue.toLong() * 1000
|
||||||
|
} catch (nfe: NumberFormatException) {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
OutcomeHandlerResult.Retry(retryAfter)
|
||||||
|
}
|
||||||
|
|
||||||
|
HttpResponseStatus.INTERNAL_SERVER_ERROR, HttpResponseStatus.SERVICE_UNAVAILABLE ->
|
||||||
|
OutcomeHandlerResult.Retry()
|
||||||
|
|
||||||
|
else -> OutcomeHandlerResult.DoNotRetry()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
is OperationOutcome.Failure -> {
|
||||||
|
OutcomeHandlerResult.Retry()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
executeWithRetry(
|
||||||
|
group,
|
||||||
|
retryPolicy.maxAttempts,
|
||||||
|
retryPolicy.initialDelayMillis.toDouble(),
|
||||||
|
retryPolicy.exp,
|
||||||
|
outcomeHandler,
|
||||||
|
operation
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
operation()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun healthCheck(nonce: ByteArray): CompletableFuture<ByteArray?> {
|
||||||
|
return executeWithRetry {
|
||||||
|
sendRequest(profile.serverURI, HttpMethod.TRACE, nonce)
|
||||||
|
}.thenApply {
|
||||||
|
val status = it.status()
|
||||||
|
if (it.status() != HttpResponseStatus.OK) {
|
||||||
|
throw HttpException(status)
|
||||||
|
} else {
|
||||||
|
it.content()
|
||||||
|
}
|
||||||
|
}.thenApply { maybeByteBuf ->
|
||||||
|
maybeByteBuf?.let {
|
||||||
|
val result = ByteArray(it.readableBytes())
|
||||||
|
it.getBytes(0, result)
|
||||||
|
result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun get(key: String): CompletableFuture<ByteArray?> {
|
||||||
|
return executeWithRetry {
|
||||||
|
sendRequest(profile.serverURI.resolve(key), HttpMethod.GET, null)
|
||||||
|
}.thenApply {
|
||||||
|
val status = it.status()
|
||||||
|
if (it.status() == HttpResponseStatus.NOT_FOUND) {
|
||||||
|
null
|
||||||
|
} else if (it.status() != HttpResponseStatus.OK) {
|
||||||
|
throw HttpException(status)
|
||||||
|
} else {
|
||||||
|
it.content()
|
||||||
|
}
|
||||||
|
}.thenApply { maybeByteBuf ->
|
||||||
|
maybeByteBuf?.let {
|
||||||
|
val result = ByteArray(it.readableBytes())
|
||||||
|
it.getBytes(0, result)
|
||||||
|
result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun put(key: String, content: ByteArray): CompletableFuture<Unit> {
|
||||||
|
return executeWithRetry {
|
||||||
|
sendRequest(profile.serverURI.resolve(key), HttpMethod.PUT, content)
|
||||||
|
}.thenApply {
|
||||||
|
val status = it.status()
|
||||||
|
if (it.status() != HttpResponseStatus.CREATED && it.status() != HttpResponseStatus.OK) {
|
||||||
|
throw HttpException(status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun sendRequest(uri: URI, method: HttpMethod, body: ByteArray?): CompletableFuture<FullHttpResponse> {
|
||||||
|
val responseFuture = CompletableFuture<FullHttpResponse>()
|
||||||
|
// Custom handler for processing responses
|
||||||
|
pool.acquire().addListener(object : GenericFutureListener<NettyFuture<Channel>> {
|
||||||
|
override fun operationComplete(channelFuture: Future<Channel>) {
|
||||||
|
if (channelFuture.isSuccess) {
|
||||||
|
val channel = channelFuture.now
|
||||||
|
val pipeline = channel.pipeline()
|
||||||
|
channel.pipeline().addLast("handler", object : SimpleChannelInboundHandler<FullHttpResponse>() {
|
||||||
|
override fun channelRead0(
|
||||||
|
ctx: ChannelHandlerContext,
|
||||||
|
response: FullHttpResponse
|
||||||
|
) {
|
||||||
|
pipeline.removeLast()
|
||||||
|
pool.release(channel)
|
||||||
|
responseFuture.complete(response)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
|
||||||
|
val ex = when (cause) {
|
||||||
|
is DecoderException -> cause.cause
|
||||||
|
else -> cause
|
||||||
|
}
|
||||||
|
responseFuture.completeExceptionally(ex)
|
||||||
|
ctx.close()
|
||||||
|
pipeline.removeLast()
|
||||||
|
pool.release(channel)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
// Prepare the HTTP request
|
||||||
|
val request: FullHttpRequest = let {
|
||||||
|
val content: ByteBuf? = body?.takeIf(ByteArray::isNotEmpty)?.let(Unpooled::wrappedBuffer)
|
||||||
|
DefaultFullHttpRequest(
|
||||||
|
HttpVersion.HTTP_1_1,
|
||||||
|
method,
|
||||||
|
uri.rawPath,
|
||||||
|
content ?: Unpooled.buffer(0)
|
||||||
|
).apply {
|
||||||
|
headers().apply {
|
||||||
|
if (content != null) {
|
||||||
|
set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.APPLICATION_OCTET_STREAM)
|
||||||
|
set(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes())
|
||||||
|
}
|
||||||
|
set(HttpHeaderNames.HOST, profile.serverURI.host)
|
||||||
|
set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE)
|
||||||
|
set(
|
||||||
|
HttpHeaderNames.ACCEPT_ENCODING,
|
||||||
|
HttpHeaderValues.GZIP.toString() + "," + HttpHeaderValues.DEFLATE.toString()
|
||||||
|
)
|
||||||
|
// Add basic auth if configured
|
||||||
|
(profile.authentication as? Configuration.Authentication.BasicAuthenticationCredentials)?.let { credentials ->
|
||||||
|
val auth = "${credentials.username}:${credentials.password}"
|
||||||
|
val encodedAuth = Base64.getEncoder().encodeToString(auth.toByteArray())
|
||||||
|
set(HttpHeaderNames.AUTHORIZATION, "Basic $encodedAuth")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set headers
|
||||||
|
// Send the request
|
||||||
|
channel.writeAndFlush(request)
|
||||||
|
} else {
|
||||||
|
responseFuture.completeExceptionally(channelFuture.cause())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return responseFuture
|
||||||
|
}
|
||||||
|
|
||||||
|
fun shutDown(): NettyFuture<*> {
|
||||||
|
return group.shutdownGracefully()
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {
|
||||||
|
shutDown().sync()
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,9 @@
|
|||||||
|
package net.woggioni.rbcs.client
|
||||||
|
|
||||||
|
import io.netty.handler.codec.http.HttpResponseStatus
|
||||||
|
|
||||||
|
class HttpException(private val status : HttpResponseStatus) : RuntimeException(status.reasonPhrase()) {
|
||||||
|
|
||||||
|
override val message: String
|
||||||
|
get() = "Http status ${status.code()}: ${status.reasonPhrase()}"
|
||||||
|
}
|
@@ -0,0 +1,108 @@
|
|||||||
|
package net.woggioni.rbcs.client.impl
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.exception.ConfigurationException
|
||||||
|
import net.woggioni.rbcs.client.RemoteBuildCacheClient
|
||||||
|
import net.woggioni.rbcs.common.Xml.Companion.asIterable
|
||||||
|
import net.woggioni.rbcs.common.Xml.Companion.renderAttribute
|
||||||
|
import org.w3c.dom.Document
|
||||||
|
import java.net.URI
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Path
|
||||||
|
import java.security.KeyStore
|
||||||
|
import java.security.PrivateKey
|
||||||
|
import java.security.cert.X509Certificate
|
||||||
|
import java.time.Duration
|
||||||
|
|
||||||
|
object Parser {
|
||||||
|
|
||||||
|
fun parse(document: Document): RemoteBuildCacheClient.Configuration {
|
||||||
|
val root = document.documentElement
|
||||||
|
val profiles = mutableMapOf<String, RemoteBuildCacheClient.Configuration.Profile>()
|
||||||
|
|
||||||
|
for (child in root.asIterable()) {
|
||||||
|
val tagName = child.localName
|
||||||
|
when (tagName) {
|
||||||
|
"profile" -> {
|
||||||
|
val name =
|
||||||
|
child.renderAttribute("name") ?: throw ConfigurationException("name attribute is required")
|
||||||
|
val uri = child.renderAttribute("base-url")?.let(::URI)
|
||||||
|
?: throw ConfigurationException("base-url attribute is required")
|
||||||
|
var authentication: RemoteBuildCacheClient.Configuration.Authentication? = null
|
||||||
|
var retryPolicy: RemoteBuildCacheClient.Configuration.RetryPolicy? = null
|
||||||
|
for (gchild in child.asIterable()) {
|
||||||
|
when (gchild.localName) {
|
||||||
|
"tls-client-auth" -> {
|
||||||
|
val keyStoreFile = gchild.renderAttribute("key-store-file")
|
||||||
|
val keyStorePassword =
|
||||||
|
gchild.renderAttribute("key-store-password")
|
||||||
|
val keyAlias = gchild.renderAttribute("key-alias")
|
||||||
|
val keyPassword = gchild.renderAttribute("key-password")
|
||||||
|
|
||||||
|
val keystore = KeyStore.getInstance("PKCS12").apply {
|
||||||
|
Files.newInputStream(Path.of(keyStoreFile)).use {
|
||||||
|
load(it, keyStorePassword?.toCharArray())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val key = keystore.getKey(keyAlias, keyPassword?.toCharArray()) as PrivateKey
|
||||||
|
val certChain = keystore.getCertificateChain(keyAlias).asSequence()
|
||||||
|
.map { it as X509Certificate }
|
||||||
|
.toList()
|
||||||
|
.toTypedArray()
|
||||||
|
authentication =
|
||||||
|
RemoteBuildCacheClient.Configuration.Authentication.TlsClientAuthenticationCredentials(
|
||||||
|
key,
|
||||||
|
certChain
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
"basic-auth" -> {
|
||||||
|
val username = gchild.renderAttribute("user")
|
||||||
|
?: throw ConfigurationException("username attribute is required")
|
||||||
|
val password = gchild.renderAttribute("password")
|
||||||
|
?: throw ConfigurationException("password attribute is required")
|
||||||
|
authentication =
|
||||||
|
RemoteBuildCacheClient.Configuration.Authentication.BasicAuthenticationCredentials(
|
||||||
|
username,
|
||||||
|
password
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
"retry-policy" -> {
|
||||||
|
val maxAttempts =
|
||||||
|
gchild.renderAttribute("max-attempts")
|
||||||
|
?.let(String::toInt)
|
||||||
|
?: throw ConfigurationException("max-attempts attribute is required")
|
||||||
|
val initialDelay =
|
||||||
|
gchild.renderAttribute("initial-delay")
|
||||||
|
?.let(Duration::parse)
|
||||||
|
?: Duration.ofSeconds(1)
|
||||||
|
val exp =
|
||||||
|
gchild.renderAttribute("exp")
|
||||||
|
?.let(String::toDouble)
|
||||||
|
?: 2.0f
|
||||||
|
retryPolicy = RemoteBuildCacheClient.Configuration.RetryPolicy(
|
||||||
|
maxAttempts,
|
||||||
|
initialDelay.toMillis(),
|
||||||
|
exp.toDouble()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val maxConnections = child.renderAttribute("max-connections")
|
||||||
|
?.let(String::toInt)
|
||||||
|
?: 50
|
||||||
|
val connectionTimeout = child.renderAttribute("connection-timeout")
|
||||||
|
?.let(Duration::parse)
|
||||||
|
profiles[name] = RemoteBuildCacheClient.Configuration.Profile(
|
||||||
|
uri,
|
||||||
|
authentication,
|
||||||
|
connectionTimeout,
|
||||||
|
maxConnections,
|
||||||
|
retryPolicy
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return RemoteBuildCacheClient.Configuration(profiles)
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,75 @@
|
|||||||
|
package net.woggioni.rbcs.client
|
||||||
|
|
||||||
|
import io.netty.util.concurrent.EventExecutorGroup
|
||||||
|
import java.util.concurrent.CompletableFuture
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
|
||||||
|
sealed class OperationOutcome<T> {
|
||||||
|
class Success<T>(val result: T) : OperationOutcome<T>()
|
||||||
|
class Failure<T>(val ex: Throwable) : OperationOutcome<T>()
|
||||||
|
}
|
||||||
|
|
||||||
|
sealed class OutcomeHandlerResult {
|
||||||
|
class Retry(val suggestedDelayMillis: Long? = null) : OutcomeHandlerResult()
|
||||||
|
class DoNotRetry : OutcomeHandlerResult()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun interface OutcomeHandler<T> {
|
||||||
|
fun shouldRetry(result: OperationOutcome<T>): OutcomeHandlerResult
|
||||||
|
}
|
||||||
|
|
||||||
|
fun <T> executeWithRetry(
|
||||||
|
eventExecutorGroup: EventExecutorGroup,
|
||||||
|
maxAttempts: Int,
|
||||||
|
initialDelay: Double,
|
||||||
|
exp: Double,
|
||||||
|
outcomeHandler: OutcomeHandler<T>,
|
||||||
|
cb: () -> CompletableFuture<T>
|
||||||
|
): CompletableFuture<T> {
|
||||||
|
val finalResult = cb()
|
||||||
|
var future = finalResult
|
||||||
|
var shortCircuit = false
|
||||||
|
for (i in 1 until maxAttempts) {
|
||||||
|
future = future.handle { result, ex ->
|
||||||
|
val operationOutcome = if (ex == null) {
|
||||||
|
OperationOutcome.Success(result)
|
||||||
|
} else {
|
||||||
|
OperationOutcome.Failure(ex.cause ?: ex)
|
||||||
|
}
|
||||||
|
if (shortCircuit) {
|
||||||
|
when(operationOutcome) {
|
||||||
|
is OperationOutcome.Failure -> throw operationOutcome.ex
|
||||||
|
is OperationOutcome.Success -> CompletableFuture.completedFuture(operationOutcome.result)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
when(val outcomeHandlerResult = outcomeHandler.shouldRetry(operationOutcome)) {
|
||||||
|
is OutcomeHandlerResult.Retry -> {
|
||||||
|
val res = CompletableFuture<T>()
|
||||||
|
val delay = run {
|
||||||
|
val scheduledDelay = (initialDelay * Math.pow(exp, i.toDouble())).toLong()
|
||||||
|
outcomeHandlerResult.suggestedDelayMillis?.coerceAtMost(scheduledDelay) ?: scheduledDelay
|
||||||
|
}
|
||||||
|
eventExecutorGroup.schedule({
|
||||||
|
cb().handle { result, ex ->
|
||||||
|
if (ex == null) {
|
||||||
|
res.complete(result)
|
||||||
|
} else {
|
||||||
|
res.completeExceptionally(ex)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, delay, TimeUnit.MILLISECONDS)
|
||||||
|
res
|
||||||
|
}
|
||||||
|
is OutcomeHandlerResult.DoNotRetry -> {
|
||||||
|
shortCircuit = true
|
||||||
|
when(operationOutcome) {
|
||||||
|
is OperationOutcome.Failure -> throw operationOutcome.ex
|
||||||
|
is OperationOutcome.Success -> CompletableFuture.completedFuture(operationOutcome.result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}.thenCompose { it }
|
||||||
|
}
|
||||||
|
return future
|
||||||
|
}
|
@@ -0,0 +1,50 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||||
|
<xs:schema targetNamespace="urn:net.woggioni.rbcs.client"
|
||||||
|
xmlns:xs="http://www.w3.org/2001/XMLSchema"
|
||||||
|
xmlns:rbcs-client="urn:net.woggioni.rbcs.client"
|
||||||
|
elementFormDefault="unqualified"
|
||||||
|
>
|
||||||
|
<xs:element name="profiles" type="rbcs-client:profilesType"/>
|
||||||
|
|
||||||
|
<xs:complexType name="profilesType">
|
||||||
|
<xs:sequence minOccurs="0">
|
||||||
|
<xs:element name="profile" type="rbcs-client:profileType" maxOccurs="unbounded"/>
|
||||||
|
</xs:sequence>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="profileType">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:choice>
|
||||||
|
<xs:element name="no-auth" type="rbcs-client:noAuthType"/>
|
||||||
|
<xs:element name="basic-auth" type="rbcs-client:basicAuthType"/>
|
||||||
|
<xs:element name="tls-client-auth" type="rbcs-client:tlsClientAuthType"/>
|
||||||
|
</xs:choice>
|
||||||
|
<xs:element name="retry-policy" type="rbcs-client:retryType" minOccurs="0"/>
|
||||||
|
</xs:sequence>
|
||||||
|
<xs:attribute name="name" type="xs:token" use="required"/>
|
||||||
|
<xs:attribute name="base-url" type="xs:anyURI" use="required"/>
|
||||||
|
<xs:attribute name="max-connections" type="xs:positiveInteger" default="50"/>
|
||||||
|
<xs:attribute name="connection-timeout" type="xs:duration"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="noAuthType"/>
|
||||||
|
|
||||||
|
<xs:complexType name="basicAuthType">
|
||||||
|
<xs:attribute name="user" type="xs:token" use="required"/>
|
||||||
|
<xs:attribute name="password" type="xs:string" use="required"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="tlsClientAuthType">
|
||||||
|
<xs:attribute name="key-store-file" type="xs:anyURI" use="required"/>
|
||||||
|
<xs:attribute name="key-store-password" type="xs:string" use="required"/>
|
||||||
|
<xs:attribute name="key-alias" type="xs:token" use="required"/>
|
||||||
|
<xs:attribute name="key-password" type="xs:string" use="optional"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="retryType">
|
||||||
|
<xs:attribute name="max-attempts" type="xs:positiveInteger" use="required"/>
|
||||||
|
<xs:attribute name="initial-delay" type="xs:duration" default="PT1S"/>
|
||||||
|
<xs:attribute name="exp" type="xs:double" default="2.0"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
</xs:schema>
|
@@ -0,0 +1,148 @@
|
|||||||
|
package net.woggioni.rbcs.client
|
||||||
|
|
||||||
|
import io.netty.util.concurrent.DefaultEventExecutorGroup
|
||||||
|
import io.netty.util.concurrent.EventExecutorGroup
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import org.junit.jupiter.api.Assertions
|
||||||
|
import org.junit.jupiter.api.extension.ExtensionContext
|
||||||
|
import org.junit.jupiter.params.ParameterizedTest
|
||||||
|
import org.junit.jupiter.params.provider.Arguments
|
||||||
|
import org.junit.jupiter.params.provider.ArgumentsProvider
|
||||||
|
import org.junit.jupiter.params.provider.ArgumentsSource
|
||||||
|
import java.util.concurrent.CompletableFuture
|
||||||
|
import java.util.stream.Stream
|
||||||
|
import kotlin.random.Random
|
||||||
|
|
||||||
|
class RetryTest {
|
||||||
|
|
||||||
|
data class TestArgs(
|
||||||
|
val seed: Int,
|
||||||
|
val maxAttempt: Int,
|
||||||
|
val initialDelay: Double,
|
||||||
|
val exp: Double,
|
||||||
|
)
|
||||||
|
|
||||||
|
class TestArguments : ArgumentsProvider {
|
||||||
|
override fun provideArguments(context: ExtensionContext): Stream<out Arguments> {
|
||||||
|
return Stream.of(
|
||||||
|
TestArgs(
|
||||||
|
seed = 101325,
|
||||||
|
maxAttempt = 5,
|
||||||
|
initialDelay = 50.0,
|
||||||
|
exp = 2.0,
|
||||||
|
),
|
||||||
|
TestArgs(
|
||||||
|
seed = 101325,
|
||||||
|
maxAttempt = 20,
|
||||||
|
initialDelay = 100.0,
|
||||||
|
exp = 1.1,
|
||||||
|
),
|
||||||
|
TestArgs(
|
||||||
|
seed = 123487,
|
||||||
|
maxAttempt = 20,
|
||||||
|
initialDelay = 100.0,
|
||||||
|
exp = 2.0,
|
||||||
|
),
|
||||||
|
TestArgs(
|
||||||
|
seed = 20082024,
|
||||||
|
maxAttempt = 10,
|
||||||
|
initialDelay = 100.0,
|
||||||
|
exp = 2.0,
|
||||||
|
)
|
||||||
|
).map {
|
||||||
|
object: Arguments {
|
||||||
|
override fun get() = arrayOf(it)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@ArgumentsSource(TestArguments::class)
|
||||||
|
@ParameterizedTest
|
||||||
|
fun test(testArgs: TestArgs) {
|
||||||
|
val log = contextLogger()
|
||||||
|
log.debug("Start")
|
||||||
|
val executor: EventExecutorGroup = DefaultEventExecutorGroup(1)
|
||||||
|
val attempts = mutableListOf<Pair<Long, OperationOutcome<Int>>>()
|
||||||
|
val outcomeHandler = OutcomeHandler<Int> { outcome ->
|
||||||
|
when(outcome) {
|
||||||
|
is OperationOutcome.Success -> {
|
||||||
|
if(outcome.result % 10 == 0) {
|
||||||
|
OutcomeHandlerResult.DoNotRetry()
|
||||||
|
} else {
|
||||||
|
OutcomeHandlerResult.Retry(null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
is OperationOutcome.Failure -> {
|
||||||
|
when(outcome.ex) {
|
||||||
|
is IllegalStateException -> {
|
||||||
|
log.debug(outcome.ex.message, outcome.ex)
|
||||||
|
OutcomeHandlerResult.Retry(null)
|
||||||
|
}
|
||||||
|
else -> {
|
||||||
|
OutcomeHandlerResult.DoNotRetry()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val random = Random(testArgs.seed)
|
||||||
|
|
||||||
|
val future =
|
||||||
|
executeWithRetry(executor, testArgs.maxAttempt, testArgs.initialDelay, testArgs.exp, outcomeHandler) {
|
||||||
|
val now = System.nanoTime()
|
||||||
|
val result = CompletableFuture<Int>()
|
||||||
|
executor.submit {
|
||||||
|
val n = random.nextInt(0, Integer.MAX_VALUE)
|
||||||
|
log.debug("Got new number: {}", n)
|
||||||
|
if(n % 3 == 0) {
|
||||||
|
val ex = IllegalStateException("Value $n can be divided by 3")
|
||||||
|
result.completeExceptionally(ex)
|
||||||
|
attempts += now to OperationOutcome.Failure(ex)
|
||||||
|
} else if(n % 7 == 0) {
|
||||||
|
val ex = RuntimeException("Value $n can be divided by 7")
|
||||||
|
result.completeExceptionally(ex)
|
||||||
|
attempts += now to OperationOutcome.Failure(ex)
|
||||||
|
} else {
|
||||||
|
result.complete(n)
|
||||||
|
attempts += now to OperationOutcome.Success(n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
Assertions.assertTrue(attempts.size <= testArgs.maxAttempt)
|
||||||
|
val result = future.handle { res, ex ->
|
||||||
|
if(ex != null) {
|
||||||
|
val err = ex.cause ?: ex
|
||||||
|
log.debug(err.message, err)
|
||||||
|
OperationOutcome.Failure(err)
|
||||||
|
} else {
|
||||||
|
OperationOutcome.Success(res)
|
||||||
|
}
|
||||||
|
}.get()
|
||||||
|
for ((index, attempt) in attempts.withIndex()) {
|
||||||
|
val (timestamp, value) = attempt
|
||||||
|
if (index > 0) {
|
||||||
|
/* Check the delay for subsequent attempts is correct */
|
||||||
|
val previousAttempt = attempts[index - 1]
|
||||||
|
val expectedTimestamp =
|
||||||
|
previousAttempt.first + testArgs.initialDelay * Math.pow(testArgs.exp, index.toDouble()) * 1e6
|
||||||
|
val actualTimestamp = timestamp
|
||||||
|
val err = Math.abs(expectedTimestamp - actualTimestamp) / expectedTimestamp
|
||||||
|
Assertions.assertTrue(err < 1e-3)
|
||||||
|
}
|
||||||
|
if (index == attempts.size - 1 && index < testArgs.maxAttempt - 1) {
|
||||||
|
/*
|
||||||
|
* If the last attempt index is lower than the maximum number of attempts, then
|
||||||
|
* check the outcome handler returns DoNotRetry
|
||||||
|
*/
|
||||||
|
Assertions.assertTrue(outcomeHandler.shouldRetry(value) is OutcomeHandlerResult.DoNotRetry)
|
||||||
|
} else if (index < attempts.size - 1) {
|
||||||
|
/*
|
||||||
|
* If the attempt is not the last attempt check the outcome handler returns Retry
|
||||||
|
*/
|
||||||
|
Assertions.assertTrue(outcomeHandler.shouldRetry(value) is OutcomeHandlerResult.Retry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
21
rbcs-client/src/test/resources/logback.xml
Normal file
21
rbcs-client/src/test/resources/logback.xml
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" ?>
|
||||||
|
<!DOCTYPE configuration>
|
||||||
|
|
||||||
|
<configuration>
|
||||||
|
<import class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"/>
|
||||||
|
<import class="ch.qos.logback.core.ConsoleAppender"/>
|
||||||
|
|
||||||
|
<appender name="console" class="ConsoleAppender">
|
||||||
|
<target>System.err</target>
|
||||||
|
<encoder class="PatternLayoutEncoder">
|
||||||
|
<pattern>%d [%highlight(%-5level)] \(%thread\) %logger{36} -%kvp- %msg %n</pattern>
|
||||||
|
</encoder>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<root level="info">
|
||||||
|
<appender-ref ref="console"/>
|
||||||
|
</root>
|
||||||
|
<logger name="io.netty" level="info"/>
|
||||||
|
<logger name="com.google.code.yanf4j" level="warn"/>
|
||||||
|
<logger name="net.rubyeye.xmemcached" level="warn"/>
|
||||||
|
</configuration>
|
@@ -0,0 +1,16 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||||
|
<rbcs-client:profiles xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xmlns:rbcs-client="urn:net.woggioni.rbcs.client"
|
||||||
|
xs:schemaLocation="urn:net.woggioni.rbcs.client jms://net.woggioni.rbcs.client/net/woggioni/rbcs/client/schema/rbcs-client.xsd"
|
||||||
|
>
|
||||||
|
<profile name="profile1" base-url="https://rbcs1.example.com/">
|
||||||
|
<tls-client-auth
|
||||||
|
key-store-file="keystore.pfx"
|
||||||
|
key-store-password="password"
|
||||||
|
key-alias="woggioni@c962475fa38"
|
||||||
|
key-password="key-password"/>
|
||||||
|
</profile>
|
||||||
|
<profile name="profile2" base-url="https://rbcs2.example.com/">
|
||||||
|
<basic-auth user="user" password="password"/>
|
||||||
|
</profile>
|
||||||
|
</rbcs-client:profiles>
|
21
rbcs-common/build.gradle
Normal file
21
rbcs-common/build.gradle
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
|
||||||
|
plugins {
|
||||||
|
id 'java-library'
|
||||||
|
id 'maven-publish'
|
||||||
|
alias catalog.plugins.kotlin.jvm
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
implementation project(':rbcs-api')
|
||||||
|
implementation catalog.slf4j.api
|
||||||
|
implementation catalog.jwo
|
||||||
|
implementation catalog.netty.buffer
|
||||||
|
}
|
||||||
|
|
||||||
|
publishing {
|
||||||
|
publications {
|
||||||
|
maven(MavenPublication) {
|
||||||
|
from(components["java"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
11
rbcs-common/src/main/java/module-info.java
Normal file
11
rbcs-common/src/main/java/module-info.java
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
module net.woggioni.rbcs.common {
|
||||||
|
requires java.xml;
|
||||||
|
requires java.logging;
|
||||||
|
requires org.slf4j;
|
||||||
|
requires kotlin.stdlib;
|
||||||
|
requires net.woggioni.jwo;
|
||||||
|
requires io.netty.buffer;
|
||||||
|
|
||||||
|
provides java.net.spi.URLStreamHandlerProvider with net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory;
|
||||||
|
exports net.woggioni.rbcs.common;
|
||||||
|
}
|
@@ -0,0 +1,25 @@
|
|||||||
|
package net.woggioni.rbcs.common
|
||||||
|
|
||||||
|
import io.netty.buffer.ByteBuf
|
||||||
|
import java.io.InputStream
|
||||||
|
|
||||||
|
class ByteBufInputStream(private val buf : ByteBuf) : InputStream() {
|
||||||
|
override fun read(): Int {
|
||||||
|
return buf.takeIf {
|
||||||
|
it.readableBytes() > 0
|
||||||
|
}?.let(ByteBuf::readByte)
|
||||||
|
?.let(Byte::toInt) ?: -1
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun read(b: ByteArray, off: Int, len: Int): Int {
|
||||||
|
val readableBytes = buf.readableBytes()
|
||||||
|
if(readableBytes == 0) return -1
|
||||||
|
val result = len.coerceAtMost(readableBytes)
|
||||||
|
buf.readBytes(b, off, result)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {
|
||||||
|
buf.release()
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,18 @@
|
|||||||
|
package net.woggioni.rbcs.common
|
||||||
|
|
||||||
|
import io.netty.buffer.ByteBuf
|
||||||
|
import java.io.OutputStream
|
||||||
|
|
||||||
|
class ByteBufOutputStream(private val buf : ByteBuf) : OutputStream() {
|
||||||
|
override fun write(b: Int) {
|
||||||
|
buf.writeByte(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun write(b: ByteArray, off: Int, len: Int) {
|
||||||
|
buf.writeBytes(b, off, len)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {
|
||||||
|
buf.release()
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,7 @@
|
|||||||
|
package net.woggioni.rbcs.common
|
||||||
|
|
||||||
|
class ResourceNotFoundException(msg : String? = null, cause: Throwable? = null) : RuntimeException(msg, cause) {
|
||||||
|
}
|
||||||
|
|
||||||
|
class ModuleNotFoundException(msg : String? = null, cause: Throwable? = null) : RuntimeException(msg, cause) {
|
||||||
|
}
|
@@ -0,0 +1,8 @@
|
|||||||
|
package net.woggioni.rbcs.common
|
||||||
|
|
||||||
|
|
||||||
|
data class HostAndPort(val host: String, val port: Int = 0) {
|
||||||
|
override fun toString(): String {
|
||||||
|
return "$host:$port"
|
||||||
|
}
|
||||||
|
}
|
@@ -1,16 +1,12 @@
|
|||||||
package net.woggioni.gbcs
|
package net.woggioni.rbcs.common
|
||||||
|
|
||||||
import io.netty.channel.ChannelHandlerContext
|
|
||||||
import org.slf4j.Logger
|
import org.slf4j.Logger
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import java.net.InetSocketAddress
|
import org.slf4j.event.Level
|
||||||
import java.nio.file.Files
|
import java.nio.file.Files
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
import java.util.logging.LogManager
|
import java.util.logging.LogManager
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
inline fun <reified T> T.contextLogger() = LoggerFactory.getLogger(T::class.java)
|
inline fun <reified T> T.contextLogger() = LoggerFactory.getLogger(T::class.java)
|
||||||
|
|
||||||
inline fun Logger.traceParam(messageBuilder : () -> Pair<String, Array<Any>>) {
|
inline fun Logger.traceParam(messageBuilder : () -> Pair<String, Array<Any>>) {
|
||||||
@@ -57,6 +53,12 @@ inline fun log(log : Logger,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inline fun Logger.log(level : Level, messageBuilder : () -> String) {
|
||||||
|
if(isEnabledForLevel(level)) {
|
||||||
|
makeLoggingEventBuilder(level).log(messageBuilder())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
inline fun Logger.trace(messageBuilder : () -> String) {
|
inline fun Logger.trace(messageBuilder : () -> String) {
|
||||||
if(isTraceEnabled) {
|
if(isTraceEnabled) {
|
||||||
trace(messageBuilder())
|
trace(messageBuilder())
|
||||||
@@ -87,31 +89,6 @@ inline fun Logger.error(messageBuilder : () -> String) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
inline fun Logger.trace(ctx : ChannelHandlerContext, messageBuilder : () -> String) {
|
|
||||||
log(this, ctx, { isTraceEnabled }, { trace(it) } , messageBuilder)
|
|
||||||
}
|
|
||||||
inline fun Logger.debug(ctx : ChannelHandlerContext, messageBuilder : () -> String) {
|
|
||||||
log(this, ctx, { isDebugEnabled }, { debug(it) } , messageBuilder)
|
|
||||||
}
|
|
||||||
inline fun Logger.info(ctx : ChannelHandlerContext, messageBuilder : () -> String) {
|
|
||||||
log(this, ctx, { isInfoEnabled }, { info(it) } , messageBuilder)
|
|
||||||
}
|
|
||||||
inline fun Logger.warn(ctx : ChannelHandlerContext, messageBuilder : () -> String) {
|
|
||||||
log(this, ctx, { isWarnEnabled }, { warn(it) } , messageBuilder)
|
|
||||||
}
|
|
||||||
inline fun Logger.error(ctx : ChannelHandlerContext, messageBuilder : () -> String) {
|
|
||||||
log(this, ctx, { isErrorEnabled }, { error(it) } , messageBuilder)
|
|
||||||
}
|
|
||||||
|
|
||||||
inline fun log(log : Logger, ctx : ChannelHandlerContext,
|
|
||||||
filter : Logger.() -> Boolean,
|
|
||||||
loggerMethod : Logger.(String) -> Unit, messageBuilder : () -> String) {
|
|
||||||
if(log.filter()) {
|
|
||||||
val clientAddress = (ctx.channel().remoteAddress() as InetSocketAddress).address.hostAddress
|
|
||||||
log.loggerMethod(clientAddress + " - " + messageBuilder())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class LoggingConfig {
|
class LoggingConfig {
|
||||||
|
|
@@ -0,0 +1,46 @@
|
|||||||
|
package net.woggioni.rbcs.common
|
||||||
|
|
||||||
|
import java.security.SecureRandom
|
||||||
|
import java.security.spec.KeySpec
|
||||||
|
import java.util.Base64
|
||||||
|
import javax.crypto.SecretKeyFactory
|
||||||
|
import javax.crypto.spec.PBEKeySpec
|
||||||
|
|
||||||
|
object PasswordSecurity {
|
||||||
|
private const val KEY_LENGTH = 256
|
||||||
|
|
||||||
|
private fun concat(arr1: ByteArray, arr2: ByteArray): ByteArray {
|
||||||
|
val result = ByteArray(arr1.size + arr2.size)
|
||||||
|
var j = 0
|
||||||
|
for(element in arr1) {
|
||||||
|
result[j] = element
|
||||||
|
j += 1
|
||||||
|
}
|
||||||
|
for(element in arr2) {
|
||||||
|
result[j] = element
|
||||||
|
j += 1
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
fun hashPassword(password : String, salt : String? = null) : String {
|
||||||
|
val actualSalt = salt?.let(Base64.getDecoder()::decode) ?: SecureRandom().run {
|
||||||
|
val result = ByteArray(16)
|
||||||
|
nextBytes(result)
|
||||||
|
result
|
||||||
|
}
|
||||||
|
val spec: KeySpec = PBEKeySpec(password.toCharArray(), actualSalt, 10, KEY_LENGTH)
|
||||||
|
val factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1")
|
||||||
|
val hash = factory.generateSecret(spec).encoded
|
||||||
|
return String(Base64.getEncoder().encode(concat(hash, actualSalt)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fun decodePasswordHash(passwordHash : String) : Pair<ByteArray, ByteArray> {
|
||||||
|
val decoded = Base64.getDecoder().decode(passwordHash)
|
||||||
|
val hash = ByteArray(KEY_LENGTH / 8)
|
||||||
|
val salt = ByteArray(decoded.size - KEY_LENGTH / 8)
|
||||||
|
System.arraycopy(decoded, 0, hash, 0, hash.size)
|
||||||
|
System.arraycopy(decoded, hash.size, salt, 0, salt.size)
|
||||||
|
return hash to salt
|
||||||
|
}
|
||||||
|
}
|
29
rbcs-common/src/main/kotlin/net/woggioni/rbcs/common/RBCS.kt
Normal file
29
rbcs-common/src/main/kotlin/net/woggioni/rbcs/common/RBCS.kt
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
package net.woggioni.rbcs.common
|
||||||
|
|
||||||
|
import net.woggioni.jwo.JWO
|
||||||
|
import java.net.URI
|
||||||
|
import java.net.URL
|
||||||
|
import java.security.MessageDigest
|
||||||
|
|
||||||
|
object RBCS {
|
||||||
|
fun String.toUrl() : URL = URL.of(URI(this), null)
|
||||||
|
|
||||||
|
const val RBCS_NAMESPACE_URI: String = "urn:net.woggioni.rbcs.server"
|
||||||
|
const val RBCS_PREFIX: String = "rbcs"
|
||||||
|
const val XML_SCHEMA_NAMESPACE_URI = "http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
|
||||||
|
fun digest(
|
||||||
|
data: ByteArray,
|
||||||
|
md: MessageDigest = MessageDigest.getInstance("MD5")
|
||||||
|
): ByteArray {
|
||||||
|
md.update(data)
|
||||||
|
return md.digest()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun digestString(
|
||||||
|
data: ByteArray,
|
||||||
|
md: MessageDigest = MessageDigest.getInstance("MD5")
|
||||||
|
): String {
|
||||||
|
return JWO.bytesToHex(digest(data, md))
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,113 @@
|
|||||||
|
package net.woggioni.rbcs.common
|
||||||
|
|
||||||
|
import java.io.IOException
|
||||||
|
import java.io.InputStream
|
||||||
|
import java.net.URL
|
||||||
|
import java.net.URLConnection
|
||||||
|
import java.net.URLStreamHandler
|
||||||
|
import java.net.spi.URLStreamHandlerProvider
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean
|
||||||
|
import java.util.stream.Collectors
|
||||||
|
|
||||||
|
|
||||||
|
class RbcsUrlStreamHandlerFactory : URLStreamHandlerProvider() {
|
||||||
|
|
||||||
|
private class ClasspathHandler(private val classLoader: ClassLoader = RbcsUrlStreamHandlerFactory::class.java.classLoader) :
|
||||||
|
URLStreamHandler() {
|
||||||
|
|
||||||
|
override fun openConnection(u: URL): URLConnection? {
|
||||||
|
return javaClass.module
|
||||||
|
?.takeIf { m: Module -> m.layer != null }
|
||||||
|
?.let {
|
||||||
|
val path = u.path
|
||||||
|
val i = path.lastIndexOf('/')
|
||||||
|
val packageName = path.substring(0, i).replace('/', '.')
|
||||||
|
val modules = packageMap[packageName]!!
|
||||||
|
ClasspathResourceURLConnection(
|
||||||
|
u,
|
||||||
|
modules
|
||||||
|
)
|
||||||
|
}
|
||||||
|
?: classLoader.getResource(u.path)?.let(URL::openConnection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class JpmsHandler : URLStreamHandler() {
|
||||||
|
|
||||||
|
override fun openConnection(u: URL): URLConnection {
|
||||||
|
val moduleName = u.host
|
||||||
|
val thisModule = javaClass.module
|
||||||
|
val sourceModule =
|
||||||
|
thisModule
|
||||||
|
?.let(Module::getLayer)
|
||||||
|
?.let { layer: ModuleLayer ->
|
||||||
|
layer.findModule(moduleName).orElse(null)
|
||||||
|
} ?: if(thisModule.layer == null) {
|
||||||
|
thisModule
|
||||||
|
} else throw ModuleNotFoundException("Module '$moduleName' not found")
|
||||||
|
|
||||||
|
return JpmsResourceURLConnection(u, sourceModule)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class JpmsResourceURLConnection(url: URL, private val module: Module) : URLConnection(url) {
|
||||||
|
override fun connect() {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Throws(IOException::class)
|
||||||
|
override fun getInputStream(): InputStream {
|
||||||
|
val resource = getURL().path
|
||||||
|
return module.getResourceAsStream(resource)
|
||||||
|
?: throw ResourceNotFoundException("Resource '$resource' not found in module '${module.name}'")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun createURLStreamHandler(protocol: String): URLStreamHandler? {
|
||||||
|
return when (protocol) {
|
||||||
|
"classpath" -> ClasspathHandler()
|
||||||
|
"jpms" -> JpmsHandler()
|
||||||
|
else -> null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class ClasspathResourceURLConnection(url: URL?, private val modules: List<Module>) :
|
||||||
|
URLConnection(url) {
|
||||||
|
override fun connect() {}
|
||||||
|
|
||||||
|
override fun getInputStream(): InputStream? {
|
||||||
|
for (module in modules) {
|
||||||
|
val result = module.getResourceAsStream(getURL().path)
|
||||||
|
if (result != null) return result
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private val installed = AtomicBoolean(false)
|
||||||
|
fun install() {
|
||||||
|
if (!installed.getAndSet(true)) {
|
||||||
|
URL.setURLStreamHandlerFactory(RbcsUrlStreamHandlerFactory())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private val packageMap: Map<String, List<Module>> by lazy {
|
||||||
|
RbcsUrlStreamHandlerFactory::class.java.module.layer
|
||||||
|
.modules()
|
||||||
|
.stream()
|
||||||
|
.flatMap { m: Module ->
|
||||||
|
m.packages.stream()
|
||||||
|
.map { p: String -> p to m }
|
||||||
|
}
|
||||||
|
.collect(
|
||||||
|
Collectors.groupingBy(
|
||||||
|
Pair<String, Module>::first,
|
||||||
|
Collectors.mapping(
|
||||||
|
Pair<String, Module>::second,
|
||||||
|
Collectors.toUnmodifiableList<Module>()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
244
rbcs-common/src/main/kotlin/net/woggioni/rbcs/common/Xml.kt
Normal file
244
rbcs-common/src/main/kotlin/net/woggioni/rbcs/common/Xml.kt
Normal file
@@ -0,0 +1,244 @@
|
|||||||
|
package net.woggioni.rbcs.common
|
||||||
|
|
||||||
|
import net.woggioni.jwo.JWO
|
||||||
|
import org.slf4j.LoggerFactory
|
||||||
|
import org.slf4j.event.Level
|
||||||
|
import org.w3c.dom.Document
|
||||||
|
import org.w3c.dom.Element
|
||||||
|
import org.w3c.dom.Node
|
||||||
|
import org.w3c.dom.NodeList
|
||||||
|
import org.xml.sax.SAXNotRecognizedException
|
||||||
|
import org.xml.sax.SAXNotSupportedException
|
||||||
|
import org.xml.sax.SAXParseException
|
||||||
|
import java.io.InputStream
|
||||||
|
import java.io.OutputStream
|
||||||
|
import java.net.URL
|
||||||
|
import javax.xml.XMLConstants.ACCESS_EXTERNAL_DTD
|
||||||
|
import javax.xml.XMLConstants.ACCESS_EXTERNAL_SCHEMA
|
||||||
|
import javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING
|
||||||
|
import javax.xml.XMLConstants.W3C_XML_SCHEMA_NS_URI
|
||||||
|
import javax.xml.parsers.DocumentBuilder
|
||||||
|
import javax.xml.parsers.DocumentBuilderFactory
|
||||||
|
import javax.xml.transform.OutputKeys
|
||||||
|
import javax.xml.transform.TransformerFactory
|
||||||
|
import javax.xml.transform.dom.DOMSource
|
||||||
|
import javax.xml.transform.stream.StreamResult
|
||||||
|
import javax.xml.transform.stream.StreamSource
|
||||||
|
import javax.xml.validation.Schema
|
||||||
|
import javax.xml.validation.SchemaFactory
|
||||||
|
import org.xml.sax.ErrorHandler as ErrHandler
|
||||||
|
|
||||||
|
|
||||||
|
class NodeListIterator(private val nodeList: NodeList) : Iterator<Node> {
|
||||||
|
private var cursor: Int = 0
|
||||||
|
override fun hasNext(): Boolean {
|
||||||
|
return cursor < nodeList.length
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun next(): Node {
|
||||||
|
return if (hasNext()) nodeList.item(cursor++) else throw NoSuchElementException()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class ElementIterator(parent: Element, name: String? = null) : Iterator<Element> {
|
||||||
|
private val it: NodeListIterator
|
||||||
|
private val name: String?
|
||||||
|
private var next: Element?
|
||||||
|
|
||||||
|
init {
|
||||||
|
it = NodeListIterator(parent.childNodes)
|
||||||
|
this.name = name
|
||||||
|
next = getNext()
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun hasNext(): Boolean {
|
||||||
|
return next != null
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun next(): Element {
|
||||||
|
val result = next ?: throw NoSuchElementException()
|
||||||
|
next = getNext()
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun getNext(): Element? {
|
||||||
|
var result: Element? = null
|
||||||
|
while (it.hasNext()) {
|
||||||
|
val node: Node = it.next()
|
||||||
|
if (node is Element && (name == null || name == node.tagName)) {
|
||||||
|
result = node
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class Xml(val doc: Document, val element: Element) {
|
||||||
|
|
||||||
|
class ErrorHandler(private val fileURL: URL) : ErrHandler {
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private val log = LoggerFactory.getLogger(ErrorHandler::class.java)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun warning(ex: SAXParseException)= err(ex, Level.WARN)
|
||||||
|
|
||||||
|
private fun err(ex: SAXParseException, level: Level) {
|
||||||
|
log.log(level) {
|
||||||
|
"Problem at ${fileURL}:${ex.lineNumber}:${ex.columnNumber} parsing deployment configuration: ${ex.message}"
|
||||||
|
}
|
||||||
|
throw ex
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun error(ex: SAXParseException) = err(ex, Level.ERROR)
|
||||||
|
override fun fatalError(ex: SAXParseException) = err(ex, Level.ERROR)
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private val dictMap: Map<String, Map<String, Any>> = sequenceOf(
|
||||||
|
"env" to System.getenv().asSequence().map { (k, v) -> k to (v as Any) }.toMap(),
|
||||||
|
"sys" to System.getProperties().asSequence().map { (k, v) -> k as String to (v as Any) }.toMap()
|
||||||
|
).toMap()
|
||||||
|
|
||||||
|
private fun renderConfigurationTemplate(template: String): String {
|
||||||
|
return JWO.renderTemplate(template, emptyMap(), dictMap).replace("$$", "$")
|
||||||
|
}
|
||||||
|
|
||||||
|
fun Element.renderAttribute(name : String, namespaceURI: String? = null) = if(namespaceURI == null) {
|
||||||
|
getAttribute(name)
|
||||||
|
} else {
|
||||||
|
getAttributeNS(name, namespaceURI)
|
||||||
|
}.takeIf(String::isNotEmpty)?.let(Companion::renderConfigurationTemplate)
|
||||||
|
|
||||||
|
|
||||||
|
fun Element.asIterable() = Iterable { ElementIterator(this, null) }
|
||||||
|
fun NodeList.asIterable() = Iterable { NodeListIterator(this) }
|
||||||
|
|
||||||
|
private fun disableProperty(dbf: DocumentBuilderFactory, propertyName: String) {
|
||||||
|
try {
|
||||||
|
dbf.setAttribute(propertyName, "")
|
||||||
|
} catch (iae: IllegalArgumentException) {
|
||||||
|
// Property not supported.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun disableProperty(sf: SchemaFactory, propertyName: String) {
|
||||||
|
try {
|
||||||
|
sf.setProperty(propertyName, "")
|
||||||
|
} catch (ex: SAXNotRecognizedException) {
|
||||||
|
// Property not supported.
|
||||||
|
} catch (ex: SAXNotSupportedException) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getSchema(schema: URL): Schema {
|
||||||
|
val sf = SchemaFactory.newInstance(W3C_XML_SCHEMA_NS_URI)
|
||||||
|
sf.setFeature(FEATURE_SECURE_PROCESSING, false)
|
||||||
|
sf.errorHandler = ErrorHandler(schema)
|
||||||
|
return sf.newSchema(schema)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getSchema(inputStream: InputStream): Schema {
|
||||||
|
val sf = SchemaFactory.newInstance(W3C_XML_SCHEMA_NS_URI)
|
||||||
|
sf.setFeature(FEATURE_SECURE_PROCESSING, true)
|
||||||
|
return sf.newSchema(StreamSource(inputStream))
|
||||||
|
}
|
||||||
|
|
||||||
|
fun newDocumentBuilderFactory(schemaResourceURL: URL?): DocumentBuilderFactory {
|
||||||
|
val dbf = DocumentBuilderFactory.newInstance()
|
||||||
|
dbf.setFeature(FEATURE_SECURE_PROCESSING, false)
|
||||||
|
dbf.setAttribute(ACCESS_EXTERNAL_SCHEMA, "all")
|
||||||
|
disableProperty(dbf, ACCESS_EXTERNAL_DTD)
|
||||||
|
dbf.isExpandEntityReferences = true
|
||||||
|
dbf.isIgnoringComments = true
|
||||||
|
dbf.isNamespaceAware = true
|
||||||
|
dbf.isValidating = schemaResourceURL == null
|
||||||
|
dbf.setFeature("http://apache.org/xml/features/validation/schema", true)
|
||||||
|
schemaResourceURL?.let {
|
||||||
|
dbf.schema = getSchema(it)
|
||||||
|
}
|
||||||
|
return dbf
|
||||||
|
}
|
||||||
|
|
||||||
|
fun newDocumentBuilder(resource: URL, schemaResourceURL: URL?): DocumentBuilder {
|
||||||
|
val db = newDocumentBuilderFactory(schemaResourceURL).newDocumentBuilder()
|
||||||
|
db.setErrorHandler(ErrorHandler(resource))
|
||||||
|
return db
|
||||||
|
}
|
||||||
|
|
||||||
|
fun parseXmlResource(resource: URL, schemaResourceURL: URL?): Document {
|
||||||
|
val db = newDocumentBuilder(resource, schemaResourceURL)
|
||||||
|
return resource.openStream().use(db::parse)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun parseXml(sourceURL: URL, sourceStream: InputStream? = null, schemaResourceURL: URL? = null): Document {
|
||||||
|
val db = newDocumentBuilder(sourceURL, schemaResourceURL)
|
||||||
|
return sourceStream?.let(db::parse) ?: sourceURL.openStream().use(db::parse)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun write(doc: Document, output: OutputStream) {
|
||||||
|
val transformerFactory = TransformerFactory.newInstance()
|
||||||
|
val transformer = transformerFactory.newTransformer()
|
||||||
|
transformer.setOutputProperty(OutputKeys.INDENT, "yes")
|
||||||
|
transformer.setOutputProperty(OutputKeys.INDENT, "yes")
|
||||||
|
transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "4")
|
||||||
|
transformer.setOutputProperty(OutputKeys.STANDALONE, "yes")
|
||||||
|
transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8")
|
||||||
|
val source = DOMSource(doc)
|
||||||
|
val result = StreamResult(output)
|
||||||
|
transformer.transform(source, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun of(
|
||||||
|
namespaceURI: String,
|
||||||
|
qualifiedName: String,
|
||||||
|
schemaResourceURL: URL? = null,
|
||||||
|
cb: Xml.(el: Element) -> Unit
|
||||||
|
): Document {
|
||||||
|
val dbf = newDocumentBuilderFactory(schemaResourceURL)
|
||||||
|
val db = dbf.newDocumentBuilder()
|
||||||
|
val doc = db.newDocument()
|
||||||
|
val root = doc.createElementNS(namespaceURI, qualifiedName)
|
||||||
|
.also(doc::appendChild)
|
||||||
|
Xml(doc, root).cb(root)
|
||||||
|
return doc
|
||||||
|
}
|
||||||
|
|
||||||
|
fun of(doc: Document, el: Element, cb: Xml.(el: Element) -> Unit): Element {
|
||||||
|
Xml(doc, el).cb(el)
|
||||||
|
return el
|
||||||
|
}
|
||||||
|
|
||||||
|
fun Element.removeChildren() {
|
||||||
|
while (true) {
|
||||||
|
removeChild(firstChild ?: break)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun node(
|
||||||
|
name: String,
|
||||||
|
namespaceURI: String? = null,
|
||||||
|
attrs: Map<String, String> = emptyMap(),
|
||||||
|
cb: Xml.(el: Element) -> Unit = {}
|
||||||
|
): Element {
|
||||||
|
val child = doc.createElementNS(namespaceURI, name)
|
||||||
|
for ((key, value) in attrs) {
|
||||||
|
child.setAttribute(key, value)
|
||||||
|
}
|
||||||
|
return child
|
||||||
|
.also {
|
||||||
|
element.appendChild(it)
|
||||||
|
Xml(doc, it).cb(it)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun attr(key: String, value: String, namespaceURI: String? = null) {
|
||||||
|
element.setAttributeNS(namespaceURI, key, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun text(txt: String) {
|
||||||
|
element.appendChild(doc.createTextNode(txt))
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1 @@
|
|||||||
|
net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory
|
68
rbcs-server-memcache/build.gradle
Normal file
68
rbcs-server-memcache/build.gradle
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
plugins {
|
||||||
|
id 'java-library'
|
||||||
|
id 'maven-publish'
|
||||||
|
alias catalog.plugins.kotlin.jvm
|
||||||
|
}
|
||||||
|
|
||||||
|
configurations {
|
||||||
|
bundle {
|
||||||
|
canBeResolved = true
|
||||||
|
canBeConsumed = false
|
||||||
|
visible = false
|
||||||
|
transitive = false
|
||||||
|
|
||||||
|
resolutionStrategy {
|
||||||
|
dependencies {
|
||||||
|
exclude group: 'org.slf4j', module: 'slf4j-api'
|
||||||
|
exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib'
|
||||||
|
exclude group: 'org.jetbrains', module: 'annotations'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
release {
|
||||||
|
transitive = false
|
||||||
|
canBeConsumed = true
|
||||||
|
canBeResolved = true
|
||||||
|
visible = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
implementation project(':rbcs-common')
|
||||||
|
implementation project(':rbcs-api')
|
||||||
|
implementation catalog.jwo
|
||||||
|
implementation catalog.slf4j.api
|
||||||
|
implementation catalog.netty.common
|
||||||
|
implementation catalog.netty.codec.memcache
|
||||||
|
|
||||||
|
bundle catalog.netty.codec.memcache
|
||||||
|
|
||||||
|
testRuntimeOnly catalog.logback.classic
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.named(JavaPlugin.TEST_TASK_NAME, Test) {
|
||||||
|
systemProperty("io.netty.leakDetectionLevel", "PARANOID")
|
||||||
|
}
|
||||||
|
|
||||||
|
Provider<Tar> bundleTask = tasks.register("bundle", Tar) {
|
||||||
|
from(tasks.named(JavaPlugin.JAR_TASK_NAME))
|
||||||
|
from(configurations.bundle)
|
||||||
|
group = BasePlugin.BUILD_GROUP
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks.named(BasePlugin.ASSEMBLE_TASK_NAME) {
|
||||||
|
dependsOn(bundleTask)
|
||||||
|
}
|
||||||
|
|
||||||
|
artifacts {
|
||||||
|
release(bundleTask)
|
||||||
|
}
|
||||||
|
|
||||||
|
publishing {
|
||||||
|
publications {
|
||||||
|
maven(MavenPublication) {
|
||||||
|
artifact bundleTask
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
19
rbcs-server-memcache/src/main/java/module-info.java
Normal file
19
rbcs-server-memcache/src/main/java/module-info.java
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
import net.woggioni.rbcs.api.CacheProvider;
|
||||||
|
|
||||||
|
module net.woggioni.rbcs.server.memcache {
|
||||||
|
requires net.woggioni.rbcs.common;
|
||||||
|
requires net.woggioni.rbcs.api;
|
||||||
|
requires net.woggioni.jwo;
|
||||||
|
requires java.xml;
|
||||||
|
requires kotlin.stdlib;
|
||||||
|
requires io.netty.transport;
|
||||||
|
requires io.netty.codec;
|
||||||
|
requires io.netty.codec.memcache;
|
||||||
|
requires io.netty.common;
|
||||||
|
requires io.netty.buffer;
|
||||||
|
requires org.slf4j;
|
||||||
|
|
||||||
|
provides CacheProvider with net.woggioni.rbcs.server.memcache.MemcacheCacheProvider;
|
||||||
|
|
||||||
|
opens net.woggioni.rbcs.server.memcache.schema;
|
||||||
|
}
|
@@ -0,0 +1,4 @@
|
|||||||
|
package net.woggioni.rbcs.server.memcache
|
||||||
|
|
||||||
|
class MemcacheException(status : Short, msg : String? = null, cause : Throwable? = null)
|
||||||
|
: RuntimeException(msg ?: "Memcached status $status", cause)
|
@@ -0,0 +1,23 @@
|
|||||||
|
package net.woggioni.rbcs.server.memcache
|
||||||
|
|
||||||
|
import io.netty.buffer.ByteBuf
|
||||||
|
import net.woggioni.rbcs.api.Cache
|
||||||
|
import net.woggioni.rbcs.server.memcache.client.MemcacheClient
|
||||||
|
import java.nio.channels.ReadableByteChannel
|
||||||
|
import java.util.concurrent.CompletableFuture
|
||||||
|
|
||||||
|
class MemcacheCache(private val cfg : MemcacheCacheConfiguration) : Cache {
|
||||||
|
private val memcacheClient = MemcacheClient(cfg)
|
||||||
|
|
||||||
|
override fun get(key: String): CompletableFuture<ReadableByteChannel?> {
|
||||||
|
return memcacheClient.get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun put(key: String, content: ByteBuf): CompletableFuture<Void> {
|
||||||
|
return memcacheClient.put(key, content, cfg.maxAge)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {
|
||||||
|
memcacheClient.close()
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,40 @@
|
|||||||
|
package net.woggioni.rbcs.server.memcache
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.common.HostAndPort
|
||||||
|
import java.time.Duration
|
||||||
|
|
||||||
|
data class MemcacheCacheConfiguration(
|
||||||
|
val servers: List<Server>,
|
||||||
|
val maxAge: Duration = Duration.ofDays(1),
|
||||||
|
val maxSize: Int = 0x100000,
|
||||||
|
val digestAlgorithm: String? = null,
|
||||||
|
val compressionMode: CompressionMode? = null,
|
||||||
|
) : Configuration.Cache {
|
||||||
|
|
||||||
|
enum class CompressionMode {
|
||||||
|
/**
|
||||||
|
* Gzip mode
|
||||||
|
*/
|
||||||
|
GZIP,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deflate mode
|
||||||
|
*/
|
||||||
|
DEFLATE
|
||||||
|
}
|
||||||
|
|
||||||
|
data class Server(
|
||||||
|
val endpoint : HostAndPort,
|
||||||
|
val connectionTimeoutMillis : Int?,
|
||||||
|
val maxConnections : Int
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
override fun materialize() = MemcacheCache(this)
|
||||||
|
|
||||||
|
override fun getNamespaceURI() = "urn:net.woggioni.rbcs.server.memcache"
|
||||||
|
|
||||||
|
override fun getTypeName() = "memcacheCacheType"
|
||||||
|
}
|
||||||
|
|
@@ -0,0 +1,101 @@
|
|||||||
|
package net.woggioni.rbcs.server.memcache
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.CacheProvider
|
||||||
|
import net.woggioni.rbcs.api.exception.ConfigurationException
|
||||||
|
import net.woggioni.rbcs.common.RBCS
|
||||||
|
import net.woggioni.rbcs.common.HostAndPort
|
||||||
|
import net.woggioni.rbcs.common.Xml
|
||||||
|
import net.woggioni.rbcs.common.Xml.Companion.asIterable
|
||||||
|
import net.woggioni.rbcs.common.Xml.Companion.renderAttribute
|
||||||
|
import org.w3c.dom.Document
|
||||||
|
import org.w3c.dom.Element
|
||||||
|
import java.time.Duration
|
||||||
|
import java.time.temporal.ChronoUnit
|
||||||
|
|
||||||
|
|
||||||
|
class MemcacheCacheProvider : CacheProvider<MemcacheCacheConfiguration> {
|
||||||
|
override fun getXmlSchemaLocation() = "jpms://net.woggioni.rbcs.server.memcache/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd"
|
||||||
|
|
||||||
|
override fun getXmlType() = "memcacheCacheType"
|
||||||
|
|
||||||
|
override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server.memcache"
|
||||||
|
|
||||||
|
val xmlNamespacePrefix : String
|
||||||
|
get() = "rbcs-memcache"
|
||||||
|
|
||||||
|
override fun deserialize(el: Element): MemcacheCacheConfiguration {
|
||||||
|
val servers = mutableListOf<MemcacheCacheConfiguration.Server>()
|
||||||
|
val maxAge = el.renderAttribute("max-age")
|
||||||
|
?.let(Duration::parse)
|
||||||
|
?: Duration.ofDays(1)
|
||||||
|
val maxSize = el.renderAttribute("max-size")
|
||||||
|
?.let(String::toInt)
|
||||||
|
?: 0x100000
|
||||||
|
val compressionMode = el.renderAttribute("compression-mode")
|
||||||
|
?.let {
|
||||||
|
when (it) {
|
||||||
|
"gzip" -> MemcacheCacheConfiguration.CompressionMode.GZIP
|
||||||
|
"deflate" -> MemcacheCacheConfiguration.CompressionMode.DEFLATE
|
||||||
|
else -> MemcacheCacheConfiguration.CompressionMode.DEFLATE
|
||||||
|
}
|
||||||
|
}
|
||||||
|
?: MemcacheCacheConfiguration.CompressionMode.DEFLATE
|
||||||
|
val digestAlgorithm = el.renderAttribute("digest")
|
||||||
|
for (child in el.asIterable()) {
|
||||||
|
when (child.nodeName) {
|
||||||
|
"server" -> {
|
||||||
|
val host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required")
|
||||||
|
val port = child.renderAttribute("port")?.toInt() ?: throw ConfigurationException("port attribute is required")
|
||||||
|
val maxConnections = child.renderAttribute("max-connections")?.toInt() ?: 1
|
||||||
|
val connectionTimeout = child.renderAttribute("connection-timeout")
|
||||||
|
?.let(Duration::parse)
|
||||||
|
?.let(Duration::toMillis)
|
||||||
|
?.let(Long::toInt)
|
||||||
|
?: 10000
|
||||||
|
servers.add(MemcacheCacheConfiguration.Server(HostAndPort(host, port), connectionTimeout, maxConnections))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return MemcacheCacheConfiguration(
|
||||||
|
servers,
|
||||||
|
maxAge,
|
||||||
|
maxSize,
|
||||||
|
digestAlgorithm,
|
||||||
|
compressionMode,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun serialize(doc: Document, cache: MemcacheCacheConfiguration) = cache.run {
|
||||||
|
val result = doc.createElement("cache")
|
||||||
|
Xml.of(doc, result) {
|
||||||
|
attr("xmlns:${xmlNamespacePrefix}", xmlNamespace, namespaceURI = "http://www.w3.org/2000/xmlns/")
|
||||||
|
|
||||||
|
attr("xs:type", "${xmlNamespacePrefix}:$xmlType", RBCS.XML_SCHEMA_NAMESPACE_URI)
|
||||||
|
for (server in servers) {
|
||||||
|
node("server") {
|
||||||
|
attr("host", server.endpoint.host)
|
||||||
|
attr("port", server.endpoint.port.toString())
|
||||||
|
server.connectionTimeoutMillis?.let { connectionTimeoutMillis ->
|
||||||
|
attr("connection-timeout", Duration.of(connectionTimeoutMillis.toLong(), ChronoUnit.MILLIS).toString())
|
||||||
|
}
|
||||||
|
attr("max-connections", server.maxConnections.toString())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
attr("max-age", maxAge.toString())
|
||||||
|
attr("max-size", maxSize.toString())
|
||||||
|
digestAlgorithm?.let { digestAlgorithm ->
|
||||||
|
attr("digest", digestAlgorithm)
|
||||||
|
}
|
||||||
|
compressionMode?.let { compressionMode ->
|
||||||
|
attr(
|
||||||
|
"compression-mode", when (compressionMode) {
|
||||||
|
MemcacheCacheConfiguration.CompressionMode.GZIP -> "gzip"
|
||||||
|
MemcacheCacheConfiguration.CompressionMode.DEFLATE -> "deflate"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,257 @@
|
|||||||
|
package net.woggioni.rbcs.server.memcache.client
|
||||||
|
|
||||||
|
|
||||||
|
import io.netty.bootstrap.Bootstrap
|
||||||
|
import io.netty.buffer.ByteBuf
|
||||||
|
import io.netty.buffer.Unpooled
|
||||||
|
import io.netty.channel.Channel
|
||||||
|
import io.netty.channel.ChannelHandlerContext
|
||||||
|
import io.netty.channel.ChannelOption
|
||||||
|
import io.netty.channel.ChannelPipeline
|
||||||
|
import io.netty.channel.SimpleChannelInboundHandler
|
||||||
|
import io.netty.channel.nio.NioEventLoopGroup
|
||||||
|
import io.netty.channel.pool.AbstractChannelPoolHandler
|
||||||
|
import io.netty.channel.pool.ChannelPool
|
||||||
|
import io.netty.channel.pool.FixedChannelPool
|
||||||
|
import io.netty.channel.socket.nio.NioSocketChannel
|
||||||
|
import io.netty.handler.codec.DecoderException
|
||||||
|
import io.netty.handler.codec.memcache.binary.BinaryMemcacheClientCodec
|
||||||
|
import io.netty.handler.codec.memcache.binary.BinaryMemcacheObjectAggregator
|
||||||
|
import io.netty.handler.codec.memcache.binary.BinaryMemcacheOpcodes
|
||||||
|
import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponseStatus
|
||||||
|
import io.netty.handler.codec.memcache.binary.DefaultFullBinaryMemcacheRequest
|
||||||
|
import io.netty.handler.codec.memcache.binary.FullBinaryMemcacheRequest
|
||||||
|
import io.netty.handler.codec.memcache.binary.FullBinaryMemcacheResponse
|
||||||
|
import io.netty.util.concurrent.GenericFutureListener
|
||||||
|
import net.woggioni.rbcs.common.ByteBufInputStream
|
||||||
|
import net.woggioni.rbcs.common.ByteBufOutputStream
|
||||||
|
import net.woggioni.rbcs.common.RBCS.digest
|
||||||
|
import net.woggioni.rbcs.common.HostAndPort
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.rbcs.server.memcache.MemcacheCacheConfiguration
|
||||||
|
import net.woggioni.rbcs.server.memcache.MemcacheException
|
||||||
|
import net.woggioni.jwo.JWO
|
||||||
|
import java.net.InetSocketAddress
|
||||||
|
import java.nio.channels.Channels
|
||||||
|
import java.nio.channels.ReadableByteChannel
|
||||||
|
import java.security.MessageDigest
|
||||||
|
import java.time.Duration
|
||||||
|
import java.time.Instant
|
||||||
|
import java.util.concurrent.CompletableFuture
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import java.util.zip.Deflater
|
||||||
|
import java.util.zip.DeflaterOutputStream
|
||||||
|
import java.util.zip.GZIPInputStream
|
||||||
|
import java.util.zip.GZIPOutputStream
|
||||||
|
import java.util.zip.InflaterInputStream
|
||||||
|
import io.netty.util.concurrent.Future as NettyFuture
|
||||||
|
|
||||||
|
|
||||||
|
class MemcacheClient(private val cfg: MemcacheCacheConfiguration) : AutoCloseable {
|
||||||
|
|
||||||
|
private companion object {
|
||||||
|
@JvmStatic
|
||||||
|
private val log = contextLogger()
|
||||||
|
}
|
||||||
|
|
||||||
|
private val group: NioEventLoopGroup
|
||||||
|
private val connectionPool: MutableMap<HostAndPort, ChannelPool> = ConcurrentHashMap()
|
||||||
|
|
||||||
|
init {
|
||||||
|
group = NioEventLoopGroup()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun newConnectionPool(server: MemcacheCacheConfiguration.Server): FixedChannelPool {
|
||||||
|
val bootstrap = Bootstrap().apply {
|
||||||
|
group(group)
|
||||||
|
channel(NioSocketChannel::class.java)
|
||||||
|
option(ChannelOption.SO_KEEPALIVE, true)
|
||||||
|
remoteAddress(InetSocketAddress(server.endpoint.host, server.endpoint.port))
|
||||||
|
server.connectionTimeoutMillis?.let {
|
||||||
|
option(ChannelOption.CONNECT_TIMEOUT_MILLIS, it)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val channelPoolHandler = object : AbstractChannelPoolHandler() {
|
||||||
|
|
||||||
|
override fun channelCreated(ch: Channel) {
|
||||||
|
val pipeline: ChannelPipeline = ch.pipeline()
|
||||||
|
pipeline.addLast(BinaryMemcacheClientCodec())
|
||||||
|
pipeline.addLast(BinaryMemcacheObjectAggregator(cfg.maxSize))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return FixedChannelPool(bootstrap, channelPoolHandler, server.maxConnections)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private fun sendRequest(request: FullBinaryMemcacheRequest): CompletableFuture<FullBinaryMemcacheResponse> {
|
||||||
|
|
||||||
|
val server = cfg.servers.let { servers ->
|
||||||
|
if (servers.size > 1) {
|
||||||
|
val key = request.key().duplicate()
|
||||||
|
var checksum = 0
|
||||||
|
while (key.readableBytes() > 4) {
|
||||||
|
val byte = key.readInt()
|
||||||
|
checksum = checksum xor byte
|
||||||
|
}
|
||||||
|
while (key.readableBytes() > 0) {
|
||||||
|
val byte = key.readByte()
|
||||||
|
checksum = checksum xor byte.toInt()
|
||||||
|
}
|
||||||
|
servers[checksum % servers.size]
|
||||||
|
} else {
|
||||||
|
servers.first()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val response = CompletableFuture<FullBinaryMemcacheResponse>()
|
||||||
|
// Custom handler for processing responses
|
||||||
|
val pool = connectionPool.computeIfAbsent(server.endpoint) {
|
||||||
|
newConnectionPool(server)
|
||||||
|
}
|
||||||
|
pool.acquire().addListener(object : GenericFutureListener<NettyFuture<Channel>> {
|
||||||
|
override fun operationComplete(channelFuture: NettyFuture<Channel>) {
|
||||||
|
if (channelFuture.isSuccess) {
|
||||||
|
val channel = channelFuture.now
|
||||||
|
val pipeline = channel.pipeline()
|
||||||
|
channel.pipeline()
|
||||||
|
.addLast("client-handler", object : SimpleChannelInboundHandler<FullBinaryMemcacheResponse>() {
|
||||||
|
override fun channelRead0(
|
||||||
|
ctx: ChannelHandlerContext,
|
||||||
|
msg: FullBinaryMemcacheResponse
|
||||||
|
) {
|
||||||
|
pipeline.removeLast()
|
||||||
|
pool.release(channel)
|
||||||
|
msg.touch("The method's caller must remember to release this")
|
||||||
|
response.complete(msg.retain())
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
|
||||||
|
val ex = when (cause) {
|
||||||
|
is DecoderException -> cause.cause!!
|
||||||
|
else -> cause
|
||||||
|
}
|
||||||
|
ctx.close()
|
||||||
|
pipeline.removeLast()
|
||||||
|
pool.release(channel)
|
||||||
|
response.completeExceptionally(ex)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
request.touch()
|
||||||
|
channel.writeAndFlush(request)
|
||||||
|
} else {
|
||||||
|
response.completeExceptionally(channelFuture.cause())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun encodeExpiry(expiry: Duration): Int {
|
||||||
|
val expirySeconds = expiry.toSeconds()
|
||||||
|
return expirySeconds.toInt().takeIf { it.toLong() == expirySeconds }
|
||||||
|
?: Instant.ofEpochSecond(expirySeconds).epochSecond.toInt()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun get(key: String): CompletableFuture<ReadableByteChannel?> {
|
||||||
|
val request = (cfg.digestAlgorithm
|
||||||
|
?.let(MessageDigest::getInstance)
|
||||||
|
?.let { md ->
|
||||||
|
digest(key.toByteArray(), md)
|
||||||
|
} ?: key.toByteArray(Charsets.UTF_8)).let { digest ->
|
||||||
|
DefaultFullBinaryMemcacheRequest(Unpooled.wrappedBuffer(digest), null).apply {
|
||||||
|
setOpcode(BinaryMemcacheOpcodes.GET)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sendRequest(request).thenApply { response ->
|
||||||
|
try {
|
||||||
|
when (val status = response.status()) {
|
||||||
|
BinaryMemcacheResponseStatus.SUCCESS -> {
|
||||||
|
val compressionMode = cfg.compressionMode
|
||||||
|
val content = response.content().retain()
|
||||||
|
content.touch()
|
||||||
|
if (compressionMode != null) {
|
||||||
|
when (compressionMode) {
|
||||||
|
MemcacheCacheConfiguration.CompressionMode.GZIP -> {
|
||||||
|
GZIPInputStream(ByteBufInputStream(content))
|
||||||
|
}
|
||||||
|
|
||||||
|
MemcacheCacheConfiguration.CompressionMode.DEFLATE -> {
|
||||||
|
InflaterInputStream(ByteBufInputStream(content))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
ByteBufInputStream(content)
|
||||||
|
}.let(Channels::newChannel)
|
||||||
|
}
|
||||||
|
|
||||||
|
BinaryMemcacheResponseStatus.KEY_ENOENT -> {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
|
||||||
|
else -> throw MemcacheException(status)
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
response.release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun put(key: String, content: ByteBuf, expiry: Duration, cas: Long? = null): CompletableFuture<Void> {
|
||||||
|
val request = (cfg.digestAlgorithm
|
||||||
|
?.let(MessageDigest::getInstance)
|
||||||
|
?.let { md ->
|
||||||
|
digest(key.toByteArray(), md)
|
||||||
|
} ?: key.toByteArray(Charsets.UTF_8)).let { digest ->
|
||||||
|
val extras = Unpooled.buffer(8, 8)
|
||||||
|
extras.writeInt(0)
|
||||||
|
extras.writeInt(encodeExpiry(expiry))
|
||||||
|
val compressionMode = cfg.compressionMode
|
||||||
|
content.retain()
|
||||||
|
val payload = if (compressionMode != null) {
|
||||||
|
val inputStream = ByteBufInputStream(content)
|
||||||
|
val buf = content.alloc().buffer()
|
||||||
|
buf.retain()
|
||||||
|
val outputStream = when (compressionMode) {
|
||||||
|
MemcacheCacheConfiguration.CompressionMode.GZIP -> {
|
||||||
|
GZIPOutputStream(ByteBufOutputStream(buf))
|
||||||
|
}
|
||||||
|
|
||||||
|
MemcacheCacheConfiguration.CompressionMode.DEFLATE -> {
|
||||||
|
DeflaterOutputStream(ByteBufOutputStream(buf), Deflater(Deflater.DEFAULT_COMPRESSION, false))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
inputStream.use { i ->
|
||||||
|
outputStream.use { o ->
|
||||||
|
JWO.copy(i, o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buf
|
||||||
|
} else {
|
||||||
|
content
|
||||||
|
}
|
||||||
|
DefaultFullBinaryMemcacheRequest(Unpooled.wrappedBuffer(digest), extras, payload).apply {
|
||||||
|
setOpcode(BinaryMemcacheOpcodes.SET)
|
||||||
|
cas?.let(this::setCas)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sendRequest(request).thenApply { response ->
|
||||||
|
try {
|
||||||
|
when (val status = response.status()) {
|
||||||
|
BinaryMemcacheResponseStatus.SUCCESS -> null
|
||||||
|
else -> throw MemcacheException(status)
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
response.release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
fun shutDown(): NettyFuture<*> {
|
||||||
|
return group.shutdownGracefully()
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {
|
||||||
|
shutDown().sync()
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1 @@
|
|||||||
|
net.woggioni.rbcs.server.memcache.MemcacheCacheProvider
|
@@ -0,0 +1,37 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||||
|
<xs:schema targetNamespace="urn:net.woggioni.rbcs.server.memcache"
|
||||||
|
xmlns:rbcs-memcache="urn:net.woggioni.rbcs.server.memcache"
|
||||||
|
xmlns:rbcs="urn:net.woggioni.rbcs.server"
|
||||||
|
xmlns:xs="http://www.w3.org/2001/XMLSchema">
|
||||||
|
|
||||||
|
<xs:import schemaLocation="jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs.xsd" namespace="urn:net.woggioni.rbcs.server"/>
|
||||||
|
|
||||||
|
<xs:complexType name="memcacheServerType">
|
||||||
|
<xs:attribute name="host" type="xs:token" use="required"/>
|
||||||
|
<xs:attribute name="port" type="xs:positiveInteger" use="required"/>
|
||||||
|
<xs:attribute name="connection-timeout" type="xs:duration"/>
|
||||||
|
<xs:attribute name="max-connections" type="xs:positiveInteger" default="1"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="memcacheCacheType">
|
||||||
|
<xs:complexContent>
|
||||||
|
<xs:extension base="rbcs:cacheType">
|
||||||
|
<xs:sequence maxOccurs="unbounded">
|
||||||
|
<xs:element name="server" type="rbcs-memcache:memcacheServerType"/>
|
||||||
|
</xs:sequence>
|
||||||
|
<xs:attribute name="max-age" type="xs:duration" default="P1D"/>
|
||||||
|
<xs:attribute name="max-size" type="xs:unsignedInt" default="1048576"/>
|
||||||
|
<xs:attribute name="digest" type="xs:token" />
|
||||||
|
<xs:attribute name="compression-mode" type="rbcs-memcache:compressionType"/>
|
||||||
|
</xs:extension>
|
||||||
|
</xs:complexContent>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:simpleType name="compressionType">
|
||||||
|
<xs:restriction base="xs:token">
|
||||||
|
<xs:enumeration value="deflate"/>
|
||||||
|
<xs:enumeration value="gzip"/>
|
||||||
|
</xs:restriction>
|
||||||
|
</xs:simpleType>
|
||||||
|
|
||||||
|
</xs:schema>
|
38
rbcs-server/build.gradle
Normal file
38
rbcs-server/build.gradle
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
plugins {
|
||||||
|
id 'java-library'
|
||||||
|
alias catalog.plugins.kotlin.jvm
|
||||||
|
id 'jacoco'
|
||||||
|
id 'maven-publish'
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
implementation catalog.jwo
|
||||||
|
implementation catalog.slf4j.api
|
||||||
|
implementation catalog.netty.codec.http
|
||||||
|
|
||||||
|
api project(':rbcs-common')
|
||||||
|
api project(':rbcs-api')
|
||||||
|
|
||||||
|
// runtimeOnly catalog.slf4j.jdk14
|
||||||
|
testRuntimeOnly catalog.logback.classic
|
||||||
|
|
||||||
|
testImplementation catalog.bcprov.jdk18on
|
||||||
|
testImplementation catalog.bcpkix.jdk18on
|
||||||
|
|
||||||
|
testRuntimeOnly project(":rbcs-server-memcache")
|
||||||
|
}
|
||||||
|
|
||||||
|
test {
|
||||||
|
systemProperty("io.netty.leakDetectionLevel", "PARANOID")
|
||||||
|
systemProperty("jdk.httpclient.redirects.retrylimit", "1")
|
||||||
|
}
|
||||||
|
|
||||||
|
publishing {
|
||||||
|
publications {
|
||||||
|
maven(MavenPublication) {
|
||||||
|
from(components["java"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
29
rbcs-server/src/main/java/module-info.java
Normal file
29
rbcs-server/src/main/java/module-info.java
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
import net.woggioni.rbcs.api.CacheProvider;
|
||||||
|
import net.woggioni.rbcs.server.cache.FileSystemCacheProvider;
|
||||||
|
import net.woggioni.rbcs.server.cache.InMemoryCacheProvider;
|
||||||
|
|
||||||
|
module net.woggioni.rbcs.server {
|
||||||
|
requires java.sql;
|
||||||
|
requires java.xml;
|
||||||
|
requires java.logging;
|
||||||
|
requires java.naming;
|
||||||
|
requires kotlin.stdlib;
|
||||||
|
requires io.netty.buffer;
|
||||||
|
requires io.netty.transport;
|
||||||
|
requires io.netty.codec.http;
|
||||||
|
requires io.netty.common;
|
||||||
|
requires io.netty.handler;
|
||||||
|
requires io.netty.codec;
|
||||||
|
requires org.slf4j;
|
||||||
|
requires net.woggioni.jwo;
|
||||||
|
requires net.woggioni.rbcs.common;
|
||||||
|
requires net.woggioni.rbcs.api;
|
||||||
|
|
||||||
|
exports net.woggioni.rbcs.server;
|
||||||
|
|
||||||
|
opens net.woggioni.rbcs.server;
|
||||||
|
opens net.woggioni.rbcs.server.schema;
|
||||||
|
|
||||||
|
uses CacheProvider;
|
||||||
|
provides CacheProvider with FileSystemCacheProvider, InMemoryCacheProvider;
|
||||||
|
}
|
@@ -0,0 +1,30 @@
|
|||||||
|
package net.woggioni.rbcs.server
|
||||||
|
|
||||||
|
import io.netty.channel.ChannelHandlerContext
|
||||||
|
import org.slf4j.Logger
|
||||||
|
import java.net.InetSocketAddress
|
||||||
|
|
||||||
|
inline fun Logger.trace(ctx : ChannelHandlerContext, messageBuilder : () -> String) {
|
||||||
|
log(this, ctx, { isTraceEnabled }, { trace(it) } , messageBuilder)
|
||||||
|
}
|
||||||
|
inline fun Logger.debug(ctx : ChannelHandlerContext, messageBuilder : () -> String) {
|
||||||
|
log(this, ctx, { isDebugEnabled }, { debug(it) } , messageBuilder)
|
||||||
|
}
|
||||||
|
inline fun Logger.info(ctx : ChannelHandlerContext, messageBuilder : () -> String) {
|
||||||
|
log(this, ctx, { isInfoEnabled }, { info(it) } , messageBuilder)
|
||||||
|
}
|
||||||
|
inline fun Logger.warn(ctx : ChannelHandlerContext, messageBuilder : () -> String) {
|
||||||
|
log(this, ctx, { isWarnEnabled }, { warn(it) } , messageBuilder)
|
||||||
|
}
|
||||||
|
inline fun Logger.error(ctx : ChannelHandlerContext, messageBuilder : () -> String) {
|
||||||
|
log(this, ctx, { isErrorEnabled }, { error(it) } , messageBuilder)
|
||||||
|
}
|
||||||
|
|
||||||
|
inline fun log(log : Logger, ctx : ChannelHandlerContext,
|
||||||
|
filter : Logger.() -> Boolean,
|
||||||
|
loggerMethod : Logger.(String) -> Unit, messageBuilder : () -> String) {
|
||||||
|
if(log.filter()) {
|
||||||
|
val clientAddress = (ctx.channel().remoteAddress() as InetSocketAddress).address.hostAddress
|
||||||
|
log.loggerMethod(clientAddress + " - " + messageBuilder())
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,432 @@
|
|||||||
|
package net.woggioni.rbcs.server
|
||||||
|
|
||||||
|
import io.netty.bootstrap.ServerBootstrap
|
||||||
|
import io.netty.buffer.ByteBuf
|
||||||
|
import io.netty.channel.Channel
|
||||||
|
import io.netty.channel.ChannelFuture
|
||||||
|
import io.netty.channel.ChannelHandler.Sharable
|
||||||
|
import io.netty.channel.ChannelHandlerContext
|
||||||
|
import io.netty.channel.ChannelInboundHandlerAdapter
|
||||||
|
import io.netty.channel.ChannelInitializer
|
||||||
|
import io.netty.channel.ChannelOption
|
||||||
|
import io.netty.channel.ChannelPromise
|
||||||
|
import io.netty.channel.nio.NioEventLoopGroup
|
||||||
|
import io.netty.channel.socket.nio.NioServerSocketChannel
|
||||||
|
import io.netty.handler.codec.compression.CompressionOptions
|
||||||
|
import io.netty.handler.codec.http.DefaultHttpContent
|
||||||
|
import io.netty.handler.codec.http.HttpContentCompressor
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderNames
|
||||||
|
import io.netty.handler.codec.http.HttpObjectAggregator
|
||||||
|
import io.netty.handler.codec.http.HttpRequest
|
||||||
|
import io.netty.handler.codec.http.HttpServerCodec
|
||||||
|
import io.netty.handler.ssl.ClientAuth
|
||||||
|
import io.netty.handler.ssl.SslContext
|
||||||
|
import io.netty.handler.ssl.SslContextBuilder
|
||||||
|
import io.netty.handler.ssl.SslHandler
|
||||||
|
import io.netty.handler.stream.ChunkedWriteHandler
|
||||||
|
import io.netty.handler.timeout.IdleState
|
||||||
|
import io.netty.handler.timeout.IdleStateEvent
|
||||||
|
import io.netty.handler.timeout.IdleStateHandler
|
||||||
|
import io.netty.util.AttributeKey
|
||||||
|
import io.netty.util.concurrent.DefaultEventExecutorGroup
|
||||||
|
import io.netty.util.concurrent.EventExecutorGroup
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.api.exception.ConfigurationException
|
||||||
|
import net.woggioni.rbcs.common.RBCS.toUrl
|
||||||
|
import net.woggioni.rbcs.common.PasswordSecurity.decodePasswordHash
|
||||||
|
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
|
||||||
|
import net.woggioni.rbcs.common.Xml
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.rbcs.common.debug
|
||||||
|
import net.woggioni.rbcs.common.info
|
||||||
|
import net.woggioni.rbcs.server.auth.AbstractNettyHttpAuthenticator
|
||||||
|
import net.woggioni.rbcs.server.auth.Authorizer
|
||||||
|
import net.woggioni.rbcs.server.auth.ClientCertificateValidator
|
||||||
|
import net.woggioni.rbcs.server.auth.RoleAuthorizer
|
||||||
|
import net.woggioni.rbcs.server.configuration.Parser
|
||||||
|
import net.woggioni.rbcs.server.configuration.Serializer
|
||||||
|
import net.woggioni.rbcs.server.exception.ExceptionHandler
|
||||||
|
import net.woggioni.rbcs.server.handler.ServerHandler
|
||||||
|
import net.woggioni.rbcs.server.throttling.ThrottlingHandler
|
||||||
|
import net.woggioni.jwo.JWO
|
||||||
|
import net.woggioni.jwo.Tuple2
|
||||||
|
import java.io.OutputStream
|
||||||
|
import java.net.InetSocketAddress
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Path
|
||||||
|
import java.security.KeyStore
|
||||||
|
import java.security.PrivateKey
|
||||||
|
import java.security.cert.X509Certificate
|
||||||
|
import java.util.Arrays
|
||||||
|
import java.util.Base64
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
import java.util.regex.Matcher
|
||||||
|
import java.util.regex.Pattern
|
||||||
|
import javax.naming.ldap.LdapName
|
||||||
|
import javax.net.ssl.SSLPeerUnverifiedException
|
||||||
|
|
||||||
|
class RemoteBuildCacheServer(private val cfg: Configuration) {
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
|
||||||
|
val userAttribute: AttributeKey<Configuration.User> = AttributeKey.valueOf("user")
|
||||||
|
val groupAttribute: AttributeKey<Set<Configuration.Group>> = AttributeKey.valueOf("group")
|
||||||
|
|
||||||
|
val DEFAULT_CONFIGURATION_URL by lazy { "classpath:net/woggioni/rbcs/server/rbcs-default.xml".toUrl() }
|
||||||
|
private const val SSL_HANDLER_NAME = "sslHandler"
|
||||||
|
|
||||||
|
fun loadConfiguration(configurationFile: Path): Configuration {
|
||||||
|
val doc = Files.newInputStream(configurationFile).use {
|
||||||
|
Xml.parseXml(configurationFile.toUri().toURL(), it)
|
||||||
|
}
|
||||||
|
return Parser.parse(doc)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun dumpConfiguration(conf: Configuration, outputStream: OutputStream) {
|
||||||
|
Xml.write(Serializer.serialize(conf), outputStream)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class HttpChunkContentCompressor(
|
||||||
|
threshold: Int,
|
||||||
|
vararg compressionOptions: CompressionOptions = emptyArray()
|
||||||
|
) : HttpContentCompressor(threshold, *compressionOptions) {
|
||||||
|
override fun write(ctx: ChannelHandlerContext, msg: Any, promise: ChannelPromise) {
|
||||||
|
var message: Any? = msg
|
||||||
|
if (message is ByteBuf) {
|
||||||
|
// convert ByteBuf to HttpContent to make it work with compression. This is needed as we use the
|
||||||
|
// ChunkedWriteHandler to send files when compression is enabled.
|
||||||
|
val buff = message
|
||||||
|
if (buff.isReadable) {
|
||||||
|
// We only encode non empty buffers, as empty buffers can be used for determining when
|
||||||
|
// the content has been flushed and it confuses the HttpContentCompressor
|
||||||
|
// if we let it go
|
||||||
|
message = DefaultHttpContent(buff)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
super.write(ctx, message, promise)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Sharable
|
||||||
|
private class ClientCertificateAuthenticator(
|
||||||
|
authorizer: Authorizer,
|
||||||
|
private val anonymousUserGroups: Set<Configuration.Group>?,
|
||||||
|
private val userExtractor: Configuration.UserExtractor?,
|
||||||
|
private val groupExtractor: Configuration.GroupExtractor?,
|
||||||
|
) : AbstractNettyHttpAuthenticator(authorizer) {
|
||||||
|
|
||||||
|
override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? {
|
||||||
|
return try {
|
||||||
|
val sslHandler = (ctx.pipeline().get(SSL_HANDLER_NAME) as? SslHandler)
|
||||||
|
?: throw ConfigurationException("Client certificate authentication cannot be used when TLS is disabled")
|
||||||
|
val sslEngine = sslHandler.engine()
|
||||||
|
sslEngine.session.peerCertificates.takeIf {
|
||||||
|
it.isNotEmpty()
|
||||||
|
}?.let { peerCertificates ->
|
||||||
|
val clientCertificate = peerCertificates.first() as X509Certificate
|
||||||
|
val user = userExtractor?.extract(clientCertificate)
|
||||||
|
val group = groupExtractor?.extract(clientCertificate)
|
||||||
|
val allGroups = ((user?.groups ?: emptySet()).asSequence() + sequenceOf(group).filterNotNull()).toSet()
|
||||||
|
AuthenticationResult(user, allGroups)
|
||||||
|
} ?: anonymousUserGroups?.let{ AuthenticationResult(null, it) }
|
||||||
|
} catch (es: SSLPeerUnverifiedException) {
|
||||||
|
anonymousUserGroups?.let{ AuthenticationResult(null, it) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Sharable
|
||||||
|
private class NettyHttpBasicAuthenticator(
|
||||||
|
private val users: Map<String, Configuration.User>, authorizer: Authorizer
|
||||||
|
) : AbstractNettyHttpAuthenticator(authorizer) {
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? {
|
||||||
|
val authorizationHeader = req.headers()[HttpHeaderNames.AUTHORIZATION] ?: let {
|
||||||
|
log.debug(ctx) {
|
||||||
|
"Missing Authorization header"
|
||||||
|
}
|
||||||
|
return users[""]?.let { AuthenticationResult(it, it.groups) }
|
||||||
|
}
|
||||||
|
val cursor = authorizationHeader.indexOf(' ')
|
||||||
|
if (cursor < 0) {
|
||||||
|
log.debug(ctx) {
|
||||||
|
"Invalid Authorization header: '$authorizationHeader'"
|
||||||
|
}
|
||||||
|
return users[""]?.let { AuthenticationResult(it, it.groups) }
|
||||||
|
}
|
||||||
|
val authenticationType = authorizationHeader.substring(0, cursor)
|
||||||
|
if ("Basic" != authenticationType) {
|
||||||
|
log.debug(ctx) {
|
||||||
|
"Invalid authentication type header: '$authenticationType'"
|
||||||
|
}
|
||||||
|
return users[""]?.let { AuthenticationResult(it, it.groups) }
|
||||||
|
}
|
||||||
|
val (username, password) = Base64.getDecoder().decode(authorizationHeader.substring(cursor + 1))
|
||||||
|
.let(::String)
|
||||||
|
.let {
|
||||||
|
val colon = it.indexOf(':')
|
||||||
|
if (colon < 0) {
|
||||||
|
log.debug(ctx) {
|
||||||
|
"Missing colon from authentication"
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
it.substring(0, colon) to it.substring(colon + 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return username.let(users::get)?.takeIf { user ->
|
||||||
|
user.password?.let { passwordAndSalt ->
|
||||||
|
val (_, salt) = decodePasswordHash(passwordAndSalt)
|
||||||
|
hashPassword(password, Base64.getEncoder().encodeToString(salt)) == passwordAndSalt
|
||||||
|
} ?: false
|
||||||
|
}?.let { user ->
|
||||||
|
AuthenticationResult(user, user.groups)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class ServerInitializer(
|
||||||
|
private val cfg: Configuration,
|
||||||
|
private val eventExecutorGroup: EventExecutorGroup
|
||||||
|
) : ChannelInitializer<Channel>() {
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private fun createSslCtx(tls: Configuration.Tls): SslContext {
|
||||||
|
val keyStore = tls.keyStore
|
||||||
|
return if (keyStore == null) {
|
||||||
|
throw IllegalArgumentException("No keystore configured")
|
||||||
|
} else {
|
||||||
|
val javaKeyStore = loadKeystore(keyStore.file, keyStore.password)
|
||||||
|
val serverKey = javaKeyStore.getKey(
|
||||||
|
keyStore.keyAlias, (keyStore.keyPassword ?: "").let(String::toCharArray)
|
||||||
|
) as PrivateKey
|
||||||
|
val serverCert: Array<X509Certificate> =
|
||||||
|
Arrays.stream(javaKeyStore.getCertificateChain(keyStore.keyAlias))
|
||||||
|
.map { it as X509Certificate }
|
||||||
|
.toArray { size -> Array<X509Certificate?>(size) { null } }
|
||||||
|
SslContextBuilder.forServer(serverKey, *serverCert).apply {
|
||||||
|
val clientAuth = tls.trustStore?.let { trustStore ->
|
||||||
|
val ts = loadKeystore(trustStore.file, trustStore.password)
|
||||||
|
trustManager(
|
||||||
|
ClientCertificateValidator.getTrustManager(ts, trustStore.isCheckCertificateStatus)
|
||||||
|
)
|
||||||
|
if(trustStore.isRequireClientCertificate) ClientAuth.REQUIRE
|
||||||
|
else ClientAuth.OPTIONAL
|
||||||
|
} ?: ClientAuth.NONE
|
||||||
|
clientAuth(clientAuth)
|
||||||
|
}.build()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun loadKeystore(file: Path, password: String?): KeyStore {
|
||||||
|
val ext = JWO.splitExtension(file)
|
||||||
|
.map(Tuple2<String, String>::get_2)
|
||||||
|
.orElseThrow {
|
||||||
|
IllegalArgumentException(
|
||||||
|
"Keystore file '${file}' must have .jks, .p12, .pfx extension"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
val keystore = when (ext.substring(1).lowercase()) {
|
||||||
|
"jks" -> KeyStore.getInstance("JKS")
|
||||||
|
"p12", "pfx" -> KeyStore.getInstance("PKCS12")
|
||||||
|
else -> throw IllegalArgumentException(
|
||||||
|
"Keystore file '${file}' must have .jks, .p12, .pfx extension"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Files.newInputStream(file).use {
|
||||||
|
keystore.load(it, password?.let(String::toCharArray))
|
||||||
|
}
|
||||||
|
return keystore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
private val serverHandler = let {
|
||||||
|
val cacheImplementation = cfg.cache.materialize()
|
||||||
|
val prefix = Path.of("/").resolve(Path.of(cfg.serverPath ?: "/"))
|
||||||
|
ServerHandler(cacheImplementation, prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
private val exceptionHandler = ExceptionHandler()
|
||||||
|
private val throttlingHandler = ThrottlingHandler(cfg)
|
||||||
|
|
||||||
|
private val authenticator = when (val auth = cfg.authentication) {
|
||||||
|
is Configuration.BasicAuthentication -> NettyHttpBasicAuthenticator(cfg.users, RoleAuthorizer())
|
||||||
|
is Configuration.ClientCertificateAuthentication -> {
|
||||||
|
ClientCertificateAuthenticator(
|
||||||
|
RoleAuthorizer(),
|
||||||
|
cfg.users[""]?.groups,
|
||||||
|
userExtractor(auth),
|
||||||
|
groupExtractor(auth)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
else -> null
|
||||||
|
}
|
||||||
|
|
||||||
|
private val sslContext: SslContext? = cfg.tls?.let(Companion::createSslCtx)
|
||||||
|
|
||||||
|
private fun userExtractor(authentication: Configuration.ClientCertificateAuthentication) =
|
||||||
|
authentication.userExtractor?.let { extractor ->
|
||||||
|
val pattern = Pattern.compile(extractor.pattern)
|
||||||
|
val rdnType = extractor.rdnType
|
||||||
|
Configuration.UserExtractor { cert: X509Certificate ->
|
||||||
|
val userName = LdapName(cert.subjectX500Principal.name).rdns.find {
|
||||||
|
it.type == rdnType
|
||||||
|
}?.let {
|
||||||
|
pattern.matcher(it.value.toString())
|
||||||
|
}?.takeIf(Matcher::matches)?.group(1)
|
||||||
|
cfg.users[userName] ?: throw java.lang.RuntimeException("Failed to extract user")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun groupExtractor(authentication: Configuration.ClientCertificateAuthentication) =
|
||||||
|
authentication.groupExtractor?.let { extractor ->
|
||||||
|
val pattern = Pattern.compile(extractor.pattern)
|
||||||
|
val rdnType = extractor.rdnType
|
||||||
|
Configuration.GroupExtractor { cert: X509Certificate ->
|
||||||
|
val groupName = LdapName(cert.subjectX500Principal.name).rdns.find {
|
||||||
|
it.type == rdnType
|
||||||
|
}?.let {
|
||||||
|
pattern.matcher(it.value.toString())
|
||||||
|
}?.takeIf(Matcher::matches)?.group(1)
|
||||||
|
cfg.groups[groupName] ?: throw java.lang.RuntimeException("Failed to extract group")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun initChannel(ch: Channel) {
|
||||||
|
log.debug {
|
||||||
|
"Created connection ${ch.id().asShortText()} with ${ch.remoteAddress()}"
|
||||||
|
}
|
||||||
|
ch.closeFuture().addListener {
|
||||||
|
log.debug {
|
||||||
|
"Closed connection ${ch.id().asShortText()} with ${ch.remoteAddress()}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val pipeline = ch.pipeline()
|
||||||
|
cfg.connection.also { conn ->
|
||||||
|
val readTimeout = conn.readTimeout.toMillis()
|
||||||
|
val writeTimeout = conn.writeTimeout.toMillis()
|
||||||
|
if(readTimeout > 0 || writeTimeout > 0) {
|
||||||
|
pipeline.addLast(
|
||||||
|
IdleStateHandler(
|
||||||
|
false,
|
||||||
|
readTimeout,
|
||||||
|
writeTimeout,
|
||||||
|
0,
|
||||||
|
TimeUnit.MILLISECONDS
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
val readIdleTimeout = conn.readIdleTimeout.toMillis()
|
||||||
|
val writeIdleTimeout = conn.writeIdleTimeout.toMillis()
|
||||||
|
val idleTimeout = conn.idleTimeout.toMillis()
|
||||||
|
if(readIdleTimeout > 0 || writeIdleTimeout > 0 || idleTimeout > 0) {
|
||||||
|
pipeline.addLast(
|
||||||
|
IdleStateHandler(
|
||||||
|
true,
|
||||||
|
readIdleTimeout,
|
||||||
|
writeIdleTimeout,
|
||||||
|
idleTimeout,
|
||||||
|
TimeUnit.MILLISECONDS
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pipeline.addLast(object : ChannelInboundHandlerAdapter() {
|
||||||
|
override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) {
|
||||||
|
if (evt is IdleStateEvent) {
|
||||||
|
when(evt.state()) {
|
||||||
|
IdleState.READER_IDLE -> log.debug {
|
||||||
|
"Read timeout reached on channel ${ch.id().asShortText()}, closing the connection"
|
||||||
|
}
|
||||||
|
IdleState.WRITER_IDLE -> log.debug {
|
||||||
|
"Write timeout reached on channel ${ch.id().asShortText()}, closing the connection"
|
||||||
|
}
|
||||||
|
IdleState.ALL_IDLE -> log.debug {
|
||||||
|
"Idle timeout reached on channel ${ch.id().asShortText()}, closing the connection"
|
||||||
|
}
|
||||||
|
null -> throw IllegalStateException("This should never happen")
|
||||||
|
}
|
||||||
|
ctx.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
sslContext?.newHandler(ch.alloc())?.also {
|
||||||
|
pipeline.addLast(SSL_HANDLER_NAME, it)
|
||||||
|
}
|
||||||
|
pipeline.addLast(HttpServerCodec())
|
||||||
|
pipeline.addLast(HttpChunkContentCompressor(1024))
|
||||||
|
pipeline.addLast(ChunkedWriteHandler())
|
||||||
|
pipeline.addLast(HttpObjectAggregator(cfg.connection.maxRequestSize))
|
||||||
|
authenticator?.let {
|
||||||
|
pipeline.addLast(it)
|
||||||
|
}
|
||||||
|
pipeline.addLast(throttlingHandler)
|
||||||
|
pipeline.addLast(eventExecutorGroup, serverHandler)
|
||||||
|
pipeline.addLast(exceptionHandler)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class ServerHandle(
|
||||||
|
httpChannelFuture: ChannelFuture,
|
||||||
|
private val executorGroups: Iterable<EventExecutorGroup>
|
||||||
|
) : AutoCloseable {
|
||||||
|
private val httpChannel: Channel = httpChannelFuture.channel()
|
||||||
|
private val closeFuture: ChannelFuture = httpChannel.closeFuture()
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
fun shutdown(): ChannelFuture {
|
||||||
|
return httpChannel.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {
|
||||||
|
try {
|
||||||
|
closeFuture.sync()
|
||||||
|
} finally {
|
||||||
|
executorGroups.forEach {
|
||||||
|
it.shutdownGracefully().sync()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info {
|
||||||
|
"RemoteBuildCacheServer has been gracefully shut down"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun run(): ServerHandle {
|
||||||
|
// Create the multithreaded event loops for the server
|
||||||
|
val bossGroup = NioEventLoopGroup(1)
|
||||||
|
val serverSocketChannel = NioServerSocketChannel::class.java
|
||||||
|
val workerGroup = NioEventLoopGroup(0)
|
||||||
|
val eventExecutorGroup = run {
|
||||||
|
val threadFactory = if (cfg.eventExecutor.isUseVirtualThreads) {
|
||||||
|
Thread.ofVirtual().factory()
|
||||||
|
} else {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
DefaultEventExecutorGroup(Runtime.getRuntime().availableProcessors(), threadFactory)
|
||||||
|
}
|
||||||
|
val bootstrap = ServerBootstrap().apply {
|
||||||
|
// Configure the server
|
||||||
|
group(bossGroup, workerGroup)
|
||||||
|
channel(serverSocketChannel)
|
||||||
|
childHandler(ServerInitializer(cfg, eventExecutorGroup))
|
||||||
|
option(ChannelOption.SO_BACKLOG, cfg.incomingConnectionsBacklogSize)
|
||||||
|
childOption(ChannelOption.SO_KEEPALIVE, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Bind and start to accept incoming connections.
|
||||||
|
val bindAddress = InetSocketAddress(cfg.host, cfg.port)
|
||||||
|
val httpChannel = bootstrap.bind(bindAddress).sync()
|
||||||
|
log.info {
|
||||||
|
"RemoteBuildCacheServer is listening on ${cfg.host}:${cfg.port}"
|
||||||
|
}
|
||||||
|
return ServerHandle(httpChannel, setOf(bossGroup, workerGroup, eventExecutorGroup))
|
||||||
|
}
|
||||||
|
}
|
@@ -1,4 +1,4 @@
|
|||||||
package net.woggioni.gbcs
|
package net.woggioni.rbcs.server.auth
|
||||||
|
|
||||||
import io.netty.buffer.Unpooled
|
import io.netty.buffer.Unpooled
|
||||||
import io.netty.channel.ChannelFutureListener
|
import io.netty.channel.ChannelFutureListener
|
||||||
@@ -11,27 +11,47 @@ import io.netty.handler.codec.http.HttpRequest
|
|||||||
import io.netty.handler.codec.http.HttpResponseStatus
|
import io.netty.handler.codec.http.HttpResponseStatus
|
||||||
import io.netty.handler.codec.http.HttpVersion
|
import io.netty.handler.codec.http.HttpVersion
|
||||||
import io.netty.util.ReferenceCountUtil
|
import io.netty.util.ReferenceCountUtil
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.api.Configuration.Group
|
||||||
|
import net.woggioni.rbcs.api.Role
|
||||||
|
import net.woggioni.rbcs.server.RemoteBuildCacheServer
|
||||||
|
|
||||||
abstract class AbstractNettyHttpAuthenticator(private val authorizer : Authorizer)
|
|
||||||
: ChannelInboundHandlerAdapter() {
|
|
||||||
|
|
||||||
private companion object {
|
abstract class AbstractNettyHttpAuthenticator(private val authorizer: Authorizer) : ChannelInboundHandlerAdapter() {
|
||||||
|
|
||||||
|
companion object {
|
||||||
private val AUTHENTICATION_FAILED: FullHttpResponse = DefaultFullHttpResponse(
|
private val AUTHENTICATION_FAILED: FullHttpResponse = DefaultFullHttpResponse(
|
||||||
HttpVersion.HTTP_1_1, HttpResponseStatus.UNAUTHORIZED, Unpooled.EMPTY_BUFFER).apply {
|
HttpVersion.HTTP_1_1, HttpResponseStatus.UNAUTHORIZED, Unpooled.EMPTY_BUFFER
|
||||||
|
).apply {
|
||||||
headers()[HttpHeaderNames.CONTENT_LENGTH] = "0"
|
headers()[HttpHeaderNames.CONTENT_LENGTH] = "0"
|
||||||
}
|
}
|
||||||
|
|
||||||
private val NOT_AUTHORIZED: FullHttpResponse = DefaultFullHttpResponse(
|
private val NOT_AUTHORIZED: FullHttpResponse = DefaultFullHttpResponse(
|
||||||
HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER).apply {
|
HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER
|
||||||
|
).apply {
|
||||||
headers()[HttpHeaderNames.CONTENT_LENGTH] = "0"
|
headers()[HttpHeaderNames.CONTENT_LENGTH] = "0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
abstract fun authenticate(ctx : ChannelHandlerContext, req : HttpRequest) : String?
|
|
||||||
|
class AuthenticationResult(val user: Configuration.User?, val groups: Set<Group>)
|
||||||
|
|
||||||
|
abstract fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult?
|
||||||
|
|
||||||
override fun channelRead(ctx: ChannelHandlerContext, msg: Any) {
|
override fun channelRead(ctx: ChannelHandlerContext, msg: Any) {
|
||||||
if (msg is HttpRequest) {
|
if (msg is HttpRequest) {
|
||||||
val user = authenticate(ctx, msg) ?: return authenticationFailure(ctx, msg)
|
val result = authenticate(ctx, msg) ?: return authenticationFailure(ctx, msg)
|
||||||
val authorized = authorizer.authorize(user, msg)
|
ctx.channel().attr(RemoteBuildCacheServer.userAttribute).set(result.user)
|
||||||
|
ctx.channel().attr(RemoteBuildCacheServer.groupAttribute).set(result.groups)
|
||||||
|
|
||||||
|
val roles = (
|
||||||
|
(result.user?.let { user ->
|
||||||
|
user.groups.asSequence().flatMap { group ->
|
||||||
|
group.roles.asSequence()
|
||||||
|
}
|
||||||
|
} ?: emptySequence<Role>()) +
|
||||||
|
result.groups.asSequence().flatMap { it.roles.asSequence() }
|
||||||
|
).toSet()
|
||||||
|
val authorized = authorizer.authorize(roles, msg)
|
||||||
if (authorized) {
|
if (authorized) {
|
||||||
super.channelRead(ctx, msg)
|
super.channelRead(ctx, msg)
|
||||||
} else {
|
} else {
|
@@ -0,0 +1,8 @@
|
|||||||
|
package net.woggioni.rbcs.server.auth
|
||||||
|
|
||||||
|
import io.netty.handler.codec.http.HttpRequest
|
||||||
|
import net.woggioni.rbcs.api.Role
|
||||||
|
|
||||||
|
fun interface Authorizer {
|
||||||
|
fun authorize(roles : Set<Role>, request: HttpRequest) : Boolean
|
||||||
|
}
|
@@ -0,0 +1,90 @@
|
|||||||
|
package net.woggioni.rbcs.server.auth
|
||||||
|
|
||||||
|
import io.netty.channel.ChannelHandlerContext
|
||||||
|
import io.netty.channel.ChannelInboundHandlerAdapter
|
||||||
|
import io.netty.handler.ssl.SslHandler
|
||||||
|
import io.netty.handler.ssl.SslHandshakeCompletionEvent
|
||||||
|
import java.security.KeyStore
|
||||||
|
import java.security.cert.CertPathValidator
|
||||||
|
import java.security.cert.CertPathValidatorException
|
||||||
|
import java.security.cert.CertificateException
|
||||||
|
import java.security.cert.CertificateFactory
|
||||||
|
import java.security.cert.PKIXParameters
|
||||||
|
import java.security.cert.PKIXRevocationChecker
|
||||||
|
import java.security.cert.X509Certificate
|
||||||
|
import java.util.EnumSet
|
||||||
|
import javax.net.ssl.SSLSession
|
||||||
|
import javax.net.ssl.TrustManagerFactory
|
||||||
|
import javax.net.ssl.X509TrustManager
|
||||||
|
|
||||||
|
|
||||||
|
class ClientCertificateValidator private constructor(
|
||||||
|
private val sslHandler: SslHandler,
|
||||||
|
private val x509TrustManager: X509TrustManager
|
||||||
|
) : ChannelInboundHandlerAdapter() {
|
||||||
|
override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) {
|
||||||
|
if (evt is SslHandshakeCompletionEvent) {
|
||||||
|
if (evt.isSuccess) {
|
||||||
|
val session: SSLSession = sslHandler.engine().session
|
||||||
|
val clientCertificateChain = session.peerCertificates as Array<X509Certificate>
|
||||||
|
val authType: String = clientCertificateChain[0].publicKey.algorithm
|
||||||
|
x509TrustManager.checkClientTrusted(clientCertificateChain, authType)
|
||||||
|
} else {
|
||||||
|
// Handle the failure, for example by closing the channel.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
super.userEventTriggered(ctx, evt)
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
fun getTrustManager(trustStore: KeyStore?, certificateRevocationEnabled: Boolean): X509TrustManager {
|
||||||
|
return if (trustStore != null) {
|
||||||
|
val certificateFactory = CertificateFactory.getInstance("X.509")
|
||||||
|
val validator = CertPathValidator.getInstance("PKIX").apply {
|
||||||
|
val rc = revocationChecker as PKIXRevocationChecker
|
||||||
|
rc.options = EnumSet.of(
|
||||||
|
PKIXRevocationChecker.Option.NO_FALLBACK
|
||||||
|
)
|
||||||
|
}
|
||||||
|
val params = PKIXParameters(trustStore).apply {
|
||||||
|
isRevocationEnabled = certificateRevocationEnabled
|
||||||
|
}
|
||||||
|
object : X509TrustManager {
|
||||||
|
override fun checkClientTrusted(chain: Array<out X509Certificate>, authType: String) {
|
||||||
|
val clientCertificateChain = certificateFactory.generateCertPath(chain.toList())
|
||||||
|
try {
|
||||||
|
validator.validate(clientCertificateChain, params)
|
||||||
|
} catch (ex: CertPathValidatorException) {
|
||||||
|
throw CertificateException(ex)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun checkServerTrusted(chain: Array<out X509Certificate>, authType: String) {
|
||||||
|
throw NotImplementedError()
|
||||||
|
}
|
||||||
|
|
||||||
|
private val acceptedIssuers = trustStore.aliases().asSequence()
|
||||||
|
.filter(trustStore::isCertificateEntry)
|
||||||
|
.map(trustStore::getCertificate)
|
||||||
|
.map { it as X509Certificate }
|
||||||
|
.toList()
|
||||||
|
.toTypedArray()
|
||||||
|
|
||||||
|
override fun getAcceptedIssuers() = acceptedIssuers
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
|
||||||
|
trustManagerFactory.trustManagers.asSequence().filter { it is X509TrustManager }
|
||||||
|
.single() as X509TrustManager
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun of(
|
||||||
|
sslHandler: SslHandler,
|
||||||
|
trustStore: KeyStore?,
|
||||||
|
certificateRevocationEnabled: Boolean
|
||||||
|
): ClientCertificateValidator {
|
||||||
|
return ClientCertificateValidator(sslHandler, getTrustManager(trustStore, certificateRevocationEnabled))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,23 @@
|
|||||||
|
package net.woggioni.rbcs.server.auth
|
||||||
|
|
||||||
|
import io.netty.handler.codec.http.HttpMethod
|
||||||
|
import io.netty.handler.codec.http.HttpRequest
|
||||||
|
import net.woggioni.rbcs.api.Role
|
||||||
|
|
||||||
|
class RoleAuthorizer : Authorizer {
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private val METHOD_MAP = mapOf(
|
||||||
|
Role.Reader to setOf(HttpMethod.GET, HttpMethod.HEAD, HttpMethod.TRACE),
|
||||||
|
Role.Writer to setOf(HttpMethod.PUT, HttpMethod.POST)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun authorize(roles: Set<Role>, request: HttpRequest) : Boolean {
|
||||||
|
val allowedMethods = roles.asSequence()
|
||||||
|
.mapNotNull(METHOD_MAP::get)
|
||||||
|
.flatten()
|
||||||
|
.toSet()
|
||||||
|
return request.method() in allowedMethods
|
||||||
|
}
|
||||||
|
}
|
130
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/FileSystemCache.kt
vendored
Normal file
130
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/FileSystemCache.kt
vendored
Normal file
@@ -0,0 +1,130 @@
|
|||||||
|
package net.woggioni.rbcs.server.cache
|
||||||
|
|
||||||
|
import io.netty.buffer.ByteBuf
|
||||||
|
import net.woggioni.rbcs.api.Cache
|
||||||
|
import net.woggioni.rbcs.common.ByteBufInputStream
|
||||||
|
import net.woggioni.rbcs.common.RBCS.digestString
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.jwo.JWO
|
||||||
|
import net.woggioni.jwo.LockFile
|
||||||
|
import java.nio.channels.Channels
|
||||||
|
import java.nio.channels.FileChannel
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Path
|
||||||
|
import java.nio.file.StandardCopyOption
|
||||||
|
import java.nio.file.StandardOpenOption
|
||||||
|
import java.nio.file.attribute.BasicFileAttributes
|
||||||
|
import java.security.MessageDigest
|
||||||
|
import java.time.Duration
|
||||||
|
import java.time.Instant
|
||||||
|
import java.util.concurrent.CompletableFuture
|
||||||
|
import java.util.concurrent.atomic.AtomicReference
|
||||||
|
import java.util.zip.Deflater
|
||||||
|
import java.util.zip.DeflaterOutputStream
|
||||||
|
import java.util.zip.Inflater
|
||||||
|
import java.util.zip.InflaterInputStream
|
||||||
|
|
||||||
|
class FileSystemCache(
|
||||||
|
val root: Path,
|
||||||
|
val maxAge: Duration,
|
||||||
|
val digestAlgorithm: String?,
|
||||||
|
val compressionEnabled: Boolean,
|
||||||
|
val compressionLevel: Int
|
||||||
|
) : Cache {
|
||||||
|
|
||||||
|
private companion object {
|
||||||
|
@JvmStatic
|
||||||
|
private val log = contextLogger()
|
||||||
|
}
|
||||||
|
|
||||||
|
init {
|
||||||
|
Files.createDirectories(root)
|
||||||
|
}
|
||||||
|
|
||||||
|
private var nextGc = AtomicReference(Instant.now().plus(maxAge))
|
||||||
|
|
||||||
|
override fun get(key: String) = (digestAlgorithm
|
||||||
|
?.let(MessageDigest::getInstance)
|
||||||
|
?.let { md ->
|
||||||
|
digestString(key.toByteArray(), md)
|
||||||
|
} ?: key).let { digest ->
|
||||||
|
root.resolve(digest).takeIf(Files::exists)
|
||||||
|
?.let { file ->
|
||||||
|
file.takeIf(Files::exists)?.let { file ->
|
||||||
|
if (compressionEnabled) {
|
||||||
|
val inflater = Inflater()
|
||||||
|
Channels.newChannel(
|
||||||
|
InflaterInputStream(
|
||||||
|
Channels.newInputStream(
|
||||||
|
FileChannel.open(
|
||||||
|
file,
|
||||||
|
StandardOpenOption.READ
|
||||||
|
)
|
||||||
|
), inflater
|
||||||
|
)
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
FileChannel.open(file, StandardOpenOption.READ)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}.also {
|
||||||
|
gc()
|
||||||
|
}.let {
|
||||||
|
CompletableFuture.completedFuture(it)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun put(key: String, content: ByteBuf): CompletableFuture<Void> {
|
||||||
|
(digestAlgorithm
|
||||||
|
?.let(MessageDigest::getInstance)
|
||||||
|
?.let { md ->
|
||||||
|
digestString(key.toByteArray(), md)
|
||||||
|
} ?: key).let { digest ->
|
||||||
|
val file = root.resolve(digest)
|
||||||
|
val tmpFile = Files.createTempFile(root, null, ".tmp")
|
||||||
|
try {
|
||||||
|
Files.newOutputStream(tmpFile).let {
|
||||||
|
if (compressionEnabled) {
|
||||||
|
val deflater = Deflater(compressionLevel)
|
||||||
|
DeflaterOutputStream(it, deflater)
|
||||||
|
} else {
|
||||||
|
it
|
||||||
|
}
|
||||||
|
}.use {
|
||||||
|
JWO.copy(ByteBufInputStream(content), it)
|
||||||
|
}
|
||||||
|
Files.move(tmpFile, file, StandardCopyOption.ATOMIC_MOVE)
|
||||||
|
} catch (t: Throwable) {
|
||||||
|
Files.delete(tmpFile)
|
||||||
|
throw t
|
||||||
|
}
|
||||||
|
}.also {
|
||||||
|
gc()
|
||||||
|
}
|
||||||
|
return CompletableFuture.completedFuture(null)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun gc() {
|
||||||
|
val now = Instant.now()
|
||||||
|
val oldValue = nextGc.getAndSet(now.plus(maxAge))
|
||||||
|
if (oldValue < now) {
|
||||||
|
actualGc(now)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Synchronized
|
||||||
|
private fun actualGc(now: Instant) {
|
||||||
|
Files.list(root).filter {
|
||||||
|
val creationTimeStamp = Files.readAttributes(it, BasicFileAttributes::class.java)
|
||||||
|
.creationTime()
|
||||||
|
.toInstant()
|
||||||
|
now > creationTimeStamp.plus(maxAge)
|
||||||
|
}.forEach { file ->
|
||||||
|
LockFile.acquire(file, false).use {
|
||||||
|
Files.delete(file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {}
|
||||||
|
}
|
27
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/FileSystemCacheConfiguration.kt
vendored
Normal file
27
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/FileSystemCacheConfiguration.kt
vendored
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
package net.woggioni.rbcs.server.cache
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.common.RBCS
|
||||||
|
import net.woggioni.jwo.Application
|
||||||
|
import java.nio.file.Path
|
||||||
|
import java.time.Duration
|
||||||
|
|
||||||
|
data class FileSystemCacheConfiguration(
|
||||||
|
val root: Path?,
|
||||||
|
val maxAge: Duration,
|
||||||
|
val digestAlgorithm : String?,
|
||||||
|
val compressionEnabled: Boolean,
|
||||||
|
val compressionLevel: Int,
|
||||||
|
) : Configuration.Cache {
|
||||||
|
override fun materialize() = FileSystemCache(
|
||||||
|
root ?: Application.builder("rbcs").build().computeCacheDirectory(),
|
||||||
|
maxAge,
|
||||||
|
digestAlgorithm,
|
||||||
|
compressionEnabled,
|
||||||
|
compressionLevel
|
||||||
|
)
|
||||||
|
|
||||||
|
override fun getNamespaceURI() = RBCS.RBCS_NAMESPACE_URI
|
||||||
|
|
||||||
|
override fun getTypeName() = "fileSystemCacheType"
|
||||||
|
}
|
63
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/FileSystemCacheProvider.kt
vendored
Normal file
63
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/FileSystemCacheProvider.kt
vendored
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
package net.woggioni.rbcs.server.cache
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.CacheProvider
|
||||||
|
import net.woggioni.rbcs.common.RBCS
|
||||||
|
import net.woggioni.rbcs.common.Xml
|
||||||
|
import net.woggioni.rbcs.common.Xml.Companion.renderAttribute
|
||||||
|
import org.w3c.dom.Document
|
||||||
|
import org.w3c.dom.Element
|
||||||
|
import java.nio.file.Path
|
||||||
|
import java.time.Duration
|
||||||
|
import java.util.zip.Deflater
|
||||||
|
|
||||||
|
class FileSystemCacheProvider : CacheProvider<FileSystemCacheConfiguration> {
|
||||||
|
|
||||||
|
override fun getXmlSchemaLocation() = "classpath:net/woggioni/rbcs/server/schema/rbcs.xsd"
|
||||||
|
|
||||||
|
override fun getXmlType() = "fileSystemCacheType"
|
||||||
|
|
||||||
|
override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server"
|
||||||
|
|
||||||
|
override fun deserialize(el: Element): FileSystemCacheConfiguration {
|
||||||
|
val path = el.renderAttribute("path")
|
||||||
|
?.let(Path::of)
|
||||||
|
val maxAge = el.renderAttribute("max-age")
|
||||||
|
?.let(Duration::parse)
|
||||||
|
?: Duration.ofDays(1)
|
||||||
|
val enableCompression = el.renderAttribute("enable-compression")
|
||||||
|
?.let(String::toBoolean)
|
||||||
|
?: true
|
||||||
|
val compressionLevel = el.renderAttribute("compression-level")
|
||||||
|
?.let(String::toInt)
|
||||||
|
?: Deflater.DEFAULT_COMPRESSION
|
||||||
|
val digestAlgorithm = el.renderAttribute("digest") ?: "MD5"
|
||||||
|
|
||||||
|
return FileSystemCacheConfiguration(
|
||||||
|
path,
|
||||||
|
maxAge,
|
||||||
|
digestAlgorithm,
|
||||||
|
enableCompression,
|
||||||
|
compressionLevel
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun serialize(doc: Document, cache : FileSystemCacheConfiguration) = cache.run {
|
||||||
|
val result = doc.createElement("cache")
|
||||||
|
Xml.of(doc, result) {
|
||||||
|
val prefix = doc.lookupPrefix(RBCS.RBCS_NAMESPACE_URI)
|
||||||
|
attr("xs:type", "${prefix}:fileSystemCacheType", RBCS.XML_SCHEMA_NAMESPACE_URI)
|
||||||
|
attr("path", root.toString())
|
||||||
|
attr("max-age", maxAge.toString())
|
||||||
|
digestAlgorithm?.let { digestAlgorithm ->
|
||||||
|
attr("digest", digestAlgorithm)
|
||||||
|
}
|
||||||
|
attr("enable-compression", compressionEnabled.toString())
|
||||||
|
compressionLevel.takeIf {
|
||||||
|
it != Deflater.DEFAULT_COMPRESSION
|
||||||
|
}?.let {
|
||||||
|
attr("compression-level", it.toString())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
}
|
150
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/InMemoryCache.kt
vendored
Normal file
150
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/InMemoryCache.kt
vendored
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
package net.woggioni.rbcs.server.cache
|
||||||
|
|
||||||
|
import io.netty.buffer.ByteBuf
|
||||||
|
import net.woggioni.rbcs.api.Cache
|
||||||
|
import net.woggioni.rbcs.common.ByteBufInputStream
|
||||||
|
import net.woggioni.rbcs.common.ByteBufOutputStream
|
||||||
|
import net.woggioni.rbcs.common.RBCS.digestString
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.jwo.JWO
|
||||||
|
import java.nio.channels.Channels
|
||||||
|
import java.security.MessageDigest
|
||||||
|
import java.time.Duration
|
||||||
|
import java.time.Instant
|
||||||
|
import java.util.concurrent.CompletableFuture
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import java.util.concurrent.PriorityBlockingQueue
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
import java.util.zip.Deflater
|
||||||
|
import java.util.zip.DeflaterOutputStream
|
||||||
|
import java.util.zip.Inflater
|
||||||
|
import java.util.zip.InflaterInputStream
|
||||||
|
|
||||||
|
class InMemoryCache(
|
||||||
|
val maxAge: Duration,
|
||||||
|
val maxSize: Long,
|
||||||
|
val digestAlgorithm: String?,
|
||||||
|
val compressionEnabled: Boolean,
|
||||||
|
val compressionLevel: Int
|
||||||
|
) : Cache {
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
@JvmStatic
|
||||||
|
private val log = contextLogger()
|
||||||
|
}
|
||||||
|
|
||||||
|
private val size = AtomicLong()
|
||||||
|
private val map = ConcurrentHashMap<String, ByteBuf>()
|
||||||
|
|
||||||
|
private class RemovalQueueElement(val key: String, val value : ByteBuf, val expiry : Instant) : Comparable<RemovalQueueElement> {
|
||||||
|
override fun compareTo(other: RemovalQueueElement) = expiry.compareTo(other.expiry)
|
||||||
|
}
|
||||||
|
|
||||||
|
private val removalQueue = PriorityBlockingQueue<RemovalQueueElement>()
|
||||||
|
|
||||||
|
private var running = true
|
||||||
|
private val garbageCollector = Thread {
|
||||||
|
while(true) {
|
||||||
|
val el = removalQueue.take()
|
||||||
|
val buf = el.value
|
||||||
|
val now = Instant.now()
|
||||||
|
if(now > el.expiry) {
|
||||||
|
val removed = map.remove(el.key, buf)
|
||||||
|
if(removed) {
|
||||||
|
updateSizeAfterRemoval(buf)
|
||||||
|
//Decrease the reference count for map
|
||||||
|
buf.release()
|
||||||
|
}
|
||||||
|
//Decrease the reference count for removalQueue
|
||||||
|
buf.release()
|
||||||
|
} else {
|
||||||
|
removalQueue.put(el)
|
||||||
|
Thread.sleep(minOf(Duration.between(now, el.expiry), Duration.ofSeconds(1)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}.apply {
|
||||||
|
start()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun removeEldest() : Long {
|
||||||
|
while(true) {
|
||||||
|
val el = removalQueue.take()
|
||||||
|
val buf = el.value
|
||||||
|
val removed = map.remove(el.key, buf)
|
||||||
|
//Decrease the reference count for removalQueue
|
||||||
|
buf.release()
|
||||||
|
if(removed) {
|
||||||
|
val newSize = updateSizeAfterRemoval(buf)
|
||||||
|
//Decrease the reference count for map
|
||||||
|
buf.release()
|
||||||
|
return newSize
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun updateSizeAfterRemoval(removed: ByteBuf) : Long {
|
||||||
|
return size.updateAndGet { currentSize : Long ->
|
||||||
|
currentSize - removed.readableBytes()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun close() {
|
||||||
|
running = false
|
||||||
|
garbageCollector.join()
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun get(key: String) =
|
||||||
|
(digestAlgorithm
|
||||||
|
?.let(MessageDigest::getInstance)
|
||||||
|
?.let { md ->
|
||||||
|
digestString(key.toByteArray(), md)
|
||||||
|
} ?: key
|
||||||
|
).let { digest ->
|
||||||
|
map[digest]
|
||||||
|
?.let { value ->
|
||||||
|
val copy = value.retainedDuplicate()
|
||||||
|
copy.touch("This has to be released by the caller of the cache")
|
||||||
|
if (compressionEnabled) {
|
||||||
|
val inflater = Inflater()
|
||||||
|
Channels.newChannel(InflaterInputStream(ByteBufInputStream(copy), inflater))
|
||||||
|
} else {
|
||||||
|
Channels.newChannel(ByteBufInputStream(copy))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}.let {
|
||||||
|
CompletableFuture.completedFuture(it)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun put(key: String, content: ByteBuf) =
|
||||||
|
(digestAlgorithm
|
||||||
|
?.let(MessageDigest::getInstance)
|
||||||
|
?.let { md ->
|
||||||
|
digestString(key.toByteArray(), md)
|
||||||
|
} ?: key).let { digest ->
|
||||||
|
content.retain()
|
||||||
|
val value = if (compressionEnabled) {
|
||||||
|
val deflater = Deflater(compressionLevel)
|
||||||
|
val buf = content.alloc().buffer()
|
||||||
|
buf.retain()
|
||||||
|
DeflaterOutputStream(ByteBufOutputStream(buf), deflater).use { outputStream ->
|
||||||
|
ByteBufInputStream(content).use { inputStream ->
|
||||||
|
JWO.copy(inputStream, outputStream)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buf
|
||||||
|
} else {
|
||||||
|
content
|
||||||
|
}
|
||||||
|
val old = map.put(digest, value)
|
||||||
|
val delta = value.readableBytes() - (old?.readableBytes() ?: 0)
|
||||||
|
var newSize = size.updateAndGet { currentSize : Long ->
|
||||||
|
currentSize + delta
|
||||||
|
}
|
||||||
|
removalQueue.put(RemovalQueueElement(digest, value.retain(), Instant.now().plus(maxAge)))
|
||||||
|
while(newSize > maxSize) {
|
||||||
|
newSize = removeEldest()
|
||||||
|
}
|
||||||
|
}.let {
|
||||||
|
CompletableFuture.completedFuture<Void>(null)
|
||||||
|
}
|
||||||
|
}
|
25
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/InMemoryCacheConfiguration.kt
vendored
Normal file
25
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/InMemoryCacheConfiguration.kt
vendored
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
package net.woggioni.rbcs.server.cache
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.common.RBCS
|
||||||
|
import java.time.Duration
|
||||||
|
|
||||||
|
data class InMemoryCacheConfiguration(
|
||||||
|
val maxAge: Duration,
|
||||||
|
val maxSize: Long,
|
||||||
|
val digestAlgorithm : String?,
|
||||||
|
val compressionEnabled: Boolean,
|
||||||
|
val compressionLevel: Int,
|
||||||
|
) : Configuration.Cache {
|
||||||
|
override fun materialize() = InMemoryCache(
|
||||||
|
maxAge,
|
||||||
|
maxSize,
|
||||||
|
digestAlgorithm,
|
||||||
|
compressionEnabled,
|
||||||
|
compressionLevel
|
||||||
|
)
|
||||||
|
|
||||||
|
override fun getNamespaceURI() = RBCS.RBCS_NAMESPACE_URI
|
||||||
|
|
||||||
|
override fun getTypeName() = "inMemoryCacheType"
|
||||||
|
}
|
63
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/InMemoryCacheProvider.kt
vendored
Normal file
63
rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/cache/InMemoryCacheProvider.kt
vendored
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
package net.woggioni.rbcs.server.cache
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.CacheProvider
|
||||||
|
import net.woggioni.rbcs.common.RBCS
|
||||||
|
import net.woggioni.rbcs.common.Xml
|
||||||
|
import net.woggioni.rbcs.common.Xml.Companion.renderAttribute
|
||||||
|
import org.w3c.dom.Document
|
||||||
|
import org.w3c.dom.Element
|
||||||
|
import java.time.Duration
|
||||||
|
import java.util.zip.Deflater
|
||||||
|
|
||||||
|
class InMemoryCacheProvider : CacheProvider<InMemoryCacheConfiguration> {
|
||||||
|
|
||||||
|
override fun getXmlSchemaLocation() = "classpath:net/woggioni/rbcs/server/schema/rbcs.xsd"
|
||||||
|
|
||||||
|
override fun getXmlType() = "inMemoryCacheType"
|
||||||
|
|
||||||
|
override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server"
|
||||||
|
|
||||||
|
override fun deserialize(el: Element): InMemoryCacheConfiguration {
|
||||||
|
val maxAge = el.renderAttribute("max-age")
|
||||||
|
?.let(Duration::parse)
|
||||||
|
?: Duration.ofDays(1)
|
||||||
|
val maxSize = el.renderAttribute("max-size")
|
||||||
|
?.let(java.lang.Long::decode)
|
||||||
|
?: 0x1000000
|
||||||
|
val enableCompression = el.renderAttribute("enable-compression")
|
||||||
|
?.let(String::toBoolean)
|
||||||
|
?: true
|
||||||
|
val compressionLevel = el.renderAttribute("compression-level")
|
||||||
|
?.let(String::toInt)
|
||||||
|
?: Deflater.DEFAULT_COMPRESSION
|
||||||
|
val digestAlgorithm = el.renderAttribute("digest") ?: "MD5"
|
||||||
|
|
||||||
|
return InMemoryCacheConfiguration(
|
||||||
|
maxAge,
|
||||||
|
maxSize,
|
||||||
|
digestAlgorithm,
|
||||||
|
enableCompression,
|
||||||
|
compressionLevel
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun serialize(doc: Document, cache : InMemoryCacheConfiguration) = cache.run {
|
||||||
|
val result = doc.createElement("cache")
|
||||||
|
Xml.of(doc, result) {
|
||||||
|
val prefix = doc.lookupPrefix(RBCS.RBCS_NAMESPACE_URI)
|
||||||
|
attr("xs:type", "${prefix}:inMemoryCacheType", RBCS.XML_SCHEMA_NAMESPACE_URI)
|
||||||
|
attr("max-age", maxAge.toString())
|
||||||
|
attr("max-size", maxSize.toString())
|
||||||
|
digestAlgorithm?.let { digestAlgorithm ->
|
||||||
|
attr("digest", digestAlgorithm)
|
||||||
|
}
|
||||||
|
attr("enable-compression", compressionEnabled.toString())
|
||||||
|
compressionLevel.takeIf {
|
||||||
|
it != Deflater.DEFAULT_COMPRESSION
|
||||||
|
}?.let {
|
||||||
|
attr("compression-level", it.toString())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,15 @@
|
|||||||
|
package net.woggioni.rbcs.server.configuration
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.CacheProvider
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import java.util.ServiceLoader
|
||||||
|
|
||||||
|
object CacheSerializers {
|
||||||
|
val index = (Configuration::class.java.module.layer?.let { layer ->
|
||||||
|
ServiceLoader.load(layer, CacheProvider::class.java)
|
||||||
|
} ?: ServiceLoader.load(CacheProvider::class.java))
|
||||||
|
.asSequence()
|
||||||
|
.map {
|
||||||
|
(it.xmlNamespace to it.xmlType) to it
|
||||||
|
}.toMap()
|
||||||
|
}
|
@@ -0,0 +1,298 @@
|
|||||||
|
package net.woggioni.rbcs.server.configuration
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.api.Configuration.Authentication
|
||||||
|
import net.woggioni.rbcs.api.Configuration.BasicAuthentication
|
||||||
|
import net.woggioni.rbcs.api.Configuration.Cache
|
||||||
|
import net.woggioni.rbcs.api.Configuration.ClientCertificateAuthentication
|
||||||
|
import net.woggioni.rbcs.api.Configuration.Group
|
||||||
|
import net.woggioni.rbcs.api.Configuration.KeyStore
|
||||||
|
import net.woggioni.rbcs.api.Configuration.Tls
|
||||||
|
import net.woggioni.rbcs.api.Configuration.TlsCertificateExtractor
|
||||||
|
import net.woggioni.rbcs.api.Configuration.TrustStore
|
||||||
|
import net.woggioni.rbcs.api.Configuration.User
|
||||||
|
import net.woggioni.rbcs.api.Role
|
||||||
|
import net.woggioni.rbcs.api.exception.ConfigurationException
|
||||||
|
import net.woggioni.rbcs.common.Xml.Companion.asIterable
|
||||||
|
import net.woggioni.rbcs.common.Xml.Companion.renderAttribute
|
||||||
|
import org.w3c.dom.Document
|
||||||
|
import org.w3c.dom.Element
|
||||||
|
import org.w3c.dom.TypeInfo
|
||||||
|
import java.nio.file.Paths
|
||||||
|
import java.time.Duration
|
||||||
|
import java.time.temporal.ChronoUnit
|
||||||
|
|
||||||
|
object Parser {
|
||||||
|
fun parse(document: Document): Configuration {
|
||||||
|
val root = document.documentElement
|
||||||
|
val anonymousUser = User("", null, emptySet(), null)
|
||||||
|
var connection: Configuration.Connection = Configuration.Connection(
|
||||||
|
Duration.of(10, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(10, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(60, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(30, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(30, ChronoUnit.SECONDS),
|
||||||
|
67108864
|
||||||
|
)
|
||||||
|
var eventExecutor: Configuration.EventExecutor = Configuration.EventExecutor(true)
|
||||||
|
var cache: Cache? = null
|
||||||
|
var host = "127.0.0.1"
|
||||||
|
var port = 11080
|
||||||
|
var users: Map<String, User> = mapOf(anonymousUser.name to anonymousUser)
|
||||||
|
var groups = emptyMap<String, Group>()
|
||||||
|
var tls: Tls? = null
|
||||||
|
val serverPath = root.renderAttribute("path")
|
||||||
|
var incomingConnectionsBacklogSize = 1024
|
||||||
|
var authentication: Authentication? = null
|
||||||
|
for (child in root.asIterable()) {
|
||||||
|
val tagName = child.localName
|
||||||
|
when (tagName) {
|
||||||
|
"authentication" -> {
|
||||||
|
for (gchild in child.asIterable()) {
|
||||||
|
when (gchild.localName) {
|
||||||
|
"basic" -> {
|
||||||
|
authentication = BasicAuthentication()
|
||||||
|
}
|
||||||
|
|
||||||
|
"client-certificate" -> {
|
||||||
|
var tlsExtractorUser: TlsCertificateExtractor? = null
|
||||||
|
var tlsExtractorGroup: TlsCertificateExtractor? = null
|
||||||
|
for (ggchild in gchild.asIterable()) {
|
||||||
|
when (ggchild.localName) {
|
||||||
|
"group-extractor" -> {
|
||||||
|
val attrName = ggchild.renderAttribute("attribute-name")
|
||||||
|
val pattern = ggchild.renderAttribute("pattern")
|
||||||
|
tlsExtractorGroup = TlsCertificateExtractor(attrName, pattern)
|
||||||
|
}
|
||||||
|
|
||||||
|
"user-extractor" -> {
|
||||||
|
val attrName = ggchild.renderAttribute("attribute-name")
|
||||||
|
val pattern = ggchild.renderAttribute("pattern")
|
||||||
|
tlsExtractorUser = TlsCertificateExtractor(attrName, pattern)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
authentication = ClientCertificateAuthentication(tlsExtractorUser, tlsExtractorGroup)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
"authorization" -> {
|
||||||
|
var knownUsers = sequenceOf(anonymousUser)
|
||||||
|
for (gchild in child.asIterable()) {
|
||||||
|
when (gchild.localName) {
|
||||||
|
"users" -> {
|
||||||
|
knownUsers += parseUsers(gchild)
|
||||||
|
}
|
||||||
|
|
||||||
|
"groups" -> {
|
||||||
|
val pair = parseGroups(gchild, knownUsers)
|
||||||
|
users = pair.first
|
||||||
|
groups = pair.second
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
"bind" -> {
|
||||||
|
host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required")
|
||||||
|
port = Integer.parseInt(child.renderAttribute("port"))
|
||||||
|
incomingConnectionsBacklogSize = child.renderAttribute("incoming-connections-backlog-size")
|
||||||
|
?.let(Integer::parseInt)
|
||||||
|
?: 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
"cache" -> {
|
||||||
|
cache = (child as TypeInfo).let { tf ->
|
||||||
|
val typeNamespace = tf.typeNamespace
|
||||||
|
val typeName = tf.typeName
|
||||||
|
CacheSerializers.index[typeNamespace to typeName]
|
||||||
|
?: throw IllegalArgumentException("Cache provider for namespace '$typeNamespace' with name '$typeName' not found")
|
||||||
|
}.deserialize(child)
|
||||||
|
}
|
||||||
|
|
||||||
|
"connection" -> {
|
||||||
|
val writeTimeout = child.renderAttribute("write-timeout")
|
||||||
|
?.let(Duration::parse) ?: Duration.of(0, ChronoUnit.SECONDS)
|
||||||
|
val readTimeout = child.renderAttribute("read-timeout")
|
||||||
|
?.let(Duration::parse) ?: Duration.of(0, ChronoUnit.SECONDS)
|
||||||
|
val idleTimeout = child.renderAttribute("idle-timeout")
|
||||||
|
?.let(Duration::parse) ?: Duration.of(30, ChronoUnit.SECONDS)
|
||||||
|
val readIdleTimeout = child.renderAttribute("read-idle-timeout")
|
||||||
|
?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS)
|
||||||
|
val writeIdleTimeout = child.renderAttribute("write-idle-timeout")
|
||||||
|
?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS)
|
||||||
|
val maxRequestSize = child.renderAttribute("max-request-size")
|
||||||
|
?.let(String::toInt) ?: 67108864
|
||||||
|
connection = Configuration.Connection(
|
||||||
|
readTimeout,
|
||||||
|
writeTimeout,
|
||||||
|
idleTimeout,
|
||||||
|
readIdleTimeout,
|
||||||
|
writeIdleTimeout,
|
||||||
|
maxRequestSize
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
"event-executor" -> {
|
||||||
|
val useVirtualThread = root.renderAttribute("use-virtual-threads")
|
||||||
|
?.let(String::toBoolean) ?: true
|
||||||
|
eventExecutor = Configuration.EventExecutor(useVirtualThread)
|
||||||
|
}
|
||||||
|
|
||||||
|
"tls" -> {
|
||||||
|
var keyStore: KeyStore? = null
|
||||||
|
var trustStore: TrustStore? = null
|
||||||
|
|
||||||
|
for (granChild in child.asIterable()) {
|
||||||
|
when (granChild.localName) {
|
||||||
|
"keystore" -> {
|
||||||
|
val keyStoreFile = Paths.get(granChild.renderAttribute("file"))
|
||||||
|
val keyStorePassword = granChild.renderAttribute("password")
|
||||||
|
val keyAlias = granChild.renderAttribute("key-alias")
|
||||||
|
val keyPassword = granChild.renderAttribute("key-password")
|
||||||
|
keyStore = KeyStore(
|
||||||
|
keyStoreFile,
|
||||||
|
keyStorePassword,
|
||||||
|
keyAlias,
|
||||||
|
keyPassword
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
"truststore" -> {
|
||||||
|
val trustStoreFile = Paths.get(granChild.renderAttribute("file"))
|
||||||
|
val trustStorePassword = granChild.renderAttribute("password")
|
||||||
|
val checkCertificateStatus = granChild.renderAttribute("check-certificate-status")
|
||||||
|
?.let(String::toBoolean)
|
||||||
|
?: false
|
||||||
|
val requireClientCertificate = child.renderAttribute("require-client-certificate")
|
||||||
|
?.let(String::toBoolean) ?: false
|
||||||
|
|
||||||
|
trustStore = TrustStore(
|
||||||
|
trustStoreFile,
|
||||||
|
trustStorePassword,
|
||||||
|
checkCertificateStatus,
|
||||||
|
requireClientCertificate
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tls = Tls(keyStore, trustStore)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Configuration.of(
|
||||||
|
host,
|
||||||
|
port,
|
||||||
|
incomingConnectionsBacklogSize,
|
||||||
|
serverPath,
|
||||||
|
eventExecutor,
|
||||||
|
connection,
|
||||||
|
users,
|
||||||
|
groups,
|
||||||
|
cache!!,
|
||||||
|
authentication,
|
||||||
|
tls,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseRoles(root: Element) = root.asIterable().asSequence().map {
|
||||||
|
when (it.localName) {
|
||||||
|
"reader" -> Role.Reader
|
||||||
|
"writer" -> Role.Writer
|
||||||
|
else -> throw UnsupportedOperationException("Illegal node '${it.localName}'")
|
||||||
|
}
|
||||||
|
}.toSet()
|
||||||
|
|
||||||
|
private fun parseUserRefs(root: Element) = root.asIterable().asSequence().map {
|
||||||
|
when (it.localName) {
|
||||||
|
"user" -> it.renderAttribute("ref")
|
||||||
|
"anonymous" -> ""
|
||||||
|
else -> ConfigurationException("Unrecognized tag '${it.localName}'")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseQuota(el: Element): Configuration.Quota {
|
||||||
|
val calls = el.renderAttribute("calls")
|
||||||
|
?.let(String::toLong)
|
||||||
|
?: throw ConfigurationException("Missing attribute 'calls'")
|
||||||
|
val maxAvailableCalls = el.renderAttribute("max-available-calls")
|
||||||
|
?.let(String::toLong)
|
||||||
|
?: calls
|
||||||
|
val initialAvailableCalls = el.renderAttribute("initial-available-calls")
|
||||||
|
?.let(String::toLong)
|
||||||
|
?: maxAvailableCalls
|
||||||
|
val period = el.renderAttribute("period")
|
||||||
|
?.let(Duration::parse)
|
||||||
|
?: throw ConfigurationException("Missing attribute 'period'")
|
||||||
|
return Configuration.Quota(calls, period, initialAvailableCalls, maxAvailableCalls)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseUsers(root: Element): Sequence<User> {
|
||||||
|
return root.asIterable().asSequence().mapNotNull { child ->
|
||||||
|
when (child.localName) {
|
||||||
|
"user" -> {
|
||||||
|
val username = child.renderAttribute("name")
|
||||||
|
val password = child.renderAttribute("password")
|
||||||
|
var quota: Configuration.Quota? = null
|
||||||
|
for (gchild in child.asIterable()) {
|
||||||
|
if (gchild.localName == "quota") {
|
||||||
|
quota = parseQuota(gchild)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
User(username, password, emptySet(), quota)
|
||||||
|
}
|
||||||
|
"anonymous" -> {
|
||||||
|
var quota: Configuration.Quota? = null
|
||||||
|
for (gchild in child.asIterable()) {
|
||||||
|
if (gchild.localName == "quota") {
|
||||||
|
quota= parseQuota(gchild)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
User("", null, emptySet(), quota)
|
||||||
|
}
|
||||||
|
else -> null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseGroups(root: Element, knownUsers: Sequence<User>): Pair<Map<String, User>, Map<String, Group>> {
|
||||||
|
val knownUsersMap = knownUsers.associateBy(User::getName)
|
||||||
|
val userGroups = mutableMapOf<String, MutableSet<String>>()
|
||||||
|
val groups = root.asIterable().asSequence().filter {
|
||||||
|
it.localName == "group"
|
||||||
|
}.map { el ->
|
||||||
|
val groupName = el.renderAttribute("name") ?: throw ConfigurationException("Group name is required")
|
||||||
|
var roles = emptySet<Role>()
|
||||||
|
var userQuota: Configuration.Quota? = null
|
||||||
|
var groupQuota: Configuration.Quota? = null
|
||||||
|
for (child in el.asIterable()) {
|
||||||
|
when (child.localName) {
|
||||||
|
"users" -> {
|
||||||
|
parseUserRefs(child).mapNotNull(knownUsersMap::get).forEach { user ->
|
||||||
|
userGroups.computeIfAbsent(user.name) {
|
||||||
|
mutableSetOf()
|
||||||
|
}.add(groupName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
"roles" -> {
|
||||||
|
roles = parseRoles(child)
|
||||||
|
}
|
||||||
|
"group-quota" -> {
|
||||||
|
userQuota = parseQuota(child)
|
||||||
|
}
|
||||||
|
"user-quota" -> {
|
||||||
|
groupQuota = parseQuota(child)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
groupName to Group(groupName, roles, userQuota, groupQuota)
|
||||||
|
}.toMap()
|
||||||
|
val users = knownUsersMap.map { (name, user) ->
|
||||||
|
name to User(name, user.password, userGroups[name]?.mapNotNull { groups[it] }?.toSet() ?: emptySet(), user.quota)
|
||||||
|
}.toMap()
|
||||||
|
return users to groups
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,186 @@
|
|||||||
|
package net.woggioni.rbcs.server.configuration
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.CacheProvider
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.common.RBCS
|
||||||
|
import net.woggioni.rbcs.common.Xml
|
||||||
|
import org.w3c.dom.Document
|
||||||
|
|
||||||
|
object Serializer {
|
||||||
|
|
||||||
|
private fun Xml.serializeQuota(quota : Configuration.Quota) {
|
||||||
|
attr("calls", quota.calls.toString())
|
||||||
|
attr("period", quota.period.toString())
|
||||||
|
attr("max-available-calls", quota.maxAvailableCalls.toString())
|
||||||
|
attr("initial-available-calls", quota.initialAvailableCalls.toString())
|
||||||
|
}
|
||||||
|
|
||||||
|
fun serialize(conf : Configuration) : Document {
|
||||||
|
val schemaLocations = CacheSerializers.index.values.asSequence().map {
|
||||||
|
it.xmlNamespace to it.xmlSchemaLocation
|
||||||
|
}.toMap()
|
||||||
|
return Xml.of(RBCS.RBCS_NAMESPACE_URI, RBCS.RBCS_PREFIX + ":server") {
|
||||||
|
// attr("xmlns:xs", GradleBuildCacheServer.XML_SCHEMA_NAMESPACE_URI)
|
||||||
|
val value = schemaLocations.asSequence().map { (k, v) -> "$k $v" }.joinToString(" ")
|
||||||
|
attr("xs:schemaLocation", value , namespaceURI = RBCS.XML_SCHEMA_NAMESPACE_URI)
|
||||||
|
|
||||||
|
conf.serverPath
|
||||||
|
?.takeIf(String::isNotEmpty)
|
||||||
|
?.let { serverPath ->
|
||||||
|
attr("path", serverPath)
|
||||||
|
}
|
||||||
|
node("bind") {
|
||||||
|
attr("host", conf.host)
|
||||||
|
attr("port", conf.port.toString())
|
||||||
|
attr("incoming-connections-backlog-size", conf.incomingConnectionsBacklogSize.toString())
|
||||||
|
}
|
||||||
|
node("connection") {
|
||||||
|
conf.connection.let { connection ->
|
||||||
|
attr("read-timeout", connection.readTimeout.toString())
|
||||||
|
attr("write-timeout", connection.writeTimeout.toString())
|
||||||
|
attr("idle-timeout", connection.idleTimeout.toString())
|
||||||
|
attr("read-idle-timeout", connection.readIdleTimeout.toString())
|
||||||
|
attr("write-idle-timeout", connection.writeIdleTimeout.toString())
|
||||||
|
attr("max-request-size", connection.maxRequestSize.toString())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
node("event-executor") {
|
||||||
|
attr("use-virtual-threads", conf.eventExecutor.isUseVirtualThreads.toString())
|
||||||
|
}
|
||||||
|
val cache = conf.cache
|
||||||
|
val serializer : CacheProvider<Configuration.Cache> =
|
||||||
|
(CacheSerializers.index[cache.namespaceURI to cache.typeName] as? CacheProvider<Configuration.Cache>) ?: throw NotImplementedError()
|
||||||
|
element.appendChild(serializer.serialize(doc, cache))
|
||||||
|
node("authorization") {
|
||||||
|
node("users") {
|
||||||
|
for(user in conf.users.values) {
|
||||||
|
if(user.name.isNotEmpty()) {
|
||||||
|
node("user") {
|
||||||
|
attr("name", user.name)
|
||||||
|
user.password?.let { password ->
|
||||||
|
attr("password", password)
|
||||||
|
}
|
||||||
|
user.quota?.let { quota ->
|
||||||
|
node("quota") {
|
||||||
|
serializeQuota(quota)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
conf.users[""]
|
||||||
|
?.let { anonymousUser ->
|
||||||
|
anonymousUser.quota?.let { quota ->
|
||||||
|
node("anonymous") {
|
||||||
|
node("quota") {
|
||||||
|
serializeQuota(quota)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
node("groups") {
|
||||||
|
val groups = conf.users.values.asSequence()
|
||||||
|
.flatMap {
|
||||||
|
user -> user.groups.map { it to user }
|
||||||
|
}.groupBy(Pair<Configuration.Group, Configuration.User>::first, Pair<Configuration.Group, Configuration.User>::second)
|
||||||
|
for(pair in groups) {
|
||||||
|
val group = pair.key
|
||||||
|
val users = pair.value
|
||||||
|
node("group") {
|
||||||
|
attr("name", group.name)
|
||||||
|
if(users.isNotEmpty()) {
|
||||||
|
node("users") {
|
||||||
|
var anonymousUser : Configuration.User? = null
|
||||||
|
for(user in users) {
|
||||||
|
if(user.name.isNotEmpty()) {
|
||||||
|
node("user") {
|
||||||
|
attr("ref", user.name)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
anonymousUser = user
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(anonymousUser != null) {
|
||||||
|
node("anonymous")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(group.roles.isNotEmpty()) {
|
||||||
|
node("roles") {
|
||||||
|
for(role in group.roles) {
|
||||||
|
node(role.toString().lowercase())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
group.userQuota?.let { quota ->
|
||||||
|
node("user-quota") {
|
||||||
|
serializeQuota(quota)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
group.groupQuota?.let { quota ->
|
||||||
|
node("group-quota") {
|
||||||
|
serializeQuota(quota)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
conf.authentication?.let { authentication ->
|
||||||
|
node("authentication") {
|
||||||
|
when(authentication) {
|
||||||
|
is Configuration.BasicAuthentication -> {
|
||||||
|
node("basic")
|
||||||
|
}
|
||||||
|
is Configuration.ClientCertificateAuthentication -> {
|
||||||
|
node("client-certificate") {
|
||||||
|
authentication.groupExtractor?.let { extractor ->
|
||||||
|
node("group-extractor") {
|
||||||
|
attr("attribute-name", extractor.rdnType)
|
||||||
|
attr("pattern", extractor.pattern)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
authentication.userExtractor?.let { extractor ->
|
||||||
|
node("user-extractor") {
|
||||||
|
attr("attribute-name", extractor.rdnType)
|
||||||
|
attr("pattern", extractor.pattern)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
conf.tls?.let { tlsConfiguration ->
|
||||||
|
node("tls") {
|
||||||
|
tlsConfiguration.keyStore?.let { keyStore ->
|
||||||
|
node("keystore") {
|
||||||
|
attr("file", keyStore.file.toString())
|
||||||
|
keyStore.password?.let { keyStorePassword ->
|
||||||
|
attr("password", keyStorePassword)
|
||||||
|
}
|
||||||
|
attr("key-alias", keyStore.keyAlias)
|
||||||
|
keyStore.keyPassword?.let { keyPassword ->
|
||||||
|
attr("key-password", keyPassword)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tlsConfiguration.trustStore?.let { trustStore ->
|
||||||
|
node("truststore") {
|
||||||
|
attr("file", trustStore.file.toString())
|
||||||
|
trustStore.password?.let { password ->
|
||||||
|
attr("password", password)
|
||||||
|
}
|
||||||
|
attr("check-certificate-status", trustStore.isCheckCertificateStatus.toString())
|
||||||
|
attr("require-client-certificate", trustStore.isRequireClientCertificate.toString())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,92 @@
|
|||||||
|
package net.woggioni.rbcs.server.exception
|
||||||
|
|
||||||
|
import io.netty.buffer.Unpooled
|
||||||
|
import io.netty.channel.ChannelDuplexHandler
|
||||||
|
import io.netty.channel.ChannelFutureListener
|
||||||
|
import io.netty.channel.ChannelHandler
|
||||||
|
import io.netty.channel.ChannelHandlerContext
|
||||||
|
import io.netty.handler.codec.DecoderException
|
||||||
|
import io.netty.handler.codec.http.DefaultFullHttpResponse
|
||||||
|
import io.netty.handler.codec.http.FullHttpResponse
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderNames
|
||||||
|
import io.netty.handler.codec.http.HttpResponseStatus
|
||||||
|
import io.netty.handler.codec.http.HttpVersion
|
||||||
|
import io.netty.handler.timeout.ReadTimeoutException
|
||||||
|
import io.netty.handler.timeout.WriteTimeoutException
|
||||||
|
import net.woggioni.rbcs.api.exception.CacheException
|
||||||
|
import net.woggioni.rbcs.api.exception.ContentTooLargeException
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.rbcs.common.debug
|
||||||
|
import javax.net.ssl.SSLPeerUnverifiedException
|
||||||
|
|
||||||
|
@ChannelHandler.Sharable
|
||||||
|
class ExceptionHandler : ChannelDuplexHandler() {
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
private val NOT_AUTHORIZED: FullHttpResponse = DefaultFullHttpResponse(
|
||||||
|
HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER
|
||||||
|
).apply {
|
||||||
|
headers()[HttpHeaderNames.CONTENT_LENGTH] = "0"
|
||||||
|
}
|
||||||
|
|
||||||
|
private val TOO_BIG: FullHttpResponse = DefaultFullHttpResponse(
|
||||||
|
HttpVersion.HTTP_1_1, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, Unpooled.EMPTY_BUFFER
|
||||||
|
).apply {
|
||||||
|
headers()[HttpHeaderNames.CONTENT_LENGTH] = "0"
|
||||||
|
}
|
||||||
|
|
||||||
|
private val NOT_AVAILABLE: FullHttpResponse = DefaultFullHttpResponse(
|
||||||
|
HttpVersion.HTTP_1_1, HttpResponseStatus.SERVICE_UNAVAILABLE, Unpooled.EMPTY_BUFFER
|
||||||
|
).apply {
|
||||||
|
headers()[HttpHeaderNames.CONTENT_LENGTH] = "0"
|
||||||
|
}
|
||||||
|
|
||||||
|
private val SERVER_ERROR: FullHttpResponse = DefaultFullHttpResponse(
|
||||||
|
HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR, Unpooled.EMPTY_BUFFER
|
||||||
|
).apply {
|
||||||
|
headers()[HttpHeaderNames.CONTENT_LENGTH] = "0"
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
|
||||||
|
when (cause) {
|
||||||
|
is DecoderException -> {
|
||||||
|
log.error(cause.message, cause)
|
||||||
|
ctx.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
is SSLPeerUnverifiedException -> {
|
||||||
|
ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate())
|
||||||
|
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE)
|
||||||
|
}
|
||||||
|
|
||||||
|
is ContentTooLargeException -> {
|
||||||
|
ctx.writeAndFlush(TOO_BIG.retainedDuplicate())
|
||||||
|
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE)
|
||||||
|
}
|
||||||
|
is ReadTimeoutException -> {
|
||||||
|
log.debug {
|
||||||
|
val channelId = ctx.channel().id().asShortText()
|
||||||
|
"Read timeout on channel $channelId, closing the connection"
|
||||||
|
}
|
||||||
|
ctx.close()
|
||||||
|
}
|
||||||
|
is WriteTimeoutException -> {
|
||||||
|
log.debug {
|
||||||
|
val channelId = ctx.channel().id().asShortText()
|
||||||
|
"Write timeout on channel $channelId, closing the connection"
|
||||||
|
}
|
||||||
|
ctx.close()
|
||||||
|
}
|
||||||
|
is CacheException -> {
|
||||||
|
log.error(cause.message, cause)
|
||||||
|
ctx.writeAndFlush(NOT_AVAILABLE.retainedDuplicate())
|
||||||
|
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE)
|
||||||
|
}
|
||||||
|
else -> {
|
||||||
|
log.error(cause.message, cause)
|
||||||
|
ctx.writeAndFlush(SERVER_ERROR.retainedDuplicate())
|
||||||
|
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,161 @@
|
|||||||
|
package net.woggioni.rbcs.server.handler
|
||||||
|
|
||||||
|
import io.netty.buffer.Unpooled
|
||||||
|
import io.netty.channel.ChannelFutureListener
|
||||||
|
import io.netty.channel.ChannelHandler
|
||||||
|
import io.netty.channel.ChannelHandlerContext
|
||||||
|
import io.netty.channel.DefaultFileRegion
|
||||||
|
import io.netty.channel.SimpleChannelInboundHandler
|
||||||
|
import io.netty.handler.codec.http.DefaultFullHttpResponse
|
||||||
|
import io.netty.handler.codec.http.DefaultHttpResponse
|
||||||
|
import io.netty.handler.codec.http.FullHttpRequest
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderNames
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderValues
|
||||||
|
import io.netty.handler.codec.http.HttpMethod
|
||||||
|
import io.netty.handler.codec.http.HttpResponseStatus
|
||||||
|
import io.netty.handler.codec.http.HttpUtil
|
||||||
|
import io.netty.handler.codec.http.LastHttpContent
|
||||||
|
import io.netty.handler.stream.ChunkedNioStream
|
||||||
|
import net.woggioni.rbcs.api.Cache
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.rbcs.server.debug
|
||||||
|
import net.woggioni.rbcs.server.warn
|
||||||
|
import java.nio.channels.FileChannel
|
||||||
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
@ChannelHandler.Sharable
|
||||||
|
class ServerHandler(private val cache: Cache, private val serverPrefix: Path) :
|
||||||
|
SimpleChannelInboundHandler<FullHttpRequest>() {
|
||||||
|
|
||||||
|
private val log = contextLogger()
|
||||||
|
|
||||||
|
override fun channelRead0(ctx: ChannelHandlerContext, msg: FullHttpRequest) {
|
||||||
|
val keepAlive: Boolean = HttpUtil.isKeepAlive(msg)
|
||||||
|
val method = msg.method()
|
||||||
|
if (method === HttpMethod.GET) {
|
||||||
|
val path = Path.of(msg.uri())
|
||||||
|
val prefix = path.parent
|
||||||
|
val key = path.fileName?.toString() ?: let {
|
||||||
|
val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.NOT_FOUND)
|
||||||
|
response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0
|
||||||
|
ctx.writeAndFlush(response)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (serverPrefix == prefix) {
|
||||||
|
cache.get(key).thenApply { channel ->
|
||||||
|
if(channel != null) {
|
||||||
|
log.debug(ctx) {
|
||||||
|
"Cache hit for key '$key'"
|
||||||
|
}
|
||||||
|
val response = DefaultHttpResponse(msg.protocolVersion(), HttpResponseStatus.OK)
|
||||||
|
response.headers()[HttpHeaderNames.CONTENT_TYPE] = HttpHeaderValues.APPLICATION_OCTET_STREAM
|
||||||
|
if (!keepAlive) {
|
||||||
|
response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE)
|
||||||
|
response.headers().set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.IDENTITY)
|
||||||
|
} else {
|
||||||
|
response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE)
|
||||||
|
response.headers().set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED)
|
||||||
|
}
|
||||||
|
ctx.write(response)
|
||||||
|
when (channel) {
|
||||||
|
is FileChannel -> {
|
||||||
|
val content = DefaultFileRegion(channel, 0, channel.size())
|
||||||
|
if (keepAlive) {
|
||||||
|
ctx.write(content)
|
||||||
|
ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT.retainedDuplicate())
|
||||||
|
} else {
|
||||||
|
ctx.writeAndFlush(content)
|
||||||
|
.addListener(ChannelFutureListener.CLOSE)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else -> {
|
||||||
|
val content = ChunkedNioStream(channel)
|
||||||
|
if (keepAlive) {
|
||||||
|
ctx.write(content).addListener {
|
||||||
|
content.close()
|
||||||
|
}
|
||||||
|
ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT.retainedDuplicate())
|
||||||
|
} else {
|
||||||
|
ctx.writeAndFlush(content)
|
||||||
|
.addListener(ChannelFutureListener.CLOSE)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.debug(ctx) {
|
||||||
|
"Cache miss for key '$key'"
|
||||||
|
}
|
||||||
|
val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.NOT_FOUND)
|
||||||
|
response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0
|
||||||
|
ctx.writeAndFlush(response)
|
||||||
|
}
|
||||||
|
}.whenComplete { _, ex -> ex?.let(ctx::fireExceptionCaught) }
|
||||||
|
} else {
|
||||||
|
log.warn(ctx) {
|
||||||
|
"Got request for unhandled path '${msg.uri()}'"
|
||||||
|
}
|
||||||
|
val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.BAD_REQUEST)
|
||||||
|
response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0
|
||||||
|
ctx.writeAndFlush(response)
|
||||||
|
}
|
||||||
|
} else if (method === HttpMethod.PUT) {
|
||||||
|
val path = Path.of(msg.uri())
|
||||||
|
val prefix = path.parent
|
||||||
|
val key = path.fileName.toString()
|
||||||
|
|
||||||
|
if (serverPrefix == prefix) {
|
||||||
|
log.debug(ctx) {
|
||||||
|
"Added value for key '$key' to build cache"
|
||||||
|
}
|
||||||
|
cache.put(key, msg.content()).thenRun {
|
||||||
|
val response = DefaultFullHttpResponse(
|
||||||
|
msg.protocolVersion(), HttpResponseStatus.CREATED,
|
||||||
|
Unpooled.copiedBuffer(key.toByteArray())
|
||||||
|
)
|
||||||
|
response.headers()[HttpHeaderNames.CONTENT_LENGTH] = response.content().readableBytes()
|
||||||
|
ctx.writeAndFlush(response)
|
||||||
|
}.whenComplete { _, ex ->
|
||||||
|
ctx.fireExceptionCaught(ex)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.warn(ctx) {
|
||||||
|
"Got request for unhandled path '${msg.uri()}'"
|
||||||
|
}
|
||||||
|
val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.BAD_REQUEST)
|
||||||
|
response.headers()[HttpHeaderNames.CONTENT_LENGTH] = "0"
|
||||||
|
ctx.writeAndFlush(response)
|
||||||
|
}
|
||||||
|
} else if(method == HttpMethod.TRACE) {
|
||||||
|
val replayedRequestHead = ctx.alloc().buffer()
|
||||||
|
replayedRequestHead.writeCharSequence("TRACE ${Path.of(msg.uri())} ${msg.protocolVersion().text()}\r\n", Charsets.US_ASCII)
|
||||||
|
msg.headers().forEach { (key, value) ->
|
||||||
|
replayedRequestHead.apply {
|
||||||
|
writeCharSequence(key, Charsets.US_ASCII)
|
||||||
|
writeCharSequence(": ", Charsets.US_ASCII)
|
||||||
|
writeCharSequence(value, Charsets.UTF_8)
|
||||||
|
writeCharSequence("\r\n", Charsets.US_ASCII)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
replayedRequestHead.writeCharSequence("\r\n", Charsets.US_ASCII)
|
||||||
|
val requestBody = msg.content()
|
||||||
|
requestBody.retain()
|
||||||
|
val responseBody = ctx.alloc().compositeBuffer(2).apply {
|
||||||
|
addComponents(true, replayedRequestHead)
|
||||||
|
addComponents(true, requestBody)
|
||||||
|
}
|
||||||
|
val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.OK, responseBody)
|
||||||
|
response.headers().apply {
|
||||||
|
set(HttpHeaderNames.CONTENT_TYPE, "message/http")
|
||||||
|
set(HttpHeaderNames.CONTENT_LENGTH, responseBody.readableBytes())
|
||||||
|
}
|
||||||
|
ctx.writeAndFlush(response)
|
||||||
|
} else {
|
||||||
|
log.warn(ctx) {
|
||||||
|
"Got request with unhandled method '${msg.method().name()}'"
|
||||||
|
}
|
||||||
|
val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.METHOD_NOT_ALLOWED)
|
||||||
|
response.headers()[HttpHeaderNames.CONTENT_LENGTH] = "0"
|
||||||
|
ctx.writeAndFlush(response)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,91 @@
|
|||||||
|
package net.woggioni.rbcs.server.throttling
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.jwo.Bucket
|
||||||
|
import java.net.InetSocketAddress
|
||||||
|
import java.util.Arrays
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import java.util.function.Function
|
||||||
|
|
||||||
|
class BucketManager private constructor(
|
||||||
|
private val bucketsByUser: Map<Configuration.User, List<Bucket>> = HashMap(),
|
||||||
|
private val bucketsByGroup: Map<Configuration.Group, Bucket> = HashMap(),
|
||||||
|
loader: Function<InetSocketAddress, Bucket>?
|
||||||
|
) {
|
||||||
|
|
||||||
|
private class BucketsByAddress(
|
||||||
|
private val map: MutableMap<ByteArrayKey, Bucket>,
|
||||||
|
private val loader: Function<InetSocketAddress, Bucket>
|
||||||
|
) {
|
||||||
|
fun getBucket(socketAddress : InetSocketAddress) = map.computeIfAbsent(ByteArrayKey(socketAddress.address.address)) {
|
||||||
|
loader.apply(socketAddress)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private val bucketsByAddress: BucketsByAddress? = loader?.let {
|
||||||
|
BucketsByAddress(ConcurrentHashMap(), it)
|
||||||
|
}
|
||||||
|
|
||||||
|
private class ByteArrayKey(val array: ByteArray) {
|
||||||
|
override fun equals(other: Any?) = (other as? ByteArrayKey)?.let { bak ->
|
||||||
|
array contentEquals bak.array
|
||||||
|
} ?: false
|
||||||
|
|
||||||
|
override fun hashCode() = Arrays.hashCode(array)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getBucketByAddress(address : InetSocketAddress) : Bucket? {
|
||||||
|
return bucketsByAddress?.getBucket(address)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getBucketByUser(user : Configuration.User) = bucketsByUser[user]
|
||||||
|
fun getBucketByGroup(group : Configuration.Group) = bucketsByGroup[group]
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
fun from(cfg : Configuration) : BucketManager {
|
||||||
|
val bucketsByUser = cfg.users.values.asSequence().map { user ->
|
||||||
|
val buckets = (
|
||||||
|
user.quota
|
||||||
|
?.let { quota ->
|
||||||
|
sequenceOf(quota)
|
||||||
|
} ?: user.groups.asSequence()
|
||||||
|
.mapNotNull(Configuration.Group::getUserQuota)
|
||||||
|
).map { quota ->
|
||||||
|
Bucket.local(
|
||||||
|
quota.maxAvailableCalls,
|
||||||
|
quota.calls,
|
||||||
|
quota.period,
|
||||||
|
quota.initialAvailableCalls
|
||||||
|
)
|
||||||
|
}.toList()
|
||||||
|
user to buckets
|
||||||
|
}.toMap()
|
||||||
|
val bucketsByGroup = cfg.groups.values.asSequence().filter {
|
||||||
|
it.groupQuota != null
|
||||||
|
}.map { group ->
|
||||||
|
val quota = group.groupQuota
|
||||||
|
val bucket = Bucket.local(
|
||||||
|
quota.maxAvailableCalls,
|
||||||
|
quota.calls,
|
||||||
|
quota.period,
|
||||||
|
quota.initialAvailableCalls
|
||||||
|
)
|
||||||
|
group to bucket
|
||||||
|
}.toMap()
|
||||||
|
return BucketManager(
|
||||||
|
bucketsByUser,
|
||||||
|
bucketsByGroup,
|
||||||
|
cfg.users[""]?.quota?.let { anonymousUserQuota ->
|
||||||
|
Function {
|
||||||
|
Bucket.local(
|
||||||
|
anonymousUserQuota.maxAvailableCalls,
|
||||||
|
anonymousUserQuota.calls,
|
||||||
|
anonymousUserQuota.period,
|
||||||
|
anonymousUserQuota.initialAvailableCalls
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,99 @@
|
|||||||
|
package net.woggioni.rbcs.server.throttling
|
||||||
|
|
||||||
|
import io.netty.channel.ChannelHandler.Sharable
|
||||||
|
import io.netty.channel.ChannelHandlerContext
|
||||||
|
import io.netty.channel.ChannelInboundHandlerAdapter
|
||||||
|
import io.netty.handler.codec.http.DefaultFullHttpResponse
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderNames
|
||||||
|
import io.netty.handler.codec.http.HttpResponseStatus
|
||||||
|
import io.netty.handler.codec.http.HttpVersion
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.common.contextLogger
|
||||||
|
import net.woggioni.rbcs.server.RemoteBuildCacheServer
|
||||||
|
import net.woggioni.jwo.Bucket
|
||||||
|
import net.woggioni.jwo.LongMath
|
||||||
|
import java.net.InetSocketAddress
|
||||||
|
import java.time.Duration
|
||||||
|
import java.time.temporal.ChronoUnit
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
|
||||||
|
|
||||||
|
@Sharable
|
||||||
|
class ThrottlingHandler(cfg: Configuration) :
|
||||||
|
ChannelInboundHandlerAdapter() {
|
||||||
|
|
||||||
|
private val log = contextLogger()
|
||||||
|
private val bucketManager = BucketManager.from(cfg)
|
||||||
|
|
||||||
|
private val connectionConfiguration = cfg.connection
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If the suggested waiting time from the bucket is lower than this
|
||||||
|
* amount, then the server will simply wait by itself before sending a response
|
||||||
|
* instead of replying with 429
|
||||||
|
*/
|
||||||
|
private val waitThreshold = minOf(
|
||||||
|
connectionConfiguration.idleTimeout,
|
||||||
|
connectionConfiguration.readIdleTimeout,
|
||||||
|
connectionConfiguration.writeIdleTimeout
|
||||||
|
).dividedBy(2)
|
||||||
|
|
||||||
|
override fun channelRead(ctx: ChannelHandlerContext, msg: Any) {
|
||||||
|
val buckets = mutableListOf<Bucket>()
|
||||||
|
val user = ctx.channel().attr(RemoteBuildCacheServer.userAttribute).get()
|
||||||
|
if (user != null) {
|
||||||
|
bucketManager.getBucketByUser(user)?.let(buckets::addAll)
|
||||||
|
}
|
||||||
|
val groups = ctx.channel().attr(RemoteBuildCacheServer.groupAttribute).get() ?: emptySet()
|
||||||
|
if (groups.isNotEmpty()) {
|
||||||
|
groups.forEach { group ->
|
||||||
|
bucketManager.getBucketByGroup(group)?.let(buckets::add)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (user == null && groups.isEmpty()) {
|
||||||
|
bucketManager.getBucketByAddress(ctx.channel().remoteAddress() as InetSocketAddress)?.let(buckets::add)
|
||||||
|
}
|
||||||
|
if (buckets.isEmpty()) {
|
||||||
|
return super.channelRead(ctx, msg)
|
||||||
|
} else {
|
||||||
|
handleBuckets(buckets, ctx, msg, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun handleBuckets(buckets : List<Bucket>, ctx : ChannelHandlerContext, msg : Any, delayResponse : Boolean) {
|
||||||
|
var nextAttempt = -1L
|
||||||
|
for (bucket in buckets) {
|
||||||
|
val bucketNextAttempt = bucket.removeTokensWithEstimate(1)
|
||||||
|
if (bucketNextAttempt > nextAttempt) {
|
||||||
|
nextAttempt = bucketNextAttempt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(nextAttempt < 0) {
|
||||||
|
super.channelRead(ctx, msg)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
val waitDuration = Duration.of(LongMath.ceilDiv(nextAttempt, 100_000_000L) * 100L, ChronoUnit.MILLIS)
|
||||||
|
if (delayResponse && waitDuration < waitThreshold) {
|
||||||
|
ctx.executor().schedule({
|
||||||
|
handleBuckets(buckets, ctx, msg, false)
|
||||||
|
}, waitDuration.toMillis(), TimeUnit.MILLISECONDS)
|
||||||
|
} else {
|
||||||
|
sendThrottledResponse(ctx, waitDuration)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun sendThrottledResponse(ctx: ChannelHandlerContext, retryAfter: Duration) {
|
||||||
|
val response = DefaultFullHttpResponse(
|
||||||
|
HttpVersion.HTTP_1_1,
|
||||||
|
HttpResponseStatus.TOO_MANY_REQUESTS
|
||||||
|
)
|
||||||
|
response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0
|
||||||
|
retryAfter.seconds.takeIf {
|
||||||
|
it > 0
|
||||||
|
}?.let {
|
||||||
|
response.headers()[HttpHeaderNames.RETRY_AFTER] = retryAfter.seconds
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.writeAndFlush(response)
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,2 @@
|
|||||||
|
net.woggioni.rbcs.server.cache.FileSystemCacheProvider
|
||||||
|
net.woggioni.rbcs.server.cache.InMemoryCacheProvider
|
@@ -0,0 +1,19 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||||
|
<rbcs:server
|
||||||
|
xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xmlns:rbcs="urn:net.woggioni.rbcs.server"
|
||||||
|
xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs.xsd">
|
||||||
|
<bind host="127.0.0.1" port="8080" incoming-connections-backlog-size="1024"/>
|
||||||
|
<connection
|
||||||
|
max-request-size="67108864"
|
||||||
|
idle-timeout="PT30S"
|
||||||
|
read-timeout="PT10S"
|
||||||
|
write-timeout="PT10S"
|
||||||
|
read-idle-timeout="PT60S"
|
||||||
|
write-idle-timeout="PT60S"/>
|
||||||
|
<event-executor use-virtual-threads="true"/>
|
||||||
|
<cache xs:type="rbcs:fileSystemCacheType" path="/tmp/rbcs" max-age="P7D"/>
|
||||||
|
<authentication>
|
||||||
|
<none/>
|
||||||
|
</authentication>
|
||||||
|
</rbcs:server>
|
@@ -0,0 +1,224 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||||
|
<xs:schema targetNamespace="urn:net.woggioni.rbcs.server"
|
||||||
|
xmlns:xs="http://www.w3.org/2001/XMLSchema"
|
||||||
|
xmlns:rbcs="urn:net.woggioni.rbcs.server"
|
||||||
|
elementFormDefault="unqualified">
|
||||||
|
<xs:element name="server" type="rbcs:serverType"/>
|
||||||
|
|
||||||
|
<xs:complexType name="serverType">
|
||||||
|
<xs:sequence minOccurs="0">
|
||||||
|
<xs:element name="bind" type="rbcs:bindType" maxOccurs="1"/>
|
||||||
|
<xs:element name="connection" type="rbcs:connectionType" minOccurs="0" maxOccurs="1"/>
|
||||||
|
<xs:element name="event-executor" type="rbcs:eventExecutorType" minOccurs="0" maxOccurs="1"/>
|
||||||
|
<xs:element name="cache" type="rbcs:cacheType" maxOccurs="1"/>
|
||||||
|
<xs:element name="authorization" type="rbcs:authorizationType" minOccurs="0">
|
||||||
|
<xs:key name="userId">
|
||||||
|
<xs:selector xpath="users/user"/>
|
||||||
|
<xs:field xpath="@name"/>
|
||||||
|
</xs:key>
|
||||||
|
<xs:keyref name="userRef" refer="rbcs:userId">
|
||||||
|
<xs:selector xpath="groups/group/users/user"/>
|
||||||
|
<xs:field xpath="@ref"/>
|
||||||
|
</xs:keyref>
|
||||||
|
</xs:element>
|
||||||
|
<xs:element name="authentication" type="rbcs:authenticationType" minOccurs="0" maxOccurs="1"/>
|
||||||
|
<xs:element name="tls" type="rbcs:tlsType" minOccurs="0" maxOccurs="1"/>
|
||||||
|
</xs:sequence>
|
||||||
|
<xs:attribute name="path" type="xs:string" use="optional"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="bindType">
|
||||||
|
<xs:attribute name="host" type="xs:token" use="required"/>
|
||||||
|
<xs:attribute name="port" type="xs:unsignedShort" use="required"/>
|
||||||
|
<xs:attribute name="incoming-connections-backlog-size" type="xs:unsignedInt" use="optional" default="1024"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="connectionType">
|
||||||
|
<xs:attribute name="read-timeout" type="xs:duration" use="optional" default="PT0S"/>
|
||||||
|
<xs:attribute name="write-timeout" type="xs:duration" use="optional" default="PT0S"/>
|
||||||
|
<xs:attribute name="idle-timeout" type="xs:duration" use="optional" default="PT30S"/>
|
||||||
|
<xs:attribute name="read-idle-timeout" type="xs:duration" use="optional" default="PT60S"/>
|
||||||
|
<xs:attribute name="write-idle-timeout" type="xs:duration" use="optional" default="PT60S"/>
|
||||||
|
<xs:attribute name="max-request-size" type="xs:unsignedInt" use="optional" default="67108864"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="eventExecutorType">
|
||||||
|
<xs:attribute name="use-virtual-threads" type="xs:boolean" use="optional" default="true"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="cacheType" abstract="true"/>
|
||||||
|
|
||||||
|
<xs:complexType name="inMemoryCacheType">
|
||||||
|
<xs:complexContent>
|
||||||
|
<xs:extension base="rbcs:cacheType">
|
||||||
|
<xs:attribute name="max-age" type="xs:duration" default="P1D"/>
|
||||||
|
<xs:attribute name="max-size" type="xs:token" default="0x1000000"/>
|
||||||
|
<xs:attribute name="digest" type="xs:token" default="MD5"/>
|
||||||
|
<xs:attribute name="enable-compression" type="xs:boolean" default="true"/>
|
||||||
|
<xs:attribute name="compression-level" type="xs:byte" default="-1"/>
|
||||||
|
</xs:extension>
|
||||||
|
</xs:complexContent>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="fileSystemCacheType">
|
||||||
|
<xs:complexContent>
|
||||||
|
<xs:extension base="rbcs:cacheType">
|
||||||
|
<xs:attribute name="path" type="xs:string" use="required"/>
|
||||||
|
<xs:attribute name="max-age" type="xs:duration" default="P1D"/>
|
||||||
|
<xs:attribute name="digest" type="xs:token" default="MD5"/>
|
||||||
|
<xs:attribute name="enable-compression" type="xs:boolean" default="true"/>
|
||||||
|
<xs:attribute name="compression-level" type="xs:byte" default="-1"/>
|
||||||
|
</xs:extension>
|
||||||
|
</xs:complexContent>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="tlsCertificateAuthorizationType">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:element name="group-extractor" type="rbcs:X500NameExtractorType" minOccurs="0"/>
|
||||||
|
<xs:element name="user-extractor" type="rbcs:X500NameExtractorType" minOccurs="0"/>
|
||||||
|
</xs:sequence>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="X500NameExtractorType">
|
||||||
|
<xs:attribute name="attribute-name" type="xs:token"/>
|
||||||
|
<xs:attribute name="pattern" type="xs:token"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="authorizationType">
|
||||||
|
<xs:all>
|
||||||
|
<xs:element name="users" type="rbcs:usersType"/>
|
||||||
|
<xs:element name="groups" type="rbcs:groupsType">
|
||||||
|
<xs:unique name="groupKey">
|
||||||
|
<xs:selector xpath="group"/>
|
||||||
|
<xs:field xpath="@name"/>
|
||||||
|
</xs:unique>
|
||||||
|
</xs:element>
|
||||||
|
</xs:all>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="authenticationType">
|
||||||
|
<xs:choice>
|
||||||
|
<xs:element name="basic"/>
|
||||||
|
<xs:element name="client-certificate" type="rbcs:tlsCertificateAuthorizationType"/>
|
||||||
|
<xs:element name="none"/>
|
||||||
|
</xs:choice>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="quotaType">
|
||||||
|
<xs:attribute name="calls" type="xs:positiveInteger" use="required"/>
|
||||||
|
<xs:attribute name="period" type="xs:duration" use="required"/>
|
||||||
|
<xs:attribute name="max-available-calls" type="xs:positiveInteger" use="optional"/>
|
||||||
|
<xs:attribute name="initial-available-calls" type="xs:unsignedInt" use="optional"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="anonymousUserType">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:element name="quota" type="rbcs:quotaType" minOccurs="0" maxOccurs="1"/>
|
||||||
|
</xs:sequence>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="userType">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:element name="quota" type="rbcs:quotaType" minOccurs="0" maxOccurs="1"/>
|
||||||
|
</xs:sequence>
|
||||||
|
<xs:attribute name="name" type="xs:token" use="required"/>
|
||||||
|
<xs:attribute name="password" type="xs:string" use="optional"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="usersType">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:element name="user" type="rbcs:userType" minOccurs="0" maxOccurs="unbounded"/>
|
||||||
|
<xs:element name="anonymous" type="rbcs:anonymousUserType" minOccurs="0" maxOccurs="1"/>
|
||||||
|
</xs:sequence>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="groupsType">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:element name="group" type="rbcs:groupType" maxOccurs="unbounded" minOccurs="0"/>
|
||||||
|
</xs:sequence>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="groupType">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:element name="users" type="rbcs:userRefsType" maxOccurs="1" minOccurs="0">
|
||||||
|
<xs:unique name="userRefWriterKey">
|
||||||
|
<xs:selector xpath="user"/>
|
||||||
|
<xs:field xpath="@ref"/>
|
||||||
|
</xs:unique>
|
||||||
|
</xs:element>
|
||||||
|
<xs:element name="roles" type="rbcs:rolesType" maxOccurs="1" minOccurs="0"/>
|
||||||
|
<xs:element name="user-quota" type="rbcs:quotaType" minOccurs="0" maxOccurs="1"/>
|
||||||
|
<xs:element name="group-quota" type="rbcs:quotaType" minOccurs="0" maxOccurs="1"/>
|
||||||
|
</xs:sequence>
|
||||||
|
<xs:attribute name="name" type="xs:token"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:simpleType name="role" final="restriction" >
|
||||||
|
<xs:restriction base="xs:token">
|
||||||
|
<xs:enumeration value="READER" />
|
||||||
|
<xs:enumeration value="WRITER" />
|
||||||
|
</xs:restriction>
|
||||||
|
</xs:simpleType>
|
||||||
|
|
||||||
|
<xs:complexType name="rolesType">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:choice maxOccurs="unbounded">
|
||||||
|
<xs:element name="writer"/>
|
||||||
|
<xs:element name="reader"/>
|
||||||
|
</xs:choice>
|
||||||
|
</xs:sequence>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="userRefsType">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:element name="user" type="rbcs:userRefType" maxOccurs="unbounded" minOccurs="0"/>
|
||||||
|
<xs:element name="anonymous" minOccurs="0" maxOccurs="1"/>
|
||||||
|
</xs:sequence>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="userRefType">
|
||||||
|
<xs:attribute name="ref" type="xs:string" use="required"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="tlsType">
|
||||||
|
<xs:all>
|
||||||
|
<xs:element name="keystore" type="rbcs:keyStoreType" />
|
||||||
|
<xs:element name="truststore" type="rbcs:trustStoreType" minOccurs="0"/>
|
||||||
|
</xs:all>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="keyStoreType">
|
||||||
|
<xs:attribute name="file" type="xs:string" use="required"/>
|
||||||
|
<xs:attribute name="password" type="xs:string"/>
|
||||||
|
<xs:attribute name="key-alias" type="xs:string" use="required"/>
|
||||||
|
<xs:attribute name="key-password" type="xs:string"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="trustStoreType">
|
||||||
|
<xs:attribute name="file" type="xs:string" use="required"/>
|
||||||
|
<xs:attribute name="password" type="xs:string"/>
|
||||||
|
<xs:attribute name="check-certificate-status" type="xs:boolean"/>
|
||||||
|
<xs:attribute name="require-client-certificate" type="xs:boolean" use="optional" default="false"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="propertiesType">
|
||||||
|
<xs:sequence>
|
||||||
|
<xs:element maxOccurs="unbounded" minOccurs="0" name="property" type="rbcs:propertyType"/>
|
||||||
|
</xs:sequence>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="propertyType">
|
||||||
|
<xs:simpleContent>
|
||||||
|
<xs:extension base="xs:string">
|
||||||
|
<xs:attribute name="key" type="xs:string" use="required"/>
|
||||||
|
</xs:extension>
|
||||||
|
</xs:simpleContent>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
<xs:complexType name="hostAndPortType">
|
||||||
|
<xs:attribute name="host" type="xs:string" use="required"/>
|
||||||
|
<xs:attribute name="port" type="xs:unsignedShort" use="required"/>
|
||||||
|
</xs:complexType>
|
||||||
|
|
||||||
|
|
||||||
|
</xs:schema>
|
@@ -0,0 +1,224 @@
|
|||||||
|
package net.woggioni.rbcs.server.test.utils;
|
||||||
|
|
||||||
|
import org.bouncycastle.asn1.DERSequence;
|
||||||
|
import org.bouncycastle.asn1.x500.X500Name;
|
||||||
|
import org.bouncycastle.asn1.x509.BasicConstraints;
|
||||||
|
import org.bouncycastle.asn1.x509.ExtendedKeyUsage;
|
||||||
|
import org.bouncycastle.asn1.x509.Extension;
|
||||||
|
import org.bouncycastle.asn1.x509.GeneralName;
|
||||||
|
import org.bouncycastle.asn1.x509.GeneralNames;
|
||||||
|
import org.bouncycastle.asn1.x509.KeyPurposeId;
|
||||||
|
import org.bouncycastle.asn1.x509.KeyUsage;
|
||||||
|
import org.bouncycastle.cert.X509v3CertificateBuilder;
|
||||||
|
import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
|
||||||
|
import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
|
||||||
|
import org.bouncycastle.operator.ContentSigner;
|
||||||
|
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.security.KeyPair;
|
||||||
|
import java.security.KeyPairGenerator;
|
||||||
|
import java.security.PrivateKey;
|
||||||
|
import java.security.SecureRandom;
|
||||||
|
import java.security.cert.X509Certificate;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.time.temporal.ChronoUnit;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
public class CertificateUtils {
|
||||||
|
|
||||||
|
public record X509Credentials(
|
||||||
|
KeyPair keyPair,
|
||||||
|
X509Certificate certificate
|
||||||
|
){ }
|
||||||
|
public static class CertificateAuthority {
|
||||||
|
private final PrivateKey privateKey;
|
||||||
|
private final X509Certificate certificate;
|
||||||
|
|
||||||
|
public CertificateAuthority(PrivateKey privateKey, X509Certificate certificate) {
|
||||||
|
this.privateKey = privateKey;
|
||||||
|
this.certificate = certificate;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PrivateKey getPrivateKey() { return privateKey; }
|
||||||
|
public X509Certificate getCertificate() { return certificate; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new Certificate Authority (CA)
|
||||||
|
* @param commonName The CA's common name
|
||||||
|
* @param validityDays How long the CA should be valid for
|
||||||
|
* @return The generated CA containing both private key and certificate
|
||||||
|
*/
|
||||||
|
public static X509Credentials createCertificateAuthority(String commonName, int validityDays)
|
||||||
|
throws Exception {
|
||||||
|
// Generate key pair
|
||||||
|
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA");
|
||||||
|
keyPairGenerator.initialize(4096);
|
||||||
|
KeyPair keyPair = keyPairGenerator.generateKeyPair();
|
||||||
|
|
||||||
|
// Prepare certificate data
|
||||||
|
X500Name issuerName = new X500Name("CN=" + commonName);
|
||||||
|
BigInteger serialNumber = new BigInteger(160, new SecureRandom());
|
||||||
|
Instant now = Instant.now();
|
||||||
|
Date startDate = Date.from(now);
|
||||||
|
Date endDate = Date.from(now.plus(validityDays, ChronoUnit.DAYS));
|
||||||
|
|
||||||
|
// Create certificate builder
|
||||||
|
X509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder(
|
||||||
|
issuerName,
|
||||||
|
serialNumber,
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
issuerName,
|
||||||
|
keyPair.getPublic()
|
||||||
|
);
|
||||||
|
|
||||||
|
// Add CA extensions
|
||||||
|
certBuilder.addExtension(
|
||||||
|
Extension.basicConstraints,
|
||||||
|
true,
|
||||||
|
new BasicConstraints(true)
|
||||||
|
);
|
||||||
|
certBuilder.addExtension(
|
||||||
|
Extension.keyUsage,
|
||||||
|
true,
|
||||||
|
new KeyUsage(KeyUsage.keyCertSign | KeyUsage.cRLSign)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Sign the certificate
|
||||||
|
ContentSigner signer = new JcaContentSignerBuilder("SHA256withRSA")
|
||||||
|
.build(keyPair.getPrivate());
|
||||||
|
X509Certificate cert = new JcaX509CertificateConverter()
|
||||||
|
.getCertificate(certBuilder.build(signer));
|
||||||
|
|
||||||
|
return new X509Credentials(keyPair, cert);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a server certificate signed by the CA
|
||||||
|
* @param ca The Certificate Authority to sign with
|
||||||
|
* @param subjectName The server's common name
|
||||||
|
* @param validityDays How long the certificate should be valid for
|
||||||
|
* @return KeyPair containing the server's private key and certificate
|
||||||
|
*/
|
||||||
|
public static X509Credentials createServerCertificate(X509Credentials ca, X500Name subjectName, int validityDays)
|
||||||
|
throws Exception {
|
||||||
|
// Generate server key pair
|
||||||
|
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA");
|
||||||
|
keyPairGenerator.initialize(2048);
|
||||||
|
KeyPair serverKeyPair = keyPairGenerator.generateKeyPair();
|
||||||
|
|
||||||
|
// Prepare certificate data
|
||||||
|
X500Name issuerName = new X500Name(ca.certificate().getSubjectX500Principal().getName());
|
||||||
|
BigInteger serialNumber = new BigInteger(160, new SecureRandom());
|
||||||
|
Instant now = Instant.now();
|
||||||
|
Date startDate = Date.from(now);
|
||||||
|
Date endDate = Date.from(now.plus(validityDays, ChronoUnit.DAYS));
|
||||||
|
|
||||||
|
// Create certificate builder
|
||||||
|
X509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder(
|
||||||
|
issuerName,
|
||||||
|
serialNumber,
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
subjectName,
|
||||||
|
serverKeyPair.getPublic()
|
||||||
|
);
|
||||||
|
|
||||||
|
// Add server certificate extensions
|
||||||
|
certBuilder.addExtension(
|
||||||
|
Extension.basicConstraints,
|
||||||
|
true,
|
||||||
|
new BasicConstraints(false)
|
||||||
|
);
|
||||||
|
certBuilder.addExtension(
|
||||||
|
Extension.keyUsage,
|
||||||
|
true,
|
||||||
|
new KeyUsage(KeyUsage.digitalSignature | KeyUsage.keyEncipherment)
|
||||||
|
);
|
||||||
|
certBuilder.addExtension(
|
||||||
|
Extension.extendedKeyUsage,
|
||||||
|
true,
|
||||||
|
new ExtendedKeyUsage(new KeyPurposeId[]{KeyPurposeId.id_kp_serverAuth})
|
||||||
|
);
|
||||||
|
GeneralNames subjectAltNames = GeneralNames.getInstance(
|
||||||
|
new DERSequence(
|
||||||
|
new GeneralName[] {
|
||||||
|
new GeneralName(GeneralName.iPAddress, "127.0.0.1")
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
certBuilder.addExtension(
|
||||||
|
Extension.subjectAlternativeName,
|
||||||
|
true,
|
||||||
|
subjectAltNames
|
||||||
|
);
|
||||||
|
|
||||||
|
// Sign the certificate
|
||||||
|
ContentSigner signer = new JcaContentSignerBuilder("SHA256withRSA")
|
||||||
|
.build(ca.keyPair().getPrivate());
|
||||||
|
X509Certificate cert = new JcaX509CertificateConverter()
|
||||||
|
.getCertificate(certBuilder.build(signer));
|
||||||
|
|
||||||
|
|
||||||
|
return new X509Credentials(serverKeyPair, cert);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a client certificate signed by the CA
|
||||||
|
* @param ca The Certificate Authority to sign with
|
||||||
|
* @param subjectName The client's common name
|
||||||
|
* @param validityDays How long the certificate should be valid for
|
||||||
|
* @return KeyPair containing the client's private key and certificate
|
||||||
|
*/
|
||||||
|
public static X509Credentials createClientCertificate(X509Credentials ca, X500Name subjectName, int validityDays)
|
||||||
|
throws Exception {
|
||||||
|
// Generate client key pair
|
||||||
|
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA");
|
||||||
|
keyPairGenerator.initialize(2048);
|
||||||
|
KeyPair clientKeyPair = keyPairGenerator.generateKeyPair();
|
||||||
|
|
||||||
|
// Prepare certificate data
|
||||||
|
X500Name issuerName = new X500Name(ca.certificate().getSubjectX500Principal().getName());
|
||||||
|
BigInteger serialNumber = new BigInteger(160, new SecureRandom());
|
||||||
|
Instant now = Instant.now();
|
||||||
|
Date startDate = Date.from(now);
|
||||||
|
Date endDate = Date.from(now.plus(validityDays, ChronoUnit.DAYS));
|
||||||
|
|
||||||
|
// Create certificate builder
|
||||||
|
X509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder(
|
||||||
|
issuerName,
|
||||||
|
serialNumber,
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
subjectName,
|
||||||
|
clientKeyPair.getPublic()
|
||||||
|
);
|
||||||
|
|
||||||
|
// Add client certificate extensions
|
||||||
|
certBuilder.addExtension(
|
||||||
|
Extension.basicConstraints,
|
||||||
|
true,
|
||||||
|
new BasicConstraints(false)
|
||||||
|
);
|
||||||
|
certBuilder.addExtension(
|
||||||
|
Extension.keyUsage,
|
||||||
|
true,
|
||||||
|
new KeyUsage(KeyUsage.digitalSignature)
|
||||||
|
);
|
||||||
|
certBuilder.addExtension(
|
||||||
|
Extension.extendedKeyUsage,
|
||||||
|
true,
|
||||||
|
new ExtendedKeyUsage(new KeyPurposeId[]{KeyPurposeId.id_kp_clientAuth})
|
||||||
|
);
|
||||||
|
|
||||||
|
// Sign the certificate
|
||||||
|
ContentSigner signer = new JcaContentSignerBuilder("SHA256withRSA")
|
||||||
|
.build(ca.keyPair().getPrivate());
|
||||||
|
X509Certificate cert = new JcaX509CertificateConverter()
|
||||||
|
.getCertificate(certBuilder.build(signer));
|
||||||
|
|
||||||
|
return new X509Credentials(clientKeyPair, cert);
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,30 @@
|
|||||||
|
package net.woggioni.rbcs.server.test.utils;
|
||||||
|
|
||||||
|
import net.woggioni.jwo.JWO;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetAddress;
|
||||||
|
import java.net.ServerSocket;
|
||||||
|
|
||||||
|
public class NetworkUtils {
|
||||||
|
|
||||||
|
private static final int MAX_ATTEMPTS = 50;
|
||||||
|
|
||||||
|
public static int getFreePort() {
|
||||||
|
int count = 0;
|
||||||
|
while(count < MAX_ATTEMPTS) {
|
||||||
|
try (ServerSocket serverSocket = new ServerSocket(0, 50, InetAddress.getLocalHost())) {
|
||||||
|
final var candidate = serverSocket.getLocalPort();
|
||||||
|
if (candidate > 0) {
|
||||||
|
return candidate;
|
||||||
|
} else {
|
||||||
|
JWO.newThrowable(RuntimeException.class, "Got invalid port number: %d", candidate);
|
||||||
|
throw new RuntimeException("Error trying to find an open port");
|
||||||
|
}
|
||||||
|
} catch (IOException ignored) {
|
||||||
|
++count;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new RuntimeException("Error trying to find an open port");
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,86 @@
|
|||||||
|
package net.woggioni.rbcs.server.test
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.api.Role
|
||||||
|
import net.woggioni.rbcs.common.Xml
|
||||||
|
import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration
|
||||||
|
import net.woggioni.rbcs.server.configuration.Serializer
|
||||||
|
import net.woggioni.rbcs.server.test.utils.NetworkUtils
|
||||||
|
import java.net.URI
|
||||||
|
import java.net.http.HttpRequest
|
||||||
|
import java.nio.charset.StandardCharsets
|
||||||
|
import java.nio.file.Path
|
||||||
|
import java.time.Duration
|
||||||
|
import java.time.temporal.ChronoUnit
|
||||||
|
import java.util.Base64
|
||||||
|
import java.util.zip.Deflater
|
||||||
|
import kotlin.random.Random
|
||||||
|
|
||||||
|
|
||||||
|
abstract class AbstractBasicAuthServerTest : AbstractServerTest() {
|
||||||
|
|
||||||
|
private lateinit var cacheDir : Path
|
||||||
|
|
||||||
|
protected val random = Random(101325)
|
||||||
|
protected val keyValuePair = newEntry(random)
|
||||||
|
protected val serverPath = "rbcs"
|
||||||
|
protected val readersGroup = Configuration.Group("readers", setOf(Role.Reader), null, null)
|
||||||
|
protected val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null, null)
|
||||||
|
|
||||||
|
abstract protected val users : List<Configuration.User>
|
||||||
|
|
||||||
|
override fun setUp() {
|
||||||
|
this.cacheDir = testDir.resolve("cache")
|
||||||
|
cfg = Configuration.of(
|
||||||
|
"127.0.0.1",
|
||||||
|
NetworkUtils.getFreePort(),
|
||||||
|
50,
|
||||||
|
serverPath,
|
||||||
|
Configuration.EventExecutor(false),
|
||||||
|
Configuration.Connection(
|
||||||
|
Duration.of(10, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(10, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(60, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(30, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(30, ChronoUnit.SECONDS),
|
||||||
|
0x1000
|
||||||
|
),
|
||||||
|
users.asSequence().map { it.name to it}.toMap(),
|
||||||
|
sequenceOf(writersGroup, readersGroup).map { it.name to it}.toMap(),
|
||||||
|
FileSystemCacheConfiguration(this.cacheDir,
|
||||||
|
maxAge = Duration.ofSeconds(3600 * 24),
|
||||||
|
digestAlgorithm = "MD5",
|
||||||
|
compressionLevel = Deflater.DEFAULT_COMPRESSION,
|
||||||
|
compressionEnabled = false
|
||||||
|
),
|
||||||
|
Configuration.BasicAuthentication(),
|
||||||
|
null,
|
||||||
|
)
|
||||||
|
Xml.write(Serializer.serialize(cfg), System.out)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun tearDown() {
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun buildAuthorizationHeader(user : Configuration.User, password : String) : String {
|
||||||
|
val b64 = Base64.getEncoder().encode("${user.name}:${password}".toByteArray(Charsets.UTF_8)).let{
|
||||||
|
String(it, StandardCharsets.UTF_8)
|
||||||
|
}
|
||||||
|
return "Basic $b64"
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun newRequestBuilder(key : String) = HttpRequest.newBuilder()
|
||||||
|
.uri(URI.create("http://${cfg.host}:${cfg.port}/$serverPath/$key"))
|
||||||
|
|
||||||
|
|
||||||
|
protected fun newEntry(random : Random) : Pair<String, ByteArray> {
|
||||||
|
val key = ByteArray(0x10).let {
|
||||||
|
random.nextBytes(it)
|
||||||
|
Base64.getUrlEncoder().encodeToString(it)
|
||||||
|
}
|
||||||
|
val value = ByteArray(0x1000).also {
|
||||||
|
random.nextBytes(it)
|
||||||
|
}
|
||||||
|
return key to value
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,50 @@
|
|||||||
|
package net.woggioni.rbcs.server.test
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.server.RemoteBuildCacheServer
|
||||||
|
import org.junit.jupiter.api.AfterAll
|
||||||
|
import org.junit.jupiter.api.BeforeAll
|
||||||
|
import org.junit.jupiter.api.MethodOrderer
|
||||||
|
import org.junit.jupiter.api.TestInstance
|
||||||
|
import org.junit.jupiter.api.TestMethodOrder
|
||||||
|
import org.junit.jupiter.api.io.TempDir
|
||||||
|
import java.nio.file.Path
|
||||||
|
|
||||||
|
|
||||||
|
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||||
|
@TestMethodOrder(MethodOrderer.OrderAnnotation::class)
|
||||||
|
abstract class AbstractServerTest {
|
||||||
|
|
||||||
|
protected lateinit var cfg : Configuration
|
||||||
|
|
||||||
|
protected lateinit var testDir : Path
|
||||||
|
|
||||||
|
private var serverHandle : RemoteBuildCacheServer.ServerHandle? = null
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
fun setUp0(@TempDir tmpDir : Path) {
|
||||||
|
this.testDir = tmpDir
|
||||||
|
setUp()
|
||||||
|
startServer(cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterAll
|
||||||
|
fun tearDown0() {
|
||||||
|
tearDown()
|
||||||
|
stopServer()
|
||||||
|
}
|
||||||
|
|
||||||
|
abstract fun setUp()
|
||||||
|
|
||||||
|
abstract fun tearDown()
|
||||||
|
|
||||||
|
private fun startServer(cfg : Configuration) {
|
||||||
|
this.serverHandle = RemoteBuildCacheServer(cfg).run()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun stopServer() {
|
||||||
|
this.serverHandle?.use {
|
||||||
|
it.shutdown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,202 @@
|
|||||||
|
package net.woggioni.rbcs.server.test
|
||||||
|
|
||||||
|
import net.woggioni.rbcs.api.Configuration
|
||||||
|
import net.woggioni.rbcs.api.Role
|
||||||
|
import net.woggioni.rbcs.common.Xml
|
||||||
|
import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration
|
||||||
|
import net.woggioni.rbcs.server.configuration.Serializer
|
||||||
|
import net.woggioni.rbcs.server.test.utils.CertificateUtils
|
||||||
|
import net.woggioni.rbcs.server.test.utils.CertificateUtils.X509Credentials
|
||||||
|
import net.woggioni.rbcs.server.test.utils.NetworkUtils
|
||||||
|
import org.bouncycastle.asn1.x500.X500Name
|
||||||
|
import java.net.URI
|
||||||
|
import java.net.http.HttpClient
|
||||||
|
import java.net.http.HttpRequest
|
||||||
|
import java.nio.charset.StandardCharsets
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Path
|
||||||
|
import java.security.KeyStore
|
||||||
|
import java.security.KeyStore.PasswordProtection
|
||||||
|
import java.time.Duration
|
||||||
|
import java.time.temporal.ChronoUnit
|
||||||
|
import java.util.Base64
|
||||||
|
import java.util.zip.Deflater
|
||||||
|
import javax.net.ssl.KeyManagerFactory
|
||||||
|
import javax.net.ssl.SSLContext
|
||||||
|
import javax.net.ssl.TrustManagerFactory
|
||||||
|
import kotlin.random.Random
|
||||||
|
|
||||||
|
|
||||||
|
abstract class AbstractTlsServerTest : AbstractServerTest() {
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private const val CA_CERTIFICATE_ENTRY = "rbcs-ca"
|
||||||
|
private const val CLIENT_CERTIFICATE_ENTRY = "rbcs-client"
|
||||||
|
private const val SERVER_CERTIFICATE_ENTRY = "rbcs-server"
|
||||||
|
private const val PASSWORD = "password"
|
||||||
|
}
|
||||||
|
|
||||||
|
private lateinit var cacheDir: Path
|
||||||
|
private lateinit var serverKeyStoreFile: Path
|
||||||
|
private lateinit var clientKeyStoreFile: Path
|
||||||
|
private lateinit var trustStoreFile: Path
|
||||||
|
private lateinit var serverKeyStore: KeyStore
|
||||||
|
private lateinit var clientKeyStore: KeyStore
|
||||||
|
private lateinit var trustStore: KeyStore
|
||||||
|
protected lateinit var ca: X509Credentials
|
||||||
|
|
||||||
|
protected val readersGroup = Configuration.Group("readers", setOf(Role.Reader), null, null)
|
||||||
|
protected val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null, null)
|
||||||
|
protected val random = Random(101325)
|
||||||
|
protected val keyValuePair = newEntry(random)
|
||||||
|
private val serverPath : String? = null
|
||||||
|
|
||||||
|
protected abstract val users : List<Configuration.User>
|
||||||
|
|
||||||
|
protected fun createKeyStoreAndTrustStore() {
|
||||||
|
ca = CertificateUtils.createCertificateAuthority(CA_CERTIFICATE_ENTRY, 30)
|
||||||
|
val serverCert = CertificateUtils.createServerCertificate(ca, X500Name("CN=$SERVER_CERTIFICATE_ENTRY"), 30)
|
||||||
|
val clientCert = CertificateUtils.createClientCertificate(ca, X500Name("CN=$CLIENT_CERTIFICATE_ENTRY"), 30)
|
||||||
|
|
||||||
|
serverKeyStore = KeyStore.getInstance("PKCS12").apply {
|
||||||
|
load(null, null)
|
||||||
|
setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null))
|
||||||
|
setEntry(
|
||||||
|
SERVER_CERTIFICATE_ENTRY,
|
||||||
|
KeyStore.PrivateKeyEntry(
|
||||||
|
serverCert.keyPair().private,
|
||||||
|
arrayOf(serverCert.certificate(), ca.certificate)
|
||||||
|
),
|
||||||
|
PasswordProtection(PASSWORD.toCharArray())
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Files.newOutputStream(this.serverKeyStoreFile).use {
|
||||||
|
serverKeyStore.store(it, null)
|
||||||
|
}
|
||||||
|
|
||||||
|
clientKeyStore = KeyStore.getInstance("PKCS12").apply {
|
||||||
|
load(null, null)
|
||||||
|
setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null))
|
||||||
|
setEntry(
|
||||||
|
CLIENT_CERTIFICATE_ENTRY,
|
||||||
|
KeyStore.PrivateKeyEntry(
|
||||||
|
clientCert.keyPair().private,
|
||||||
|
arrayOf(clientCert.certificate(), ca.certificate)
|
||||||
|
),
|
||||||
|
PasswordProtection(PASSWORD.toCharArray())
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Files.newOutputStream(this.clientKeyStoreFile).use {
|
||||||
|
clientKeyStore.store(it, null)
|
||||||
|
}
|
||||||
|
|
||||||
|
trustStore = KeyStore.getInstance("PKCS12").apply {
|
||||||
|
load(null, null)
|
||||||
|
setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null))
|
||||||
|
}
|
||||||
|
Files.newOutputStream(this.trustStoreFile).use {
|
||||||
|
trustStore.store(it, null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun getClientKeyStore(ca: X509Credentials, subject: X500Name) = KeyStore.getInstance("PKCS12").apply {
|
||||||
|
val clientCert = CertificateUtils.createClientCertificate(ca, subject, 30)
|
||||||
|
|
||||||
|
load(null, null)
|
||||||
|
setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null))
|
||||||
|
setEntry(
|
||||||
|
CLIENT_CERTIFICATE_ENTRY,
|
||||||
|
KeyStore.PrivateKeyEntry(clientCert.keyPair().private, arrayOf(clientCert.certificate(), ca.certificate)),
|
||||||
|
PasswordProtection(PASSWORD.toCharArray())
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun getHttpClient(clientKeyStore: KeyStore?): HttpClient {
|
||||||
|
val kmf = clientKeyStore?.let {
|
||||||
|
KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()).apply {
|
||||||
|
init(it, PASSWORD.toCharArray())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Set up trust manager factory with the truststore
|
||||||
|
val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
|
||||||
|
tmf.init(trustStore)
|
||||||
|
|
||||||
|
// Create SSL context with the key and trust managers
|
||||||
|
val sslContext = SSLContext.getInstance("TLS").apply {
|
||||||
|
init(kmf?.keyManagers ?: emptyArray(), tmf.trustManagers, null)
|
||||||
|
}
|
||||||
|
return HttpClient.newBuilder().sslContext(sslContext).build()
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun setUp() {
|
||||||
|
this.clientKeyStoreFile = testDir.resolve("client-keystore.p12")
|
||||||
|
this.serverKeyStoreFile = testDir.resolve("server-keystore.p12")
|
||||||
|
this.trustStoreFile = testDir.resolve("truststore.p12")
|
||||||
|
this.cacheDir = testDir.resolve("cache")
|
||||||
|
createKeyStoreAndTrustStore()
|
||||||
|
cfg = Configuration(
|
||||||
|
"127.0.0.1",
|
||||||
|
NetworkUtils.getFreePort(),
|
||||||
|
100,
|
||||||
|
serverPath,
|
||||||
|
Configuration.EventExecutor(false),
|
||||||
|
Configuration.Connection(
|
||||||
|
Duration.of(10, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(10, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(60, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(30, ChronoUnit.SECONDS),
|
||||||
|
Duration.of(30, ChronoUnit.SECONDS),
|
||||||
|
0x1000
|
||||||
|
),
|
||||||
|
users.asSequence().map { it.name to it }.toMap(),
|
||||||
|
sequenceOf(writersGroup, readersGroup).map { it.name to it }.toMap(),
|
||||||
|
FileSystemCacheConfiguration(this.cacheDir,
|
||||||
|
maxAge = Duration.ofSeconds(3600 * 24),
|
||||||
|
compressionEnabled = true,
|
||||||
|
compressionLevel = Deflater.DEFAULT_COMPRESSION,
|
||||||
|
digestAlgorithm = "MD5"
|
||||||
|
),
|
||||||
|
// InMemoryCacheConfiguration(
|
||||||
|
// maxAge = Duration.ofSeconds(3600 * 24),
|
||||||
|
// compressionEnabled = true,
|
||||||
|
// compressionLevel = Deflater.DEFAULT_COMPRESSION,
|
||||||
|
// digestAlgorithm = "MD5"
|
||||||
|
// ),
|
||||||
|
Configuration.ClientCertificateAuthentication(
|
||||||
|
Configuration.TlsCertificateExtractor("CN", "(.*)"),
|
||||||
|
null
|
||||||
|
),
|
||||||
|
Configuration.Tls(
|
||||||
|
Configuration.KeyStore(this.serverKeyStoreFile, null, SERVER_CERTIFICATE_ENTRY, PASSWORD),
|
||||||
|
Configuration.TrustStore(this.trustStoreFile, null, false, false),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
Xml.write(Serializer.serialize(cfg), System.out)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun tearDown() {
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun newRequestBuilder(key: String) = HttpRequest.newBuilder()
|
||||||
|
.uri(URI.create("https://${cfg.host}:${cfg.port}/${serverPath ?: ""}/$key"))
|
||||||
|
|
||||||
|
private fun buildAuthorizationHeader(user: Configuration.User, password: String): String {
|
||||||
|
val b64 = Base64.getEncoder().encode("${user.name}:${password}".toByteArray(Charsets.UTF_8)).let {
|
||||||
|
String(it, StandardCharsets.UTF_8)
|
||||||
|
}
|
||||||
|
return "Basic $b64"
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun newEntry(random: Random): Pair<String, ByteArray> {
|
||||||
|
val key = ByteArray(0x10).let {
|
||||||
|
random.nextBytes(it)
|
||||||
|
Base64.getUrlEncoder().encodeToString(it)
|
||||||
|
}
|
||||||
|
val value = ByteArray(0x1000).also {
|
||||||
|
random.nextBytes(it)
|
||||||
|
}
|
||||||
|
return key to value
|
||||||
|
}
|
||||||
|
}
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user