Compare commits

..

26 Commits

Author SHA1 Message Date
woggioni b4a97845ca tmp
CI / build (push) Failing after 1m26s
2026-06-09 22:32:40 +08:00
woggioni 77cc044d0e restored docker image environmental variables
CI / build (push) Successful in 3m44s
2026-05-30 12:56:03 +08:00
opencode 9a7a2566fa Generalize OTEL API and add memcache tracing support
CI / build (push) Successful in 6m22s
- Rename RedisSpan -> SpanHandle for generic span handling
- Generalize TelemetryController methods: startSpan/endSpan with dbSystem param
- Rename RedisOtelSpan -> OtelSpanHandle in rbcs-server-otel
- Update Redis cache handler to use new generic API
- Add OpenTelemetry tracing for memcache GET and SET commands
- Add channel property to MemcacheRequestController for server address attribution
- Add uses TelemetryController directive in memcache module-info

Memcache spans follow the same pattern as Redis:
db.system=memcache, db.operation=GET|SET, server.address, server.port
2026-05-27 23:37:50 +08:00
opencode f154bbd33c Add OpenTelemetry tracing support for Redis commands
- Add RedisSpan interface in rbcs-api for opaque span handles
- Extend TelemetryController with startRedisSpan/endRedisSpan methods
- Implement Redis tracing in rbcs-server-otel via OtelController and RedisOtelSpan
- Instrument RedisCacheHandler to create spans around GET and SET commands
- Add uses directive in rbcs-server-redis module-info for ServiceLoader discovery

Redis spans are created as CLIENT spans with attributes:
db.system=redis, db.operation=GET|SET, server.address, server.port
2026-05-23 23:46:37 +08:00
woggioni 316f9e61b0 minor fix 2026-05-21 07:07:07 +08:00
woggioni 953d687651 optimized imports 2026-05-21 06:58:17 +08:00
woggioni 9c9f98cd72 fixed Dockerfile
CI / build (push) Successful in 3m21s
2026-05-20 22:43:12 +08:00
woggioni df7f747168 updated lys-catalog to 2026.05.16 2026-05-20 22:39:20 +08:00
woggioni 4d9a424528 removed telemetry switch from configuration
CI / build (push) Successful in 16m7s
2026-05-20 22:20:29 +08:00
woggioni ab2a06e810 refactor
CI / build (push) Successful in 3m4s
2026-04-30 02:15:34 +08:00
woggioni 1d938b7ea3 Add optional OpenTelemetry Netty server instrumentation
CI / build (push) Successful in 3m50s
- Update lys.version to 2026.04.14

- Add optional compileOnly dependency on opentelemetry-netty-4.1 in rbcs-server

- Add runtime guard to only activate instrumentation when OTel classes are on classpath

- Insert OTel combined handler after HttpServerCodec in the Netty pipeline

- Add requires-static JPMS directives for optional module support
2026-04-29 02:59:51 +08:00
woggioni 5d190d81ab version bump to 0.5.0
CI / build (push) Successful in 2m54s
2026-04-13 22:28:33 +08:00
woggioni e6f35f4340 Added support for client certificate forwarding
CI / build (push) Has been cancelled
2026-04-13 22:19:12 +08:00
woggioni 6d214eb066 uniformed Docker images 2026-04-13 22:19:12 +08:00
woggioni 0a50ae0643 improved error handling 2026-04-13 22:19:12 +08:00
woggioni 742c025fa5 Update netty to 4.2.12
CI / build (push) Successful in 3m32s
2026-03-26 20:23:44 +08:00
woggioni e3a3f21721 renamed docker image tags 2026-03-26 20:18:38 +08:00
woggioni a696eebbf9 added redis-enabled docker image 2026-03-26 20:03:34 +08:00
woggioni c9390ea51d added experimental redis support
CI / build (push) Failing after 4m7s
2026-03-03 02:59:48 +08:00
woggioni 43fdf131fa Added AGENT.md and updated docs
CI / build (push) Successful in 4m19s
2026-02-22 05:12:45 +08:00
woggioni b58462a085 improved logging 2026-02-21 20:47:18 +08:00
woggioni e9f9f23f91 Update Gradle to 9.3.1, Netty to 4.2.10 and Kotlin to 2.3.10
CI / build (push) Failing after 8s
2026-02-20 06:55:57 +08:00
woggioni 5854a632f8 Added server support for proxy protocol 2025-12-29 22:21:34 +08:00
woggioni 9a9cb4ed2c bump Netty 4.2.9 and Kotlin 2.3.0 2025-12-26 17:14:31 +08:00
woggioni 03a3dafecc updated dependencies
CI / build (push) Successful in 6m12s
2025-11-21 22:12:27 +08:00
woggioni 1ffe938c22 update to JDK 25
CI / build (push) Successful in 4m21s
2025-10-24 07:00:25 +08:00
87 changed files with 2441 additions and 240 deletions
+32 -15
View File
@@ -9,8 +9,6 @@ jobs:
steps: steps:
- name: Checkout sources - name: Checkout sources
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v3
- name: Execute Gradle build - name: Execute Gradle build
run: ./gradlew build run: ./gradlew build
- name: Prepare Docker image build - name: Prepare Docker image build
@@ -18,12 +16,6 @@ jobs:
- name: Get project version - name: Get project version
id: retrieve-version id: retrieve-version
run: ./gradlew -q version >> "$GITHUB_OUTPUT" run: ./gradlew -q version >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: Login to Gitea container registry - name: Login to Gitea container registry
uses: docker/login-action@v3 uses: docker/login-action@v3
with: with:
@@ -34,47 +26,72 @@ jobs:
name: Build rbcs Docker image name: Build rbcs Docker image
uses: docker/build-push-action@v5.3.0 uses: docker/build-push-action@v5.3.0
with: with:
builder: "multiplatform-builder"
context: "docker/build/docker" context: "docker/build/docker"
platforms: linux/amd64,linux/arm64 platforms: linux/amd64,linux/arm64
push: true push: true
pull: true pull: true
tags: | tags: |
gitea.woggioni.net/woggioni/rbcs:vanilla-dev gitea.woggioni.net/woggioni/rbcs:dev-vanilla
target: release-vanilla target: release-vanilla
cache-from: type=registry,ref=gitea.woggioni.net/woggioni/rbcs:buildx
- -
name: Build rbcs memcache Docker image name: Build rbcs memcache Docker image
uses: docker/build-push-action@v5.3.0 uses: docker/build-push-action@v5.3.0
with: with:
builder: "multiplatform-builder"
context: "docker/build/docker" context: "docker/build/docker"
platforms: linux/amd64,linux/arm64 platforms: linux/amd64,linux/arm64
push: true push: true
pull: true pull: true
tags: | tags: |
gitea.woggioni.net/woggioni/rbcs:memcache-dev gitea.woggioni.net/woggioni/rbcs:dev-memcache
target: release-memcache target: release-memcache
cache-from: type=registry,ref=gitea.woggioni.net/woggioni/rbcs:buildx -
cache-to: type=registry,mode=max,compression=zstd,image-manifest=true,oci-mediatypes=true,ref=gitea.woggioni.net/woggioni/rbcs:buildx name: Build rbcs redis Docker image
uses: docker/build-push-action@v5.3.0
with:
builder: "multiplatform-builder"
context: "docker/build/docker"
platforms: linux/amd64,linux/arm64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:dev-redis
target: release-redis
-
name: Build rbcs full Docker image
uses: docker/build-push-action@v5.3.0
with:
builder: "multiplatform-builder"
context: "docker/build/docker"
platforms: linux/amd64,linux/arm64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:dev-full
target: release-full
- -
name: Build rbcs native Docker image name: Build rbcs native Docker image
uses: docker/build-push-action@v5.3.0 uses: docker/build-push-action@v5.3.0
with: with:
builder: "multiplatform-builder"
context: "docker/build/docker" context: "docker/build/docker"
platforms: linux/amd64 platforms: linux/amd64
push: true push: true
pull: true pull: true
tags: | tags: |
gitea.woggioni.net/woggioni/rbcs:native-dev gitea.woggioni.net/woggioni/rbcs:dev-native
target: release-native target: release-native
- -
name: Build rbcs jlink Docker image name: Build rbcs jlink Docker image
uses: docker/build-push-action@v5.3.0 uses: docker/build-push-action@v5.3.0
with: with:
builder: "multiplatform-builder"
context: "docker/build/docker" context: "docker/build/docker"
platforms: linux/amd64 platforms: linux/amd64
push: true push: true
pull: true pull: true
tags: | tags: |
gitea.woggioni.net/woggioni/rbcs:jlink-dev gitea.woggioni.net/woggioni/rbcs:dev-jlink
target: release-jlink target: release-jlink
+44 -27
View File
@@ -5,12 +5,10 @@ on:
- '*' - '*'
jobs: jobs:
build: build:
runs-on: hostinger runs-on: woryzen
steps: steps:
- name: Checkout sources - name: Checkout sources
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v3
- name: Execute Gradle build - name: Execute Gradle build
run: ./gradlew build run: ./gradlew build
- name: Prepare Docker image build - name: Prepare Docker image build
@@ -18,12 +16,6 @@ jobs:
- name: Get project version - name: Get project version
id: retrieve-version id: retrieve-version
run: ./gradlew -q version >> "$GITHUB_OUTPUT" run: ./gradlew -q version >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: Login to Gitea container registry - name: Login to Gitea container registry
uses: docker/login-action@v3 uses: docker/login-action@v3
with: with:
@@ -34,19 +26,7 @@ jobs:
name: Build rbcs Docker image name: Build rbcs Docker image
uses: docker/build-push-action@v5.3.0 uses: docker/build-push-action@v5.3.0
with: with:
context: "docker/build/docker" builder: "multiplatform-builder"
platforms: linux/amd64,linux/arm64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:vanilla
gitea.woggioni.net/woggioni/rbcs:vanilla-${{ steps.retrieve-version.outputs.VERSION }}
target: release-vanilla
cache-from: type=registry,ref=gitea.woggioni.net/woggioni/rbcs:buildx
-
name: Build rbcs memcache Docker image
uses: docker/build-push-action@v5.3.0
with:
context: "docker/build/docker" context: "docker/build/docker"
platforms: linux/amd64,linux/arm64 platforms: linux/amd64,linux/arm64
push: true push: true
@@ -54,34 +34,71 @@ jobs:
tags: | tags: |
gitea.woggioni.net/woggioni/rbcs:latest gitea.woggioni.net/woggioni/rbcs:latest
gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }} gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}
target: release-vanilla
-
name: Build rbcs memcache Docker image
uses: docker/build-push-action@v5.3.0
with:
builder: "multiplatform-builder"
context: "docker/build/docker"
platforms: linux/amd64,linux/arm64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:memcache gitea.woggioni.net/woggioni/rbcs:memcache
gitea.woggioni.net/woggioni/rbcs:memcache-${{ steps.retrieve-version.outputs.VERSION }} gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}-memcache
target: release-memcache target: release-memcache
cache-from: type=registry,ref=gitea.woggioni.net/woggioni/rbcs:buildx -
cache-to: type=registry,mode=max,compression=zstd,image-manifest=true,oci-mediatypes=true,ref=gitea.woggioni.net/woggioni/rbcs:buildx name: Build rbcs redis Docker image
uses: docker/build-push-action@v5.3.0
with:
builder: "multiplatform-builder"
context: "docker/build/docker"
platforms: linux/amd64,linux/arm64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:redis
gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}-redis
target: release-redis
-
name: Build rbcs full Docker image
uses: docker/build-push-action@v5.3.0
with:
builder: "multiplatform-builder"
context: "docker/build/docker"
platforms: linux/amd64,linux/arm64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:full
gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}-full
target: release-full
- -
name: Build rbcs native Docker image name: Build rbcs native Docker image
uses: docker/build-push-action@v5.3.0 uses: docker/build-push-action@v5.3.0
with: with:
builder: "multiplatform-builder"
context: "docker/build/docker" context: "docker/build/docker"
platforms: linux/amd64 platforms: linux/amd64
push: true push: true
pull: true pull: true
tags: | tags: |
gitea.woggioni.net/woggioni/rbcs:native gitea.woggioni.net/woggioni/rbcs:native
gitea.woggioni.net/woggioni/rbcs:native-${{ steps.retrieve-version.outputs.VERSION }} gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}-native
target: release-native target: release-native
- -
name: Build rbcs jlink Docker image name: Build rbcs jlink Docker image
uses: docker/build-push-action@v5.3.0 uses: docker/build-push-action@v5.3.0
with: with:
builder: "multiplatform-builder"
context: "docker/build/docker" context: "docker/build/docker"
platforms: linux/amd64 platforms: linux/amd64
push: true push: true
pull: true pull: true
tags: | tags: |
gitea.woggioni.net/woggioni/rbcs:jlink gitea.woggioni.net/woggioni/rbcs:jlink
gitea.woggioni.net/woggioni/rbcs:jlink-${{ steps.retrieve-version.outputs.VERSION }}-jlink gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}-jlink
target: release-jlink target: release-jlink
- name: Publish artifacts - name: Publish artifacts
env: env:
+5
View File
@@ -5,3 +5,8 @@
build build
rbcs-cli/native-image/*.json rbcs-cli/native-image/*.json
# Ignore JDTLS files
.classpath
.project
.settings
+173
View File
@@ -0,0 +1,173 @@
# AGENTS.md — Coding Agent Guide for RBCS
## Project Overview
RBCS (Remote Build Cache Server) is a Kotlin/Java multi-module Gradle project built on Netty.
It serves as a remote build cache for Gradle and Maven. Java is used for the public API layer
(with Lombok); Kotlin is used for all implementation modules. All modules use JPMS
(`module-info.java`). The project is built on Netty with Java 25.
**Modules:** `rbcs-api`, `rbcs-common`, `rbcs-server`, `rbcs-server-memcache`, `rbcs-client`,
`rbcs-cli`, `rbcs-servlet`, `docker`
## Build Commands
```bash
./gradlew build # Full build: compile + test + assemble
./gradlew assemble # Build JARs without running tests
./gradlew compileJava compileKotlin # Compile only
./gradlew clean # Clean build outputs
./gradlew :rbcs-server:compileKotlin # Compile a single module
./gradlew -q version # Print project version
```
## Test Commands
```bash
./gradlew test # Run all tests (all modules)
./gradlew :rbcs-server:test # Run tests for a single module
./gradlew :rbcs-server:test --tests "net.woggioni.rbcs.server.test.NoAuthServerTest"
# Run a single test class
./gradlew :rbcs-server:test --tests "net.woggioni.rbcs.server.test.NoAuthServerTest.putWithNoAuthorizationHeader"
# Run a single test method
./gradlew test --tests "*TlsServer*" # Run tests matching a pattern
./gradlew :rbcs-server:jacocoTestReport # Generate code coverage (rbcs-server only)
```
**Test framework:** JUnit 5 (Jupiter). Tests are integration-style — they start real Netty
servers and use `java.net.http.HttpClient` to make HTTP requests. Netty leak detection is
set to `PARANOID` in test configurations.
**Test locations:** `src/test/kotlin/` and `src/test/java/` following standard Gradle layout.
Test resources (XML configs, logback) are in `src/test/resources/`.
## Lint / Format
No linting or formatting tools are configured (no Checkstyle, Detekt, ktlint, Spotless, or
similar). Follow the existing code style described below.
## Code Style Guidelines
### Language Split
- **Java** for `rbcs-api` (public API consumed by Java clients) with Lombok annotations
- **Kotlin** for all implementation modules
- **`module-info.java`** in every module (JPMS)
### Import Ordering (Kotlin)
Three groups separated by blank lines, each alphabetically sorted:
1. External/third-party (`io.netty.*`, `org.slf4j.*`)
2. Java standard library (`java.*`, `javax.*`)
3. Internal project (`net.woggioni.rbcs.*`)
Import aliases are rare; used only to resolve conflicts:
`import io.netty.util.concurrent.Future as NettyFuture`
### Naming Conventions
| Element | Convention | Examples |
|---------------------|-------------------------|---------------------------------------------------|
| Classes/interfaces | PascalCase | `RemoteBuildCacheServer`, `CacheHandler` |
| Abstract classes | `Abstract` prefix | `AbstractServerTest`, `AbstractNettyHttpAuthenticator` |
| Functions/methods | camelCase | `loadConfiguration()`, `sendShutdownSignal()` |
| Variables/properties| camelCase | `bucketManager`, `sslContext` |
| Constants | SCREAMING_SNAKE_CASE | `SSL_HANDLER_NAME`, `RBCS_NAMESPACE_URI` |
| Handler names | `val NAME = ...::class.java.name` in companion object |
| Packages | lowercase dot-separated | `net.woggioni.rbcs.server.throttling` |
| Enum values | PascalCase | `Role.Reader`, `Role.Writer` |
| Kotlin files | PascalCase matching primary class; lowercase for files with only top-level functions |
### Error Handling
- Custom unchecked exception hierarchy rooted at `RbcsException extends RuntimeException`
- Domain subclasses: `CacheException`, `ConfigurationException`, `ContentTooLargeException`
- Async errors propagated via `CompletableFuture.completeExceptionally()`
- Synchronous errors thrown directly: `throw IllegalArgumentException(...)`
- Kotlin null safety idioms preferred over null checks: `?.let`, `?:`, `takeIf`
- `ExceptionHandler` maps exception types to HTTP responses via exhaustive `when`
### Type Annotations
- **Kotlin:** Heavy use of type inference for local variables. Explicit types required on:
- Class properties: `private val sslContext: SslContext?`
- Non-trivial return types: `fun run(): ServerHandle`
- `lateinit var` in tests: `protected lateinit var cfg: Configuration`
- **Java:** Lombok `@Value` for immutable data classes; modern pattern matching with `instanceof`
- No `typealias` declarations are used in this project
### Async Patterns
- Primary abstraction: `CompletableFuture<T>`**no Kotlin coroutines**
- Netty event-driven callbacks (`ChannelFuture`, `GenericFutureListener`)
- Custom `AsyncCloseable` interface wraps `CompletableFuture<Void>` for async shutdown
- Retry logic uses `CompletableFuture` composition with exponential backoff + jitter
- Virtual threads used selectively (background GC, configurable event executors)
- Connection pooling via Netty `FixedChannelPool`
### Logging
- SLF4J via custom Kotlin lazy-evaluation extension functions (defined in `rbcs-common`)
- Logger creation: `private val log = createLogger<ClassName>()` (in companion object)
- Lazy log calls: `log.debug { "message with ${interpolation}" }` (lambda only evaluated if enabled)
- Channel-aware variants: `log.debug(ctx) { "message" }` (adds MDC: channel-id, remote-address)
- Java classes use Lombok `@Slf4j`
- String templates with `${}` for Kotlin log messages
### Kotlin Idioms
- `apply` as builder pattern: `ServerBootstrap().apply { group(...); option(...) }`
- `also` for side effects, `let` for transformation, `run` for scoping
- Trailing commas in constructor parameter lists and multi-line function calls
- `sealed class`/`sealed interface` for algebraic types (e.g., `OperationOutcome`, `Authentication`)
- `data class` for value types; `companion object` for static members and constants
- No trailing semicolons
### Java Idioms (API module)
- Lombok annotations: `@Value`, `@Getter`, `@RequiredArgsConstructor`, `@EqualsAndHashCode.Include`
- `sealed interface` with `final` permitted subtypes (e.g., `CacheMessage`)
- `@FunctionalInterface` on single-method interfaces
- JPMS `module-info.java` in every module with `requires`, `exports`, `uses`, `provides`
### Testing Patterns
- `@TestInstance(Lifecycle.PER_CLASS)` — single instance per test class
- `@TestMethodOrder(MethodOrderer.OrderAnnotation)` with `@Order(n)` for sequential execution
- Abstract base class hierarchy: `AbstractServerTest``AbstractBasicAuthServerTest` → concrete
- Server lifecycle in `@BeforeAll` / `@AfterAll` (start/stop real Netty server)
- `@TempDir` for temporary directories
- `@ParameterizedTest` with `@ValueSource` and `@ArgumentsSource` for parameterized tests
- Assertions via `org.junit.jupiter.api.Assertions` (`assertEquals`, `assertArrayEquals`)
- No mocking framework — all tests are integration-style against real servers
### Documentation
Minimal doc comments in the codebase. Inline comments used sparingly for clarification.
When adding new public API, follow existing style — doc comments are not enforced but
are welcome on complex logic.
## Module Dependency Graph
```
rbcs-api → (standalone, Lombok, Netty types)
rbcs-common → Netty, Kotlin stdlib
rbcs-server → rbcs-api, rbcs-common
rbcs-server-memcache → rbcs-api, rbcs-common
rbcs-client → rbcs-api, rbcs-common
rbcs-cli → rbcs-client, rbcs-server (Picocli for CLI)
rbcs-servlet → rbcs-api, rbcs-common (Jakarta Servlet/CDI)
docker → rbcs-cli, rbcs-server-memcache
```
## Plugin System
Cache backends use the `ServiceLoader` pattern via JPMS `provides`/`uses` directives.
To add a new cache provider, implement `CacheProvider<T extends Configuration.Cache>` and
register it in your module's `module-info.java`.
## Configuration
- XML-based with XSD schema validation
- Schemas in `src/main/resources/.../schema/*.xsd`
- Default config loaded from JPMS resources: `jpms://net.woggioni.rbcs.server/...`
+4 -4
View File
@@ -28,7 +28,7 @@ allprojects { subproject ->
mavenCentral() mavenCentral()
} }
pluginManager.withPlugin('java-library') { pluginManager.withPlugin('java') {
ext { ext {
jpmsModuleName = subproject.group + '.' + subproject.name.replace('-', '.') jpmsModuleName = subproject.group + '.' + subproject.name.replace('-', '.')
@@ -38,7 +38,7 @@ allprojects { subproject ->
withSourcesJar() withSourcesJar()
modularity.inferModulePath = true modularity.inferModulePath = true
toolchain { toolchain {
languageVersion = JavaLanguageVersion.of(21) languageVersion = JavaLanguageVersion.of(25)
} }
} }
@@ -55,7 +55,7 @@ allprojects { subproject ->
tasks.withType(JavaCompile) { tasks.withType(JavaCompile) {
modularity.inferModulePath = true modularity.inferModulePath = true
options.release = 21 options.release = 25
} }
tasks.named(JavaPlugin.COMPILE_JAVA_TASK_NAME, JavaCompile) { tasks.named(JavaPlugin.COMPILE_JAVA_TASK_NAME, JavaCompile) {
@@ -80,7 +80,7 @@ allprojects { subproject ->
pluginManager.withPlugin(catalog.plugins.kotlin.jvm.get().pluginId) { pluginManager.withPlugin(catalog.plugins.kotlin.jvm.get().pluginId) {
tasks.withType(KotlinCompile.class) { tasks.withType(KotlinCompile.class) {
compilerOptions.jvmTarget = JvmTarget.JVM_21 compilerOptions.jvmTarget = JvmTarget.JVM_25
} }
} }
+20 -1
View File
@@ -14,8 +14,21 @@ Configures server socket settings.
**Attributes:** **Attributes:**
- `host` (required): Server bind address - `host` (required): Server bind address
- `port` (required): Server port number - `port` (required): Server port number
- `proxy-protocol` (optional, default: false): Enable [HAProxy proxy protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) support. When enabled, the server decodes proxy protocol headers to extract the real client IP address from proxied connections.
- `incoming-connections-backlog-size` (optional, default: 1024): Maximum queue length for incoming connection indications - `incoming-connections-backlog-size` (optional, default: 1024): Maximum queue length for incoming connection indications
**Child Elements:**
##### `<trusted-proxies>`
Restricts which proxy servers are trusted to provide accurate client IP information via the proxy protocol. Only used when `proxy-protocol` is set to `true`.
If omitted or empty, all proxies are trusted. When specified, only connections originating from the listed CIDR ranges will have their forwarded client IP honored.
- Contains `<allow>` elements:
**Attributes:**
- `cidr` (required): An IPv4 or IPv6 CIDR range identifying a trusted proxy address (e.g. `192.168.0.0/24`, `::1/128`)
#### `<connection>` #### `<connection>`
Configures connection handling parameters. Configures connection handling parameters.
@@ -126,8 +139,14 @@ Configures TLS encryption.
<rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance" <rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server" xmlns:rbcs="urn:net.woggioni.rbcs.server"
xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd" xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"
path="/my/custom/path"
> >
<bind host="0.0.0.0" port="8080" incoming-connections-backlog-size="1024"/> <bind host="0.0.0.0" port="8080" incoming-connections-backlog-size="1024" proxy-protocol="true">
<trusted-proxies>
<allow cidr="192.168.0.11/32"/>
<allow cidr="::1/128"/>
</trusted-proxies>
</bind>
<connection <connection
max-request-size="67108864" max-request-size="67108864"
idle-timeout="PT10S" idle-timeout="PT10S"
+44 -18
View File
@@ -1,42 +1,68 @@
FROM eclipse-temurin:21-jre-alpine AS base-release FROM eclipse-temurin:25-jre-alpine AS base-release
RUN adduser -D luser RUN adduser -D rbcs
USER luser USER rbcs
WORKDIR /home/luser ENV RBCS_CONFIGURATION_DIR="/etc/rbcs"
WORKDIR /var/lib/rbcs
FROM base-release AS release-vanilla FROM base-release AS release-vanilla
ADD rbcs-cli-envelope-*.jar rbcs.jar ADD rbcs-cli-envelope-*.jar rbcs.jar
ENTRYPOINT ["java", "-Dlogback.configurationFile=logback.xml", "-XX:MaxRAMPercentage=70", "-XX:GCTimeRatio=24", "-XX:+UseZGC", "-XX:+ZGenerational", "-jar", "/home/luser/rbcs.jar"] ADD logback.xml /etc/rbcs/logback.xml
ENTRYPOINT ["java", "-jar", "/var/lib/rbcs/rbcs.jar"]
FROM base-release AS release-memcache FROM base-release AS release-memcache
ADD --chown=luser:luser rbcs-cli-envelope-*.jar rbcs.jar ADD --chown=rbcs:rbcs rbcs-cli-envelope-*.jar rbcs.jar
RUN mkdir plugins RUN mkdir plugins
WORKDIR /home/luser/plugins WORKDIR /var/lib/rbcs/plugins
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-memcache*.tar RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-memcache*.tar
WORKDIR /home/luser WORKDIR /var/lib/rbcs
ADD logback.xml . ADD logback.xml /etc/rbcs/logback.xml
ENTRYPOINT ["java", "-Dlogback.configurationFile=logback.xml", "-XX:MaxRAMPercentage=70", "-XX:GCTimeRatio=24", "-XX:+UseZGC", "-XX:+ZGenerational", "-jar", "/home/luser/rbcs.jar"] ENTRYPOINT ["java", "-jar", "/var/lib/rbcs/rbcs.jar"]
FROM base-release AS release-redis
ADD --chown=rbcs:rbcs rbcs-cli-envelope-*.jar rbcs.jar
RUN mkdir plugins
WORKDIR /var/lib/rbcs/plugins
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-redis*.tar
WORKDIR /var/lib/rbcs
ADD logback.xml /etc/rbcs/logback.xml
ENTRYPOINT ["java", "-jar", "/var/lib/rbcs/rbcs.jar"]
FROM base-release AS release-full
ADD --chown=rbcs:rbcs rbcs-cli-envelope-*.jar rbcs.jar
RUN mkdir plugins
WORKDIR /var/lib/rbcs/plugins
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-memcache*.tar
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-redis*.tar
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-otel*.tar
WORKDIR /var/lib/rbcs
ADD logback.xml /etc/rbcs/logback.xml
ENV OTEL_SDK_DISABLED="true"
ENTRYPOINT ["java", "-jar", "/var/lib/rbcs/rbcs.jar"]
FROM busybox:musl AS base-native FROM busybox:musl AS base-native
RUN mkdir -p /var/lib/rbcs /etc/rbcs RUN mkdir -p /var/lib/rbcs /var/tmp/rbcs /etc/rbcs
RUN adduser -D -u 1000 rbcs -h /var/lib/rbcs RUN adduser -D -u 1000 rbcs -h /var/lib/rbcs
RUN chown rbcs:rbcs /var/tmp/rbcs
FROM scratch AS release-native FROM scratch AS release-native
COPY --from=base-native /etc/passwd /etc/passwd COPY --from=base-native /etc/passwd /etc/passwd
COPY --from=base-native /etc/rbcs /etc/rbcs COPY --from=base-native /etc/rbcs /etc/rbcs
COPY --from=base-native /var/lib/rbcs /var/lib/rbcs COPY --from=base-native /var/lib/rbcs /var/lib/rbcs
COPY --from=base-native /var/tmp/rbcs /var/tmp/rbcs
ADD rbcs-cli.upx /usr/bin/rbcs-cli ADD rbcs-cli.upx /usr/bin/rbcs-cli
ENV RBCS_CONFIGURATION_DIR="/etc/rbcs"
USER rbcs USER rbcs
WORKDIR /var/lib/rbcs WORKDIR /var/lib/rbcs
ENTRYPOINT ["/usr/bin/rbcs-cli", "-XX:MaximumHeapSizePercent=70"] ENV RBCS_CONFIGURATION_DIR="/etc/rbcs"
ENTRYPOINT ["/usr/bin/rbcs-cli", "-XX:MaximumHeapSizePercent=70", "-Dio.netty.tmpdir=/var/tmp/rbcs", "-Dlogback.configurationFile=/etc/rbcs/logback.xml"]
FROM debian:12-slim AS release-jlink FROM debian:12-slim AS release-jlink
RUN mkdir -p /usr/share/java/rbcs RUN mkdir -p /usr/share/java/rbcs
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-cli*.tar -C /usr/share/java/rbcs RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-cli*.tar -C /usr/share/java/rbcs
RUN chmod 755 /usr/share/java/rbcs/bin/*
ADD --chmod=755 rbcs-cli.sh /usr/local/bin/rbcs-cli ADD --chmod=755 rbcs-cli.sh /usr/local/bin/rbcs-cli
RUN adduser -u 1000 luser RUN adduser -u 1000 rbcs
USER luser USER rbcs
WORKDIR /home/luser WORKDIR /var/lib/rbcs
ADD logback.xml . ADD logback.xml /etc/rbcs/logback.xml
ENV JAVA_OPTS=-XX:-UseJVMCICompiler\ -Dlogback.configurationFile=logback.xml\ -XX:MaxRAMPercentage=70\ -XX:GCTimeRatio=24\ -XX:+UseZGC\ -XX:+ZGenerational ENV RBCS_CONFIGURATION_DIR="/etc/rbcs"
ENTRYPOINT ["/usr/local/bin/rbcs-cli"] ENTRYPOINT ["/usr/local/bin/rbcs-cli"]
+1 -1
View File
@@ -5,7 +5,7 @@ There are 3 image flavours:
- native - native
The `vanilla` image only contains the envelope The `vanilla` image only contains the envelope
jar file with no plugins and is based on `eclipse-temurin:21-jre-alpine` jar file with no plugins and is based on `eclipse-temurin:25-jre-alpine`
The `memcache` image is similar to the `vanilla` image, except that it also contains The `memcache` image is similar to the `vanilla` image, except that it also contains
the `rbcs-server-memcache` plugin in the `plugins` folder, use this image if you don't want to use the `native` the `rbcs-server-memcache` plugin in the `plugins` folder, use this image if you don't want to use the `native`
+2
View File
@@ -20,6 +20,8 @@ configurations {
dependencies { dependencies {
docker project(path: ':rbcs-cli', configuration: 'release') docker project(path: ':rbcs-cli', configuration: 'release')
docker project(path: ':rbcs-server-memcache', configuration: 'release') docker project(path: ':rbcs-server-memcache', configuration: 'release')
docker project(path: ':rbcs-server-redis', configuration: 'release')
docker project(path: ':rbcs-server-otel', configuration: 'release')
} }
Provider<Task> cleanTaskProvider = tasks.named(BasePlugin.CLEAN_TASK_NAME) {} Provider<Task> cleanTaskProvider = tasks.named(BasePlugin.CLEAN_TASK_NAME) {}
+1 -1
View File
@@ -1,3 +1,3 @@
#!/bin/sh #!/bin/sh
DIR=/usr/share/java/rbcs DIR=/usr/share/java/rbcs
$DIR/bin/java $JAVA_OPTS -m net.woggioni.rbcs.cli "$@" exec $DIR/bin/java $JAVA_OPTS -m net.woggioni.rbcs.cli $@
+2 -2
View File
@@ -2,9 +2,9 @@ org.gradle.configuration-cache=false
org.gradle.parallel=true org.gradle.parallel=true
org.gradle.caching=true org.gradle.caching=true
rbcs.version = 0.3.4 rbcs.version = 0.5.1
lys.version = 2025.09.30 lys.version = 2026.06.08
gitea.maven.url = https://gitea.woggioni.net/api/packages/woggioni/maven gitea.maven.url = https://gitea.woggioni.net/api/packages/woggioni/maven
docker.registry.url=gitea.woggioni.net docker.registry.url=gitea.woggioni.net
Binary file not shown.
+1 -1
View File
@@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-9.3.1-bin.zip
networkTimeout=10000 networkTimeout=10000
validateDistributionUrl=true validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
Vendored
+1 -4
View File
@@ -1,7 +1,7 @@
#!/bin/sh #!/bin/sh
# #
# Copyright © 2015-2021 the original authors. # Copyright © 2015 the original authors.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
@@ -114,7 +114,6 @@ case "$( uname )" in #(
NONSTOP* ) nonstop=true ;; NONSTOP* ) nonstop=true ;;
esac esac
CLASSPATH="\\\"\\\""
# Determine the Java command to use to start the JVM. # Determine the Java command to use to start the JVM.
@@ -172,7 +171,6 @@ fi
# For Cygwin or MSYS, switch paths to Windows format before running java # For Cygwin or MSYS, switch paths to Windows format before running java
if "$cygwin" || "$msys" ; then if "$cygwin" || "$msys" ; then
APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
JAVACMD=$( cygpath --unix "$JAVACMD" ) JAVACMD=$( cygpath --unix "$JAVACMD" )
@@ -212,7 +210,6 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
set -- \ set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \ "-Dorg.gradle.appname=$APP_BASE_NAME" \
-classpath "$CLASSPATH" \
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
"$@" "$@"
Vendored
+1 -2
View File
@@ -70,11 +70,10 @@ goto fail
:execute :execute
@rem Setup the command line @rem Setup the command line
set CLASSPATH=
@rem Execute Gradle @rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %*
:end :end
@rem End local scope for the variables with windows NT shell @rem End local scope for the variables with windows NT shell
+1
View File
@@ -11,6 +11,7 @@ dependencies {
api catalog.netty.buffer api catalog.netty.buffer
api catalog.netty.handler api catalog.netty.handler
api catalog.netty.codec.http api catalog.netty.codec.http
api catalog.jetbrains.annotations
} }
publishing { publishing {
+1 -1
View File
@@ -8,7 +8,7 @@ module net.woggioni.rbcs.api {
requires io.netty.buffer; requires io.netty.buffer;
requires org.slf4j; requires org.slf4j;
requires java.xml; requires java.xml;
requires org.jetbrains.annotations;
exports net.woggioni.rbcs.api; exports net.woggioni.rbcs.api;
exports net.woggioni.rbcs.api.exception; exports net.woggioni.rbcs.api.exception;
@@ -4,10 +4,12 @@ package net.woggioni.rbcs.api;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.NonNull; import lombok.NonNull;
import lombok.Value; import lombok.Value;
import net.woggioni.rbcs.common.Cidr;
import java.nio.file.Path; import java.nio.file.Path;
import java.security.cert.X509Certificate; import java.security.cert.X509Certificate;
import java.time.Duration; import java.time.Duration;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -16,8 +18,10 @@ import java.util.stream.Collectors;
public class Configuration { public class Configuration {
String host; String host;
int port; int port;
int incomingConnectionsBacklogSize;
String serverPath; String serverPath;
boolean proxyProtocolEnabled;
List<Cidr> trustedProxyIPs;
int incomingConnectionsBacklogSize;
@NonNull @NonNull
EventExecutor eventExecutor; EventExecutor eventExecutor;
@NonNull @NonNull
@@ -30,6 +34,7 @@ public class Configuration {
Authentication authentication; Authentication authentication;
Tls tls; Tls tls;
@Value @Value
public static class RateLimiter { public static class RateLimiter {
boolean delayRequest; boolean delayRequest;
@@ -131,6 +136,13 @@ public class Configuration {
TlsCertificateExtractor groupExtractor; TlsCertificateExtractor groupExtractor;
} }
@Value
public static class ForwardedClientCertificateAuthentication implements Authentication {
String headerName;
TlsCertificateExtractor userExtractor;
TlsCertificateExtractor groupExtractor;
}
public interface Cache { public interface Cache {
CacheHandlerFactory materialize(); CacheHandlerFactory materialize();
String getNamespaceURI(); String getNamespaceURI();
@@ -140,6 +152,8 @@ public class Configuration {
public static Configuration of( public static Configuration of(
String host, String host,
int port, int port,
boolean proxyProtocolEnabled,
List<Cidr> trustedProxyIPs,
int incomingConnectionsBacklogSize, int incomingConnectionsBacklogSize,
String serverPath, String serverPath,
EventExecutor eventExecutor, EventExecutor eventExecutor,
@@ -154,8 +168,10 @@ public class Configuration {
return new Configuration( return new Configuration(
host, host,
port, port,
incomingConnectionsBacklogSize,
serverPath != null && !serverPath.isEmpty() && !serverPath.equals("/") ? serverPath : null, serverPath != null && !serverPath.isEmpty() && !serverPath.equals("/") ? serverPath : null,
proxyProtocolEnabled,
trustedProxyIPs,
incomingConnectionsBacklogSize,
eventExecutor, eventExecutor,
rateLimiter, rateLimiter,
connection, connection,
@@ -0,0 +1,13 @@
package net.woggioni.rbcs.api;
import org.jetbrains.annotations.NotNull;
public interface SpanHandle {
void setAttribute(@NotNull String key, @NotNull String value);
void setAttribute(@NotNull String key, long value);
void setAttribute(@NotNull String key, boolean value);
}
@@ -0,0 +1,18 @@
package net.woggioni.rbcs.api;
import io.netty.channel.ChannelHandler;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Map;
public interface TelemetryController {
void initialize();
@NotNull ChannelHandler createHandler();
@Nullable SpanHandle startSpan(@NotNull String command);
void endSpan(@Nullable SpanHandle span);
void endSpan(@Nullable SpanHandle span, @NotNull Throwable error);
}
+17 -11
View File
@@ -30,7 +30,6 @@ configurations {
transitive = false transitive = false
canBeConsumed = true canBeConsumed = true
canBeResolved = true canBeResolved = true
visible = true
} }
configureNativeImageImplementation { configureNativeImageImplementation {
@@ -50,6 +49,8 @@ configurations {
dependencies { dependencies {
configureNativeImageImplementation project configureNativeImageImplementation project
configureNativeImageImplementation project(':rbcs-server-memcache') configureNativeImageImplementation project(':rbcs-server-memcache')
configureNativeImageImplementation project(':rbcs-server-redis')
// configureNativeImageImplementation project(':rbcs-server-otel')
implementation catalog.jwo implementation catalog.jwo
implementation catalog.slf4j.api implementation catalog.slf4j.api
@@ -58,10 +59,9 @@ dependencies {
implementation project(':rbcs-client') implementation project(':rbcs-client')
implementation project(':rbcs-server') implementation project(':rbcs-server')
// runtimeOnly catalog.slf4j.jdk14
runtimeOnly catalog.logback.classic runtimeOnly catalog.logback.classic
// runtimeOnly catalog.slf4j.simple
nativeImage project(':rbcs-server-memcache') nativeImage project(':rbcs-server-memcache')
nativeImage project(':rbcs-server-redis')
} }
@@ -89,8 +89,8 @@ Provider<EnvelopeJarTask> envelopeJarTaskProvider = tasks.named(EnvelopePlugin.E
tasks.named(NativeImagePlugin.CONFIGURE_NATIVE_IMAGE_TASK_NAME, NativeImageConfigurationTask) { tasks.named(NativeImagePlugin.CONFIGURE_NATIVE_IMAGE_TASK_NAME, NativeImageConfigurationTask) {
toolchain { toolchain {
languageVersion = JavaLanguageVersion.of(21) languageVersion = JavaLanguageVersion.of(25)
vendor = JvmVendorSpec.GRAAL_VM vendor = JvmVendorSpec.ORACLE
} }
mainClass = "net.woggioni.rbcs.cli.graal.GraalNativeImageConfiguration" mainClass = "net.woggioni.rbcs.cli.graal.GraalNativeImageConfiguration"
classpath = project.files( classpath = project.files(
@@ -107,11 +107,11 @@ tasks.named(NativeImagePlugin.CONFIGURE_NATIVE_IMAGE_TASK_NAME, NativeImageConfi
nativeImage { nativeImage {
toolchain { toolchain {
languageVersion = JavaLanguageVersion.of(23) languageVersion = JavaLanguageVersion.of(25)
vendor = JvmVendorSpec.GRAAL_VM vendor = JvmVendorSpec.ORACLE
} }
mainClass = mainClassName mainClass = mainClassName
// mainModule = mainModuleName mainModule = mainModuleName
useMusl = true useMusl = true
buildStaticImage = true buildStaticImage = true
linkAtBuildTime = false linkAtBuildTime = false
@@ -126,8 +126,8 @@ Provider<UpxTask> upxTaskProvider = tasks.named(NativeImagePlugin.UPX_TASK_NAME,
Provider<JlinkTask> jlinkTaskProvider = tasks.named(JlinkPlugin.JLINK_TASK_NAME, JlinkTask) { Provider<JlinkTask> jlinkTaskProvider = tasks.named(JlinkPlugin.JLINK_TASK_NAME, JlinkTask) {
toolchain { toolchain {
languageVersion = JavaLanguageVersion.of(21) languageVersion = JavaLanguageVersion.of(25)
vendor = JvmVendorSpec.GRAAL_VM vendor = JvmVendorSpec.ORACLE
} }
mainClass = mainClassName mainClass = mainClassName
@@ -138,8 +138,14 @@ Provider<JlinkTask> jlinkTaskProvider = tasks.named(JlinkPlugin.JLINK_TASK_NAME,
) )
additionalModules = [ additionalModules = [
'net.woggioni.rbcs.server.memcache', 'net.woggioni.rbcs.server.memcache',
'net.woggioni.rbcs.server.redis',
'ch.qos.logback.classic', 'ch.qos.logback.classic',
'jdk.crypto.ec' 'jdk.crypto.ec',
// 'io.opentelemetry.api',
// 'io.opentelemetry.instrumentation.netty_4_1',
// 'io.opentelemetry.sdk.autoconfigure',
// 'io.opentelemetry.instrumentation.logback_appender_1_0',
// 'io.opentelemetry.extension.trace.propagation'
] ]
compressionLevel = 2 compressionLevel = 2
stripDebug = false stripDebug = false
+53
View File
@@ -0,0 +1,53 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server"
xmlns:rbcs-redis="urn:net.woggioni.rbcs.server.redis"
xs:schemaLocation="urn:net.woggioni.rbcs.server.redis jpms://net.woggioni.rbcs.server.redis/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd urn:net.woggioni.rbcs.server.memcache jpms://net.woggioni.rbcs.server.memcache/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"
>
<bind host="127.0.0.1" port="8080" incoming-connections-backlog-size="1024"/>
<connection
max-request-size="67108864"
idle-timeout="PT10S"
read-idle-timeout="PT20S"
write-idle-timeout="PT20S"/>
<event-executor use-virtual-threads="true"/>
<cache xs:type="rbcs-redis:redisCacheType" max-age="P7D" digest="MD5">
<server host="127.0.0.1" port="6379" max-connections="256"/>
</cache>
<!--cache xs:type="rbcs:inMemoryCacheType" max-age="P7D" enable-compression="false" max-size="0x10000000" /-->
<!--cache xs:type="rbcs:fileSystemCacheType" max-age="P7D" enable-compression="false" /-->
<authorization>
<users>
<user name="woggioni" password="II+qeNLft2pZ/JVNo9F7jpjM/BqEcfsJW27NZ6dPVs8tAwHbxrJppKYsbL7J/SMl">
<quota calls="100" period="PT1S"/>
</user>
<user name="gitea" password="v6T9+q6/VNpvLknji3ixPiyz2YZCQMXj2FN7hvzbfc2Ig+IzAHO0iiBCH9oWuBDq"/>
<anonymous>
<quota calls="10" period="PT60S" initial-available-calls="10" max-available-calls="10"/>
</anonymous>
</users>
<groups>
<group name="readers">
<users>
<anonymous/>
</users>
<roles>
<reader/>
</roles>
</group>
<group name="writers">
<users>
<user ref="woggioni"/>
<user ref="gitea"/>
</users>
<roles>
<reader/>
<writer/>
</roles>
</group>
</groups>
</authorization>
<authentication>
<none/>
</authentication>
</rbcs:server>
@@ -1,2 +1,2 @@
Args=-O3 -march=x86-64-v2 --gc=serial --install-exit-handlers --initialize-at-run-time=io.netty --enable-url-protocols=jpms --initialize-at-build-time=net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory,net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory$JpmsHandler Args=-O3 -march=x86-64-v2 --gc=serial --initialize-at-run-time=io.netty --enable-url-protocols=jpms --emit build-report -H:+UnlockExperimentalVMOptions -H:+SharedArenaSupport --initialize-at-build-time=net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory,net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory$JpmsHandler --add-opens=io.netty.common/io.netty.util=org.graalvm.nativeimage.builder --add-opens=io.netty.common/io.netty.util.internal.svm=org.graalvm.nativeimage.builder
#-H:TraceClassInitialization=io.netty.handler.ssl.BouncyCastleAlpnSslUtils #-H:TraceClassInitialization=io.netty.handler.ssl.BouncyCastleAlpnSslUtils
+13 -7
View File
@@ -183,9 +183,6 @@
"name":"io.netty.channel.SimpleChannelInboundHandler", "name":"io.netty.channel.SimpleChannelInboundHandler",
"methods":[{"name":"channelRead","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }] "methods":[{"name":"channelRead","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }]
}, },
{
"name":"io.netty.channel.embedded.EmbeddedChannel$2"
},
{ {
"name":"io.netty.channel.pool.SimpleChannelPool$1" "name":"io.netty.channel.pool.SimpleChannelPool$1"
}, },
@@ -195,7 +192,7 @@
}, },
{ {
"name":"io.netty.handler.codec.ByteToMessageDecoder", "name":"io.netty.handler.codec.ByteToMessageDecoder",
"methods":[{"name":"channelInactive","parameterTypes":["io.netty.channel.ChannelHandlerContext"] }, {"name":"channelRead","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }, {"name":"channelReadComplete","parameterTypes":["io.netty.channel.ChannelHandlerContext"] }, {"name":"userEventTriggered","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }] "methods":[{"name":"channelInactive","parameterTypes":["io.netty.channel.ChannelHandlerContext"] }, {"name":"channelRead","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }, {"name":"userEventTriggered","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }]
}, },
{ {
"name":"io.netty.handler.codec.MessageAggregator", "name":"io.netty.handler.codec.MessageAggregator",
@@ -214,7 +211,8 @@
"methods":[{"name":"channelRead","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }] "methods":[{"name":"channelRead","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }]
}, },
{ {
"name":"io.netty.handler.codec.compression.JdkZlibDecoder" "name":"io.netty.handler.codec.compression.JdkZlibDecoder",
"methods":[{"name":"channelReadComplete","parameterTypes":["io.netty.channel.ChannelHandlerContext"] }]
}, },
{ {
"name":"io.netty.handler.codec.compression.JdkZlibEncoder", "name":"io.netty.handler.codec.compression.JdkZlibEncoder",
@@ -227,6 +225,10 @@
"name":"io.netty.handler.codec.http.HttpContentDecoder", "name":"io.netty.handler.codec.http.HttpContentDecoder",
"methods":[{"name":"channelInactive","parameterTypes":["io.netty.channel.ChannelHandlerContext"] }, {"name":"channelReadComplete","parameterTypes":["io.netty.channel.ChannelHandlerContext"] }] "methods":[{"name":"channelInactive","parameterTypes":["io.netty.channel.ChannelHandlerContext"] }, {"name":"channelReadComplete","parameterTypes":["io.netty.channel.ChannelHandlerContext"] }]
}, },
{
"name":"io.netty.handler.codec.http.HttpContentDecoder$ByteBufForwarder",
"methods":[{"name":"channelRead","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }]
},
{ {
"name":"io.netty.handler.codec.http.HttpContentDecompressor" "name":"io.netty.handler.codec.http.HttpContentDecompressor"
}, },
@@ -278,9 +280,13 @@
"name":"io.netty.util.concurrent.DefaultPromise", "name":"io.netty.util.concurrent.DefaultPromise",
"fields":[{"name":"result"}] "fields":[{"name":"result"}]
}, },
{
"name":"io.netty.util.concurrent.MpscIntQueue$MpscAtomicIntegerArrayQueue",
"fields":[{"name":"consumerIndex"}, {"name":"producerIndex"}, {"name":"producerLimit"}]
},
{ {
"name":"io.netty.util.concurrent.SingleThreadEventExecutor", "name":"io.netty.util.concurrent.SingleThreadEventExecutor",
"fields":[{"name":"state"}, {"name":"threadProperties"}] "fields":[{"name":"accumulatedActiveTimeNanos"}, {"name":"consecutiveBusyCycles"}, {"name":"consecutiveIdleCycles"}, {"name":"state"}, {"name":"threadProperties"}]
}, },
{ {
"name":"io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields", "name":"io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields",
@@ -566,7 +572,7 @@
}, },
{ {
"name":"net.woggioni.rbcs.server.handler.ReadTriggerDuplexHandler", "name":"net.woggioni.rbcs.server.handler.ReadTriggerDuplexHandler",
"methods":[{"name":"channelRead","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }, {"name":"write","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object","io.netty.channel.ChannelPromise"] }] "methods":[{"name":"channelInactive","parameterTypes":["io.netty.channel.ChannelHandlerContext"] }, {"name":"channelRead","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }, {"name":"write","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object","io.netty.channel.ChannelPromise"] }]
}, },
{ {
"name":"net.woggioni.rbcs.server.handler.ServerHandler", "name":"net.woggioni.rbcs.server.handler.ServerHandler",
@@ -36,6 +36,8 @@
"pattern":"\\Qnet/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd\\E" "pattern":"\\Qnet/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd\\E"
}, { }, {
"pattern":"\\Qnet/woggioni/rbcs/server/schema/rbcs-server.xsd\\E" "pattern":"\\Qnet/woggioni/rbcs/server/schema/rbcs-server.xsd\\E"
}, {
"pattern":"jdk.jfr:\\Qjdk/jfr/internal/types/metadata.bin\\E"
}]}, }]},
"bundles":[{ "bundles":[{
"name":"com.sun.org.apache.xerces.internal.impl.xpath.regex.message", "name":"com.sun.org.apache.xerces.internal.impl.xpath.regex.message",
@@ -7,6 +7,8 @@ import java.time.Duration
import java.time.temporal.ChronoUnit import java.time.temporal.ChronoUnit
import java.util.concurrent.ExecutionException import java.util.concurrent.ExecutionException
import java.util.zip.Deflater import java.util.zip.Deflater
import net.woggioni.rbcs.client.Configuration as ClientConfiguration
import net.woggioni.rbcs.client.impl.Parser as ClientConfigurationParser
import net.woggioni.jwo.NullOutputStream import net.woggioni.jwo.NullOutputStream
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.api.Configuration.User import net.woggioni.rbcs.api.Configuration.User
@@ -16,8 +18,6 @@ import net.woggioni.rbcs.cli.impl.commands.BenchmarkCommand
import net.woggioni.rbcs.cli.impl.commands.GetCommand import net.woggioni.rbcs.cli.impl.commands.GetCommand
import net.woggioni.rbcs.cli.impl.commands.HealthCheckCommand import net.woggioni.rbcs.cli.impl.commands.HealthCheckCommand
import net.woggioni.rbcs.cli.impl.commands.PutCommand import net.woggioni.rbcs.cli.impl.commands.PutCommand
import net.woggioni.rbcs.client.Configuration as ClientConfiguration
import net.woggioni.rbcs.client.impl.Parser as ClientConfigurationParser
import net.woggioni.rbcs.common.HostAndPort import net.woggioni.rbcs.common.HostAndPort
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
import net.woggioni.rbcs.common.RBCS import net.woggioni.rbcs.common.RBCS
@@ -27,16 +27,27 @@ import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration
import net.woggioni.rbcs.server.cache.InMemoryCacheConfiguration import net.woggioni.rbcs.server.cache.InMemoryCacheConfiguration
import net.woggioni.rbcs.server.configuration.Parser import net.woggioni.rbcs.server.configuration.Parser
import net.woggioni.rbcs.server.memcache.MemcacheCacheConfiguration import net.woggioni.rbcs.server.memcache.MemcacheCacheConfiguration
import net.woggioni.rbcs.server.redis.RedisCacheConfiguration
object GraalNativeImageConfiguration { object GraalNativeImageConfiguration {
@JvmStatic @JvmStatic
fun main(vararg args : String) { fun main(vararg args : String) {
let {
val serverURL = URI.create("file:conf/rbcs-server.xml").toURL() val serverURL = URI.create("file:conf/rbcs-server.xml").toURL()
val serverDoc = serverURL.openStream().use { val serverDoc = serverURL.openStream().use {
Xml.parseXml(serverURL, it) Xml.parseXml(serverURL, it)
} }
Parser.parse(serverDoc) Parser.parse(serverDoc)
}
let {
val serverURL = URI.create("file:conf/rbcs-server-redis.xml").toURL()
val serverDoc = serverURL.openStream().use {
Xml.parseXml(serverURL, it)
}
Parser.parse(serverDoc)
}
val url = URI.create("file:conf/rbcs-client.xml").toURL() val url = URI.create("file:conf/rbcs-client.xml").toURL()
val clientDoc = url.openStream().use { val clientDoc = url.openStream().use {
@@ -90,6 +101,18 @@ object GraalNativeImageConfiguration {
"MD5", "MD5",
null, null,
1, 1,
),
RedisCacheConfiguration(
listOf(RedisCacheConfiguration.Server(
HostAndPort("127.0.0.1", 6379),
1000,
4)
),
Duration.ofSeconds(60),
"someCustomPrefix",
"MD5",
null,
1,
) )
) )
@@ -97,8 +120,10 @@ object GraalNativeImageConfiguration {
val serverConfiguration = Configuration( val serverConfiguration = Configuration(
"127.0.0.1", "127.0.0.1",
serverPort, serverPort,
100,
null, null,
false,
emptyList(),
100,
Configuration.EventExecutor(true), Configuration.EventExecutor(true),
Configuration.RateLimiter( Configuration.RateLimiter(
false, 0x100000, 10 false, 0x100000, 10
+3 -1
View File
@@ -1,11 +1,11 @@
module net.woggioni.rbcs.cli { module net.woggioni.rbcs.cli {
requires org.slf4j; requires org.slf4j;
requires net.woggioni.jwo;
requires net.woggioni.rbcs.server; requires net.woggioni.rbcs.server;
requires info.picocli; requires info.picocli;
requires net.woggioni.rbcs.common; requires net.woggioni.rbcs.common;
requires net.woggioni.rbcs.client; requires net.woggioni.rbcs.client;
requires kotlin.stdlib; requires kotlin.stdlib;
requires net.woggioni.jwo;
requires net.woggioni.rbcs.api; requires net.woggioni.rbcs.api;
exports net.woggioni.rbcs.cli.impl.converters to info.picocli; exports net.woggioni.rbcs.cli.impl.converters to info.picocli;
@@ -14,4 +14,6 @@ module net.woggioni.rbcs.cli {
opens net.woggioni.rbcs.cli to info.picocli, net.woggioni.rbcs.common; opens net.woggioni.rbcs.cli to info.picocli, net.woggioni.rbcs.common;
exports net.woggioni.rbcs.cli; exports net.woggioni.rbcs.cli;
uses net.woggioni.rbcs.api.TelemetryController;
} }
@@ -1,6 +1,8 @@
package net.woggioni.rbcs.cli package net.woggioni.rbcs.cli
import net.woggioni.jwo.Application import net.woggioni.jwo.Application
import net.woggioni.jwo.LoggerController
import net.woggioni.rbcs.api.TelemetryController
import net.woggioni.rbcs.cli.impl.AbstractVersionProvider import net.woggioni.rbcs.cli.impl.AbstractVersionProvider
import net.woggioni.rbcs.cli.impl.RbcsCommand import net.woggioni.rbcs.cli.impl.RbcsCommand
import net.woggioni.rbcs.cli.impl.commands.BenchmarkCommand import net.woggioni.rbcs.cli.impl.commands.BenchmarkCommand
@@ -10,6 +12,7 @@ import net.woggioni.rbcs.cli.impl.commands.HealthCheckCommand
import net.woggioni.rbcs.cli.impl.commands.PasswordHashCommand import net.woggioni.rbcs.cli.impl.commands.PasswordHashCommand
import net.woggioni.rbcs.cli.impl.commands.PutCommand import net.woggioni.rbcs.cli.impl.commands.PutCommand
import net.woggioni.rbcs.cli.impl.commands.ServerCommand import net.woggioni.rbcs.cli.impl.commands.ServerCommand
import net.woggioni.rbcs.common.RBCS.loadService
import net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory import net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory
import net.woggioni.rbcs.common.createLogger import net.woggioni.rbcs.common.createLogger
import picocli.CommandLine import picocli.CommandLine
@@ -61,6 +64,10 @@ class RemoteBuildCacheServerCli : RbcsCommand() {
@JvmStatic @JvmStatic
fun main(vararg args: String) { fun main(vararg args: String) {
loadService(TelemetryController::class.java)
.firstOrNull()
?.initialize()
LoggerController.initializeLoggers()
System.exit(createCommandLine().execute(*args)) System.exit(createCommandLine().execute(*args))
} }
} }
@@ -1,5 +1,18 @@
package net.woggioni.rbcs.client package net.woggioni.rbcs.client
import java.io.IOException
import java.net.InetSocketAddress
import java.net.URI
import java.security.cert.X509Certificate
import java.util.Base64
import java.util.concurrent.CompletableFuture
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import java.util.concurrent.atomic.AtomicInteger
import javax.net.ssl.TrustManagerFactory
import javax.net.ssl.X509TrustManager
import kotlin.random.Random
import io.netty.util.concurrent.Future as NettyFuture
import io.netty.bootstrap.Bootstrap import io.netty.bootstrap.Bootstrap
import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBuf
import io.netty.buffer.Unpooled import io.netty.buffer.Unpooled
@@ -34,20 +47,7 @@ import io.netty.handler.timeout.IdleState
import io.netty.handler.timeout.IdleStateEvent import io.netty.handler.timeout.IdleStateEvent
import io.netty.handler.timeout.IdleStateHandler import io.netty.handler.timeout.IdleStateHandler
import io.netty.util.concurrent.Future import io.netty.util.concurrent.Future
import io.netty.util.concurrent.Future as NettyFuture
import io.netty.util.concurrent.GenericFutureListener import io.netty.util.concurrent.GenericFutureListener
import java.io.IOException
import java.net.InetSocketAddress
import java.net.URI
import java.security.cert.X509Certificate
import java.util.Base64
import java.util.concurrent.CompletableFuture
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import java.util.concurrent.atomic.AtomicInteger
import javax.net.ssl.TrustManagerFactory
import javax.net.ssl.X509TrustManager
import kotlin.random.Random
import net.woggioni.rbcs.api.CacheValueMetadata import net.woggioni.rbcs.api.CacheValueMetadata
import net.woggioni.rbcs.common.RBCS.loadKeystore import net.woggioni.rbcs.common.RBCS.loadKeystore
import net.woggioni.rbcs.common.createLogger import net.woggioni.rbcs.common.createLogger
@@ -1,10 +1,10 @@
package net.woggioni.rbcs.client package net.woggioni.rbcs.client
import io.netty.util.concurrent.EventExecutorGroup
import java.util.concurrent.CompletableFuture import java.util.concurrent.CompletableFuture
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import kotlin.math.pow import kotlin.math.pow
import kotlin.random.Random import kotlin.random.Random
import io.netty.util.concurrent.EventExecutorGroup
sealed class OperationOutcome<T> { sealed class OperationOutcome<T> {
class Success<T>(val result: T) : OperationOutcome<T>() class Success<T>(val result: T) : OperationOutcome<T>()
@@ -1,10 +1,10 @@
package net.woggioni.rbcs.client package net.woggioni.rbcs.client
import io.netty.util.concurrent.DefaultEventExecutorGroup
import io.netty.util.concurrent.EventExecutorGroup
import java.util.concurrent.CompletableFuture import java.util.concurrent.CompletableFuture
import java.util.stream.Stream import java.util.stream.Stream
import kotlin.random.Random import kotlin.random.Random
import io.netty.util.concurrent.DefaultEventExecutorGroup
import io.netty.util.concurrent.EventExecutorGroup
import net.woggioni.rbcs.common.contextLogger import net.woggioni.rbcs.common.contextLogger
import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.extension.ExtensionContext import org.junit.jupiter.api.extension.ExtensionContext
@@ -1,7 +1,7 @@
package net.woggioni.rbcs.common package net.woggioni.rbcs.common
import io.netty.buffer.ByteBuf
import java.io.InputStream import java.io.InputStream
import io.netty.buffer.ByteBuf
class ByteBufInputStream(private val buf : ByteBuf) : InputStream() { class ByteBufInputStream(private val buf : ByteBuf) : InputStream() {
override fun read(): Int { override fun read(): Int {
@@ -1,7 +1,7 @@
package net.woggioni.rbcs.common package net.woggioni.rbcs.common
import io.netty.buffer.ByteBuf
import java.io.OutputStream import java.io.OutputStream
import io.netty.buffer.ByteBuf
class ByteBufOutputStream(private val buf : ByteBuf) : OutputStream() { class ByteBufOutputStream(private val buf : ByteBuf) : OutputStream() {
override fun write(b: Int) { override fun write(b: Int) {
@@ -0,0 +1,62 @@
package net.woggioni.rbcs.common
import java.net.InetAddress
data class Cidr private constructor(
val networkAddress: InetAddress,
val prefixLength: Int
) {
companion object {
fun from(cidr: String) : Cidr {
val separator = cidr.indexOf("/")
if(separator < 0) {
throw IllegalArgumentException("Invalid CIDR format: $cidr")
}
val networkAddress = InetAddress.getByName(cidr.substring(0, separator))
val prefixLength = cidr.substring(separator + 1, cidr.length).toInt()
// Validate prefix length
val maxPrefix = if (networkAddress.address.size == 4) 32 else 128
require(prefixLength in 0..maxPrefix) { "Invalid prefix length: $prefixLength" }
return Cidr(networkAddress, prefixLength)
}
}
fun contains(address: InetAddress): Boolean {
val networkBytes = networkAddress.address
val addressBytes = address.address
if (networkBytes.size != addressBytes.size) {
return false
}
// Calculate how many full bytes and remaining bits to check
val fullBytes = prefixLength / 8
val remainingBits = prefixLength % 8
// Check full bytes
for (i in 0..<fullBytes) {
if (networkBytes[i] != addressBytes[i]) {
return false
}
}
// Check remaining bits if any
if (remainingBits > 0 && fullBytes < networkBytes.size) {
val mask = (0xFF shl (8 - remainingBits)).toByte()
if ((networkBytes[fullBytes].toInt() and mask.toInt()) != (addressBytes[fullBytes].toInt() and mask.toInt())) {
return false
}
}
return true
}
override fun toString(): String {
return networkAddress.hostAddress + "/" + prefixLength
}
}
@@ -1,18 +1,20 @@
package net.woggioni.rbcs.common package net.woggioni.rbcs.common
import io.netty.channel.Channel
import io.netty.channel.ChannelHandlerContext
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
import java.util.logging.LogManager import java.util.logging.LogManager
import io.netty.channel.Channel
import io.netty.channel.ChannelHandlerContext
import net.woggioni.jwo.LoggerController
import org.slf4j.Logger import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.slf4j.MDC import org.slf4j.MDC
import org.slf4j.event.Level import org.slf4j.event.Level
import org.slf4j.spi.LoggingEventBuilder import org.slf4j.spi.LoggingEventBuilder
inline fun <reified T> T.contextLogger() = LoggerFactory.getLogger(T::class.java) fun <T> lazyLogger(cls: Class<T>) = LoggerController.lazyLogger(cls)
inline fun <reified T> createLogger() = LoggerFactory.getLogger(T::class.java)
inline fun <reified T> T.contextLogger() = lazyLogger(T::class.java)
inline fun <reified T> createLogger() = lazyLogger(T::class.java)
inline fun Logger.traceParam(messageBuilder: () -> Pair<String, Array<Any>>) { inline fun Logger.traceParam(messageBuilder: () -> Pair<String, Array<Any>>) {
if (isTraceEnabled) { if (isTraceEnabled) {
@@ -81,9 +83,6 @@ inline fun Logger.log(level: Level, channel: Channel, crossinline messageBuilder
) )
withMDC(params) { withMDC(params) {
val builder = makeLoggingEventBuilder(level) val builder = makeLoggingEventBuilder(level)
// for ((key, value) in params) {
// builder.addKeyValue(key, value)
// }
messageBuilder(builder) messageBuilder(builder)
builder.log() builder.log()
} }
@@ -17,6 +17,7 @@ import java.security.cert.PKIXParameters
import java.security.cert.PKIXRevocationChecker import java.security.cert.PKIXRevocationChecker
import java.security.cert.X509Certificate import java.security.cert.X509Certificate
import java.util.EnumSet import java.util.EnumSet
import java.util.ServiceLoader
import javax.net.ssl.TrustManagerFactory import javax.net.ssl.TrustManagerFactory
import javax.net.ssl.X509TrustManager import javax.net.ssl.X509TrustManager
import net.woggioni.jwo.JWO import net.woggioni.jwo.JWO
@@ -164,4 +165,10 @@ object RBCS {
.single() as X509TrustManager .single() as X509TrustManager
} }
} }
inline fun <T, reified U> U.loadService(serviceClass : Class<T>): Sequence<T> {
return (U::class.java.module.layer?.let { layer ->
ServiceLoader.load(layer, serviceClass)
} ?: ServiceLoader.load(serviceClass)).asSequence()
}
} }
@@ -16,13 +16,13 @@ import javax.xml.transform.stream.StreamResult
import javax.xml.transform.stream.StreamSource import javax.xml.transform.stream.StreamSource
import javax.xml.validation.Schema import javax.xml.validation.Schema
import javax.xml.validation.SchemaFactory import javax.xml.validation.SchemaFactory
import org.xml.sax.ErrorHandler as ErrHandler
import net.woggioni.jwo.JWO import net.woggioni.jwo.JWO
import org.slf4j.event.Level import org.slf4j.event.Level
import org.w3c.dom.Document import org.w3c.dom.Document
import org.w3c.dom.Element import org.w3c.dom.Element
import org.w3c.dom.Node import org.w3c.dom.Node
import org.w3c.dom.NodeList import org.w3c.dom.NodeList
import org.xml.sax.ErrorHandler as ErrHandler
import org.xml.sax.SAXNotRecognizedException import org.xml.sax.SAXNotRecognizedException
import org.xml.sax.SAXNotSupportedException import org.xml.sax.SAXNotSupportedException
import org.xml.sax.SAXParseException import org.xml.sax.SAXParseException
@@ -0,0 +1,17 @@
package net.woggioni.rbcs.common
import java.net.InetAddress
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Test
class CidrTest {
class CidrTest {
@Test
fun test() {
val cidr = Cidr.from("2a02:4780:12:368b::1/128")
Assertions.assertTrue {
cidr.contains(InetAddress.ofLiteral("2a02:4780:12:368b::1"))
}
}
}
}
@@ -1,4 +1,5 @@
import net.woggioni.rbcs.api.CacheProvider; import net.woggioni.rbcs.api.CacheProvider;
import net.woggioni.rbcs.api.TelemetryController;
module net.woggioni.rbcs.server.memcache { module net.woggioni.rbcs.server.memcache {
requires net.woggioni.rbcs.common; requires net.woggioni.rbcs.common;
@@ -16,5 +17,7 @@ module net.woggioni.rbcs.server.memcache {
provides CacheProvider with net.woggioni.rbcs.server.memcache.MemcacheCacheProvider; provides CacheProvider with net.woggioni.rbcs.server.memcache.MemcacheCacheProvider;
uses TelemetryController;
opens net.woggioni.rbcs.server.memcache.schema; opens net.woggioni.rbcs.server.memcache.schema;
} }
@@ -1,15 +1,15 @@
package net.woggioni.rbcs.server.memcache package net.woggioni.rbcs.server.memcache
import io.netty.channel.ChannelFactory
import io.netty.channel.EventLoopGroup
import io.netty.channel.pool.FixedChannelPool
import io.netty.channel.socket.DatagramChannel
import io.netty.channel.socket.SocketChannel
import java.time.Duration import java.time.Duration
import java.util.concurrent.CompletableFuture import java.util.concurrent.CompletableFuture
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.atomic.AtomicReference
import io.netty.channel.ChannelFactory
import io.netty.channel.EventLoopGroup
import io.netty.channel.pool.FixedChannelPool
import io.netty.channel.socket.DatagramChannel
import io.netty.channel.socket.SocketChannel
import net.woggioni.rbcs.api.CacheHandler import net.woggioni.rbcs.api.CacheHandler
import net.woggioni.rbcs.api.CacheHandlerFactory import net.woggioni.rbcs.api.CacheHandlerFactory
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
@@ -1,18 +1,5 @@
package net.woggioni.rbcs.server.memcache package net.woggioni.rbcs.server.memcache
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import io.netty.buffer.CompositeByteBuf
import io.netty.channel.Channel as NettyChannel
import io.netty.channel.ChannelHandlerContext
import io.netty.handler.codec.memcache.DefaultLastMemcacheContent
import io.netty.handler.codec.memcache.DefaultMemcacheContent
import io.netty.handler.codec.memcache.LastMemcacheContent
import io.netty.handler.codec.memcache.MemcacheContent
import io.netty.handler.codec.memcache.binary.BinaryMemcacheOpcodes
import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponse
import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponseStatus
import io.netty.handler.codec.memcache.binary.DefaultBinaryMemcacheRequest
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import java.io.ObjectInputStream import java.io.ObjectInputStream
import java.io.ObjectOutputStream import java.io.ObjectOutputStream
@@ -22,12 +9,27 @@ import java.nio.channels.FileChannel
import java.nio.channels.ReadableByteChannel import java.nio.channels.ReadableByteChannel
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.StandardOpenOption import java.nio.file.StandardOpenOption
import java.net.InetSocketAddress
import java.time.Duration import java.time.Duration
import java.time.Instant import java.time.Instant
import java.util.concurrent.CompletableFuture import java.util.concurrent.CompletableFuture
import java.util.concurrent.atomic.AtomicReference
import java.util.zip.Deflater import java.util.zip.Deflater
import java.util.zip.DeflaterOutputStream import java.util.zip.DeflaterOutputStream
import java.util.zip.InflaterOutputStream import java.util.zip.InflaterOutputStream
import io.netty.channel.Channel as NettyChannel
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import io.netty.buffer.CompositeByteBuf
import io.netty.channel.ChannelHandlerContext
import io.netty.handler.codec.memcache.DefaultLastMemcacheContent
import io.netty.handler.codec.memcache.DefaultMemcacheContent
import io.netty.handler.codec.memcache.LastMemcacheContent
import io.netty.handler.codec.memcache.MemcacheContent
import io.netty.handler.codec.memcache.binary.BinaryMemcacheOpcodes
import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponse
import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponseStatus
import io.netty.handler.codec.memcache.binary.DefaultBinaryMemcacheRequest
import net.woggioni.rbcs.api.CacheHandler import net.woggioni.rbcs.api.CacheHandler
import net.woggioni.rbcs.api.CacheValueMetadata import net.woggioni.rbcs.api.CacheValueMetadata
import net.woggioni.rbcs.api.exception.ContentTooLargeException import net.woggioni.rbcs.api.exception.ContentTooLargeException
@@ -39,8 +41,11 @@ import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse
import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse
import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse
import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent
import net.woggioni.rbcs.api.SpanHandle
import net.woggioni.rbcs.api.TelemetryController
import net.woggioni.rbcs.common.ByteBufInputStream import net.woggioni.rbcs.common.ByteBufInputStream
import net.woggioni.rbcs.common.ByteBufOutputStream import net.woggioni.rbcs.common.ByteBufOutputStream
import net.woggioni.rbcs.common.RBCS.loadService
import net.woggioni.rbcs.common.RBCS.processCacheKey import net.woggioni.rbcs.common.RBCS.processCacheKey
import net.woggioni.rbcs.common.RBCS.toIntOrNull import net.woggioni.rbcs.common.RBCS.toIntOrNull
import net.woggioni.rbcs.common.createLogger import net.woggioni.rbcs.common.createLogger
@@ -70,6 +75,10 @@ class MemcacheCacheHandler(
} }
} }
private val telemetryController by lazy {
loadService(TelemetryController::class.java).firstOrNull()
}
private interface InProgressRequest { private interface InProgressRequest {
} }
@@ -153,7 +162,9 @@ class MemcacheCacheHandler(
metadata: CacheValueMetadata, metadata: CacheValueMetadata,
val digest: ByteBuf, val digest: ByteBuf,
val requestController: CompletableFuture<MemcacheRequestController>, val requestController: CompletableFuture<MemcacheRequestController>,
private val alloc: ByteBufAllocator private val alloc: ByteBufAllocator,
val entryKey: String,
val memcacheSpanRef: AtomicReference<SpanHandle?>,
) : InProgressRequest { ) : InProgressRequest {
private var totalSize = 0 private var totalSize = 0
private var tmpFile: FileChannel? = null private var tmpFile: FileChannel? = null
@@ -251,6 +262,17 @@ class MemcacheCacheHandler(
val key = ctx.alloc().buffer().also { val key = ctx.alloc().buffer().also {
it.writeBytes(processCacheKey(msg.key, keyPrefix, digestAlgorithm)) it.writeBytes(processCacheKey(msg.key, keyPrefix, digestAlgorithm))
} }
val memcacheSpan = telemetryController?.startSpan("GET")?.apply {
setAttribute("db.system", "memcache")
setAttribute("db.operation.name", "GET")
val remoteAddr = ctx.channel().remoteAddress()
if (remoteAddr is InetSocketAddress) {
remoteAddr.hostString?.let {
setAttribute("server.address", it)
}
setAttribute("server.port", remoteAddr.port.toLong())
}
}
val responseHandler = object : MemcacheResponseHandler { val responseHandler = object : MemcacheResponseHandler {
override fun responseReceived(response: BinaryMemcacheResponse) { override fun responseReceived(response: BinaryMemcacheResponse) {
val status = response.status() val status = response.status()
@@ -266,8 +288,15 @@ class MemcacheCacheHandler(
log.debug(ctx) { log.debug(ctx) {
"Cache miss for key ${msg.key} on memcache" "Cache miss for key ${msg.key} on memcache"
} }
telemetryController?.endSpan(memcacheSpan)
sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key)) sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key))
} }
else -> {
val ex = MemcacheException(status)
telemetryController?.endSpan(memcacheSpan, ex)
this@MemcacheCacheHandler.exceptionCaught(ctx, ex)
}
} }
} }
@@ -282,11 +311,13 @@ class MemcacheCacheHandler(
if (content is LastMemcacheContent) { if (content is LastMemcacheContent) {
inProgressRequest = null inProgressRequest = null
inProgressGetRequest.commit() inProgressGetRequest.commit()
telemetryController?.endSpan(memcacheSpan)
} }
} }
} }
override fun exceptionCaught(ex: Throwable) { override fun exceptionCaught(ex: Throwable) {
telemetryController?.endSpan(memcacheSpan, ex)
(inProgressRequest as? InProgressGetRequest).let { inProgressGetRequest -> (inProgressRequest as? InProgressGetRequest).let { inProgressGetRequest ->
inProgressGetRequest?.let { inProgressGetRequest?.let {
inProgressRequest = null inProgressRequest = null
@@ -312,6 +343,7 @@ class MemcacheCacheHandler(
val key = ctx.alloc().buffer().also { val key = ctx.alloc().buffer().also {
it.writeBytes(processCacheKey(msg.key, keyPrefix, digestAlgorithm)) it.writeBytes(processCacheKey(msg.key, keyPrefix, digestAlgorithm))
} }
val memcacheSpanRef = AtomicReference<SpanHandle?>(null)
val responseHandler = object : MemcacheResponseHandler { val responseHandler = object : MemcacheResponseHandler {
override fun responseReceived(response: BinaryMemcacheResponse) { override fun responseReceived(response: BinaryMemcacheResponse) {
val status = response.status() val status = response.status()
@@ -320,16 +352,22 @@ class MemcacheCacheHandler(
log.debug(ctx) { log.debug(ctx) {
"Inserted key ${msg.key} into memcache" "Inserted key ${msg.key} into memcache"
} }
telemetryController?.endSpan(memcacheSpanRef.get())
sendMessageAndFlush(ctx, CachePutResponse(msg.key)) sendMessageAndFlush(ctx, CachePutResponse(msg.key))
} }
else -> this@MemcacheCacheHandler.exceptionCaught(ctx, MemcacheException(status)) else -> {
val ex = MemcacheException(status)
telemetryController?.endSpan(memcacheSpanRef.get(), ex)
this@MemcacheCacheHandler.exceptionCaught(ctx, ex)
}
} }
} }
override fun contentReceived(content: MemcacheContent) {} override fun contentReceived(content: MemcacheContent) {}
override fun exceptionCaught(ex: Throwable) { override fun exceptionCaught(ex: Throwable) {
telemetryController?.endSpan(memcacheSpanRef.get(), ex)
this@MemcacheCacheHandler.exceptionCaught(ctx, ex) this@MemcacheCacheHandler.exceptionCaught(ctx, ex)
} }
} }
@@ -339,7 +377,7 @@ class MemcacheCacheHandler(
this@MemcacheCacheHandler.exceptionCaught(ctx, ex) this@MemcacheCacheHandler.exceptionCaught(ctx, ex)
} }
} }
inProgressRequest = InProgressPutRequest(ctx.channel(), msg.metadata, key, requestController, ctx.alloc()) inProgressRequest = InProgressPutRequest(ctx.channel(), msg.metadata, key, requestController, ctx.alloc(), msg.key, memcacheSpanRef)
} }
private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) { private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) {
@@ -362,22 +400,41 @@ class MemcacheCacheHandler(
val request = inProgressRequest val request = inProgressRequest
when (request) { when (request) {
is InProgressPutRequest -> { is InProgressPutRequest -> {
val putRequest = request
inProgressRequest = null inProgressRequest = null
log.trace(ctx) { log.trace(ctx) {
"Received last chunk of ${msg.content().readableBytes()} bytes for memcache" "Received last chunk of ${msg.content().readableBytes()} bytes for memcache"
} }
request.write(msg.content()) putRequest.write(msg.content())
val key = request.digest.retainedDuplicate() val memcacheSpan = telemetryController?.startSpan("SET",
val (payloadSize, payloadSource) = request.commit() )?.apply {
setAttribute("db.system", "memcache")
setAttribute("db.operation.name", "SET")
val remoteAddr = ctx.channel().remoteAddress()
if (remoteAddr is InetSocketAddress) {
remoteAddr.hostString?.let {
setAttribute("server.address", it)
}
setAttribute("server.port", remoteAddr.port.toLong())
}
}
putRequest.memcacheSpanRef.set(memcacheSpan)
val key = putRequest.digest.retainedDuplicate()
val (payloadSize, payloadSource) = putRequest.commit()
val extras = ctx.alloc().buffer(8, 8) val extras = ctx.alloc().buffer(8, 8)
extras.writeInt(0) extras.writeInt(0)
extras.writeInt(encodeExpiry(maxAge)) extras.writeInt(encodeExpiry(maxAge))
val totalBodyLength = request.digest.readableBytes() + extras.readableBytes() + payloadSize val totalBodyLength = putRequest.digest.readableBytes() + extras.readableBytes() + payloadSize
log.trace(ctx) { log.trace(ctx) {
"Trying to send SET request to memcache" "Trying to send SET request to memcache"
} }
request.requestController.whenComplete { requestController, ex -> putRequest.requestController.whenComplete { requestController, ex ->
if (ex == null) { if (ex == null) {
val remoteAddr = requestController.channel.remoteAddress()
if (remoteAddr is InetSocketAddress) {
remoteAddr.hostString?.let { memcacheSpan?.setAttribute("server.address", it) }
memcacheSpan?.setAttribute("server.port", remoteAddr.port.toLong())
}
log.trace(ctx) { log.trace(ctx) {
"Sending SET request to memcache" "Sending SET request to memcache"
} }
@@ -1,6 +1,11 @@
package net.woggioni.rbcs.server.memcache.client package net.woggioni.rbcs.server.memcache.client
import java.io.IOException
import java.net.InetSocketAddress
import java.util.concurrent.CompletableFuture
import java.util.concurrent.ConcurrentHashMap
import io.netty.util.concurrent.Future as NettyFuture
import io.netty.bootstrap.Bootstrap import io.netty.bootstrap.Bootstrap
import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBuf
import io.netty.channel.Channel import io.netty.channel.Channel
@@ -20,12 +25,7 @@ import io.netty.handler.codec.memcache.MemcacheObject
import io.netty.handler.codec.memcache.binary.BinaryMemcacheClientCodec import io.netty.handler.codec.memcache.binary.BinaryMemcacheClientCodec
import io.netty.handler.codec.memcache.binary.BinaryMemcacheRequest import io.netty.handler.codec.memcache.binary.BinaryMemcacheRequest
import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponse import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponse
import io.netty.util.concurrent.Future as NettyFuture
import io.netty.util.concurrent.GenericFutureListener import io.netty.util.concurrent.GenericFutureListener
import java.io.IOException
import java.net.InetSocketAddress
import java.util.concurrent.CompletableFuture
import java.util.concurrent.ConcurrentHashMap
import net.woggioni.rbcs.common.HostAndPort import net.woggioni.rbcs.common.HostAndPort
import net.woggioni.rbcs.common.createLogger import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.trace import net.woggioni.rbcs.common.trace
@@ -147,6 +147,8 @@ class MemcacheClient(
channel.pipeline().addLast(handler) channel.pipeline().addLast(handler)
response.complete(object : MemcacheRequestController { response.complete(object : MemcacheRequestController {
override val channel: Channel = channel
private var channelReleased = false private var channelReleased = false
override fun sendRequest(request: BinaryMemcacheRequest) { override fun sendRequest(request: BinaryMemcacheRequest) {
@@ -1,10 +1,13 @@
package net.woggioni.rbcs.server.memcache.client package net.woggioni.rbcs.server.memcache.client
import io.netty.channel.Channel
import io.netty.handler.codec.memcache.MemcacheContent import io.netty.handler.codec.memcache.MemcacheContent
import io.netty.handler.codec.memcache.binary.BinaryMemcacheRequest import io.netty.handler.codec.memcache.binary.BinaryMemcacheRequest
interface MemcacheRequestController { interface MemcacheRequestController {
val channel: Channel
fun sendRequest(request : BinaryMemcacheRequest) fun sendRequest(request : BinaryMemcacheRequest)
fun sendContent(content : MemcacheContent) fun sendContent(content : MemcacheContent)
@@ -1,11 +1,11 @@
package net.woggioni.rbcs.server.memcache.client package net.woggioni.rbcs.server.memcache.client
import io.netty.buffer.ByteBufUtil
import io.netty.buffer.Unpooled
import java.io.ByteArrayInputStream import java.io.ByteArrayInputStream
import java.nio.ByteBuffer import java.nio.ByteBuffer
import java.nio.channels.Channels import java.nio.channels.Channels
import kotlin.random.Random import kotlin.random.Random
import io.netty.buffer.ByteBufUtil
import io.netty.buffer.Unpooled
import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
+81
View File
@@ -0,0 +1,81 @@
plugins {
id 'java-library'
id 'maven-publish'
alias catalog.plugins.kotlin.jvm
}
configurations {
bundle {
canBeResolved = false
canBeConsumed = false
transitive = true
}
filteredBundle {
canBeResolved = true
canBeConsumed = false
transitive = true
extendsFrom bundle
resolutionStrategy {
dependencies {
exclude group: 'org.slf4j', module: 'slf4j-api'
exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib'
exclude group: 'org.jetbrains', module: 'annotations'
}
}
}
release {
transitive = false
canBeConsumed = true
canBeResolved = true
}
compileOnly {
extendsFrom bundle
}
}
dependencies {
compileOnly project(':rbcs-common')
compileOnly project(':rbcs-api')
compileOnly catalog.netty.transport
compileOnly catalog.slf4j.api
compileOnly catalog.kotlin.stdlib.jdk8
compileOnly catalog.logback.core
compileOnly catalog.logback.classic
bundle catalog.opentelemetry.netty.'4'.'1'
bundle catalog.opentelemetry.sdk.extension.autoconfigure
bundle catalog.opentelemetry.logback.appender.'1'.'0'
bundle catalog.opentelemetry.logback.mdc.'1'.'0'
bundle catalog.opentelemetry.extension.trace.propagators
bundle catalog.opentelemetry.exporter.otlp
bundle catalog.opentelemetry.runtime.telemetry
}
Provider<Tar> bundleTask = tasks.register("bundle", Tar) {
from(tasks.named(JavaPlugin.JAR_TASK_NAME))
from(configurations.filteredBundle)
group = BasePlugin.BUILD_GROUP
}
tasks.named(BasePlugin.ASSEMBLE_TASK_NAME) {
dependsOn(bundleTask)
}
artifacts {
release(bundleTask)
}
publishing {
publications {
maven(MavenPublication) {
artifact bundleTask
}
}
}
@@ -0,0 +1,20 @@
module net.woggioni.rbcs.server.otel {
requires net.woggioni.rbcs.common;
requires kotlin.stdlib;
requires io.netty.transport;
requires io.netty.common;
requires io.netty.buffer;
requires org.slf4j;
requires ch.qos.logback.core;
requires ch.qos.logback.classic;
requires io.opentelemetry.api;
requires io.opentelemetry.sdk.autoconfigure;
requires io.opentelemetry.instrumentation.runtime_telemetry;
requires io.opentelemetry.instrumentation.netty_4_1;
requires io.opentelemetry.instrumentation.logback_appender_1_0;
requires io.opentelemetry.extension.trace.propagation;
requires net.woggioni.rbcs.api;
provides net.woggioni.rbcs.api.TelemetryController with net.woggioni.rbcs.server.otel.OtelController;
}
@@ -0,0 +1,65 @@
package net.woggioni.rbcs.server.otel
import io.netty.channel.ChannelHandler
import io.opentelemetry.api.GlobalOpenTelemetry
import io.opentelemetry.api.common.AttributeKey
import io.opentelemetry.api.trace.SpanKind
import io.opentelemetry.api.trace.StatusCode
import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender
import io.opentelemetry.instrumentation.netty.v4_1.NettyServerTelemetry
import io.opentelemetry.instrumentation.runtimetelemetry.RuntimeTelemetry
import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk
import net.woggioni.rbcs.api.SpanHandle
import net.woggioni.rbcs.api.TelemetryController
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.info
class OtelController : TelemetryController {
private val log = createLogger<OtelController>()
private val tracer by lazy {
GlobalOpenTelemetry.getTracer("net.woggioni.rbcs.server.redis", "0.5.0")
}
override fun initialize() {
log.info { "Initializing OpenTelemetry SDK with auto-configuration" }
val sdk = AutoConfiguredOpenTelemetrySdk.builder()
.setResultAsGlobal()
.build()
.openTelemetrySdk
RuntimeTelemetry.create(sdk)
runCatching {
OpenTelemetryAppender.install(sdk)
log.info { "OpenTelemetry logback appender installed" }
}.onFailure { ex ->
val msg = ex.localizedMessage ?: ex.javaClass.name
log.info { "Failed to install OpenTelemetry logback appender: $msg" }
}
log.info { "OpenTelemetry SDK initialized successfully" }
}
override fun createHandler(): ChannelHandler {
return NettyServerTelemetry.create(GlobalOpenTelemetry.get()).createCombinedHandler()
}
override fun startSpan(name: String): SpanHandle {
val span = tracer.spanBuilder(name)
.setSpanKind(SpanKind.CLIENT)
.startSpan()
return OtelSpanHandle(span)
}
override fun endSpan(span: SpanHandle?) {
(span as? OtelSpanHandle)?.delegate?.end()
}
override fun endSpan(span: SpanHandle?, error: Throwable) {
val s = (span as? OtelSpanHandle)?.delegate ?: return
s.recordException(error)
s.setStatus(StatusCode.ERROR)
s.end()
}
}
@@ -0,0 +1,21 @@
package net.woggioni.rbcs.server.otel
import io.opentelemetry.api.trace.Span
import net.woggioni.rbcs.api.SpanHandle
internal class OtelSpanHandle(
val delegate: Span,
) : SpanHandle {
override fun setAttribute(key: String, value: String) {
delegate.setAttribute(key, value)
}
override fun setAttribute(key: String, value: Long) {
delegate.setAttribute(key, value)
}
override fun setAttribute(key: String, value: Boolean) {
delegate.setAttribute(key, value)
}
}
+69
View File
@@ -0,0 +1,69 @@
plugins {
id 'java-library'
id 'maven-publish'
alias catalog.plugins.kotlin.jvm
}
configurations {
bundle {
canBeResolved = true
canBeConsumed = false
visible = false
transitive = false
resolutionStrategy {
dependencies {
exclude group: 'org.slf4j', module: 'slf4j-api'
exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib'
exclude group: 'org.jetbrains', module: 'annotations'
}
}
}
release {
transitive = false
canBeConsumed = true
canBeResolved = true
visible = true
}
}
dependencies {
implementation project(':rbcs-common')
implementation project(':rbcs-api')
implementation catalog.jwo
implementation catalog.slf4j.api
implementation catalog.netty.common
implementation catalog.netty.handler
implementation catalog.netty.codec.redis
bundle catalog.netty.codec.redis
testRuntimeOnly catalog.logback.classic
}
tasks.named(JavaPlugin.TEST_TASK_NAME, Test) {
systemProperty("io.netty.leakDetectionLevel", "PARANOID")
}
Provider<Tar> bundleTask = tasks.register("bundle", Tar) {
from(tasks.named(JavaPlugin.JAR_TASK_NAME))
from(configurations.bundle)
group = BasePlugin.BUILD_GROUP
}
tasks.named(BasePlugin.ASSEMBLE_TASK_NAME) {
dependsOn(bundleTask)
}
artifacts {
release(bundleTask)
}
publishing {
publications {
maven(MavenPublication) {
artifact bundleTask
}
}
}
@@ -0,0 +1,23 @@
import net.woggioni.rbcs.api.CacheProvider;
import net.woggioni.rbcs.api.TelemetryController;
module net.woggioni.rbcs.server.redis {
requires net.woggioni.rbcs.common;
requires net.woggioni.rbcs.api;
requires net.woggioni.jwo;
requires java.xml;
requires kotlin.stdlib;
requires io.netty.transport;
requires io.netty.codec;
requires io.netty.codec.redis;
requires io.netty.common;
requires io.netty.buffer;
requires io.netty.handler;
requires org.slf4j;
provides CacheProvider with net.woggioni.rbcs.server.redis.RedisCacheProvider;
uses TelemetryController;
opens net.woggioni.rbcs.server.redis.schema;
}
@@ -0,0 +1,4 @@
package net.woggioni.rbcs.server.redis
class RedisException(msg: String, cause: Throwable? = null)
: RuntimeException(msg, cause)
@@ -0,0 +1,105 @@
package net.woggioni.rbcs.server.redis
import java.time.Duration
import java.util.concurrent.CompletableFuture
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicReference
import io.netty.channel.ChannelFactory
import io.netty.channel.EventLoopGroup
import io.netty.channel.pool.FixedChannelPool
import io.netty.channel.socket.DatagramChannel
import io.netty.channel.socket.SocketChannel
import net.woggioni.rbcs.api.CacheHandler
import net.woggioni.rbcs.api.CacheHandlerFactory
import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.common.HostAndPort
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.server.redis.client.RedisClient
data class RedisCacheConfiguration(
val servers: List<Server>,
val maxAge: Duration = Duration.ofDays(1),
val keyPrefix: String? = null,
val digestAlgorithm: String? = null,
val compressionMode: CompressionMode? = null,
val compressionLevel: Int,
) : Configuration.Cache {
companion object {
private val log = createLogger<RedisCacheConfiguration>()
}
enum class CompressionMode {
/**
* Deflate mode
*/
DEFLATE
}
data class Server(
val endpoint: HostAndPort,
val connectionTimeoutMillis: Int?,
val maxConnections: Int,
val password: String? = null,
)
override fun materialize() = object : CacheHandlerFactory {
private val connectionPoolMap = ConcurrentHashMap<HostAndPort, FixedChannelPool>()
override fun newHandler(
cfg: Configuration,
eventLoop: EventLoopGroup,
socketChannelFactory: ChannelFactory<SocketChannel>,
datagramChannelFactory: ChannelFactory<DatagramChannel>,
): CacheHandler {
return RedisCacheHandler(
RedisClient(
this@RedisCacheConfiguration.servers,
cfg.connection.chunkSize,
eventLoop,
socketChannelFactory,
connectionPoolMap
),
keyPrefix,
digestAlgorithm,
compressionMode != null,
compressionLevel,
cfg.connection.chunkSize,
maxAge
)
}
override fun asyncClose() = object : CompletableFuture<Void>() {
init {
val failure = AtomicReference<Throwable>(null)
val pools = connectionPoolMap.values.toList()
val npools = pools.size
val finished = AtomicInteger(0)
if (pools.isEmpty()) {
complete(null)
} else {
pools.forEach { pool ->
pool.closeAsync().addListener {
if (!it.isSuccess) {
failure.compareAndSet(null, it.cause())
}
if (finished.incrementAndGet() == npools) {
when (val ex = failure.get()) {
null -> complete(null)
else -> completeExceptionally(ex)
}
}
}
}
}
}
}
}
override fun getNamespaceURI() = "urn:net.woggioni.rbcs.server.redis"
override fun getTypeName() = "redisCacheType"
}
@@ -0,0 +1,482 @@
package net.woggioni.rbcs.server.redis
import java.io.ByteArrayOutputStream
import java.io.ObjectInputStream
import java.io.ObjectOutputStream
import java.net.InetSocketAddress
import java.nio.ByteBuffer
import java.nio.channels.Channels
import java.nio.channels.FileChannel
import java.nio.channels.ReadableByteChannel
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.StandardOpenOption
import java.time.Duration
import java.util.zip.Deflater
import java.util.zip.DeflaterOutputStream
import java.util.zip.InflaterOutputStream
import io.netty.channel.Channel as NettyChannel
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import io.netty.buffer.CompositeByteBuf
import io.netty.channel.ChannelHandlerContext
import io.netty.handler.codec.redis.ArrayRedisMessage
import io.netty.handler.codec.redis.ErrorRedisMessage
import io.netty.handler.codec.redis.FullBulkStringRedisMessage
import io.netty.handler.codec.redis.RedisMessage
import io.netty.handler.codec.redis.SimpleStringRedisMessage
import net.woggioni.rbcs.api.CacheHandler
import net.woggioni.rbcs.api.CacheValueMetadata
import net.woggioni.rbcs.api.exception.ContentTooLargeException
import net.woggioni.rbcs.api.message.CacheMessage
import net.woggioni.rbcs.api.message.CacheMessage.CacheContent
import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest
import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest
import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse
import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse
import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse
import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent
import net.woggioni.rbcs.api.TelemetryController
import net.woggioni.rbcs.common.ByteBufInputStream
import net.woggioni.rbcs.common.ByteBufOutputStream
import net.woggioni.rbcs.common.RBCS.loadService
import net.woggioni.rbcs.common.RBCS.processCacheKey
import net.woggioni.rbcs.common.RBCS.toIntOrNull
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.debug
import net.woggioni.rbcs.common.extractChunk
import net.woggioni.rbcs.common.trace
import net.woggioni.rbcs.common.warn
import net.woggioni.rbcs.server.redis.client.RedisClient
import net.woggioni.rbcs.server.redis.client.RedisResponseHandler
class RedisCacheHandler(
private val client: RedisClient,
private val keyPrefix: String?,
private val digestAlgorithm: String?,
private val compressionEnabled: Boolean,
private val compressionLevel: Int,
private val chunkSize: Int,
private val maxAge: Duration,
) : CacheHandler() {
companion object {
private val log = createLogger<RedisCacheHandler>()
}
private val telemetryController by lazy {
loadService(TelemetryController::class.java).firstOrNull()
}
private interface InProgressRequest
private inner class InProgressGetRequest(
val key: String,
private val ctx: ChannelHandlerContext,
) : InProgressRequest {
private val chunk = ctx.alloc().compositeBuffer()
private val outputStream = ByteBufOutputStream(chunk).let {
if (compressionEnabled) {
InflaterOutputStream(it)
} else {
it
}
}
fun processResponse(data: ByteBuf) {
if (data.readableBytes() < Int.SIZE_BYTES) {
log.debug(ctx) {
"Received empty or corrupt data from Redis for key $key"
}
sendMessageAndFlush(ctx, CacheValueNotFoundResponse(key))
data.release()
return
}
val metadataSize = data.readInt()
if (data.readableBytes() < metadataSize) {
log.debug(ctx) {
"Received incomplete metadata from Redis for key $key"
}
sendMessageAndFlush(ctx, CacheValueNotFoundResponse(key))
data.release()
return
}
val metadata = ObjectInputStream(ByteBufInputStream(data)).use {
data.retain()
it.readObject() as CacheValueMetadata
}
data.readerIndex(Int.SIZE_BYTES + metadataSize)
log.trace(ctx) {
"Sending response from cache"
}
sendMessageAndFlush(ctx, CacheValueFoundResponse(key, metadata))
// Decompress and stream the remaining payload
data.readBytes(outputStream, data.readableBytes())
data.release()
commit()
}
private fun flush(last: Boolean) {
val toSend = extractChunk(chunk, ctx.alloc())
val msg = if (last) {
log.trace(ctx) {
"Sending last chunk to client"
}
LastCacheContent(toSend)
} else {
log.trace(ctx) {
"Sending chunk to client"
}
CacheContent(toSend)
}
sendMessageAndFlush(ctx, msg)
}
fun commit() {
chunk.retain()
outputStream.close()
flush(true)
chunk.release()
}
fun rollback() {
outputStream.close()
}
}
private inner class InProgressPutRequest(
private val ch: NettyChannel,
metadata: CacheValueMetadata,
val keyString: String,
val keyBytes: ByteBuf,
private val alloc: ByteBufAllocator,
) : InProgressRequest {
private var totalSize = 0
private var tmpFile: FileChannel? = null
private val accumulator = alloc.compositeBuffer()
private val stream = ByteBufOutputStream(accumulator).let {
if (compressionEnabled) {
DeflaterOutputStream(it, Deflater(compressionLevel))
} else {
it
}
}
init {
ByteArrayOutputStream().let { baos ->
ObjectOutputStream(baos).use {
it.writeObject(metadata)
}
val serializedBytes = baos.toByteArray()
accumulator.writeInt(serializedBytes.size)
accumulator.writeBytes(serializedBytes)
}
}
fun write(buf: ByteBuf) {
totalSize += buf.readableBytes()
buf.readBytes(stream, buf.readableBytes())
tmpFile?.let {
flushToDisk(it, accumulator)
}
if (accumulator.readableBytes() > 0x100000) {
log.debug(ch) {
"Entry is too big, buffering it into a file"
}
val opts = arrayOf(
StandardOpenOption.DELETE_ON_CLOSE,
StandardOpenOption.READ,
StandardOpenOption.WRITE,
StandardOpenOption.TRUNCATE_EXISTING
)
FileChannel.open(Files.createTempFile("rbcs-server-redis", ".tmp"), *opts).let { fc ->
tmpFile = fc
flushToDisk(fc, accumulator)
}
}
}
private fun flushToDisk(fc: FileChannel, buf: CompositeByteBuf) {
val chunk = extractChunk(buf, alloc)
fc.write(chunk.nioBuffer())
chunk.release()
}
fun commit(): Pair<Int, ReadableByteChannel> {
keyBytes.release()
accumulator.retain()
stream.close()
val fileChannel = tmpFile
return if (fileChannel != null) {
flushToDisk(fileChannel, accumulator)
accumulator.release()
fileChannel.position(0)
val fileSize = fileChannel.size().toIntOrNull() ?: let {
fileChannel.close()
throw ContentTooLargeException("Request body is too large", null)
}
fileSize to fileChannel
} else {
accumulator.readableBytes() to Channels.newChannel(ByteBufInputStream(accumulator))
}
}
fun rollback() {
stream.close()
keyBytes.release()
tmpFile?.close()
}
}
private var inProgressRequest: InProgressRequest? = null
override fun channelRead0(ctx: ChannelHandlerContext, msg: CacheMessage) {
when (msg) {
is CacheGetRequest -> handleGetRequest(ctx, msg)
is CachePutRequest -> handlePutRequest(ctx, msg)
is LastCacheContent -> handleLastCacheContent(ctx, msg)
is CacheContent -> handleCacheContent(ctx, msg)
else -> ctx.fireChannelRead(msg)
}
}
private fun handleGetRequest(ctx: ChannelHandlerContext, msg: CacheGetRequest) {
log.debug(ctx) {
"Fetching ${msg.key} from Redis"
}
val keyBytes = processCacheKey(msg.key, keyPrefix, digestAlgorithm)
val keyString = String(keyBytes, StandardCharsets.UTF_8)
val redisSpan = telemetryController?.startSpan("GET")?.apply {
setAttribute("db.system", "redis")
setAttribute("db.operation.name", "GET")
val remoteAddr = ctx.channel().remoteAddress()
if (remoteAddr is InetSocketAddress) {
remoteAddr.hostString?.let {
setAttribute("server.address", it)
}
setAttribute("server.port", remoteAddr.port.toLong())
}
}
val responseHandler = object : RedisResponseHandler {
override fun responseReceived(response: RedisMessage) {
when (response) {
is FullBulkStringRedisMessage -> {
if (response === FullBulkStringRedisMessage.NULL_INSTANCE || response.content().readableBytes() == 0) {
log.debug(ctx) {
"Cache miss for key ${msg.key} on Redis"
}
telemetryController?.endSpan(redisSpan)
sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key))
} else {
log.debug(ctx) {
"Cache hit for key ${msg.key} on Redis"
}
telemetryController?.endSpan(redisSpan)
val getRequest = InProgressGetRequest(msg.key, ctx)
inProgressRequest = getRequest
getRequest.processResponse(response.content())
inProgressRequest = null
}
}
is ErrorRedisMessage -> {
val ex = RedisException("Redis error for GET ${msg.key}: ${response.content()}")
telemetryController?.endSpan(redisSpan, ex)
this@RedisCacheHandler.exceptionCaught(ctx, ex)
}
else -> {
log.warn(ctx) {
"Unexpected response type from Redis for key ${msg.key}: ${response.javaClass.name}"
}
telemetryController?.endSpan(redisSpan)
sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key))
}
}
}
override fun exceptionCaught(ex: Throwable) {
telemetryController?.endSpan(redisSpan, ex)
this@RedisCacheHandler.exceptionCaught(ctx, ex)
}
}
client.sendCommand(keyBytes, ctx.alloc(), responseHandler).thenAccept { channel ->
val remoteAddr = channel.remoteAddress()
if (remoteAddr is InetSocketAddress) {
remoteAddr.hostString?.let { redisSpan?.setAttribute("server.address", it) }
redisSpan?.setAttribute("server.port", remoteAddr.port.toLong())
}
log.trace(ctx) {
"Sending GET request for key ${msg.key} to Redis"
}
val cmd = buildRedisCommand(ctx.alloc(), "GET", keyString)
channel.writeAndFlush(cmd)
}
}
private fun handlePutRequest(ctx: ChannelHandlerContext, msg: CachePutRequest) {
val keyBytes = processCacheKey(msg.key, keyPrefix, digestAlgorithm)
val keyBuf = ctx.alloc().buffer().also {
it.writeBytes(keyBytes)
}
inProgressRequest = InProgressPutRequest(ctx.channel(), msg.metadata, msg.key, keyBuf, ctx.alloc())
}
private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) {
val request = inProgressRequest
when (request) {
is InProgressPutRequest -> {
log.trace(ctx) {
"Received chunk of ${msg.content().readableBytes()} bytes for Redis"
}
request.write(msg.content())
}
is InProgressGetRequest -> {
msg.release()
}
}
}
private fun handleLastCacheContent(ctx: ChannelHandlerContext, msg: LastCacheContent) {
val request = inProgressRequest
when (request) {
is InProgressPutRequest -> {
inProgressRequest = null
log.trace(ctx) {
"Received last chunk of ${msg.content().readableBytes()} bytes for Redis"
}
request.write(msg.content())
val keyBytes = processCacheKey(request.keyString, keyPrefix, digestAlgorithm)
val keyString = String(keyBytes, StandardCharsets.UTF_8)
val (payloadSize, payloadSource) = request.commit()
// Read the entire payload into a single ByteBuf for the SET command
val valueBuf = ctx.alloc().buffer(payloadSize)
payloadSource.use { source ->
val bb = ByteBuffer.allocate(chunkSize)
while (true) {
val read = source.read(bb)
if (read < 0) break
bb.flip()
valueBuf.writeBytes(bb)
bb.clear()
}
}
val expirySeconds = maxAge.toSeconds().toString()
val redisSpan = telemetryController?.startSpan("SET")?.apply {
setAttribute("db.system", "redis")
setAttribute("db.operation.name", "SET")
val remoteAddr = ctx.channel().remoteAddress()
if (remoteAddr is InetSocketAddress) {
remoteAddr.hostString?.let {
setAttribute("server.address", it)
}
setAttribute("server.port", remoteAddr.port.toLong())
}
}
val responseHandler = object : RedisResponseHandler {
override fun responseReceived(response: RedisMessage) {
when (response) {
is SimpleStringRedisMessage -> {
log.debug(ctx) {
"Inserted key ${request.keyString} into Redis"
}
telemetryController?.endSpan(redisSpan)
sendMessageAndFlush(ctx, CachePutResponse(request.keyString))
}
is ErrorRedisMessage -> {
val ex = RedisException("Redis error for SET ${request.keyString}: ${response.content()}")
telemetryController?.endSpan(redisSpan, ex)
this@RedisCacheHandler.exceptionCaught(ctx, ex)
}
else -> {
val ex = RedisException("Unexpected response for SET ${request.keyString}: ${response.javaClass.name}")
telemetryController?.endSpan(redisSpan, ex)
this@RedisCacheHandler.exceptionCaught(ctx, ex)
}
}
}
override fun exceptionCaught(ex: Throwable) {
telemetryController?.endSpan(redisSpan, ex)
this@RedisCacheHandler.exceptionCaught(ctx, ex)
}
}
// Use a ByteBuf key for server selection
client.sendCommand(keyBytes, ctx.alloc(), responseHandler).thenAccept { channel ->
val remoteAddr = channel.remoteAddress()
if (remoteAddr is InetSocketAddress) {
remoteAddr.hostString?.let { redisSpan?.setAttribute("server.address", it) }
redisSpan?.setAttribute("server.port", remoteAddr.port.toLong())
}
log.trace(ctx) {
"Sending SET request to Redis"
}
// Build SET key value EX seconds
val cmd = buildRedisSetCommand(ctx.alloc(), keyString, valueBuf, expirySeconds)
channel.writeAndFlush(cmd)
}.whenComplete { _, ex ->
if (ex != null) {
valueBuf.release()
this@RedisCacheHandler.exceptionCaught(ctx, ex)
}
}
}
}
}
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
val request = inProgressRequest
when (request) {
is InProgressPutRequest -> {
inProgressRequest = null
request.rollback()
}
is InProgressGetRequest -> {
inProgressRequest = null
request.rollback()
}
}
super.exceptionCaught(ctx, cause)
}
private fun buildRedisCommand(alloc: ByteBufAllocator, vararg args: String): ArrayRedisMessage {
val children = args.map { arg ->
FullBulkStringRedisMessage(
alloc.buffer(arg.toByteArray(StandardCharsets.UTF_8))
)
}
return ArrayRedisMessage(children)
}
private fun ByteBufAllocator.buffer(bytes : ByteArray) = buffer().apply {
writeBytes(bytes)
}
private fun buildRedisSetCommand(
alloc: ByteBufAllocator,
key: String,
value: ByteBuf,
expirySeconds: String,
): ArrayRedisMessage {
val children = listOf(
FullBulkStringRedisMessage(alloc.buffer("SET".toByteArray(StandardCharsets.UTF_8))),
FullBulkStringRedisMessage(alloc.buffer(key.toByteArray(StandardCharsets.UTF_8))),
FullBulkStringRedisMessage(value),
FullBulkStringRedisMessage(alloc.buffer("EX".toByteArray(StandardCharsets.UTF_8))),
FullBulkStringRedisMessage(alloc.buffer(expirySeconds.toByteArray(StandardCharsets.UTF_8))),
)
return ArrayRedisMessage(children)
}
}
@@ -0,0 +1,106 @@
package net.woggioni.rbcs.server.redis
import java.time.Duration
import java.time.temporal.ChronoUnit
import net.woggioni.rbcs.api.CacheProvider
import net.woggioni.rbcs.api.exception.ConfigurationException
import net.woggioni.rbcs.common.HostAndPort
import net.woggioni.rbcs.common.RBCS
import net.woggioni.rbcs.common.Xml
import net.woggioni.rbcs.common.Xml.Companion.asIterable
import net.woggioni.rbcs.common.Xml.Companion.renderAttribute
import org.w3c.dom.Document
import org.w3c.dom.Element
class RedisCacheProvider : CacheProvider<RedisCacheConfiguration> {
override fun getXmlSchemaLocation() = "jpms://net.woggioni.rbcs.server.redis/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd"
override fun getXmlType() = "redisCacheType"
override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server.redis"
val xmlNamespacePrefix: String
get() = "rbcs-redis"
override fun deserialize(el: Element): RedisCacheConfiguration {
val servers = mutableListOf<RedisCacheConfiguration.Server>()
val maxAge = el.renderAttribute("max-age")
?.let(Duration::parse)
?: Duration.ofDays(1)
val compressionLevel = el.renderAttribute("compression-level")
?.let(Integer::decode)
?: -1
val compressionMode = el.renderAttribute("compression-mode")
?.let {
when (it) {
"deflate" -> RedisCacheConfiguration.CompressionMode.DEFLATE
else -> RedisCacheConfiguration.CompressionMode.DEFLATE
}
}
val keyPrefix = el.renderAttribute("key-prefix")
val digestAlgorithm = el.renderAttribute("digest")
for (child in el.asIterable()) {
when (child.nodeName) {
"server" -> {
val host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required")
val port = child.renderAttribute("port")?.toInt() ?: throw ConfigurationException("port attribute is required")
val maxConnections = child.renderAttribute("max-connections")?.toInt() ?: 1
val connectionTimeout = child.renderAttribute("connection-timeout")
?.let(Duration::parse)
?.let(Duration::toMillis)
?.let(Long::toInt)
?: 10000
val password = child.renderAttribute("password")
servers.add(RedisCacheConfiguration.Server(HostAndPort(host, port), connectionTimeout, maxConnections, password))
}
}
}
return RedisCacheConfiguration(
servers,
maxAge,
keyPrefix,
digestAlgorithm,
compressionMode,
compressionLevel
)
}
override fun serialize(doc: Document, cache: RedisCacheConfiguration) = cache.run {
val result = doc.createElement("cache")
Xml.of(doc, result) {
attr("xmlns:${xmlNamespacePrefix}", xmlNamespace, namespaceURI = "http://www.w3.org/2000/xmlns/")
attr("xs:type", "${xmlNamespacePrefix}:$xmlType", RBCS.XML_SCHEMA_NAMESPACE_URI)
for (server in servers) {
node("server") {
attr("host", server.endpoint.host)
attr("port", server.endpoint.port.toString())
server.connectionTimeoutMillis?.let { connectionTimeoutMillis ->
attr("connection-timeout", Duration.of(connectionTimeoutMillis.toLong(), ChronoUnit.MILLIS).toString())
}
attr("max-connections", server.maxConnections.toString())
server.password?.let { password ->
attr("password", password)
}
}
}
attr("max-age", maxAge.toString())
keyPrefix?.let {
attr("key-prefix", it)
}
digestAlgorithm?.let { digestAlgorithm ->
attr("digest", digestAlgorithm)
}
compressionMode?.let { compressionMode ->
attr(
"compression-mode", when (compressionMode) {
RedisCacheConfiguration.CompressionMode.DEFLATE -> "deflate"
}
)
}
attr("compression-level", compressionLevel.toString())
}
result
}
}
@@ -0,0 +1,202 @@
package net.woggioni.rbcs.server.redis.client
import java.io.IOException
import java.net.InetSocketAddress
import java.nio.charset.StandardCharsets
import java.util.concurrent.CompletableFuture
import java.util.concurrent.ConcurrentHashMap
import io.netty.util.concurrent.Future as NettyFuture
import io.netty.bootstrap.Bootstrap
import io.netty.buffer.ByteBufAllocator
import io.netty.buffer.Unpooled
import io.netty.channel.Channel
import io.netty.channel.ChannelFactory
import io.netty.channel.ChannelFutureListener
import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelOption
import io.netty.channel.ChannelPipeline
import io.netty.channel.EventLoopGroup
import io.netty.channel.SimpleChannelInboundHandler
import io.netty.channel.pool.AbstractChannelPoolHandler
import io.netty.channel.pool.FixedChannelPool
import io.netty.channel.socket.SocketChannel
import io.netty.handler.codec.redis.ArrayRedisMessage
import io.netty.handler.codec.redis.ErrorRedisMessage
import io.netty.handler.codec.redis.FullBulkStringRedisMessage
import io.netty.handler.codec.redis.RedisArrayAggregator
import io.netty.handler.codec.redis.RedisBulkStringAggregator
import io.netty.handler.codec.redis.RedisDecoder
import io.netty.handler.codec.redis.RedisEncoder
import io.netty.handler.codec.redis.RedisMessage
import io.netty.util.concurrent.GenericFutureListener
import net.woggioni.rbcs.common.HostAndPort
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.trace
import net.woggioni.rbcs.server.redis.RedisCacheConfiguration
import net.woggioni.rbcs.server.redis.RedisCacheHandler
class RedisClient(
private val servers: List<RedisCacheConfiguration.Server>,
private val chunkSize: Int,
private val group: EventLoopGroup,
private val channelFactory: ChannelFactory<SocketChannel>,
private val connectionPool: ConcurrentHashMap<HostAndPort, FixedChannelPool>,
) : AutoCloseable {
private companion object {
private val log = createLogger<RedisCacheHandler>()
}
private fun newConnectionPool(server: RedisCacheConfiguration.Server): FixedChannelPool {
val bootstrap = Bootstrap().apply {
group(group)
channelFactory(channelFactory)
option(ChannelOption.SO_KEEPALIVE, true)
remoteAddress(InetSocketAddress(server.endpoint.host, server.endpoint.port))
server.connectionTimeoutMillis?.let {
option(ChannelOption.CONNECT_TIMEOUT_MILLIS, it)
}
}
val channelPoolHandler = object : AbstractChannelPoolHandler() {
override fun channelCreated(ch: Channel) {
val pipeline: ChannelPipeline = ch.pipeline()
pipeline.addLast(RedisEncoder())
pipeline.addLast(RedisDecoder())
pipeline.addLast(RedisBulkStringAggregator())
pipeline.addLast(RedisArrayAggregator())
server.password?.let { password ->
// Send AUTH command synchronously on new connections
val authCmd = buildCommand("AUTH", password)
ch.writeAndFlush(authCmd).addListener(ChannelFutureListener { future ->
if (!future.isSuccess) {
ch.close()
}
})
// Install a one-shot handler to consume the AUTH response
pipeline.addLast(object : SimpleChannelInboundHandler<RedisMessage>() {
override fun channelRead0(ctx: ChannelHandlerContext, msg: RedisMessage) {
when (msg) {
is ErrorRedisMessage -> {
ctx.close()
}
else -> {
// AUTH succeeded, remove this one-shot handler
ctx.pipeline().remove(this)
}
}
}
})
}
}
}
return FixedChannelPool(bootstrap, channelPoolHandler, server.maxConnections)
}
private fun buildCommand(vararg args: String): ArrayRedisMessage {
val children = args.map { arg ->
FullBulkStringRedisMessage(
Unpooled.wrappedBuffer(arg.toByteArray(StandardCharsets.UTF_8))
)
}
return ArrayRedisMessage(children)
}
fun sendCommand(
key: ByteArray,
alloc: ByteBufAllocator,
responseHandler: RedisResponseHandler,
): CompletableFuture<Channel> {
val server = if (servers.size > 1) {
val keyBuffer = alloc.buffer(key.size)
keyBuffer.writeBytes(key)
var checksum = 0
while (keyBuffer.readableBytes() > 4) {
val byte = keyBuffer.readInt()
checksum = checksum xor byte
}
while (keyBuffer.readableBytes() > 0) {
val byte = keyBuffer.readByte()
checksum = checksum xor byte.toInt()
}
keyBuffer.release()
servers[Math.floorMod(checksum, servers.size)]
} else {
servers.first()
}
val response = CompletableFuture<Channel>()
val pool = connectionPool.computeIfAbsent(server.endpoint) {
newConnectionPool(server)
}
pool.acquire().addListener(object : GenericFutureListener<NettyFuture<Channel>> {
override fun operationComplete(channelFuture: NettyFuture<Channel>) {
if (channelFuture.isSuccess) {
val channel = channelFuture.now
var connectionClosedByTheRemoteServer = true
val closeCallback = {
if (connectionClosedByTheRemoteServer) {
val ex = IOException("The Redis server closed the connection")
val completed = response.completeExceptionally(ex)
if (!completed) responseHandler.exceptionCaught(ex)
}
}
val closeListener = ChannelFutureListener {
closeCallback()
}
channel.closeFuture().addListener(closeListener)
val pipeline = channel.pipeline()
val handler = object : SimpleChannelInboundHandler<RedisMessage>(false) {
override fun handlerAdded(ctx: ChannelHandlerContext) {
channel.closeFuture().removeListener(closeListener)
}
override fun channelRead0(
ctx: ChannelHandlerContext,
msg: RedisMessage,
) {
pipeline.remove(this)
pool.release(channel)
log.trace(channel) {
"Channel released"
}
responseHandler.responseReceived(msg)
}
override fun channelInactive(ctx: ChannelHandlerContext) {
closeCallback()
ctx.fireChannelInactive()
}
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
connectionClosedByTheRemoteServer = false
pipeline.remove(this)
ctx.close()
pool.release(channel)
log.trace(channel) {
"Channel released after exception"
}
responseHandler.exceptionCaught(cause)
}
}
channel.pipeline().addLast(handler)
response.complete(channel)
} else {
response.completeExceptionally(channelFuture.cause())
}
}
})
return response
}
fun shutDown(): NettyFuture<*> {
return group.shutdownGracefully()
}
override fun close() {
shutDown().sync()
}
}
@@ -0,0 +1,10 @@
package net.woggioni.rbcs.server.redis.client
import io.netty.handler.codec.redis.RedisMessage
interface RedisResponseHandler {
fun responseReceived(response: RedisMessage)
fun exceptionCaught(ex: Throwable)
}
@@ -0,0 +1 @@
net.woggioni.rbcs.server.redis.RedisCacheProvider
@@ -0,0 +1,52 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<xs:schema targetNamespace="urn:net.woggioni.rbcs.server.redis"
xmlns:rbcs-redis="urn:net.woggioni.rbcs.server.redis"
xmlns:rbcs="urn:net.woggioni.rbcs.server"
xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:import schemaLocation="jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd" namespace="urn:net.woggioni.rbcs.server"/>
<xs:complexType name="redisServerType">
<xs:attribute name="host" type="xs:token" use="required"/>
<xs:attribute name="port" type="xs:positiveInteger" use="required"/>
<xs:attribute name="connection-timeout" type="xs:duration"/>
<xs:attribute name="max-connections" type="xs:positiveInteger" default="1"/>
<xs:attribute name="password" type="xs:string" use="optional">
<xs:annotation>
<xs:documentation>
Password for Redis AUTH command, used when the Redis server requires authentication
</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
<xs:complexType name="redisCacheType">
<xs:complexContent>
<xs:extension base="rbcs:cacheType">
<xs:sequence maxOccurs="unbounded">
<xs:element name="server" type="rbcs-redis:redisServerType"/>
</xs:sequence>
<xs:attribute name="max-age" type="xs:duration" default="P1D"/>
<xs:attribute name="chunk-size" type="rbcs:byteSizeType" default="0x10000"/>
<xs:attribute name="key-prefix" type="xs:string" use="optional">
<xs:annotation>
<xs:documentation>
Prepend this string to all the keys inserted in Redis,
useful in case the caching backend is shared with other applications
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="digest" type="xs:token"/>
<xs:attribute name="compression-mode" type="rbcs-redis:compressionType"/>
<xs:attribute name="compression-level" type="rbcs:compressionLevelType" default="-1"/>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:simpleType name="compressionType">
<xs:restriction base="xs:token">
<xs:enumeration value="deflate"/>
</xs:restriction>
</xs:simpleType>
</xs:schema>
+7
View File
@@ -12,6 +12,12 @@ dependencies {
implementation catalog.netty.handler implementation catalog.netty.handler
implementation catalog.netty.buffer implementation catalog.netty.buffer
implementation catalog.netty.transport implementation catalog.netty.transport
implementation catalog.netty.codec.haproxy
compileOnly catalog.opentelemetry.netty['4']['1']
compileOnly catalog.opentelemetry.sdk.extension.autoconfigure
compileOnly catalog.opentelemetry.logback.appender['1']['0']
compileOnly catalog.opentelemetry.extension.trace.propagators
compileOnly catalog.logback.classic
api project(':rbcs-common') api project(':rbcs-common')
api project(':rbcs-api') api project(':rbcs-api')
@@ -23,6 +29,7 @@ dependencies {
testImplementation catalog.bcpkix.jdk18on testImplementation catalog.bcpkix.jdk18on
testRuntimeOnly project(":rbcs-server-memcache") testRuntimeOnly project(":rbcs-server-memcache")
testRuntimeOnly project(":rbcs-server-redis")
} }
test { test {
@@ -16,6 +16,7 @@ module net.woggioni.rbcs.server {
requires io.netty.buffer; requires io.netty.buffer;
requires io.netty.common; requires io.netty.common;
requires io.netty.codec; requires io.netty.codec;
requires io.netty.codec.haproxy;
requires org.slf4j; requires org.slf4j;
exports net.woggioni.rbcs.server; exports net.woggioni.rbcs.server;
@@ -25,5 +26,6 @@ module net.woggioni.rbcs.server {
uses CacheProvider; uses CacheProvider;
uses net.woggioni.rbcs.api.TelemetryController;
provides CacheProvider with FileSystemCacheProvider, InMemoryCacheProvider; provides CacheProvider with FileSystemCacheProvider, InMemoryCacheProvider;
} }
@@ -1,5 +1,23 @@
package net.woggioni.rbcs.server package net.woggioni.rbcs.server
import java.io.OutputStream
import java.net.InetSocketAddress
import java.nio.file.Files
import java.nio.file.Path
import java.security.PrivateKey
import java.security.cert.X509Certificate
import java.time.Duration
import java.time.Instant
import java.util.Arrays
import java.util.Base64
import java.util.concurrent.CompletableFuture
import java.util.concurrent.Future
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import java.util.regex.Matcher
import java.util.regex.Pattern
import javax.naming.ldap.LdapName
import javax.net.ssl.SSLPeerUnverifiedException
import io.netty.bootstrap.ServerBootstrap import io.netty.bootstrap.ServerBootstrap
import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBuf
import io.netty.channel.Channel import io.netty.channel.Channel
@@ -20,6 +38,7 @@ import io.netty.channel.socket.nio.NioDatagramChannel
import io.netty.channel.socket.nio.NioServerSocketChannel import io.netty.channel.socket.nio.NioServerSocketChannel
import io.netty.channel.socket.nio.NioSocketChannel import io.netty.channel.socket.nio.NioSocketChannel
import io.netty.handler.codec.compression.CompressionOptions import io.netty.handler.codec.compression.CompressionOptions
import io.netty.handler.codec.haproxy.HAProxyMessageDecoder
import io.netty.handler.codec.http.DefaultHttpContent import io.netty.handler.codec.http.DefaultHttpContent
import io.netty.handler.codec.http.HttpContentCompressor import io.netty.handler.codec.http.HttpContentCompressor
import io.netty.handler.codec.http.HttpDecoderConfig import io.netty.handler.codec.http.HttpDecoderConfig
@@ -36,31 +55,16 @@ import io.netty.handler.timeout.IdleStateEvent
import io.netty.handler.timeout.IdleStateHandler import io.netty.handler.timeout.IdleStateHandler
import io.netty.util.AttributeKey import io.netty.util.AttributeKey
import io.netty.util.concurrent.EventExecutorGroup import io.netty.util.concurrent.EventExecutorGroup
import java.io.OutputStream
import java.net.InetSocketAddress
import java.nio.file.Files
import java.nio.file.Path
import java.security.PrivateKey
import java.security.cert.X509Certificate
import java.time.Duration
import java.time.Instant
import java.util.Arrays
import java.util.Base64
import java.util.concurrent.CompletableFuture
import java.util.concurrent.Future
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import java.util.regex.Matcher
import java.util.regex.Pattern
import javax.naming.ldap.LdapName
import javax.net.ssl.SSLPeerUnverifiedException
import net.woggioni.rbcs.api.AsyncCloseable import net.woggioni.rbcs.api.AsyncCloseable
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.api.TelemetryController
import net.woggioni.rbcs.api.exception.ConfigurationException import net.woggioni.rbcs.api.exception.ConfigurationException
import net.woggioni.rbcs.common.Cidr
import net.woggioni.rbcs.common.PasswordSecurity.decodePasswordHash import net.woggioni.rbcs.common.PasswordSecurity.decodePasswordHash
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
import net.woggioni.rbcs.common.RBCS.getTrustManager import net.woggioni.rbcs.common.RBCS.getTrustManager
import net.woggioni.rbcs.common.RBCS.loadKeystore import net.woggioni.rbcs.common.RBCS.loadKeystore
import net.woggioni.rbcs.common.RBCS.loadService
import net.woggioni.rbcs.common.RBCS.toUrl import net.woggioni.rbcs.common.RBCS.toUrl
import net.woggioni.rbcs.common.Xml import net.woggioni.rbcs.common.Xml
import net.woggioni.rbcs.common.createLogger import net.woggioni.rbcs.common.createLogger
@@ -73,6 +77,7 @@ import net.woggioni.rbcs.server.configuration.Parser
import net.woggioni.rbcs.server.configuration.Serializer import net.woggioni.rbcs.server.configuration.Serializer
import net.woggioni.rbcs.server.exception.ExceptionHandler import net.woggioni.rbcs.server.exception.ExceptionHandler
import net.woggioni.rbcs.server.handler.MaxRequestSizeHandler import net.woggioni.rbcs.server.handler.MaxRequestSizeHandler
import net.woggioni.rbcs.server.handler.ProxyProtocolHandler
import net.woggioni.rbcs.server.handler.ReadTriggerDuplexHandler import net.woggioni.rbcs.server.handler.ReadTriggerDuplexHandler
import net.woggioni.rbcs.server.handler.ServerHandler import net.woggioni.rbcs.server.handler.ServerHandler
import net.woggioni.rbcs.server.throttling.BucketManager import net.woggioni.rbcs.server.throttling.BucketManager
@@ -85,6 +90,7 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
val userAttribute: AttributeKey<Configuration.User> = AttributeKey.valueOf("user") val userAttribute: AttributeKey<Configuration.User> = AttributeKey.valueOf("user")
val groupAttribute: AttributeKey<Set<Configuration.Group>> = AttributeKey.valueOf("group") val groupAttribute: AttributeKey<Set<Configuration.Group>> = AttributeKey.valueOf("group")
val clientIp: AttributeKey<InetSocketAddress> = AttributeKey.valueOf("client-ip")
val DEFAULT_CONFIGURATION_URL by lazy { "jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/rbcs-default.xml".toUrl() } val DEFAULT_CONFIGURATION_URL by lazy { "jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/rbcs-default.xml".toUrl() }
private const val SSL_HANDLER_NAME = "sslHandler" private const val SSL_HANDLER_NAME = "sslHandler"
@@ -145,12 +151,68 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
((user?.groups ?: emptySet()).asSequence() + sequenceOf(group).filterNotNull()).toSet() ((user?.groups ?: emptySet()).asSequence() + sequenceOf(group).filterNotNull()).toSet()
AuthenticationResult(user, allGroups) AuthenticationResult(user, allGroups)
} ?: anonymousUserGroups?.let { AuthenticationResult(null, it) } } ?: anonymousUserGroups?.let { AuthenticationResult(null, it) }
} catch (es: SSLPeerUnverifiedException) { } catch (ex: SSLPeerUnverifiedException) {
log.debug(ctx) {
ex.message ?: "Error witch client certificate authentication"
}
anonymousUserGroups?.let { AuthenticationResult(null, it) } anonymousUserGroups?.let { AuthenticationResult(null, it) }
} }
} }
} }
@Sharable
private class ForwardedClientCertificateAuthenticator(
authorizer: Authorizer,
private val anonymousUserGroups: Set<Configuration.Group>?,
private val subjectDnUserExtractor: SubjectDnExtractor?,
private val subjectDnGroupExtractor: SubjectDnExtractor?,
private val headerName: String,
private val trustedProxyIPs: List<Cidr>,
private val users: Map<String, Configuration.User>,
private val groups: Map<String, Configuration.Group>,
) : AbstractNettyHttpAuthenticator(authorizer) {
companion object {
private val log = createLogger<ForwardedClientCertificateAuthenticator>()
}
override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? {
val clientIp = ctx.channel().attr(clientIp).get()
if (clientIp == null || trustedProxyIPs.none { it.contains(clientIp.address) }) {
log.debug(ctx) {
"Rejecting forwarded client certificate authentication from untrusted address: $clientIp"
}
return null
}
val subjectDn = req.headers()[headerName]
?: return anonymousUserGroups?.let { AuthenticationResult(null, it) }
val ldapName = try {
LdapName(subjectDn)
} catch (_: Exception) {
log.debug(ctx) {
"Invalid subject DN in header $headerName: $subjectDn"
}
return anonymousUserGroups?.let { AuthenticationResult(null, it) }
}
val user = subjectDnUserExtractor?.extract(ldapName)?.let { userName ->
users[userName] ?: throw RuntimeException("Failed to extract user '$userName'")
}
val group = subjectDnGroupExtractor?.extract(ldapName)?.let { groupName ->
groups[groupName] ?: throw RuntimeException("Failed to extract group '$groupName'")
}
val allGroups = ((user?.groups ?: emptySet()).asSequence() + sequenceOf(group).filterNotNull()).toSet()
return AuthenticationResult(user, allGroups)
}
}
private data class SubjectDnExtractor(val rdnType: String, val pattern: Pattern) {
fun extract(ldapName: LdapName): String? {
return ldapName.rdns.find { it.type == rdnType }
?.let { pattern.matcher(it.value.toString()) }
?.takeIf(Matcher::matches)?.group(1)
}
}
@Sharable @Sharable
private class NettyHttpBasicAuthenticator( private class NettyHttpBasicAuthenticator(
private val users: Map<String, Configuration.User>, authorizer: Authorizer private val users: Map<String, Configuration.User>, authorizer: Authorizer
@@ -234,6 +296,7 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
else ClientAuth.OPTIONAL else ClientAuth.OPTIONAL
} ?: ClientAuth.NONE } ?: ClientAuth.NONE
clientAuth(clientAuth) clientAuth(clientAuth)
}.build() }.build()
} }
} }
@@ -256,9 +319,29 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
) )
} }
is Configuration.ForwardedClientCertificateAuthentication -> {
ForwardedClientCertificateAuthenticator(
RoleAuthorizer(),
cfg.users[""]?.groups,
auth.userExtractor?.let { extractor ->
SubjectDnExtractor(extractor.rdnType, Pattern.compile(extractor.pattern))
},
auth.groupExtractor?.let { extractor ->
SubjectDnExtractor(extractor.rdnType, Pattern.compile(extractor.pattern))
},
auth.headerName,
cfg.trustedProxyIPs,
cfg.users,
cfg.groups,
)
}
else -> null else -> null
} }
private val proxyProtocolEnabled: Boolean = cfg.isProxyProtocolEnabled
private val trustedProxyIPs: List<Cidr> = cfg.trustedProxyIPs
private val sslContext: SslContext? = cfg.tls?.let(Companion::createSslCtx) private val sslContext: SslContext? = cfg.tls?.let(Companion::createSslCtx)
private fun userExtractor(authentication: Configuration.ClientCertificateAuthentication) = private fun userExtractor(authentication: Configuration.ClientCertificateAuthentication) =
@@ -271,7 +354,7 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
}?.let { }?.let {
pattern.matcher(it.value.toString()) pattern.matcher(it.value.toString())
}?.takeIf(Matcher::matches)?.group(1) }?.takeIf(Matcher::matches)?.group(1)
cfg.users[userName] ?: throw java.lang.RuntimeException("Failed to extract user") cfg.users[userName] ?: throw RuntimeException("Failed to extract user")
} }
} }
@@ -285,11 +368,12 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
}?.let { }?.let {
pattern.matcher(it.value.toString()) pattern.matcher(it.value.toString())
}?.takeIf(Matcher::matches)?.group(1) }?.takeIf(Matcher::matches)?.group(1)
cfg.groups[groupName] ?: throw java.lang.RuntimeException("Failed to extract group") cfg.groups[groupName] ?: throw RuntimeException("Failed to extract group")
} }
} }
override fun initChannel(ch: Channel) { override fun initChannel(ch: Channel) {
ch.attr(clientIp).set(ch.remoteAddress() as InetSocketAddress)
log.debug { log.debug {
"Created connection ${ch.id().asShortText()} with ${ch.remoteAddress()}" "Created connection ${ch.id().asShortText()} with ${ch.remoteAddress()}"
} }
@@ -338,6 +422,10 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
} }
} }
}) })
if(proxyProtocolEnabled) {
pipeline.addLast(HAProxyMessageDecoder())
pipeline.addLast(ProxyProtocolHandler(trustedProxyIPs))
}
sslContext?.newHandler(ch.alloc())?.also { sslContext?.newHandler(ch.alloc())?.also {
pipeline.addLast(SSL_HANDLER_NAME, it) pipeline.addLast(SSL_HANDLER_NAME, it)
} }
@@ -345,6 +433,10 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
maxChunkSize = cfg.connection.chunkSize maxChunkSize = cfg.connection.chunkSize
} }
pipeline.addLast(HttpServerCodec(httpDecoderConfig)) pipeline.addLast(HttpServerCodec(httpDecoderConfig))
loadService(TelemetryController::class.java)
.firstOrNull()
?.createHandler()
?.let { pipeline.addLast(it) }
pipeline.addLast(ReadTriggerDuplexHandler.NAME, ReadTriggerDuplexHandler()) pipeline.addLast(ReadTriggerDuplexHandler.NAME, ReadTriggerDuplexHandler())
pipeline.addLast(MaxRequestSizeHandler.NAME, MaxRequestSizeHandler(cfg.connection.maxRequestSize)) pipeline.addLast(MaxRequestSizeHandler.NAME, MaxRequestSizeHandler(cfg.connection.maxRequestSize))
pipeline.addLast(HttpChunkContentCompressor(1024)) pipeline.addLast(HttpChunkContentCompressor(1024))
@@ -1,5 +1,6 @@
package net.woggioni.rbcs.server.auth package net.woggioni.rbcs.server.auth
import java.net.InetSocketAddress
import io.netty.buffer.Unpooled import io.netty.buffer.Unpooled
import io.netty.channel.ChannelFutureListener import io.netty.channel.ChannelFutureListener
import io.netty.channel.ChannelHandlerContext import io.netty.channel.ChannelHandlerContext
@@ -15,12 +16,15 @@ import io.netty.util.ReferenceCountUtil
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.api.Configuration.Group import net.woggioni.rbcs.api.Configuration.Group
import net.woggioni.rbcs.api.Role import net.woggioni.rbcs.api.Role
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.debug
import net.woggioni.rbcs.server.RemoteBuildCacheServer import net.woggioni.rbcs.server.RemoteBuildCacheServer
abstract class AbstractNettyHttpAuthenticator(private val authorizer: Authorizer) : ChannelInboundHandlerAdapter() { abstract class AbstractNettyHttpAuthenticator(private val authorizer: Authorizer) : ChannelInboundHandlerAdapter() {
companion object { companion object {
private val log = createLogger<AbstractNettyHttpAuthenticator>()
private val AUTHENTICATION_FAILED: FullHttpResponse = DefaultFullHttpResponse( private val AUTHENTICATION_FAILED: FullHttpResponse = DefaultFullHttpResponse(
HttpVersion.HTTP_1_1, HttpResponseStatus.UNAUTHORIZED, Unpooled.EMPTY_BUFFER HttpVersion.HTTP_1_1, HttpResponseStatus.UNAUTHORIZED, Unpooled.EMPTY_BUFFER
).apply { ).apply {
@@ -53,6 +57,18 @@ abstract class AbstractNettyHttpAuthenticator(private val authorizer: Authorizer
result.groups.asSequence().flatMap { it.roles.asSequence() } result.groups.asSequence().flatMap { it.roles.asSequence() }
).toSet() ).toSet()
val authorized = authorizer.authorize(roles, msg) val authorized = authorizer.authorize(roles, msg)
log.debug {
val authorizedMessage = if (authorized) {
"Authorized"
} else {
"Forbidden"
}
val clientAddress = ctx.channel().attr(RemoteBuildCacheServer.clientIp).get()
val roleString = "[" + roles.asSequence().map { "\"" + it + "\"" }.joinToString(", ") + "]"
result.user?.name?.takeUnless(String::isEmpty)?.let { username ->
"$authorizedMessage ${msg.method()} request from user $username with address $clientAddress, granted roles $roleString"
} ?: "$authorizedMessage anonymous ${msg.method()} request with address $clientAddress, granted roles $roleString"
}
if (authorized) { if (authorized) {
super.channelRead(ctx, msg) super.channelRead(ctx, msg)
} else { } else {
@@ -72,5 +88,4 @@ abstract class AbstractNettyHttpAuthenticator(private val authorizer: Authorizer
ReferenceCountUtil.release(msg) ReferenceCountUtil.release(msg)
ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate()).addListener(ChannelFutureListener.CLOSE_ON_FAILURE) ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate()).addListener(ChannelFutureListener.CLOSE_ON_FAILURE)
} }
} }
@@ -1,11 +1,11 @@
package net.woggioni.rbcs.server.cache package net.woggioni.rbcs.server.cache
import java.nio.file.Path
import java.time.Duration
import io.netty.channel.ChannelFactory import io.netty.channel.ChannelFactory
import io.netty.channel.EventLoopGroup import io.netty.channel.EventLoopGroup
import io.netty.channel.socket.DatagramChannel import io.netty.channel.socket.DatagramChannel
import io.netty.channel.socket.SocketChannel import io.netty.channel.socket.SocketChannel
import java.nio.file.Path
import java.time.Duration
import net.woggioni.jwo.Application import net.woggioni.jwo.Application
import net.woggioni.rbcs.api.CacheHandlerFactory import net.woggioni.rbcs.api.CacheHandlerFactory
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
@@ -1,14 +1,14 @@
package net.woggioni.rbcs.server.cache package net.woggioni.rbcs.server.cache
import io.netty.buffer.ByteBuf
import io.netty.channel.ChannelHandlerContext
import io.netty.handler.codec.http.LastHttpContent
import io.netty.handler.stream.ChunkedNioFile
import java.nio.channels.Channels import java.nio.channels.Channels
import java.util.Base64 import java.util.Base64
import java.util.zip.Deflater import java.util.zip.Deflater
import java.util.zip.DeflaterOutputStream import java.util.zip.DeflaterOutputStream
import java.util.zip.InflaterInputStream import java.util.zip.InflaterInputStream
import io.netty.buffer.ByteBuf
import io.netty.channel.ChannelHandlerContext
import io.netty.handler.codec.http.LastHttpContent
import io.netty.handler.stream.ChunkedNioFile
import net.woggioni.rbcs.api.CacheHandler import net.woggioni.rbcs.api.CacheHandler
import net.woggioni.rbcs.api.message.CacheMessage import net.woggioni.rbcs.api.message.CacheMessage
import net.woggioni.rbcs.api.message.CacheMessage.CacheContent import net.woggioni.rbcs.api.message.CacheMessage.CacheContent
@@ -1,10 +1,10 @@
package net.woggioni.rbcs.server.cache package net.woggioni.rbcs.server.cache
import java.time.Duration
import io.netty.channel.ChannelFactory import io.netty.channel.ChannelFactory
import io.netty.channel.EventLoopGroup import io.netty.channel.EventLoopGroup
import io.netty.channel.socket.DatagramChannel import io.netty.channel.socket.DatagramChannel
import io.netty.channel.socket.SocketChannel import io.netty.channel.socket.SocketChannel
import java.time.Duration
import net.woggioni.rbcs.api.CacheHandlerFactory import net.woggioni.rbcs.api.CacheHandlerFactory
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.common.RBCS import net.woggioni.rbcs.common.RBCS
@@ -1,12 +1,10 @@
package net.woggioni.rbcs.server.cache package net.woggioni.rbcs.server.cache
import io.netty.buffer.ByteBuf
import io.netty.channel.ChannelHandlerContext
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.util.zip.Deflater import java.util.zip.Deflater
import java.util.zip.DeflaterOutputStream import java.util.zip.DeflaterOutputStream
import java.util.zip.InflaterOutputStream import java.util.zip.InflaterOutputStream
import io.netty.buffer.ByteBuf
import io.netty.channel.ChannelHandlerContext
import net.woggioni.rbcs.api.CacheHandler import net.woggioni.rbcs.api.CacheHandler
import net.woggioni.rbcs.api.message.CacheMessage import net.woggioni.rbcs.api.message.CacheMessage
import net.woggioni.rbcs.api.message.CacheMessage.CacheContent import net.woggioni.rbcs.api.message.CacheMessage.CacheContent
@@ -8,6 +8,7 @@ import net.woggioni.rbcs.api.Configuration.Authentication
import net.woggioni.rbcs.api.Configuration.BasicAuthentication import net.woggioni.rbcs.api.Configuration.BasicAuthentication
import net.woggioni.rbcs.api.Configuration.Cache import net.woggioni.rbcs.api.Configuration.Cache
import net.woggioni.rbcs.api.Configuration.ClientCertificateAuthentication import net.woggioni.rbcs.api.Configuration.ClientCertificateAuthentication
import net.woggioni.rbcs.api.Configuration.ForwardedClientCertificateAuthentication
import net.woggioni.rbcs.api.Configuration.Group import net.woggioni.rbcs.api.Configuration.Group
import net.woggioni.rbcs.api.Configuration.KeyStore import net.woggioni.rbcs.api.Configuration.KeyStore
import net.woggioni.rbcs.api.Configuration.Tls import net.woggioni.rbcs.api.Configuration.Tls
@@ -16,6 +17,7 @@ import net.woggioni.rbcs.api.Configuration.TrustStore
import net.woggioni.rbcs.api.Configuration.User import net.woggioni.rbcs.api.Configuration.User
import net.woggioni.rbcs.api.Role import net.woggioni.rbcs.api.Role
import net.woggioni.rbcs.api.exception.ConfigurationException import net.woggioni.rbcs.api.exception.ConfigurationException
import net.woggioni.rbcs.common.Cidr
import net.woggioni.rbcs.common.Xml.Companion.asIterable import net.woggioni.rbcs.common.Xml.Companion.asIterable
import net.woggioni.rbcs.common.Xml.Companion.renderAttribute import net.woggioni.rbcs.common.Xml.Companion.renderAttribute
import org.w3c.dom.Document import org.w3c.dom.Document
@@ -38,6 +40,8 @@ object Parser {
var cache: Cache? = null var cache: Cache? = null
var host = "127.0.0.1" var host = "127.0.0.1"
var port = 11080 var port = 11080
var proxyProtocolEnabled = false
var trustedProxies = emptyList<Cidr>()
var users: Map<String, User> = mapOf(anonymousUser.name to anonymousUser) var users: Map<String, User> = mapOf(anonymousUser.name to anonymousUser)
var groups = emptyMap<String, Group>() var groups = emptyMap<String, Group>()
var tls: Tls? = null var tls: Tls? = null
@@ -74,6 +78,28 @@ object Parser {
} }
authentication = ClientCertificateAuthentication(tlsExtractorUser, tlsExtractorGroup) authentication = ClientCertificateAuthentication(tlsExtractorUser, tlsExtractorGroup)
} }
"forwarded-client-certificate" -> {
val headerName = gchild.renderAttribute("header-name") ?: "X-Client-Cert-Subject-DN"
var tlsExtractorUser: TlsCertificateExtractor? = null
var tlsExtractorGroup: TlsCertificateExtractor? = null
for (ggchild in gchild.asIterable()) {
when (ggchild.localName) {
"group-extractor" -> {
val attrName = ggchild.renderAttribute("attribute-name")
val pattern = ggchild.renderAttribute("pattern")
tlsExtractorGroup = TlsCertificateExtractor(attrName, pattern)
}
"user-extractor" -> {
val attrName = ggchild.renderAttribute("attribute-name")
val pattern = ggchild.renderAttribute("pattern")
tlsExtractorUser = TlsCertificateExtractor(attrName, pattern)
}
}
}
authentication = ForwardedClientCertificateAuthentication(headerName, tlsExtractorUser, tlsExtractorGroup)
}
} }
} }
} }
@@ -98,9 +124,23 @@ object Parser {
"bind" -> { "bind" -> {
host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required") host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required")
port = Integer.parseInt(child.renderAttribute("port")) port = Integer.parseInt(child.renderAttribute("port"))
proxyProtocolEnabled = child.renderAttribute("proxy-protocol")
?.let(String::toBoolean) ?: false
incomingConnectionsBacklogSize = child.renderAttribute("incoming-connections-backlog-size") incomingConnectionsBacklogSize = child.renderAttribute("incoming-connections-backlog-size")
?.let(Integer::parseInt) ?.let(Integer::parseInt)
?: 1024 ?: 1024
for(grandChild in child.asIterable()) {
when(grandChild.localName) {
"trusted-proxies" -> {
trustedProxies = parseTrustedProxies(grandChild)
}
}
}
child.asIterable().filter {
it.localName == "trusted-proxies"
}.firstOrNull()?.let(::parseTrustedProxies)
} }
"cache" -> { "cache" -> {
@@ -195,6 +235,8 @@ object Parser {
return Configuration.of( return Configuration.of(
host, host,
port, port,
proxyProtocolEnabled,
trustedProxies,
incomingConnectionsBacklogSize, incomingConnectionsBacklogSize,
serverPath, serverPath,
eventExecutor, eventExecutor,
@@ -217,6 +259,15 @@ object Parser {
} }
}.toSet() }.toSet()
private fun parseTrustedProxies(root: Element) = root.asIterable().asSequence().map {
when (it.localName) {
"allow" -> it.renderAttribute("cidr")
?.let(Cidr::from)
?: throw ConfigurationException("Missing 'cidr' attribute")
else -> throw ConfigurationException("Unrecognized tag '${it.localName}'")
}
}.toList()
private fun parseUserRefs(root: Element) = root.asIterable().asSequence().map { private fun parseUserRefs(root: Element) = root.asIterable().asSequence().map {
when (it.localName) { when (it.localName) {
"user" -> it.renderAttribute("ref") "user" -> it.renderAttribute("ref")
@@ -293,14 +344,14 @@ object Parser {
roles = parseRoles(child) roles = parseRoles(child)
} }
"group-quota" -> { "group-quota" -> {
userQuota = parseQuota(child)
}
"user-quota" -> {
groupQuota = parseQuota(child) groupQuota = parseQuota(child)
} }
"user-quota" -> {
userQuota = parseQuota(child)
} }
} }
groupName to Group(groupName, roles, userQuota, groupQuota) }
groupName to Group(groupName, roles, groupQuota, userQuota)
}.toMap() }.toMap()
val users = knownUsersMap.map { (name, user) -> val users = knownUsersMap.map { (name, user) ->
name to User(name, user.password, userGroups[name]?.mapNotNull { groups[it] }?.toSet() ?: emptySet(), user.quota) name to User(name, user.password, userGroups[name]?.mapNotNull { groups[it] }?.toSet() ?: emptySet(), user.quota)
@@ -33,6 +33,17 @@ object Serializer {
attr("host", conf.host) attr("host", conf.host)
attr("port", conf.port.toString()) attr("port", conf.port.toString())
attr("incoming-connections-backlog-size", conf.incomingConnectionsBacklogSize.toString()) attr("incoming-connections-backlog-size", conf.incomingConnectionsBacklogSize.toString())
attr("proxy-protocol", conf.isProxyProtocolEnabled.toString())
if (conf.trustedProxyIPs.isNotEmpty()) {
node("trusted-proxies") {
for(trustedProxy in conf.trustedProxyIPs) {
node("allow") {
attr("cidr", trustedProxy.toString())
}
}
}
}
} }
node("connection") { node("connection") {
conf.connection.let { connection -> conf.connection.let { connection ->
@@ -154,6 +165,23 @@ object Serializer {
} }
} }
} }
is Configuration.ForwardedClientCertificateAuthentication -> {
node("forwarded-client-certificate") {
attr("header-name", authentication.headerName)
authentication.groupExtractor?.let { extractor ->
node("group-extractor") {
attr("attribute-name", extractor.rdnType)
attr("pattern", extractor.pattern)
}
}
authentication.userExtractor?.let { extractor ->
node("user-extractor") {
attr("attribute-name", extractor.rdnType)
attr("pattern", extractor.pattern)
}
}
}
}
} }
} }
} }
@@ -1,5 +1,9 @@
package net.woggioni.rbcs.server.exception package net.woggioni.rbcs.server.exception
import java.net.ConnectException
import java.net.SocketException
import javax.net.ssl.SSLException
import javax.net.ssl.SSLPeerUnverifiedException
import io.netty.buffer.Unpooled import io.netty.buffer.Unpooled
import io.netty.channel.ChannelDuplexHandler import io.netty.channel.ChannelDuplexHandler
import io.netty.channel.ChannelFutureListener import io.netty.channel.ChannelFutureListener
@@ -13,10 +17,6 @@ import io.netty.handler.codec.http.HttpResponseStatus
import io.netty.handler.codec.http.HttpVersion import io.netty.handler.codec.http.HttpVersion
import io.netty.handler.timeout.ReadTimeoutException import io.netty.handler.timeout.ReadTimeoutException
import io.netty.handler.timeout.WriteTimeoutException import io.netty.handler.timeout.WriteTimeoutException
import java.net.ConnectException
import java.net.SocketException
import javax.net.ssl.SSLException
import javax.net.ssl.SSLPeerUnverifiedException
import net.woggioni.rbcs.api.exception.CacheException import net.woggioni.rbcs.api.exception.CacheException
import net.woggioni.rbcs.api.exception.ContentTooLargeException import net.woggioni.rbcs.api.exception.ContentTooLargeException
import net.woggioni.rbcs.common.contextLogger import net.woggioni.rbcs.common.contextLogger
@@ -74,6 +74,7 @@ object ExceptionHandler : ChannelDuplexHandler() {
} }
is SSLPeerUnverifiedException -> { is SSLPeerUnverifiedException -> {
log.debug(cause.message, cause)
ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate()) ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate())
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE) .addListener(ChannelFutureListener.CLOSE_ON_FAILURE)
} }
@@ -0,0 +1,45 @@
package net.woggioni.rbcs.server.handler
import java.net.InetAddress
import java.net.InetSocketAddress
import io.netty.channel.ChannelHandlerContext
import io.netty.channel.SimpleChannelInboundHandler
import io.netty.handler.codec.haproxy.HAProxyMessage
import net.woggioni.rbcs.common.Cidr
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.trace
import net.woggioni.rbcs.server.RemoteBuildCacheServer
class ProxyProtocolHandler(private val trustedProxyIPs : List<Cidr>) : SimpleChannelInboundHandler<HAProxyMessage>() {
companion object {
private val log = createLogger<ProxyProtocolHandler>()
}
override fun channelRead0(
ctx: ChannelHandlerContext,
msg: HAProxyMessage
) {
val sourceAddress = ctx.channel().remoteAddress()
if (sourceAddress is InetSocketAddress &&
(trustedProxyIPs.isEmpty() ||
trustedProxyIPs.any { it.contains(sourceAddress.address) }.also {
if(!it) {
log.trace {
"Received a proxied connection request from $sourceAddress which is not a trusted proxy address, " +
"the proxy server address will be used instead"
}
}
})) {
val proxiedClientAddress = InetSocketAddress(
InetAddress.ofLiteral(msg.sourceAddress()),
msg.sourcePort()
)
log.trace {
"Received proxied connection request from $sourceAddress forwarded for $proxiedClientAddress"
}
ctx.channel().attr(RemoteBuildCacheServer.clientIp).set(proxiedClientAddress)
}
}
}
@@ -1,5 +1,6 @@
package net.woggioni.rbcs.server.handler package net.woggioni.rbcs.server.handler
import java.nio.file.Path
import io.netty.channel.ChannelDuplexHandler import io.netty.channel.ChannelDuplexHandler
import io.netty.channel.ChannelHandler import io.netty.channel.ChannelHandler
import io.netty.channel.ChannelHandlerContext import io.netty.channel.ChannelHandlerContext
@@ -18,7 +19,6 @@ import io.netty.handler.codec.http.HttpResponseStatus
import io.netty.handler.codec.http.HttpUtil import io.netty.handler.codec.http.HttpUtil
import io.netty.handler.codec.http.HttpVersion import io.netty.handler.codec.http.HttpVersion
import io.netty.handler.codec.http.LastHttpContent import io.netty.handler.codec.http.LastHttpContent
import java.nio.file.Path
import net.woggioni.rbcs.api.CacheValueMetadata import net.woggioni.rbcs.api.CacheValueMetadata
import net.woggioni.rbcs.api.message.CacheMessage import net.woggioni.rbcs.api.message.CacheMessage
import net.woggioni.rbcs.api.message.CacheMessage.CacheContent import net.woggioni.rbcs.api.message.CacheMessage.CacheContent
@@ -171,7 +171,6 @@ class ServerHandler(private val serverPrefix: Path, private val cacheHandlerSupp
ctx.pipeline().addBefore(ExceptionHandler.NAME, null, cacheHandler) ctx.pipeline().addBefore(ExceptionHandler.NAME, null, cacheHandler)
key.let(::CacheGetRequest) key.let(::CacheGetRequest)
.let(ctx::fireChannelRead) .let(ctx::fireChannelRead)
?: ctx.channel().write(CacheValueNotFoundResponse(key))
} else { } else {
cacheRequestInProgress = false cacheRequestInProgress = false
log.warn(ctx) { log.warn(ctx) {
@@ -1,5 +1,6 @@
package net.woggioni.rbcs.server.handler package net.woggioni.rbcs.server.handler
import java.nio.file.Path
import io.netty.channel.ChannelHandler.Sharable import io.netty.channel.ChannelHandler.Sharable
import io.netty.channel.ChannelHandlerContext import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelInboundHandlerAdapter import io.netty.channel.ChannelInboundHandlerAdapter
@@ -10,7 +11,6 @@ import io.netty.handler.codec.http.HttpHeaderValues
import io.netty.handler.codec.http.HttpRequest import io.netty.handler.codec.http.HttpRequest
import io.netty.handler.codec.http.HttpResponseStatus import io.netty.handler.codec.http.HttpResponseStatus
import io.netty.handler.codec.http.LastHttpContent import io.netty.handler.codec.http.LastHttpContent
import java.nio.file.Path
@Sharable @Sharable
object TraceHandler : ChannelInboundHandlerAdapter() { object TraceHandler : ChannelInboundHandlerAdapter() {
@@ -1,5 +1,10 @@
package net.woggioni.rbcs.server.throttling package net.woggioni.rbcs.server.throttling
import java.net.InetSocketAddress
import java.time.Duration
import java.time.temporal.ChronoUnit
import java.util.ArrayDeque
import java.util.concurrent.TimeUnit
import io.netty.buffer.ByteBufHolder import io.netty.buffer.ByteBufHolder
import io.netty.channel.ChannelHandlerContext import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelInboundHandlerAdapter import io.netty.channel.ChannelInboundHandlerAdapter
@@ -10,11 +15,6 @@ import io.netty.handler.codec.http.HttpRequest
import io.netty.handler.codec.http.HttpResponseStatus import io.netty.handler.codec.http.HttpResponseStatus
import io.netty.handler.codec.http.HttpVersion import io.netty.handler.codec.http.HttpVersion
import io.netty.handler.codec.http.LastHttpContent import io.netty.handler.codec.http.LastHttpContent
import java.net.InetSocketAddress
import java.time.Duration
import java.time.temporal.ChronoUnit
import java.util.ArrayDeque
import java.util.concurrent.TimeUnit
import net.woggioni.jwo.Bucket import net.woggioni.jwo.Bucket
import net.woggioni.jwo.LongMath import net.woggioni.jwo.LongMath
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
@@ -197,7 +197,8 @@ class ThrottlingHandler(
} }
} }
if (user == null && groups.isEmpty()) { if (user == null && groups.isEmpty()) {
bucketManager.getBucketByAddress(ctx.channel().remoteAddress() as InetSocketAddress)?.let(buckets::add) val clientAddress = ctx.channel().attr<InetSocketAddress>(RemoteBuildCacheServer.clientIp).get()
bucketManager.getBucketByAddress(clientAddress)?.let(buckets::add)
} }
var nextAttempt = -1L var nextAttempt = -1L
@@ -62,6 +62,9 @@
</xs:complexType> </xs:complexType>
<xs:complexType name="bindType"> <xs:complexType name="bindType">
<xs:sequence minOccurs="0">
<xs:element name="trusted-proxies" type="rbcs:trustedProxiesType"/>
</xs:sequence>
<xs:attribute name="host" type="xs:token" use="required"> <xs:attribute name="host" type="xs:token" use="required">
<xs:annotation> <xs:annotation>
<xs:documentation>Server bind address</xs:documentation> <xs:documentation>Server bind address</xs:documentation>
@@ -72,6 +75,12 @@
<xs:documentation>Server port number</xs:documentation> <xs:documentation>Server port number</xs:documentation>
</xs:annotation> </xs:annotation>
</xs:attribute> </xs:attribute>
<xs:attribute name="proxy-protocol" type="xs:boolean" use="optional">
<xs:annotation>
<xs:documentation>Enable proxy protocol</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="incoming-connections-backlog-size" type="xs:unsignedInt" use="optional" default="1024"> <xs:attribute name="incoming-connections-backlog-size" type="xs:unsignedInt" use="optional" default="1024">
<xs:annotation> <xs:annotation>
<xs:documentation> <xs:documentation>
@@ -83,6 +92,16 @@
</xs:attribute> </xs:attribute>
</xs:complexType> </xs:complexType>
<xs:complexType name="trustedProxiesType">
<xs:sequence minOccurs="0">
<xs:element name="allow" type="rbcs:allowType" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="allowType">
<xs:attribute name="cidr" type="rbcs:cidr"/>
</xs:complexType>
<xs:complexType name="connectionType"> <xs:complexType name="connectionType">
<xs:attribute name="idle-timeout" type="xs:duration" use="optional" default="PT30S"> <xs:attribute name="idle-timeout" type="xs:duration" use="optional" default="PT30S">
<xs:annotation> <xs:annotation>
@@ -292,6 +311,45 @@
</xs:sequence> </xs:sequence>
</xs:complexType> </xs:complexType>
<xs:complexType name="forwardedClientCertificateAuthorizationType">
<xs:annotation>
<xs:documentation>
Authenticate clients based on a custom HTTP header containing the client TLS certificate
subject DN, forwarded by a reverse proxy that performs TLS termination. The proxy must be
listed in the trusted-proxies configuration for the header to be accepted.
</xs:documentation>
</xs:annotation>
<xs:sequence>
<xs:element name="group-extractor" type="rbcs:X500NameExtractorType" minOccurs="0">
<xs:annotation>
<xs:documentation>
A regex based extractor that will be used to determine which group the client belongs to,
based on the X.500 name of the subject DN forwarded by the reverse proxy.
When this is set RBAC works even if the user isn't listed in the &lt;users/&gt; section as
the client will be assigned role solely based on the group he is found to belong to.
Note that this does not allow for a client to be part of multiple groups.
</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element name="user-extractor" type="rbcs:X500NameExtractorType" minOccurs="0">
<xs:annotation>
<xs:documentation>
A regex based extractor that will be used to assign a user to a connected client,
based on the X.500 name of the subject DN forwarded by the reverse proxy.
</xs:documentation>
</xs:annotation>
</xs:element>
</xs:sequence>
<xs:attribute name="header-name" type="xs:token">
<xs:annotation>
<xs:documentation>
Name of the HTTP header containing the client certificate subject DN
forwarded by the reverse proxy. Defaults to "X-Client-Cert-Subject-DN".
</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
<xs:complexType name="X500NameExtractorType"> <xs:complexType name="X500NameExtractorType">
<xs:annotation> <xs:annotation>
<xs:documentation> <xs:documentation>
@@ -361,6 +419,15 @@
</xs:documentation> </xs:documentation>
</xs:annotation> </xs:annotation>
</xs:element> </xs:element>
<xs:element name="forwarded-client-certificate" type="rbcs:forwardedClientCertificateAuthorizationType">
<xs:annotation>
<xs:documentation>
Enable forwarded client certificate authentication. Authenticates clients based on
a custom HTTP header containing the client certificate subject DN, forwarded by a
reverse proxy that performs TLS termination. Requires trusted-proxies to be configured.
</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element name="none"> <xs:element name="none">
<xs:annotation> <xs:annotation>
<xs:documentation> <xs:documentation>
@@ -681,4 +748,20 @@
</xs:restriction> </xs:restriction>
</xs:simpleType> </xs:simpleType>
<xs:simpleType name="cidrIPv4">
<xs:restriction base="xs:string">
<xs:pattern value="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/(3[0-2]|[12]?[0-9])" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="cidrIPv6">
<xs:restriction base="xs:string">
<xs:pattern value="((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(:([0-9A-Fa-f]{1,4}){1,7}|:)))(%[\p{L}\p{N}_-]+)?\/(12[0-8]|1[01][0-9]|[1-9]?[0-9])" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="cidr">
<xs:union memberTypes="rbcs:cidrIPv4 rbcs:cidrIPv6" />
</xs:simpleType>
</xs:schema> </xs:schema>
@@ -34,6 +34,8 @@ abstract class AbstractBasicAuthServerTest : AbstractServerTest() {
cfg = Configuration.of( cfg = Configuration.of(
"127.0.0.1", "127.0.0.1",
getFreePort(), getFreePort(),
false,
emptyList(),
50, 50,
serverPath, serverPath,
Configuration.EventExecutor(false), Configuration.EventExecutor(false),
@@ -140,8 +140,10 @@ abstract class AbstractTlsServerTest : AbstractServerTest() {
cfg = Configuration( cfg = Configuration(
"127.0.0.1", "127.0.0.1",
getFreePort(), getFreePort(),
100,
serverPath, serverPath,
false,
emptyList(),
100,
Configuration.EventExecutor(false), Configuration.EventExecutor(false),
Configuration.RateLimiter(true, 0x100000, 50), Configuration.RateLimiter(true, 0x100000, 50),
Configuration.Connection( Configuration.Connection(
@@ -1,11 +1,11 @@
package net.woggioni.rbcs.server.test package net.woggioni.rbcs.server.test
import io.netty.handler.codec.http.HttpResponseStatus
import java.net.http.HttpClient import java.net.http.HttpClient
import java.net.http.HttpRequest import java.net.http.HttpRequest
import java.net.http.HttpResponse import java.net.http.HttpResponse
import java.time.Duration import java.time.Duration
import java.time.temporal.ChronoUnit import java.time.temporal.ChronoUnit
import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.api.Role import net.woggioni.rbcs.api.Role
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
@@ -21,6 +21,8 @@ class ConfigurationTest {
"classpath:net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml", "classpath:net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml",
"classpath:net/woggioni/rbcs/server/test/valid/rbcs-tls.xml", "classpath:net/woggioni/rbcs/server/test/valid/rbcs-tls.xml",
"classpath:net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml", "classpath:net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml",
"classpath:net/woggioni/rbcs/server/test/valid/rbcs-redis.xml",
"classpath:net/woggioni/rbcs/server/test/valid/rbcs-redis-tls.xml",
] ]
) )
@ParameterizedTest @ParameterizedTest
@@ -1,9 +1,9 @@
package net.woggioni.rbcs.server.test package net.woggioni.rbcs.server.test
import io.netty.handler.codec.http.HttpResponseStatus
import java.net.http.HttpClient import java.net.http.HttpClient
import java.net.http.HttpRequest import java.net.http.HttpRequest
import java.net.http.HttpResponse import java.net.http.HttpResponse
import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions
@@ -1,9 +1,9 @@
package net.woggioni.rbcs.server.test package net.woggioni.rbcs.server.test
import io.netty.handler.codec.http.HttpResponseStatus
import java.net.http.HttpClient import java.net.http.HttpClient
import java.net.http.HttpRequest import java.net.http.HttpRequest
import java.net.http.HttpResponse import java.net.http.HttpResponse
import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Order import org.junit.jupiter.api.Order
@@ -1,6 +1,5 @@
package net.woggioni.rbcs.server.test package net.woggioni.rbcs.server.test
import io.netty.handler.codec.http.HttpResponseStatus
import java.net.URI import java.net.URI
import java.net.http.HttpClient import java.net.http.HttpClient
import java.net.http.HttpRequest import java.net.http.HttpRequest
@@ -11,6 +10,7 @@ import java.time.temporal.ChronoUnit
import java.util.Base64 import java.util.Base64
import java.util.zip.Deflater import java.util.zip.Deflater
import kotlin.random.Random import kotlin.random.Random
import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.common.RBCS.getFreePort import net.woggioni.rbcs.common.RBCS.getFreePort
import net.woggioni.rbcs.common.Xml import net.woggioni.rbcs.common.Xml
@@ -34,8 +34,10 @@ class NoAuthServerTest : AbstractServerTest() {
cfg = Configuration( cfg = Configuration(
"127.0.0.1", "127.0.0.1",
getFreePort(), getFreePort(),
100,
serverPath, serverPath,
false,
emptyList(),
100,
Configuration.EventExecutor(false), Configuration.EventExecutor(false),
Configuration.RateLimiter(true, 0x100000, 50), Configuration.RateLimiter(true, 0x100000, 50),
Configuration.Connection( Configuration.Connection(
@@ -1,9 +1,9 @@
package net.woggioni.rbcs.server.test package net.woggioni.rbcs.server.test
import io.netty.handler.codec.http.HttpResponseStatus
import java.net.http.HttpClient import java.net.http.HttpClient
import java.net.http.HttpRequest import java.net.http.HttpRequest
import java.net.http.HttpResponse import java.net.http.HttpResponse
import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.rbcs.api.Configuration import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.api.Role import net.woggioni.rbcs.api.Role
import org.bouncycastle.asn1.x500.X500Name import org.bouncycastle.asn1.x500.X500Name
@@ -1,8 +1,15 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance" <rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server" xmlns:rbcs="urn:net.woggioni.rbcs.server"
xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"> xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"
<bind host="127.0.0.1" port="11443" incoming-connections-backlog-size="22"/> path="/my/custom/path">
<bind host="127.0.0.1" port="11443" incoming-connections-backlog-size="22" proxy-protocol="true">
<trusted-proxies>
<allow cidr="192.168.0.11/32"/>
<allow cidr="::1/128"/>
<allow cidr="fda7:9b54:5678::2f9/128"/>
</trusted-proxies>
</bind>
<connection <connection
read-idle-timeout="PT10M" read-idle-timeout="PT10M"
write-idle-timeout="PT11M" write-idle-timeout="PT11M"
@@ -0,0 +1,53 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server"
xmlns:rbcs-redis="urn:net.woggioni.rbcs.server.redis"
xs:schemaLocation="urn:net.woggioni.rbcs.server.redis jpms://net.woggioni.rbcs.server.redis/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"
>
<bind host="0.0.0.0" port="8443" incoming-connections-backlog-size="4096"/>
<connection
max-request-size="67108864"
idle-timeout="PT30S"
read-idle-timeout="PT60S"
write-idle-timeout="PT60S"
chunk-size="123"/>
<event-executor use-virtual-threads="true"/>
<rate-limiter delay-response="false" message-buffer-size="12000" max-queued-messages="53"/>
<cache xs:type="rbcs-redis:redisCacheType" max-age="P7D" key-prefix="some-prefix-string">
<server host="redis-server" port="6379" password="secret123"/>
</cache>
<authorization>
<users>
<user name="woggioni">
<quota calls="1000" period="PT1S"/>
</user>
<user name="gitea">
<quota calls="10" period="PT1S" initial-available-calls="100" max-available-calls="100"/>
</user>
<anonymous>
<quota calls="2" period="PT5S"/>
</anonymous>
</users>
<groups>
<group name="writers">
<users>
<user ref="woggioni"/>
<user ref="gitea"/>
</users>
<roles>
<reader/>
<writer/>
</roles>
</group>
</groups>
</authorization>
<authentication>
<client-certificate>
<user-extractor attribute-name="CN" pattern="(.*)"/>
</client-certificate>
</authentication>
<tls>
<keystore file="/home/luser/ssl/rbcs.woggioni.net.pfx" key-alias="rbcs.woggioni.net" password="KEYSTORE_PASSWOR" key-password="KEY_PASSWORD"/>
<truststore file="/home/luser/ssl/woggioni.net.pfx" check-certificate-status="false" password="TRUSTSTORE_PASSWORD"/>
</tls>
</rbcs:server>
@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server"
xmlns:rbcs-redis="urn:net.woggioni.rbcs.server.redis"
xs:schemaLocation="urn:net.woggioni.rbcs.server.redis jpms://net.woggioni.rbcs.server.redis/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd">
<bind host="127.0.0.1" port="11443" incoming-connections-backlog-size="50"/>
<connection
read-idle-timeout="PT10M"
write-idle-timeout="PT11M"
idle-timeout="PT30M"
max-request-size="101325"
chunk-size="456"/>
<event-executor use-virtual-threads="false"/>
<rate-limiter delay-response="true" message-buffer-size="65432" max-queued-messages="21"/>
<cache xs:type="rbcs-redis:redisCacheType" max-age="P7D" key-prefix="some-prefix-string" digest="SHA-256" compression-mode="deflate" compression-level="7">
<server host="127.0.0.1" port="6379" max-connections="10" connection-timeout="PT20S"/>
</cache>
<authentication>
<none/>
</authentication>
</rbcs:server>
@@ -1,12 +1,5 @@
package net.woggioni.rbcs.servlet package net.woggioni.rbcs.servlet
import jakarta.annotation.PreDestroy
import jakarta.enterprise.context.ApplicationScoped
import jakarta.inject.Inject
import jakarta.servlet.annotation.WebServlet
import jakarta.servlet.http.HttpServlet
import jakarta.servlet.http.HttpServletRequest
import jakarta.servlet.http.HttpServletResponse
import java.io.ByteArrayInputStream import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import java.nio.file.Path import java.nio.file.Path
@@ -17,6 +10,13 @@ import java.util.concurrent.PriorityBlockingQueue
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicLong
import java.util.logging.Logger import java.util.logging.Logger
import jakarta.annotation.PreDestroy
import jakarta.enterprise.context.ApplicationScoped
import jakarta.inject.Inject
import jakarta.servlet.annotation.WebServlet
import jakarta.servlet.http.HttpServlet
import jakarta.servlet.http.HttpServletRequest
import jakarta.servlet.http.HttpServletResponse
import net.woggioni.jwo.HttpClient.HttpStatus import net.woggioni.jwo.HttpClient.HttpStatus
import net.woggioni.jwo.JWO import net.woggioni.jwo.JWO
+3 -8
View File
@@ -3,12 +3,8 @@ pluginManagement {
// mavenLocal { // mavenLocal {
// content { // content {
// includeGroup 'net.woggioni.gradle' // includeGroup 'net.woggioni.gradle'
// includeGroup 'net.woggioni.gradle.jpms-check'
// includeGroup 'net.woggioni.gradle.lombok' // includeGroup 'net.woggioni.gradle.lombok'
// includeGroup 'net.woggioni.gradle.jdeps' // includeGroup 'net.woggioni.gradle.finalguard'
// includeGroup 'net.woggioni.gradle.sambal'
// includeGroup 'net.woggioni.gradle.graalvm.jlink'
// includeGroup 'net.woggioni.gradle.graalvm.native-image'
// } // }
// } // }
maven { maven {
@@ -30,8 +26,6 @@ dependencyResolutionManagement {
versionCatalogs { versionCatalogs {
catalog { catalog {
from group: 'com.lys', name: 'lys-catalog', version: getProperty('lys.version') from group: 'com.lys', name: 'lys-catalog', version: getProperty('lys.version')
// version('my-gradle-plugins', '2025.04.16')
// version('junit', '5.12.0')
} }
} }
} }
@@ -41,9 +35,10 @@ rootProject.name = 'rbcs'
include 'rbcs-api' include 'rbcs-api'
include 'rbcs-common' include 'rbcs-common'
include 'rbcs-server-memcache' include 'rbcs-server-memcache'
include 'rbcs-server-redis'
include 'rbcs-cli' include 'rbcs-cli'
include 'rbcs-client' include 'rbcs-client'
include 'rbcs-server' include 'rbcs-server'
include 'rbcs-servlet' include 'rbcs-servlet'
include 'rbcs-server-otel'
include 'docker' include 'docker'
//include 'bug'