Add optional OpenTelemetry Netty server instrumentation
All checks were successful
CI / build (push) Successful in 3m50s

- Update lys.version to 2026.04.14

- Add optional compileOnly dependency on opentelemetry-netty-4.1 in rbcs-server

- Add runtime guard to only activate instrumentation when OTel classes are on classpath

- Insert OTel combined handler after HttpServerCodec in the Netty pipeline

- Add requires-static JPMS directives for optional module support
This commit is contained in:
2026-04-29 02:50:55 +08:00
parent 5d190d81ab
commit 1d938b7ea3
24 changed files with 230 additions and 51 deletions

View File

@@ -58,6 +58,18 @@ jobs:
tags: |
gitea.woggioni.net/woggioni/rbcs:dev-redis
target: release-redis
-
name: Build rbcs full Docker image
uses: docker/build-push-action@v5.3.0
with:
builder: "multiplatform-builder"
context: "docker/build/docker"
platforms: linux/amd64,linux/arm64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:dev-full
target: release-full
-
name: Build rbcs native Docker image
uses: docker/build-push-action@v5.3.0

View File

@@ -61,6 +61,19 @@ jobs:
gitea.woggioni.net/woggioni/rbcs:redis
gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}-redis
target: release-redis
-
name: Build rbcs full Docker image
uses: docker/build-push-action@v5.3.0
with:
builder: "multiplatform-builder"
context: "docker/build/docker"
platforms: linux/amd64,linux/arm64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:full
gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}-full
target: release-full
-
name: Build rbcs native Docker image
uses: docker/build-push-action@v5.3.0

5
.gitignore vendored
View File

@@ -5,3 +5,8 @@
build
rbcs-cli/native-image/*.json
# Ignore JDTLS files
.classpath
.project
.settings

View File

@@ -5,6 +5,7 @@ The root element that contains all server configuration.
**Attributes:**
- `path` (optional): URI path prefix for cache requests. Example: if set to "cache", requests would be made to "http://www.example.com/cache/KEY"
- `enable-telemetry` (optional): If set to "true" it will enable opentelemetry integration (you will need to make sure the `rbcs-server-otel` has been unpacked in the `plugins` folder)
#### Child Elements
@@ -139,6 +140,7 @@ Configures TLS encryption.
<rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server"
xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"
path="/my/custom/path" enable-telemetry="true"
>
<bind host="0.0.0.0" port="8080" incoming-connections-backlog-size="1024" proxy-protocol="true">
<trusted-proxies>

View File

@@ -29,6 +29,18 @@ ADD logback.xml /etc/rbcs/logback.xml
ENV RBCS_CONFIGURATION_DIR="/etc/rbcs"
ENTRYPOINT ["java", "-Dlogback.configurationFile=/etc/rbcs/logback.xml", "-XX:MaxRAMPercentage=70", "-XX:GCTimeRatio=24", "-XX:+UseZGC", "-jar", "/var/lib/rbcs/rbcs.jar"]
FROM base-release AS release-full
ADD --chown=rbcs:rbcs rbcs-cli-envelope-*.jar rbcs.jar
RUN mkdir plugins
WORKDIR /var/lib/rbcs/plugins
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-memcache*.tar
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-redis*.tar
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-otel*.tar
WORKDIR /var/lib/rbcs
ADD logback.xml /etc/rbcs/logback.xml
ENV RBCS_CONFIGURATION_DIR="/etc/rbcs"
ENTRYPOINT ["java", "-Dlogback.configurationFile=/etc/rbcs/logback.xml", "-XX:MaxRAMPercentage=70", "-XX:GCTimeRatio=24", "-XX:+UseZGC", "-jar", "/var/lib/rbcs/rbcs.jar"]
FROM busybox:musl AS base-native
RUN mkdir -p /var/lib/rbcs /var/tmp/rbcs /etc/rbcs
RUN adduser -D -u 1000 rbcs -h /var/lib/rbcs

View File

@@ -21,6 +21,7 @@ dependencies {
docker project(path: ':rbcs-cli', configuration: 'release')
docker project(path: ':rbcs-server-memcache', configuration: 'release')
docker project(path: ':rbcs-server-redis', configuration: 'release')
docker project(path: ':rbcs-server-otel', configuration: 'release')
}
Provider<Task> cleanTaskProvider = tasks.named(BasePlugin.CLEAN_TASK_NAME) {}

View File

@@ -4,7 +4,7 @@ org.gradle.caching=true
rbcs.version = 0.5.0
lys.version = 2026.03.26
lys.version = 2026.04.28
gitea.maven.url = https://gitea.woggioni.net/api/packages/woggioni/maven
docker.registry.url=gitea.woggioni.net

View File

@@ -18,10 +18,11 @@ import java.util.stream.Collectors;
public class Configuration {
String host;
int port;
String serverPath;
boolean proxyProtocolEnabled;
List<Cidr> trustedProxyIPs;
boolean enableTelemetry;
int incomingConnectionsBacklogSize;
String serverPath;
@NonNull
EventExecutor eventExecutor;
@NonNull
@@ -150,6 +151,7 @@ public class Configuration {
}
public static Configuration of(
boolean enableTelemetry,
String host,
int port,
boolean proxyProtocolEnabled,
@@ -168,10 +170,11 @@ public class Configuration {
return new Configuration(
host,
port,
serverPath != null && !serverPath.isEmpty() && !serverPath.equals("/") ? serverPath : null,
proxyProtocolEnabled,
trustedProxyIPs,
enableTelemetry,
incomingConnectionsBacklogSize,
serverPath != null && !serverPath.isEmpty() && !serverPath.equals("/") ? serverPath : null,
eventExecutor,
rateLimiter,
connection,

View File

@@ -30,7 +30,6 @@ configurations {
transitive = false
canBeConsumed = true
canBeResolved = true
visible = true
}
configureNativeImageImplementation {
@@ -51,6 +50,7 @@ dependencies {
configureNativeImageImplementation project
configureNativeImageImplementation project(':rbcs-server-memcache')
configureNativeImageImplementation project(':rbcs-server-redis')
// configureNativeImageImplementation project(':rbcs-server-otel')
implementation catalog.jwo
implementation catalog.slf4j.api
@@ -59,9 +59,7 @@ dependencies {
implementation project(':rbcs-client')
implementation project(':rbcs-server')
// runtimeOnly catalog.slf4j.jdk14
runtimeOnly catalog.logback.classic
// runtimeOnly catalog.slf4j.simple
nativeImage project(':rbcs-server-memcache')
nativeImage project(':rbcs-server-redis')
@@ -142,7 +140,12 @@ Provider<JlinkTask> jlinkTaskProvider = tasks.named(JlinkPlugin.JLINK_TASK_NAME,
'net.woggioni.rbcs.server.memcache',
'net.woggioni.rbcs.server.redis',
'ch.qos.logback.classic',
'jdk.crypto.ec'
'jdk.crypto.ec',
// 'io.opentelemetry.api',
// 'io.opentelemetry.instrumentation.netty_4_1',
// 'io.opentelemetry.sdk.autoconfigure',
// 'io.opentelemetry.instrumentation.logback_appender_1_0',
// 'io.opentelemetry.extension.trace.propagation'
]
compressionLevel = 2
stripDebug = false

View File

@@ -120,10 +120,11 @@ object GraalNativeImageConfiguration {
val serverConfiguration = Configuration(
"127.0.0.1",
serverPort,
null,
false,
emptyList(),
false,
100,
null,
Configuration.EventExecutor(true),
Configuration.RateLimiter(
false, 0x100000, 10

View File

@@ -0,0 +1,60 @@
plugins {
id 'java-library'
id 'maven-publish'
alias catalog.plugins.kotlin.jvm
}
configurations {
bundle {
canBeResolved = true
canBeConsumed = false
transitive = true
resolutionStrategy {
dependencies {
exclude group: 'org.slf4j', module: 'slf4j-api'
exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib'
exclude group: 'org.jetbrains', module: 'annotations'
}
}
}
implementation {
extendsFrom bundle
}
release {
transitive = false
canBeConsumed = true
canBeResolved = true
}
}
dependencies {
bundle catalog.opentelemetry.netty['4']['1']
bundle catalog.opentelemetry.sdk.extension.autoconfigure
bundle catalog.opentelemetry.logback.appender['1']['0']
bundle catalog.opentelemetry.extension.trace.propagators
bundle catalog.opentelemetry.exporter.otlp
}
Provider<Tar> bundleTask = tasks.register("bundle", Tar) {
from(configurations.bundle)
group = BasePlugin.BUILD_GROUP
}
tasks.named(BasePlugin.ASSEMBLE_TASK_NAME) {
dependsOn(bundleTask)
}
artifacts {
release(bundleTask)
}
publishing {
publications {
maven(MavenPublication) {
artifact bundleTask
}
}
}

View File

@@ -13,6 +13,11 @@ dependencies {
implementation catalog.netty.buffer
implementation catalog.netty.transport
implementation catalog.netty.codec.haproxy
compileOnly catalog.opentelemetry.netty['4']['1']
compileOnly catalog.opentelemetry.sdk.extension.autoconfigure
compileOnly catalog.opentelemetry.logback.appender['1']['0']
compileOnly catalog.opentelemetry.extension.trace.propagators
compileOnly catalog.logback.classic
api project(':rbcs-common')
api project(':rbcs-api')

View File

@@ -17,6 +17,11 @@ module net.woggioni.rbcs.server {
requires io.netty.common;
requires io.netty.codec;
requires io.netty.codec.haproxy;
requires static io.opentelemetry.api;
requires static io.opentelemetry.instrumentation.netty_4_1;
requires static io.opentelemetry.sdk.autoconfigure;
requires static io.opentelemetry.instrumentation.logback_appender_1_0;
requires static io.opentelemetry.extension.trace.propagation;
requires org.slf4j;
exports net.woggioni.rbcs.server;

View File

@@ -2,16 +2,8 @@ package net.woggioni.rbcs.server
import io.netty.bootstrap.ServerBootstrap
import io.netty.buffer.ByteBuf
import io.netty.channel.Channel
import io.netty.channel.ChannelFactory
import io.netty.channel.ChannelFuture
import io.netty.channel.*
import io.netty.channel.ChannelHandler.Sharable
import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelInboundHandlerAdapter
import io.netty.channel.ChannelInitializer
import io.netty.channel.ChannelOption
import io.netty.channel.ChannelPromise
import io.netty.channel.MultiThreadIoEventLoopGroup
import io.netty.channel.nio.NioIoHandler
import io.netty.channel.socket.DatagramChannel
import io.netty.channel.socket.ServerSocketChannel
@@ -21,12 +13,7 @@ import io.netty.channel.socket.nio.NioServerSocketChannel
import io.netty.channel.socket.nio.NioSocketChannel
import io.netty.handler.codec.compression.CompressionOptions
import io.netty.handler.codec.haproxy.HAProxyMessageDecoder
import io.netty.handler.codec.http.DefaultHttpContent
import io.netty.handler.codec.http.HttpContentCompressor
import io.netty.handler.codec.http.HttpDecoderConfig
import io.netty.handler.codec.http.HttpHeaderNames
import io.netty.handler.codec.http.HttpRequest
import io.netty.handler.codec.http.HttpServerCodec
import io.netty.handler.codec.http.*
import io.netty.handler.ssl.ClientAuth
import io.netty.handler.ssl.SslContext
import io.netty.handler.ssl.SslContextBuilder
@@ -37,37 +24,15 @@ import io.netty.handler.timeout.IdleStateEvent
import io.netty.handler.timeout.IdleStateHandler
import io.netty.util.AttributeKey
import io.netty.util.concurrent.EventExecutorGroup
import java.io.OutputStream
import java.net.InetSocketAddress
import java.nio.file.Files
import java.nio.file.Path
import java.security.PrivateKey
import java.security.cert.X509Certificate
import java.time.Duration
import java.time.Instant
import java.util.Arrays
import java.util.Base64
import java.util.concurrent.CompletableFuture
import java.util.concurrent.Future
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import java.util.regex.Matcher
import java.util.regex.Pattern
import javax.naming.ldap.LdapName
import javax.net.ssl.SSLPeerUnverifiedException
import net.woggioni.rbcs.api.AsyncCloseable
import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.api.exception.ConfigurationException
import net.woggioni.rbcs.common.Cidr
import net.woggioni.rbcs.common.*
import net.woggioni.rbcs.common.PasswordSecurity.decodePasswordHash
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
import net.woggioni.rbcs.common.RBCS.getTrustManager
import net.woggioni.rbcs.common.RBCS.loadKeystore
import net.woggioni.rbcs.common.RBCS.toUrl
import net.woggioni.rbcs.common.Xml
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.debug
import net.woggioni.rbcs.common.info
import net.woggioni.rbcs.server.auth.AbstractNettyHttpAuthenticator
import net.woggioni.rbcs.server.auth.Authorizer
import net.woggioni.rbcs.server.auth.RoleAuthorizer
@@ -78,8 +43,26 @@ import net.woggioni.rbcs.server.handler.MaxRequestSizeHandler
import net.woggioni.rbcs.server.handler.ProxyProtocolHandler
import net.woggioni.rbcs.server.handler.ReadTriggerDuplexHandler
import net.woggioni.rbcs.server.handler.ServerHandler
import net.woggioni.rbcs.server.otel.OtelSdkIntegration
import net.woggioni.rbcs.server.throttling.BucketManager
import net.woggioni.rbcs.server.throttling.ThrottlingHandler
import java.io.OutputStream
import java.net.InetSocketAddress
import java.nio.file.Files
import java.nio.file.Path
import java.security.PrivateKey
import java.security.cert.X509Certificate
import java.time.Duration
import java.time.Instant
import java.util.*
import java.util.concurrent.CompletableFuture
import java.util.concurrent.Future
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import java.util.regex.Matcher
import java.util.regex.Pattern
import javax.naming.ldap.LdapName
import javax.net.ssl.SSLPeerUnverifiedException
class RemoteBuildCacheServer(private val cfg: Configuration) {
@@ -431,6 +414,9 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
maxChunkSize = cfg.connection.chunkSize
}
pipeline.addLast(HttpServerCodec(httpDecoderConfig))
if(cfg.isEnableTelemetry) {
OtelSdkIntegration.createHandler().let { pipeline.addLast(it) }
}
pipeline.addLast(ReadTriggerDuplexHandler.NAME, ReadTriggerDuplexHandler())
pipeline.addLast(MaxRequestSizeHandler.NAME, MaxRequestSizeHandler(cfg.connection.maxRequestSize))
pipeline.addLast(HttpChunkContentCompressor(1024))
@@ -525,6 +511,9 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
}
fun run(): ServerHandle {
if(cfg.isEnableTelemetry) {
OtelSdkIntegration.initialize()
}
// Create the multithreaded event loops for the server
val bossGroup = MultiThreadIoEventLoopGroup(1, NioIoHandler.newFactory())
val channelFactory = ChannelFactory<SocketChannel> { NioSocketChannel() }

View File

@@ -46,6 +46,8 @@ object Parser {
var groups = emptyMap<String, Group>()
var tls: Tls? = null
val serverPath = root.renderAttribute("path")
var enableTelemetry = root.renderAttribute("enable-telemetry")
?.let(String::toBoolean) ?: false
var incomingConnectionsBacklogSize = 1024
var authentication: Authentication? = null
for (child in root.asIterable()) {
@@ -233,6 +235,7 @@ object Parser {
}
}
return Configuration.of(
enableTelemetry,
host,
port,
proxyProtocolEnabled,

View File

@@ -29,6 +29,7 @@ object Serializer {
?.let { serverPath ->
attr("path", serverPath)
}
attr("enable-telemetry", conf.isEnableTelemetry.toString())
node("bind") {
attr("host", conf.host)
attr("port", conf.port.toString())

View File

@@ -0,0 +1,51 @@
package net.woggioni.rbcs.server.otel
import io.netty.channel.ChannelHandler
import io.opentelemetry.api.GlobalOpenTelemetry
import io.opentelemetry.instrumentation.netty.v4_1.NettyServerTelemetry
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.info
object OtelSdkIntegration {
private val log = createLogger<OtelSdkIntegration>()
private val appenderAvailable: Boolean by lazy {
runCatching {
Class.forName("io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender")
}.fold(
onSuccess = { true },
onFailure = {
log.info { "OpenTelemetry logback appender not on classpath" }
false
},
)
}
fun initialize() {
log.info { "Initializing OpenTelemetry SDK with auto-configuration" }
val sdk = io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk.builder()
.setResultAsGlobal()
.build()
.openTelemetrySdk
if (appenderAvailable) {
runCatching {
val clazz = Class.forName("io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender")
clazz.getMethod("install", Class.forName("io.opentelemetry.api.OpenTelemetry"))
.invoke(null, sdk)
log.info { "OpenTelemetry logback appender installed" }
}.onFailure { ex ->
val msg = ex.localizedMessage ?: ex.javaClass.name
log.info { "Failed to install OpenTelemetry logback appender: $msg" }
}
}
log.info { "OpenTelemetry SDK initialized successfully" }
}
fun createHandler(): ChannelHandler {
return NettyServerTelemetry.create(GlobalOpenTelemetry.get()).createCombinedHandler()
}
}

View File

@@ -2,7 +2,8 @@
<rbcs:server
xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server"
xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd">
xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"
enable-telemetry="false">
<bind host="127.0.0.1" port="8080" incoming-connections-backlog-size="1024"/>
<cache xs:type="rbcs:fileSystemCacheType" path="${sys:java.io.tmpdir}/rbcs" max-age="P7D"/>
</rbcs:server>

View File

@@ -59,6 +59,14 @@
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="enable-telemetry" type="xs:boolean" use="optional" default="false">
<xs:annotation>
<xs:documentation>
Enable OpenTelemetry distributed tracing for the server.
Even when enabled, telemetry only activates if OpenTelemetry classes are present on the classpath.
</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
<xs:complexType name="bindType">

View File

@@ -32,6 +32,7 @@ abstract class AbstractBasicAuthServerTest : AbstractServerTest() {
override fun setUp() {
this.cacheDir = testDir.resolve("cache")
cfg = Configuration.of(
false,
"127.0.0.1",
getFreePort(),
false,

View File

@@ -140,11 +140,11 @@ abstract class AbstractTlsServerTest : AbstractServerTest() {
cfg = Configuration(
"127.0.0.1",
getFreePort(),
serverPath,
false,
emptyList(),
false,
100,
serverPath,
Configuration.EventExecutor(false),
Configuration.RateLimiter(true, 0x100000, 50),
Configuration.Connection(

View File

@@ -34,10 +34,11 @@ class NoAuthServerTest : AbstractServerTest() {
cfg = Configuration(
"127.0.0.1",
getFreePort(),
serverPath,
false,
emptyList(),
false,
100,
serverPath,
Configuration.EventExecutor(false),
Configuration.RateLimiter(true, 0x100000, 50),
Configuration.Connection(

View File

@@ -1,7 +1,8 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server"
xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd">
xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"
enable-telemetry="true" path="/my/custom/path">
<bind host="127.0.0.1" port="11443" incoming-connections-backlog-size="22" proxy-protocol="true">
<trusted-proxies>
<allow cidr="192.168.0.11/32"/>

View File

@@ -33,4 +33,5 @@ include 'rbcs-cli'
include 'rbcs-client'
include 'rbcs-server'
include 'rbcs-servlet'
include 'rbcs-server-otel'
include 'docker'