Add optional OpenTelemetry Netty server instrumentation

- Update lys.version to 2026.04.14

- Add optional compileOnly dependency on opentelemetry-netty-4.1 in rbcs-server

- Add runtime guard to only activate instrumentation when OTel classes are on classpath

- Insert OTel combined handler after HttpServerCodec in the Netty pipeline

- Add requires-static JPMS directives for optional module support

- Add enableTelemetry config attribute to rbcs:server with default false

- Update Configuration DTO, XSD schema, Parser, Serializer, and all tests
This commit is contained in:
OpenCode
2026-04-28 14:59:08 +00:00
parent 70eccf83a8
commit ee7bc7e850
198 changed files with 11040 additions and 4 deletions

28
.project Normal file
View File

@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>rbcs</name>
<comment>Project rbcs created by Buildship.</comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
</natures>
<filteredResources>
<filter>
<id>1777370776929</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>

View File

@@ -0,0 +1,13 @@
arguments=--init-script /var/lib/opencode/.eclipse/2139989682_linux_gtk_x86_64/configuration/org.eclipse.osgi/58/0/.cp/gradle/init/init.gradle
auto.sync=false
build.scans.enabled=false
connection.gradle.distribution=GRADLE_DISTRIBUTION(WRAPPER)
connection.project.dir=
eclipse.preferences.version=1
gradle.user.home=
java.home=/usr/lib/jvm/java-26-openjdk
jvm.arguments=
offline.mode=false
override.workspace.settings=true
show.console.view=true
show.executions.view=true

28
docker/.project Normal file
View File

@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>docker</name>
<comment>Project docker created by Buildship.</comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
</natures>
<filteredResources>
<filter>
<id>1777370776861</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>

View File

@@ -0,0 +1,2 @@
connection.project.dir=..
eclipse.preferences.version=1

12
rbcs-api/.classpath Normal file
View File

@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="bin/main" path="src/main/java">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-25/"/>
<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
<classpathentry kind="output" path="bin/default"/>
</classpath>

34
rbcs-api/.project Normal file
View File

@@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>rbcs-api</name>
<comment>Project rbcs-api created by Buildship.</comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
</natures>
<filteredResources>
<filter>
<id>1777370776935</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>

View File

@@ -0,0 +1,2 @@
connection.project.dir=..
eclipse.preferences.version=1

View File

@@ -0,0 +1,4 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=25
org.eclipse.jdt.core.compiler.compliance=25
org.eclipse.jdt.core.compiler.source=25

Binary file not shown.

Binary file not shown.

View File

@@ -16,6 +16,7 @@ import java.util.stream.Collectors;
@Value @Value
public class Configuration { public class Configuration {
boolean enableTelemetry;
String host; String host;
int port; int port;
boolean proxyProtocolEnabled; boolean proxyProtocolEnabled;
@@ -150,6 +151,7 @@ public class Configuration {
} }
public static Configuration of( public static Configuration of(
boolean enableTelemetry,
String host, String host,
int port, int port,
boolean proxyProtocolEnabled, boolean proxyProtocolEnabled,
@@ -166,6 +168,7 @@ public class Configuration {
Tls tls Tls tls
) { ) {
return new Configuration( return new Configuration(
enableTelemetry,
host, host,
port, port,
proxyProtocolEnabled, proxyProtocolEnabled,

24
rbcs-cli/.classpath Normal file
View File

@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="bin/main" path="src/main/java">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/main" path="src/main/kotlin">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/configureNativeImage" path="src/configureNativeImage/kotlin">
<attributes>
<attribute name="gradle_scope" value="configureNativeImage"/>
<attribute name="gradle_used_by_scope" value="configureNativeImage"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-25/"/>
<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
<classpathentry kind="output" path="bin/default"/>
</classpath>

34
rbcs-cli/.project Normal file
View File

@@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>rbcs-cli</name>
<comment>Project rbcs-cli created by Buildship.</comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
</natures>
<filteredResources>
<filter>
<id>1777370776944</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>

View File

@@ -0,0 +1,2 @@
connection.project.dir=..
eclipse.preferences.version=1

View File

@@ -0,0 +1,4 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=25
org.eclipse.jdt.core.compiler.compliance=25
org.eclipse.jdt.core.compiler.source=25

View File

@@ -0,0 +1,200 @@
package net.woggioni.rbcs.cli.graal
import java.io.ByteArrayInputStream
import java.net.URI
import java.nio.file.Path
import java.time.Duration
import java.time.temporal.ChronoUnit
import java.util.concurrent.ExecutionException
import java.util.zip.Deflater
import net.woggioni.jwo.NullOutputStream
import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.api.Configuration.User
import net.woggioni.rbcs.api.Role
import net.woggioni.rbcs.cli.RemoteBuildCacheServerCli
import net.woggioni.rbcs.cli.impl.commands.BenchmarkCommand
import net.woggioni.rbcs.cli.impl.commands.GetCommand
import net.woggioni.rbcs.cli.impl.commands.HealthCheckCommand
import net.woggioni.rbcs.cli.impl.commands.PutCommand
import net.woggioni.rbcs.client.Configuration as ClientConfiguration
import net.woggioni.rbcs.client.impl.Parser as ClientConfigurationParser
import net.woggioni.rbcs.common.HostAndPort
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
import net.woggioni.rbcs.common.RBCS
import net.woggioni.rbcs.common.Xml
import net.woggioni.rbcs.server.RemoteBuildCacheServer
import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration
import net.woggioni.rbcs.server.cache.InMemoryCacheConfiguration
import net.woggioni.rbcs.server.configuration.Parser
import net.woggioni.rbcs.server.memcache.MemcacheCacheConfiguration
import net.woggioni.rbcs.server.redis.RedisCacheConfiguration
object GraalNativeImageConfiguration {
@JvmStatic
fun main(vararg args : String) {
let {
val serverURL = URI.create("file:conf/rbcs-server.xml").toURL()
val serverDoc = serverURL.openStream().use {
Xml.parseXml(serverURL, it)
}
Parser.parse(serverDoc)
}
let {
val serverURL = URI.create("file:conf/rbcs-server-redis.xml").toURL()
val serverDoc = serverURL.openStream().use {
Xml.parseXml(serverURL, it)
}
Parser.parse(serverDoc)
}
val url = URI.create("file:conf/rbcs-client.xml").toURL()
val clientDoc = url.openStream().use {
Xml.parseXml(url, it)
}
ClientConfigurationParser.parse(clientDoc)
val PASSWORD = "password"
val readersGroup = Configuration.Group("readers", setOf(Role.Reader, Role.Healthcheck), null, null)
val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null, null)
val users = listOf(
User("user1", hashPassword(PASSWORD), setOf(readersGroup), null),
User("user2", hashPassword(PASSWORD), setOf(writersGroup), null),
User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup), null),
User("", null, setOf(readersGroup), null),
User("user4", hashPassword(PASSWORD), setOf(readersGroup),
Configuration.Quota(1, Duration.of(1, ChronoUnit.DAYS), 0, 1)
),
User("user5", hashPassword(PASSWORD), setOf(readersGroup),
Configuration.Quota(1, Duration.of(5, ChronoUnit.SECONDS), 0, 1)
)
)
val serverPort = RBCS.getFreePort()
val caches = listOf<Configuration.Cache>(
InMemoryCacheConfiguration(
maxAge = Duration.ofSeconds(3600),
digestAlgorithm = "MD5",
compressionLevel = Deflater.DEFAULT_COMPRESSION,
compressionEnabled = false,
maxSize = 0x1000000,
),
FileSystemCacheConfiguration(
Path.of(System.getProperty("java.io.tmpdir")).resolve("rbcs"),
maxAge = Duration.ofSeconds(3600),
digestAlgorithm = "MD5",
compressionLevel = Deflater.DEFAULT_COMPRESSION,
compressionEnabled = false,
),
MemcacheCacheConfiguration(
listOf(MemcacheCacheConfiguration.Server(
HostAndPort("127.0.0.1", 11211),
1000,
4)
),
Duration.ofSeconds(60),
"someCustomPrefix",
"MD5",
null,
1,
),
RedisCacheConfiguration(
listOf(RedisCacheConfiguration.Server(
HostAndPort("127.0.0.1", 6379),
1000,
4)
),
Duration.ofSeconds(60),
"someCustomPrefix",
"MD5",
null,
1,
)
)
for (cache in caches) {
val serverConfiguration = Configuration(
"127.0.0.1",
serverPort,
false,
emptyList(),
100,
null,
Configuration.EventExecutor(true),
Configuration.RateLimiter(
false, 0x100000, 10
),
Configuration.Connection(
Duration.ofSeconds(10),
Duration.ofSeconds(15),
Duration.ofSeconds(15),
0x10000,
0x1000
),
users.asSequence().map { it.name to it }.toMap(),
sequenceOf(writersGroup, readersGroup).map { it.name to it }.toMap(),
cache,
Configuration.BasicAuthentication(),
null,
)
val serverHandle = RemoteBuildCacheServer(serverConfiguration).run()
val clientProfile = ClientConfiguration.Profile(
URI.create("http://127.0.0.1:$serverPort/"),
ClientConfiguration.Connection(
Duration.ofSeconds(5),
Duration.ofSeconds(5),
Duration.ofSeconds(7),
true,
),
ClientConfiguration.Authentication.BasicAuthenticationCredentials("user3", PASSWORD),
Duration.ofSeconds(3),
10,
true,
ClientConfiguration.RetryPolicy(
3,
1000,
1.2
),
ClientConfiguration.TrustStore(null, null, false, false)
)
HealthCheckCommand.execute(clientProfile)
BenchmarkCommand.execute(
clientProfile,
1000,
0x100,
true
)
PutCommand.execute(
clientProfile,
"some-file.bin",
ByteArrayInputStream(ByteArray(0x1000) { it.toByte() }),
"application/octet-setream",
"attachment; filename=\"some-file.bin\""
)
GetCommand.execute(
clientProfile,
"some-file.bin",
NullOutputStream()
)
serverHandle.sendShutdownSignal()
try {
serverHandle.get()
} catch (ee : ExecutionException) {
}
}
System.setProperty("net.woggioni.rbcs.conf.dir", System.getProperty("gradle.tmp.dir"))
RemoteBuildCacheServerCli.createCommandLine().execute("--version")
RemoteBuildCacheServerCli.createCommandLine().execute("server", "-t", "PT10S")
}
}

Binary file not shown.

View File

@@ -0,0 +1,79 @@
package net.woggioni.rbcs.cli
import net.woggioni.jwo.Application
import net.woggioni.rbcs.cli.impl.AbstractVersionProvider
import net.woggioni.rbcs.cli.impl.RbcsCommand
import net.woggioni.rbcs.cli.impl.commands.BenchmarkCommand
import net.woggioni.rbcs.cli.impl.commands.ClientCommand
import net.woggioni.rbcs.cli.impl.commands.GetCommand
import net.woggioni.rbcs.cli.impl.commands.HealthCheckCommand
import net.woggioni.rbcs.cli.impl.commands.PasswordHashCommand
import net.woggioni.rbcs.cli.impl.commands.PutCommand
import net.woggioni.rbcs.cli.impl.commands.ServerCommand
import net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory
import net.woggioni.rbcs.common.createLogger
import picocli.CommandLine
import picocli.CommandLine.Model.CommandSpec
@CommandLine.Command(
name = "rbcs", versionProvider = RemoteBuildCacheServerCli.VersionProvider::class
)
class RemoteBuildCacheServerCli : RbcsCommand() {
class VersionProvider : AbstractVersionProvider()
companion object {
private fun setPropertyIfNotPresent(key: String, value: String) {
System.getProperty(key) ?: System.setProperty(key, value)
}
fun createCommandLine() : CommandLine {
setPropertyIfNotPresent("logback.configurationFile", "net/woggioni/rbcs/cli/logback.xml")
setPropertyIfNotPresent("io.netty.leakDetectionLevel", "DISABLED")
val currentClassLoader = RemoteBuildCacheServerCli::class.java.classLoader
Thread.currentThread().contextClassLoader = currentClassLoader
if(currentClassLoader.javaClass.name == "net.woggioni.envelope.loader.ModuleClassLoader") {
//We're running in an envelope jar and custom URL protocols won't work
RbcsUrlStreamHandlerFactory.install()
}
val log = createLogger<RemoteBuildCacheServerCli>()
val app = Application.builder("rbcs")
.configurationDirectoryEnvVar("RBCS_CONFIGURATION_DIR")
.configurationDirectoryPropertyKey("net.woggioni.rbcs.conf.dir")
.build()
val rbcsCli = RemoteBuildCacheServerCli()
val commandLine = CommandLine(rbcsCli)
commandLine.setExecutionExceptionHandler { ex, cl, parseResult ->
log.error(ex.message, ex)
CommandLine.ExitCode.SOFTWARE
}
commandLine.addSubcommand(ServerCommand(app))
commandLine.addSubcommand(PasswordHashCommand())
commandLine.addSubcommand(
CommandLine(ClientCommand(app)).apply {
addSubcommand(BenchmarkCommand())
addSubcommand(PutCommand())
addSubcommand(GetCommand())
addSubcommand(HealthCheckCommand())
})
return commandLine
}
@JvmStatic
fun main(vararg args: String) {
System.exit(createCommandLine().execute(*args))
}
}
@CommandLine.Option(names = ["-V", "--version"], versionHelp = true)
var versionHelp = false
private set
@CommandLine.Spec
private lateinit var spec: CommandSpec
override fun run() {
spec.commandLine().usage(System.out);
}
}

View File

@@ -0,0 +1,30 @@
package net.woggioni.rbcs.cli.impl
import java.util.jar.Attributes
import java.util.jar.JarFile
import java.util.jar.Manifest
import picocli.CommandLine
abstract class AbstractVersionProvider : CommandLine.IVersionProvider {
private val version: String
private val vcsHash: String
init {
val mf = Manifest()
javaClass.module.getResourceAsStream(JarFile.MANIFEST_NAME).use { `is` ->
mf.read(`is`)
}
val mainAttributes = mf.mainAttributes
version = mainAttributes.getValue(Attributes.Name.SPECIFICATION_VERSION) ?: throw RuntimeException("Version information not found in manifest")
vcsHash = mainAttributes.getValue(Attributes.Name.IMPLEMENTATION_VERSION) ?: throw RuntimeException("Version information not found in manifest")
}
override fun getVersion(): Array<String?> {
return if (version.endsWith("-SNAPSHOT")) {
arrayOf(version, vcsHash)
} else {
arrayOf(version)
}
}
}

View File

@@ -0,0 +1,19 @@
package net.woggioni.rbcs.cli.impl
import java.nio.file.Path
import net.woggioni.jwo.Application
import picocli.CommandLine
abstract class RbcsCommand : Runnable {
@CommandLine.Option(names = ["-h", "--help"], usageHelp = true)
var usageHelp = false
private set
protected fun findConfigurationFile(app: Application, fileName : String): Path {
val confDir = app.computeConfigurationDirectory(false)
val configurationFile = confDir.resolve(fileName)
return configurationFile
}
}

View File

@@ -0,0 +1,199 @@
package net.woggioni.rbcs.cli.impl.commands
import java.security.SecureRandom
import java.time.Duration
import java.time.Instant
import java.time.temporal.ChronoUnit
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.Semaphore
import java.util.concurrent.atomic.AtomicLong
import kotlin.random.Random
import net.woggioni.jwo.JWO
import net.woggioni.jwo.LongMath
import net.woggioni.rbcs.api.CacheValueMetadata
import net.woggioni.rbcs.cli.impl.RbcsCommand
import net.woggioni.rbcs.cli.impl.converters.ByteSizeConverter
import net.woggioni.rbcs.client.Configuration
import net.woggioni.rbcs.client.RemoteBuildCacheClient
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.debug
import net.woggioni.rbcs.common.error
import net.woggioni.rbcs.common.info
import picocli.CommandLine
@CommandLine.Command(
name = "benchmark",
description = ["Run a load test against the server"],
showDefaultValues = true
)
class BenchmarkCommand : RbcsCommand() {
companion object {
private val log = createLogger<BenchmarkCommand>()
fun execute(profile : Configuration.Profile,
numberOfEntries : Int,
entrySize : Int,
useRandomValue : Boolean,
) {
val progressThreshold = LongMath.ceilDiv(numberOfEntries.toLong(), 20)
RemoteBuildCacheClient(profile).use { client ->
val entryGenerator = sequence {
val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong())
while (true) {
val key = JWO.bytesToHex(random.nextBytes(16))
val value = if (useRandomValue) {
random.nextBytes(entrySize)
} else {
val byteValue = random.nextInt().toByte()
ByteArray(entrySize) { _ -> byteValue }
}
yield(key to value)
}
}
log.info {
"Starting insertion"
}
val entries = let {
val completionCounter = AtomicLong(0)
val completionQueue = LinkedBlockingQueue<Pair<String, ByteArray>>(numberOfEntries)
val start = Instant.now()
val semaphore = Semaphore(profile.maxConnections * 5)
val iterator = entryGenerator.take(numberOfEntries).iterator()
while (completionCounter.get() < numberOfEntries) {
if (iterator.hasNext()) {
val entry = iterator.next()
semaphore.acquire()
val future =
client.put(entry.first, entry.second, CacheValueMetadata(null, null)).thenApply { entry }
future.whenComplete { result, ex ->
if (ex != null) {
log.error(ex.message, ex)
} else {
completionQueue.put(result)
}
semaphore.release()
val completed = completionCounter.incrementAndGet()
if (completed.mod(progressThreshold) == 0L) {
log.debug {
"Inserted $completed / $numberOfEntries"
}
}
}
} else {
Thread.sleep(Duration.of(500, ChronoUnit.MILLIS))
}
}
val inserted = completionQueue.toList()
val end = Instant.now()
log.info {
val elapsed = Duration.between(start, end).toMillis()
val opsPerSecond = String.format("%.2f", numberOfEntries.toDouble() / elapsed * 1000)
"Insertion rate: $opsPerSecond ops/s"
}
inserted
}
log.info {
"Inserted ${entries.size} entries"
}
log.info {
"Starting retrieval"
}
if (entries.isNotEmpty()) {
val errorCounter = AtomicLong(0)
val completionCounter = AtomicLong(0)
val semaphore = Semaphore(profile.maxConnections * 5)
val start = Instant.now()
val it = entries.iterator()
while (completionCounter.get() < entries.size) {
if (it.hasNext()) {
val entry = it.next()
semaphore.acquire()
val future = client.get(entry.first).handle { response, ex ->
if(ex != null) {
errorCounter.incrementAndGet()
log.error(ex.message, ex)
} else if (response == null) {
errorCounter.incrementAndGet()
log.error {
"Missing entry for key '${entry.first}'"
}
} else if (!entry.second.contentEquals(response)) {
errorCounter.incrementAndGet()
log.error {
"Retrieved a value different from what was inserted for key '${entry.first}': " +
"expected '${JWO.bytesToHex(entry.second)}', got '${JWO.bytesToHex(response)}' instead"
}
}
}
future.whenComplete { _, _ ->
val completed = completionCounter.incrementAndGet()
if (completed.mod(progressThreshold) == 0L) {
log.debug {
"Retrieved $completed / ${entries.size}"
}
}
semaphore.release()
}
} else {
Thread.sleep(Duration.of(500, ChronoUnit.MILLIS))
}
}
val end = Instant.now()
val errors = errorCounter.get()
val successfulRetrievals = entries.size - errors
val successRate = successfulRetrievals.toDouble() / entries.size
log.info {
"Successfully retrieved ${entries.size - errors}/${entries.size} (${String.format("%.1f", successRate * 100)}%)"
}
log.info {
val elapsed = Duration.between(start, end).toMillis()
val opsPerSecond = String.format("%.2f", entries.size.toDouble() / elapsed * 1000)
"Retrieval rate: $opsPerSecond ops/s"
}
} else {
log.error("Skipping retrieval benchmark as it was not possible to insert any entry in the cache")
}
}
}
}
@CommandLine.Spec
private lateinit var spec: CommandLine.Model.CommandSpec
@CommandLine.Option(
names = ["-e", "--entries"],
description = ["Total number of elements to be added to the cache"],
paramLabel = "NUMBER_OF_ENTRIES"
)
private var numberOfEntries = 1000
@CommandLine.Option(
names = ["-s", "--size"],
description = ["Size of a cache value in bytes"],
paramLabel = "SIZE",
converter = [ByteSizeConverter::class]
)
private var size = 0x1000
@CommandLine.Option(
names = ["-r", "--random"],
description = ["Insert completely random byte values"]
)
private var randomValues = false
override fun run() {
val clientCommand = spec.parent().userObject() as ClientCommand
val profile = clientCommand.profileName.let { profileName ->
clientCommand.configuration.profiles[profileName]
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
}
execute(
profile,
numberOfEntries,
size,
randomValues
)
}
}

View File

@@ -0,0 +1,48 @@
package net.woggioni.rbcs.cli.impl.commands
import java.nio.file.Path
import net.woggioni.jwo.Application
import net.woggioni.rbcs.cli.impl.RbcsCommand
import net.woggioni.rbcs.client.Configuration
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.debug
import picocli.CommandLine
@CommandLine.Command(
name = "client",
description = ["RBCS client"],
showDefaultValues = true
)
class ClientCommand(app : Application) : RbcsCommand() {
@CommandLine.Option(
names = ["-c", "--configuration"],
description = ["Path to the client configuration file"],
paramLabel = "CONFIGURATION_FILE"
)
private var configurationFile : Path = findConfigurationFile(app, "rbcs-client.xml")
@CommandLine.Option(
names = ["-p", "--profile"],
description = ["Name of the client profile to be used"],
paramLabel = "PROFILE",
required = false
)
var profileName : String? = null
get() = field ?: throw IllegalArgumentException("A profile name must be specified using the '-p' command line parameter")
val configuration : Configuration by lazy {
Configuration.parse(configurationFile)
}
override fun run() {
val log = createLogger<ClientCommand>()
log.debug {
"Using configuration file '$configurationFile'"
}
println("Available profiles:")
configuration.profiles.forEach { (profileName, _) ->
println(profileName)
}
}
}

View File

@@ -0,0 +1,59 @@
package net.woggioni.rbcs.cli.impl.commands
import java.io.OutputStream
import java.nio.file.Files
import java.nio.file.Path
import net.woggioni.rbcs.cli.impl.RbcsCommand
import net.woggioni.rbcs.client.Configuration
import net.woggioni.rbcs.client.RemoteBuildCacheClient
import net.woggioni.rbcs.common.createLogger
import picocli.CommandLine
@CommandLine.Command(
name = "get",
description = ["Fetch a value from the cache with the specified key"],
showDefaultValues = true
)
class GetCommand : RbcsCommand() {
companion object {
private val log = createLogger<GetCommand>()
fun execute(profile : Configuration.Profile, key : String, outputStream: OutputStream) {
RemoteBuildCacheClient(profile).use { client ->
client.get(key).thenApply { value ->
value?.let {
outputStream.use {
it.write(value)
}
} ?: throw NoSuchElementException("No value found for key $key")
}.get()
}
}
}
@CommandLine.Spec
private lateinit var spec: CommandLine.Model.CommandSpec
@CommandLine.Option(
names = ["-k", "--key"],
description = ["The key for the new value"],
paramLabel = "KEY"
)
private var key : String = ""
@CommandLine.Option(
names = ["-v", "--value"],
description = ["Path to a file where the retrieved value will be written (defaults to stdout)"],
paramLabel = "VALUE_FILE",
)
private var output : Path? = null
override fun run() {
val clientCommand = spec.parent().userObject() as ClientCommand
val profile = clientCommand.profileName.let { profileName ->
clientCommand.configuration.profiles[profileName]
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
}
execute(profile, key, (output?.let(Files::newOutputStream) ?: System.out))
}
}

View File

@@ -0,0 +1,53 @@
package net.woggioni.rbcs.cli.impl.commands
import java.security.SecureRandom
import kotlin.random.Random
import net.woggioni.rbcs.cli.impl.RbcsCommand
import net.woggioni.rbcs.client.Configuration
import net.woggioni.rbcs.client.RemoteBuildCacheClient
import net.woggioni.rbcs.common.createLogger
import picocli.CommandLine
@CommandLine.Command(
name = "health",
description = ["Check server health"],
showDefaultValues = true
)
class HealthCheckCommand : RbcsCommand() {
companion object{
private val log = createLogger<HealthCheckCommand>()
fun execute(profile : Configuration.Profile) {
RemoteBuildCacheClient(profile).use { client ->
val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong())
val nonce = ByteArray(0xa0)
random.nextBytes(nonce)
client.healthCheck(nonce).thenApply { value ->
if(value == null) {
throw IllegalStateException("Empty response from server")
}
val offset = value.size - nonce.size
for(i in 0 until nonce.size) {
val a = nonce[i]
val b = value[offset + i]
if(a != b) {
throw IllegalStateException("Server nonce does not match")
}
}
}.get()
}
}
}
@CommandLine.Spec
private lateinit var spec: CommandLine.Model.CommandSpec
override fun run() {
val clientCommand = spec.parent().userObject() as ClientCommand
val profile = clientCommand.profileName.let { profileName ->
clientCommand.configuration.profiles[profileName]
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
}
execute(profile)
}
}

View File

@@ -0,0 +1,37 @@
package net.woggioni.rbcs.cli.impl.commands
import java.io.OutputStream
import java.io.OutputStreamWriter
import java.io.PrintWriter
import net.woggioni.jwo.UncloseableOutputStream
import net.woggioni.rbcs.cli.impl.RbcsCommand
import net.woggioni.rbcs.cli.impl.converters.OutputStreamConverter
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
import picocli.CommandLine
@CommandLine.Command(
name = "password",
description = ["Generate a password hash to add to RBCS configuration file"],
showDefaultValues = true
)
class PasswordHashCommand : RbcsCommand() {
@CommandLine.Option(
names = ["-o", "--output-file"],
description = ["Write the output to a file instead of stdout"],
converter = [OutputStreamConverter::class],
showDefaultValue = CommandLine.Help.Visibility.NEVER,
paramLabel = "OUTPUT_FILE"
)
private var outputStream: OutputStream = UncloseableOutputStream(System.out)
override fun run() {
val password1 = String(System.console().readPassword("Type your password:"))
val password2 = String(System.console().readPassword("Type your password again for confirmation:"))
if(password1 != password2) throw IllegalArgumentException("Passwords do not match")
PrintWriter(OutputStreamWriter(outputStream, Charsets.UTF_8)).use {
it.println(hashPassword(password1))
}
}
}

View File

@@ -0,0 +1,113 @@
package net.woggioni.rbcs.cli.impl.commands
import java.io.InputStream
import java.nio.file.Files
import java.nio.file.Path
import java.util.UUID
import net.woggioni.jwo.Hash
import net.woggioni.jwo.JWO
import net.woggioni.jwo.NullOutputStream
import net.woggioni.rbcs.api.CacheValueMetadata
import net.woggioni.rbcs.cli.impl.RbcsCommand
import net.woggioni.rbcs.client.Configuration
import net.woggioni.rbcs.client.RemoteBuildCacheClient
import net.woggioni.rbcs.common.createLogger
import picocli.CommandLine
@CommandLine.Command(
name = "put",
description = ["Add or replace a value to the cache with the specified key"],
showDefaultValues = true
)
class PutCommand : RbcsCommand() {
companion object {
private val log = createLogger<PutCommand>()
fun execute(profile: Configuration.Profile,
actualKey : String,
inputStream: InputStream,
mimeType : String?,
contentDisposition: String?
) {
RemoteBuildCacheClient(profile).use { client ->
inputStream.use {
client.put(actualKey, it.readAllBytes(), CacheValueMetadata(contentDisposition, mimeType))
}.get()
println(profile.serverURI.resolve(actualKey))
}
}
}
@CommandLine.Spec
private lateinit var spec: CommandLine.Model.CommandSpec
@CommandLine.Option(
names = ["-k", "--key"],
description = ["The key for the new value, randomly generated if omitted"],
paramLabel = "KEY"
)
private var key : String? = null
@CommandLine.Option(
names = ["-i", "--inline"],
description = ["File is to be displayed in the browser"],
paramLabel = "INLINE",
)
private var inline : Boolean = false
@CommandLine.Option(
names = ["-t", "--type"],
description = ["File mime type"],
paramLabel = "MIME_TYPE",
)
private var mimeType : String? = null
@CommandLine.Option(
names = ["-v", "--value"],
description = ["Path to a file containing the value to be added (defaults to stdin)"],
paramLabel = "VALUE_FILE",
)
private var value : Path? = null
override fun run() {
val clientCommand = spec.parent().userObject() as ClientCommand
val profile = clientCommand.profileName.let { profileName ->
clientCommand.configuration.profiles[profileName]
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
}
RemoteBuildCacheClient(profile).use { client ->
val inputStream : InputStream
val mimeType : String?
val contentDisposition : String?
val valuePath = value
val actualKey : String?
if(valuePath != null) {
inputStream = Files.newInputStream(valuePath)
mimeType = this.mimeType ?: Files.probeContentType(valuePath)
contentDisposition = if(inline) {
"inline"
} else {
"attachment; filename=\"${valuePath.fileName}\""
}
actualKey = key ?: let {
val md = Hash.Algorithm.SHA512.newInputStream(Files.newInputStream(valuePath)).use {
JWO.copy(it, NullOutputStream())
it.messageDigest
}
UUID.nameUUIDFromBytes(md.digest()).toString()
}
} else {
inputStream = System.`in`
mimeType = this.mimeType
contentDisposition = if(inline) {
"inline"
} else {
null
}
actualKey = key ?: UUID.randomUUID().toString()
}
execute(profile, actualKey, inputStream, mimeType, contentDisposition)
}
}
}

View File

@@ -0,0 +1,90 @@
package net.woggioni.rbcs.cli.impl.commands
import java.io.ByteArrayOutputStream
import java.nio.file.Files
import java.nio.file.Path
import java.time.Duration
import java.util.concurrent.TimeUnit
import net.woggioni.jwo.Application
import net.woggioni.jwo.JWO
import net.woggioni.rbcs.cli.impl.RbcsCommand
import net.woggioni.rbcs.cli.impl.converters.DurationConverter
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.debug
import net.woggioni.rbcs.common.info
import net.woggioni.rbcs.server.RemoteBuildCacheServer
import net.woggioni.rbcs.server.RemoteBuildCacheServer.Companion.DEFAULT_CONFIGURATION_URL
import picocli.CommandLine
@CommandLine.Command(
name = "server",
description = ["RBCS server"],
showDefaultValues = true
)
class ServerCommand(app : Application) : RbcsCommand() {
companion object {
private val log = createLogger<ServerCommand>()
}
private fun createDefaultConfigurationFile(configurationFile: Path) {
log.info {
"Creating default configuration file at '$configurationFile'"
}
val defaultConfigurationFileResource = DEFAULT_CONFIGURATION_URL
Files.newOutputStream(configurationFile).use { outputStream ->
defaultConfigurationFileResource.openStream().use { inputStream ->
JWO.copy(inputStream, outputStream)
}
}
}
@CommandLine.Option(
names = ["-t", "--timeout"],
description = ["Exit after the specified time"],
paramLabel = "TIMEOUT",
converter = [DurationConverter::class]
)
private var timeout: Duration? = null
@CommandLine.Option(
names = ["-c", "--config-file"],
description = ["Read the application configuration from this file"],
paramLabel = "CONFIG_FILE"
)
private var configurationFile: Path = findConfigurationFile(app, "rbcs-server.xml")
override fun run() {
if (!Files.exists(configurationFile)) {
Files.createDirectories(configurationFile.parent)
createDefaultConfigurationFile(configurationFile)
}
log.debug {
"Using configuration file '$configurationFile'"
}
val configuration = RemoteBuildCacheServer.loadConfiguration(configurationFile)
log.debug {
ByteArrayOutputStream().also {
RemoteBuildCacheServer.dumpConfiguration(configuration, it)
}.let {
"Server configuration:\n${String(it.toByteArray())}"
}
}
val server = RemoteBuildCacheServer(configuration)
val handle = server.run()
val shutdownHook = Thread.ofPlatform().unstarted {
handle.sendShutdownSignal()
try {
handle.get(60, TimeUnit.SECONDS)
} catch (ex : Throwable) {
log.warn(ex.message, ex)
}
}
Runtime.getRuntime().addShutdownHook(shutdownHook)
if(timeout != null) {
Thread.sleep(timeout)
handle.sendShutdownSignal()
}
handle.get()
}
}

View File

@@ -0,0 +1,10 @@
package net.woggioni.rbcs.cli.impl.converters
import picocli.CommandLine
class ByteSizeConverter : CommandLine.ITypeConverter<Int> {
override fun convert(value: String): Int {
return Integer.decode(value)
}
}

View File

@@ -0,0 +1,11 @@
package net.woggioni.rbcs.cli.impl.converters
import java.time.Duration
import picocli.CommandLine
class DurationConverter : CommandLine.ITypeConverter<Duration> {
override fun convert(value: String): Duration {
return Duration.parse(value)
}
}

View File

@@ -0,0 +1,13 @@
package net.woggioni.rbcs.cli.impl.converters
import java.io.InputStream
import java.nio.file.Files
import java.nio.file.Paths
import picocli.CommandLine
class InputStreamConverter : CommandLine.ITypeConverter<InputStream> {
override fun convert(value: String): InputStream {
return Files.newInputStream(Paths.get(value))
}
}

View File

@@ -0,0 +1,13 @@
package net.woggioni.rbcs.cli.impl.converters
import java.io.OutputStream
import java.nio.file.Files
import java.nio.file.Paths
import picocli.CommandLine
class OutputStreamConverter : CommandLine.ITypeConverter<OutputStream> {
override fun convert(value: String): OutputStream {
return Files.newOutputStream(Paths.get(value))
}
}

38
rbcs-client/.classpath Normal file
View File

@@ -0,0 +1,38 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="bin/main" path="src/main/java">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/main" path="src/main/kotlin">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/main" path="src/main/resources">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/test" path="src/test/kotlin">
<attributes>
<attribute name="gradle_scope" value="test"/>
<attribute name="gradle_used_by_scope" value="test"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/test" path="src/test/resources">
<attributes>
<attribute name="gradle_scope" value="test"/>
<attribute name="gradle_used_by_scope" value="test"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-25/"/>
<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
<classpathentry kind="output" path="bin/default"/>
</classpath>

34
rbcs-client/.project Normal file
View File

@@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>rbcs-client</name>
<comment>Project rbcs-client created by Buildship.</comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
</natures>
<filteredResources>
<filter>
<id>1777370776950</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>

View File

@@ -0,0 +1,2 @@
connection.project.dir=..
eclipse.preferences.version=1

View File

@@ -0,0 +1,4 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=25
org.eclipse.jdt.core.compiler.compliance=25
org.eclipse.jdt.core.compiler.source=25

Binary file not shown.

View File

@@ -0,0 +1,62 @@
package net.woggioni.rbcs.client
import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
import java.security.PrivateKey
import java.security.cert.X509Certificate
import java.time.Duration
import net.woggioni.rbcs.client.impl.Parser
import net.woggioni.rbcs.common.Xml
data class Configuration(
val profiles: Map<String, Profile>
) {
sealed class Authentication {
data class TlsClientAuthenticationCredentials(
val key: PrivateKey,
val certificateChain: Array<X509Certificate>
) : Authentication()
data class BasicAuthenticationCredentials(val username: String, val password: String) : Authentication()
}
class TrustStore (
var file: Path?,
var password: String?,
var checkCertificateStatus: Boolean = false,
var verifyServerCertificate: Boolean = true,
)
class RetryPolicy(
val maxAttempts: Int,
val initialDelayMillis: Long,
val exp: Double
)
class Connection(
val readIdleTimeout: Duration,
val writeIdleTimeout: Duration,
val idleTimeout: Duration,
val requestPipelining : Boolean,
)
data class Profile(
val serverURI: URI,
val connection: Connection,
val authentication: Authentication?,
val connectionTimeout: Duration?,
val maxConnections: Int,
val compressionEnabled: Boolean,
val retryPolicy: RetryPolicy?,
val tlsTruststore : TrustStore?
)
companion object {
fun parse(path: Path): Configuration {
return Files.newInputStream(path).use {
Xml.parseXml(path.toUri().toURL(), it)
}.let(Parser::parse)
}
}
}

View File

@@ -0,0 +1,9 @@
package net.woggioni.rbcs.client
import io.netty.handler.codec.http.HttpResponseStatus
class HttpException(private val status : HttpResponseStatus) : RuntimeException(status.reasonPhrase()) {
override val message: String
get() = "Http status ${status.code()}: ${status.reasonPhrase()}"
}

View File

@@ -0,0 +1,433 @@
package net.woggioni.rbcs.client
import io.netty.bootstrap.Bootstrap
import io.netty.buffer.ByteBuf
import io.netty.buffer.Unpooled
import io.netty.channel.Channel
import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelOption
import io.netty.channel.ChannelPipeline
import io.netty.channel.IoEventLoopGroup
import io.netty.channel.MultiThreadIoEventLoopGroup
import io.netty.channel.SimpleChannelInboundHandler
import io.netty.channel.nio.NioIoHandler
import io.netty.channel.pool.AbstractChannelPoolHandler
import io.netty.channel.pool.ChannelPool
import io.netty.channel.pool.FixedChannelPool
import io.netty.channel.socket.nio.NioSocketChannel
import io.netty.handler.codec.DecoderException
import io.netty.handler.codec.http.DefaultFullHttpRequest
import io.netty.handler.codec.http.FullHttpRequest
import io.netty.handler.codec.http.FullHttpResponse
import io.netty.handler.codec.http.HttpClientCodec
import io.netty.handler.codec.http.HttpContentDecompressor
import io.netty.handler.codec.http.HttpHeaderNames
import io.netty.handler.codec.http.HttpHeaderValues
import io.netty.handler.codec.http.HttpMethod
import io.netty.handler.codec.http.HttpObjectAggregator
import io.netty.handler.codec.http.HttpResponseStatus
import io.netty.handler.codec.http.HttpVersion
import io.netty.handler.ssl.SslContext
import io.netty.handler.ssl.SslContextBuilder
import io.netty.handler.stream.ChunkedWriteHandler
import io.netty.handler.timeout.IdleState
import io.netty.handler.timeout.IdleStateEvent
import io.netty.handler.timeout.IdleStateHandler
import io.netty.util.concurrent.Future
import io.netty.util.concurrent.Future as NettyFuture
import io.netty.util.concurrent.GenericFutureListener
import java.io.IOException
import java.net.InetSocketAddress
import java.net.URI
import java.security.cert.X509Certificate
import java.util.Base64
import java.util.concurrent.CompletableFuture
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import java.util.concurrent.atomic.AtomicInteger
import javax.net.ssl.TrustManagerFactory
import javax.net.ssl.X509TrustManager
import kotlin.random.Random
import net.woggioni.rbcs.api.CacheValueMetadata
import net.woggioni.rbcs.common.RBCS.loadKeystore
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.debug
import net.woggioni.rbcs.common.trace
class RemoteBuildCacheClient(private val profile: Configuration.Profile) : AutoCloseable {
companion object {
private val log = createLogger<RemoteBuildCacheClient>()
}
private val group: IoEventLoopGroup
private val sslContext: SslContext
private val pool: ChannelPool
init {
group = MultiThreadIoEventLoopGroup(NioIoHandler.newFactory())
sslContext = SslContextBuilder.forClient().also { builder ->
(profile.authentication as? Configuration.Authentication.TlsClientAuthenticationCredentials)?.let { tlsClientAuthenticationCredentials ->
builder.apply {
keyManager(
tlsClientAuthenticationCredentials.key,
*tlsClientAuthenticationCredentials.certificateChain
)
profile.tlsTruststore?.let { trustStore ->
if (!trustStore.verifyServerCertificate) {
trustManager(object : X509TrustManager {
override fun checkClientTrusted(certChain: Array<out X509Certificate>, p1: String?) {
}
override fun checkServerTrusted(certChain: Array<out X509Certificate>, p1: String?) {
}
override fun getAcceptedIssuers() = null
})
} else {
trustStore.file?.let {
val ts = loadKeystore(it, trustStore.password)
val trustManagerFactory: TrustManagerFactory =
TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
trustManagerFactory.init(ts)
trustManager(trustManagerFactory)
}
}
}
}
}
}.build()
val (scheme, host, port) = profile.serverURI.run {
Triple(
if (scheme == null) "http" else profile.serverURI.scheme,
host,
port.takeIf { it > 0 } ?: if ("https" == scheme.lowercase()) 443 else 80
)
}
val bootstrap = Bootstrap().apply {
group(group)
channel(NioSocketChannel::class.java)
option(ChannelOption.TCP_NODELAY, true)
option(ChannelOption.SO_KEEPALIVE, true)
remoteAddress(InetSocketAddress(host, port))
profile.connectionTimeout?.let {
option(ChannelOption.CONNECT_TIMEOUT_MILLIS, it.toMillis().toInt())
}
}
val channelPoolHandler = object : AbstractChannelPoolHandler() {
@Volatile
private var connectionCount = AtomicInteger()
@Volatile
private var leaseCount = AtomicInteger()
override fun channelReleased(ch: Channel) {
val activeLeases = leaseCount.decrementAndGet()
log.trace {
"Released channel ${ch.id().asShortText()}, number of active leases: $activeLeases"
}
}
override fun channelAcquired(ch: Channel) {
val activeLeases = leaseCount.getAndIncrement()
log.trace {
"Acquired channel ${ch.id().asShortText()}, number of active leases: $activeLeases"
}
}
override fun channelCreated(ch: Channel) {
val connectionId = connectionCount.incrementAndGet()
log.debug {
"Created connection ${ch.id().asShortText()}, total number of active connections: $connectionId"
}
ch.closeFuture().addListener {
val activeConnections = connectionCount.decrementAndGet()
log.debug {
"Closed connection ${
ch.id().asShortText()
}, total number of active connections: $activeConnections"
}
}
val pipeline: ChannelPipeline = ch.pipeline()
profile.connection.also { conn ->
val readIdleTimeout = conn.readIdleTimeout.toMillis()
val writeIdleTimeout = conn.writeIdleTimeout.toMillis()
val idleTimeout = conn.idleTimeout.toMillis()
if (readIdleTimeout > 0 || writeIdleTimeout > 0 || idleTimeout > 0) {
pipeline.addLast(
IdleStateHandler(
true,
readIdleTimeout,
writeIdleTimeout,
idleTimeout,
TimeUnit.MILLISECONDS
)
)
}
}
// Add SSL handler if needed
if ("https".equals(scheme, ignoreCase = true)) {
pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port))
}
// HTTP handlers
pipeline.addLast("codec", HttpClientCodec())
if (profile.compressionEnabled) {
pipeline.addLast("decompressor", HttpContentDecompressor())
}
pipeline.addLast("aggregator", HttpObjectAggregator(134217728))
pipeline.addLast("chunked", ChunkedWriteHandler())
}
}
pool = FixedChannelPool(bootstrap, channelPoolHandler, profile.maxConnections)
}
private fun executeWithRetry(operation: () -> CompletableFuture<FullHttpResponse>): CompletableFuture<FullHttpResponse> {
val retryPolicy = profile.retryPolicy
return if (retryPolicy != null) {
val outcomeHandler = OutcomeHandler<FullHttpResponse> { outcome ->
when (outcome) {
is OperationOutcome.Success -> {
val response = outcome.result
val status = response.status()
when (status) {
HttpResponseStatus.TOO_MANY_REQUESTS -> {
val retryAfter = response.headers()[HttpHeaderNames.RETRY_AFTER]?.let { headerValue ->
try {
headerValue.toLong() * 1000
} catch (nfe: NumberFormatException) {
null
}
}
OutcomeHandlerResult.Retry(retryAfter)
}
HttpResponseStatus.INTERNAL_SERVER_ERROR, HttpResponseStatus.SERVICE_UNAVAILABLE ->
OutcomeHandlerResult.Retry()
else -> OutcomeHandlerResult.DoNotRetry()
}
}
is OperationOutcome.Failure -> {
OutcomeHandlerResult.Retry()
}
}
}
executeWithRetry(
group,
retryPolicy.maxAttempts,
retryPolicy.initialDelayMillis.toDouble(),
retryPolicy.exp,
outcomeHandler,
Random.Default,
operation
)
} else {
operation()
}
}
fun healthCheck(nonce: ByteArray): CompletableFuture<ByteArray?> {
return executeWithRetry {
sendRequest(profile.serverURI, HttpMethod.TRACE, nonce)
}.thenApply {
val status = it.status()
if (it.status() != HttpResponseStatus.OK) {
throw HttpException(status)
} else {
it.content()
}
}.thenApply { maybeByteBuf ->
maybeByteBuf?.let {
val result = ByteArray(it.readableBytes())
it.getBytes(0, result)
result
}
}
}
fun get(key: String): CompletableFuture<ByteArray?> {
return executeWithRetry {
sendRequest(profile.serverURI.resolve(key), HttpMethod.GET, null)
}.thenApply { response ->
val status = response.status()
if (response.status() == HttpResponseStatus.NOT_FOUND) {
response.release()
null
} else if (response.status() != HttpResponseStatus.OK) {
response.release()
throw HttpException(status)
} else {
response.content().also {
it.retain()
response.release()
}
}
}.thenApply { maybeByteBuf ->
maybeByteBuf?.let { buf ->
val result = ByteArray(buf.readableBytes())
buf.getBytes(0, result)
buf.release()
result
}
}
}
fun put(key: String, content: ByteArray, metadata: CacheValueMetadata): CompletableFuture<Unit> {
return executeWithRetry {
val extraHeaders = sequenceOf(
metadata.mimeType?.let { HttpHeaderNames.CONTENT_TYPE to it },
metadata.contentDisposition?.let { HttpHeaderNames.CONTENT_DISPOSITION to it }
).filterNotNull()
sendRequest(profile.serverURI.resolve(key), HttpMethod.PUT, content, extraHeaders.asIterable())
}.thenApply {
val status = it.status()
if (it.status() != HttpResponseStatus.CREATED && it.status() != HttpResponseStatus.OK) {
throw HttpException(status)
}
}
}
private fun sendRequest(
uri: URI,
method: HttpMethod,
body: ByteArray?,
extraHeaders: Iterable<Pair<CharSequence, CharSequence>>? = null
): CompletableFuture<FullHttpResponse> {
val responseFuture = CompletableFuture<FullHttpResponse>()
// Custom handler for processing responses
pool.acquire().addListener(object : GenericFutureListener<NettyFuture<Channel>> {
override fun operationComplete(channelFuture: Future<Channel>) {
if (channelFuture.isSuccess) {
val channel = channelFuture.now
val pipeline = channel.pipeline()
val closeListener = GenericFutureListener<Future<Void>> {
responseFuture.completeExceptionally(IOException("The remote server closed the connection"))
}
channel.closeFuture().addListener(closeListener)
val responseHandler = object : SimpleChannelInboundHandler<FullHttpResponse>() {
override fun handlerAdded(ctx: ChannelHandlerContext) {
channel.closeFuture().removeListener(closeListener)
}
override fun channelRead0(
ctx: ChannelHandlerContext,
response: FullHttpResponse
) {
pipeline.remove(this)
responseFuture.complete(response.retainedDuplicate())
if (!profile.connection.requestPipelining) {
pool.release(channel)
}
}
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
ctx.newPromise()
val ex = when (cause) {
is DecoderException -> cause.cause
else -> cause
}
responseFuture.completeExceptionally(ex)
ctx.close()
}
override fun channelInactive(ctx: ChannelHandlerContext) {
responseFuture.completeExceptionally(IOException("The remote server closed the connection"))
super.channelInactive(ctx)
pool.release(channel)
}
override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) {
if (evt is IdleStateEvent) {
val te = when (evt.state()) {
IdleState.READER_IDLE -> TimeoutException("Read timeout")
IdleState.WRITER_IDLE -> TimeoutException("Write timeout")
IdleState.ALL_IDLE -> TimeoutException("Idle timeout")
null -> throw IllegalStateException("This should never happen")
}
responseFuture.completeExceptionally(te)
super.userEventTriggered(ctx, evt)
if (this === pipeline.last()) {
ctx.close()
}
if (!profile.connection.requestPipelining) {
pool.release(channel)
}
} else {
super.userEventTriggered(ctx, evt)
}
}
}
pipeline.addLast(responseHandler)
// Prepare the HTTP request
val request: FullHttpRequest = let {
val content: ByteBuf? = body?.takeIf(ByteArray::isNotEmpty)?.let(Unpooled::wrappedBuffer)
DefaultFullHttpRequest(
HttpVersion.HTTP_1_1,
method,
uri.rawPath,
content ?: Unpooled.buffer(0)
).apply {
// Set headers
headers().apply {
if (content != null) {
set(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes())
}
set(HttpHeaderNames.HOST, profile.serverURI.host)
set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE)
if (profile.compressionEnabled) {
set(
HttpHeaderNames.ACCEPT_ENCODING,
HttpHeaderValues.GZIP.toString() + "," + HttpHeaderValues.DEFLATE.toString()
)
}
extraHeaders?.forEach { (k, v) ->
add(k, v)
}
// Add basic auth if configured
(profile.authentication as? Configuration.Authentication.BasicAuthenticationCredentials)?.let { credentials ->
val auth = "${credentials.username}:${credentials.password}"
val encodedAuth = Base64.getEncoder().encodeToString(auth.toByteArray())
set(HttpHeaderNames.AUTHORIZATION, "Basic $encodedAuth")
}
}
}
}
// Send the request
channel.writeAndFlush(request).addListener {
if (!it.isSuccess) {
val ex = it.cause()
log.warn(ex.message, ex)
}
if (profile.connection.requestPipelining) {
pool.release(channel)
}
}
} else {
responseFuture.completeExceptionally(channelFuture.cause())
}
}
})
return responseFuture
}
fun shutDown(): NettyFuture<*> {
return group.shutdownGracefully()
}
override fun close() {
shutDown().sync()
}
}

View File

@@ -0,0 +1,151 @@
package net.woggioni.rbcs.client.impl
import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
import java.security.KeyStore
import java.security.PrivateKey
import java.security.cert.X509Certificate
import java.time.Duration
import java.time.temporal.ChronoUnit
import net.woggioni.rbcs.api.exception.ConfigurationException
import net.woggioni.rbcs.client.Configuration
import net.woggioni.rbcs.common.Xml.Companion.asIterable
import net.woggioni.rbcs.common.Xml.Companion.renderAttribute
import org.w3c.dom.Document
object Parser {
fun parse(document: Document): Configuration {
val root = document.documentElement
val profiles = mutableMapOf<String, Configuration.Profile>()
for (child in root.asIterable()) {
val tagName = child.localName
when (tagName) {
"profile" -> {
val name =
child.renderAttribute("name") ?: throw ConfigurationException("name attribute is required")
val uri = child.renderAttribute("base-url")?.let(::URI)
?: throw ConfigurationException("base-url attribute is required")
var authentication: Configuration.Authentication? = null
var retryPolicy: Configuration.RetryPolicy? = null
var connection : Configuration.Connection = Configuration.Connection(
Duration.ofSeconds(60),
Duration.ofSeconds(60),
Duration.ofSeconds(30),
false
)
var trustStore : Configuration.TrustStore? = null
for (gchild in child.asIterable()) {
when (gchild.localName) {
"tls-client-auth" -> {
val keyStoreFile = gchild.renderAttribute("key-store-file")
val keyStorePassword =
gchild.renderAttribute("key-store-password")
val keyAlias = gchild.renderAttribute("key-alias")
val keyPassword = gchild.renderAttribute("key-password")
val keystore = KeyStore.getInstance("PKCS12").apply {
Files.newInputStream(Path.of(keyStoreFile)).use {
load(it, keyStorePassword?.toCharArray())
}
}
val key = keystore.getKey(keyAlias, keyPassword?.toCharArray()) as PrivateKey
val certChain = keystore.getCertificateChain(keyAlias).asSequence()
.map { it as X509Certificate }
.toList()
.toTypedArray()
authentication =
Configuration.Authentication.TlsClientAuthenticationCredentials(
key,
certChain
)
}
"basic-auth" -> {
val username = gchild.renderAttribute("user")
?: throw ConfigurationException("username attribute is required")
val password = gchild.renderAttribute("password")
?: throw ConfigurationException("password attribute is required")
authentication =
Configuration.Authentication.BasicAuthenticationCredentials(
username,
password
)
}
"retry-policy" -> {
val maxAttempts =
gchild.renderAttribute("max-attempts")
?.let(String::toInt)
?: throw ConfigurationException("max-attempts attribute is required")
val initialDelay =
gchild.renderAttribute("initial-delay")
?.let(Duration::parse)
?: Duration.ofSeconds(1)
val exp =
gchild.renderAttribute("exp")
?.let(String::toDouble)
?: 2.0f
retryPolicy = Configuration.RetryPolicy(
maxAttempts,
initialDelay.toMillis(),
exp.toDouble()
)
}
"connection" -> {
val idleTimeout = gchild.renderAttribute("idle-timeout")
?.let(Duration::parse) ?: Duration.of(30, ChronoUnit.SECONDS)
val readIdleTimeout = gchild.renderAttribute("read-idle-timeout")
?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS)
val writeIdleTimeout = gchild.renderAttribute("write-idle-timeout")
?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS)
val requestPipelining = gchild.renderAttribute("request-pipelining")
?.let(String::toBoolean) ?: false
connection = Configuration.Connection(
readIdleTimeout,
writeIdleTimeout,
idleTimeout,
requestPipelining
)
}
"tls-trust-store" -> {
val file = gchild.renderAttribute("file")
?.let(Path::of)
val password = gchild.renderAttribute("password")
val checkCertificateStatus = gchild.renderAttribute("check-certificate-status")
?.let(String::toBoolean) ?: false
val verifyServerCertificate = gchild.renderAttribute("verify-server-certificate")
?.let(String::toBoolean) ?: true
trustStore = Configuration.TrustStore(file, password, checkCertificateStatus, verifyServerCertificate)
}
}
}
val maxConnections = child.renderAttribute("max-connections")
?.let(String::toInt)
?: 50
val connectionTimeout = child.renderAttribute("connection-timeout")
?.let(Duration::parse)
val compressionEnabled = child.renderAttribute("enable-compression")
?.let(String::toBoolean)
?: true
profiles[name] = Configuration.Profile(
uri,
connection,
authentication,
connectionTimeout,
maxConnections,
compressionEnabled,
retryPolicy,
trustStore
)
}
}
}
return Configuration(profiles)
}
}

View File

@@ -0,0 +1,79 @@
package net.woggioni.rbcs.client
import io.netty.util.concurrent.EventExecutorGroup
import java.util.concurrent.CompletableFuture
import java.util.concurrent.TimeUnit
import kotlin.math.pow
import kotlin.random.Random
sealed class OperationOutcome<T> {
class Success<T>(val result: T) : OperationOutcome<T>()
class Failure<T>(val ex: Throwable) : OperationOutcome<T>()
}
sealed class OutcomeHandlerResult {
class Retry(val suggestedDelayMillis: Long? = null) : OutcomeHandlerResult()
class DoNotRetry : OutcomeHandlerResult()
}
fun interface OutcomeHandler<T> {
fun shouldRetry(result: OperationOutcome<T>): OutcomeHandlerResult
}
fun <T> executeWithRetry(
eventExecutorGroup: EventExecutorGroup,
maxAttempts: Int,
initialDelay: Double,
exp: Double,
outcomeHandler: OutcomeHandler<T>,
randomizer : Random?,
cb: () -> CompletableFuture<T>
): CompletableFuture<T> {
val finalResult = cb()
var future = finalResult
var shortCircuit = false
for (i in 1 until maxAttempts) {
future = future.handle { result, ex ->
val operationOutcome = if (ex == null) {
OperationOutcome.Success(result)
} else {
OperationOutcome.Failure(ex.cause ?: ex)
}
if (shortCircuit) {
when(operationOutcome) {
is OperationOutcome.Failure -> throw operationOutcome.ex
is OperationOutcome.Success -> CompletableFuture.completedFuture(operationOutcome.result)
}
} else {
when(val outcomeHandlerResult = outcomeHandler.shouldRetry(operationOutcome)) {
is OutcomeHandlerResult.Retry -> {
val res = CompletableFuture<T>()
val delay = run {
val scheduledDelay = (initialDelay * exp.pow(i.toDouble()) * (1.0 + (randomizer?.nextDouble(-0.5, 0.5) ?: 0.0))).toLong()
outcomeHandlerResult.suggestedDelayMillis?.coerceAtMost(scheduledDelay) ?: scheduledDelay
}
eventExecutorGroup.schedule({
cb().handle { result, ex ->
if (ex == null) {
res.complete(result)
} else {
res.completeExceptionally(ex)
}
}
}, delay, TimeUnit.MILLISECONDS)
res
}
is OutcomeHandlerResult.DoNotRetry -> {
shortCircuit = true
when(operationOutcome) {
is OperationOutcome.Failure -> throw operationOutcome.ex
is OperationOutcome.Success -> CompletableFuture.completedFuture(operationOutcome.result)
}
}
}
}
}.thenCompose { it }
}
return future
}

View File

@@ -0,0 +1,260 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<xs:schema targetNamespace="urn:net.woggioni.rbcs.client"
xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns:rbcs-client="urn:net.woggioni.rbcs.client"
elementFormDefault="unqualified"
>
<xs:element name="profiles" type="rbcs-client:profilesType"/>
<xs:complexType name="profilesType">
<xs:sequence minOccurs="0">
<xs:element name="profile" type="rbcs-client:profileType" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="profileType">
<xs:sequence>
<xs:choice>
<xs:element name="no-auth" type="rbcs-client:noAuthType">
<xs:annotation>
<xs:documentation>
Disable authentication.
</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element name="basic-auth" type="rbcs-client:basicAuthType">
<xs:annotation>
<xs:documentation>
Enable HTTP basic authentication.
</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element name="tls-client-auth" type="rbcs-client:tlsClientAuthType">
<xs:annotation>
<xs:documentation>
Enable TLS certificate authentication.
</xs:documentation>
</xs:annotation>
</xs:element>
</xs:choice>
<xs:element name="connection" type="rbcs-client:connectionType" minOccurs="0" >
<xs:annotation>
<xs:documentation>
Set inactivity timeouts for connections to this server,
if not present, connections are only closed on network errors.
</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element name="retry-policy" type="rbcs-client:retryType" minOccurs="0">
<xs:annotation>
<xs:documentation>
Set a retry policy for this server, if not present requests won't be retried
</xs:documentation>
</xs:annotation>
</xs:element>
<xs:element name="tls-trust-store" type="rbcs-client:trustStoreType" minOccurs="0">
<xs:annotation>
<xs:documentation>
If set, specify an alternative truststore to validate the server certificate.
If not present the system truststore is used.
</xs:documentation>
</xs:annotation>
</xs:element>
</xs:sequence>
<xs:attribute name="name" type="xs:token" use="required">
<xs:annotation>
<xs:documentation>
Name of this server profile, to be referred to from rbcs-cli with the '-p' parameter
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="base-url" type="xs:anyURI" use="required">
<xs:annotation>
<xs:documentation>
RBCs server URL
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="max-connections" type="xs:positiveInteger" default="50">
<xs:annotation>
<xs:documentation>
Maximum number of concurrent TCP connection to open with this server
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="connection-timeout" type="xs:duration">
<xs:annotation>
<xs:documentation>
Enable HTTP compression when communicating to this server
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="enable-compression" type="xs:boolean" default="true">
<xs:annotation>
<xs:documentation>
Enable HTTP compression when communicating to this server
</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
<xs:complexType name="connectionType">
<xs:attribute name="idle-timeout" type="xs:duration" use="optional" default="PT30S">
<xs:annotation>
<xs:documentation>
The client will close the connection with the server
when neither a read nor a write was performed for the specified period of time.
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="read-idle-timeout" type="xs:duration" use="optional" default="PT60S">
<xs:annotation>
<xs:documentation>
The client will close the connection with the server
when no read was performed for the specified period of time.
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="write-idle-timeout" type="xs:duration" use="optional" default="PT60S">
<xs:annotation>
<xs:documentation>
The client will close the connection with the server
when no write was performed for the specified period of time.
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="request-pipelining" type="xs:boolean" use="optional" default="false">
<xs:annotation>
<xs:documentation>
Enables HTTP/1.1 request pipelining
</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
<xs:complexType name="noAuthType">
<xs:annotation>
<xs:documentation>
Add this tag to not use any type of authentication when talking to the RBCS server
</xs:documentation>
</xs:annotation>
</xs:complexType>
<xs:complexType name="basicAuthType">
<xs:annotation>
<xs:documentation>
Add this tag to enable HTTP basic authentication for the communication to this server,
mind that HTTP basic authentication sends credentials directly over the network, so make sure
your communication is protected by TLS (i.e. your server's URL starts with "https")
</xs:documentation>
</xs:annotation>
<xs:attribute name="user" type="xs:token" use="required">
<xs:annotation>
<xs:documentation>
Username for HTTP basic authentication
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="password" type="xs:string" use="required">
<xs:annotation>
<xs:documentation>
Password used for HTTP basic authentication
</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
<xs:complexType name="tlsClientAuthType">
<xs:attribute name="key-store-file" type="xs:anyURI" use="required">
<xs:annotation>
<xs:documentation>
System path to the keystore file
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="key-store-password" type="xs:string" use="required">
<xs:annotation>
<xs:documentation>
Password to open they keystore file
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="key-alias" type="xs:token" use="required">
<xs:annotation>
<xs:documentation>
Alias of the keystore entry containing the private key
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="key-password" type="xs:string" use="optional">
<xs:annotation>
<xs:documentation>
Private key entry's encryption password
</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
<xs:complexType name="retryType">
<xs:annotation>
<xs:documentation>
Retry policy to use in case of failures, based on exponential backoff
https://en.wikipedia.org/wiki/Exponential_backoff
</xs:documentation>
</xs:annotation>
<xs:attribute name="max-attempts" type="xs:positiveInteger" use="required">
<xs:annotation>
<xs:documentation>
Maximum number of attempts, after which the call will result in an error,
throwing an exception related to the last received failure
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="initial-delay" type="xs:duration" default="PT1S">
<xs:annotation>
<xs:documentation>
Delay to apply before retrying after the first failed call
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="exp" type="xs:double" default="2.0">
<xs:annotation>
<xs:documentation>
Exponent to apply to compute the next delay
</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
<xs:complexType name="trustStoreType">
<xs:attribute name="file" type="xs:string" use="required">
<xs:annotation>
<xs:documentation>
Path to the truststore file
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="password" type="xs:string">
<xs:annotation>
<xs:documentation>
Truststore file password
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="check-certificate-status" type="xs:boolean">
<xs:annotation>
<xs:documentation>
Whether or not check the server certificate validity using CRL/OCSP
</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="verify-server-certificate" type="xs:boolean" use="optional" default="true">
<xs:annotation>
<xs:documentation>
If false, the client will blindly trust the certificate provided by the server
</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
</xs:schema>

View File

@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration>
<configuration>
<import class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"/>
<import class="ch.qos.logback.core.ConsoleAppender"/>
<appender name="console" class="ConsoleAppender">
<target>System.err</target>
<encoder class="PatternLayoutEncoder">
<pattern>%d [%highlight(%-5level)] \(%thread\) %logger{36} -%kvp- %msg %n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="console"/>
</root>
<logger name="io.netty" level="info"/>
<logger name="com.google.code.yanf4j" level="warn"/>
<logger name="net.rubyeye.xmemcached" level="warn"/>
</configuration>

View File

@@ -0,0 +1,152 @@
package net.woggioni.rbcs.client
import io.netty.util.concurrent.DefaultEventExecutorGroup
import io.netty.util.concurrent.EventExecutorGroup
import java.util.concurrent.CompletableFuture
import java.util.stream.Stream
import kotlin.random.Random
import net.woggioni.rbcs.common.contextLogger
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.extension.ExtensionContext
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.Arguments
import org.junit.jupiter.params.provider.ArgumentsProvider
import org.junit.jupiter.params.provider.ArgumentsSource
import org.junit.jupiter.params.support.ParameterDeclarations
class RetryTest {
data class TestArgs(
val seed: Int,
val maxAttempt: Int,
val initialDelay: Double,
val exp: Double,
)
class TestArguments : ArgumentsProvider {
override fun provideArguments(
parameters: ParameterDeclarations,
context: ExtensionContext
): Stream<out Arguments> {
return Stream.of(
TestArgs(
seed = 101325,
maxAttempt = 5,
initialDelay = 50.0,
exp = 2.0,
),
TestArgs(
seed = 101325,
maxAttempt = 20,
initialDelay = 100.0,
exp = 1.1,
),
TestArgs(
seed = 123487,
maxAttempt = 20,
initialDelay = 100.0,
exp = 2.0,
),
TestArgs(
seed = 20082024,
maxAttempt = 10,
initialDelay = 100.0,
exp = 2.0,
)
).map {
object: Arguments {
override fun get() = arrayOf(it)
}
}
}
}
@ArgumentsSource(TestArguments::class)
@ParameterizedTest
fun test(testArgs: TestArgs) {
val log = contextLogger()
log.debug("Start")
val executor: EventExecutorGroup = DefaultEventExecutorGroup(1)
val attempts = mutableListOf<Pair<Long, OperationOutcome<Int>>>()
val outcomeHandler = OutcomeHandler<Int> { outcome ->
when(outcome) {
is OperationOutcome.Success -> {
if(outcome.result % 10 == 0) {
OutcomeHandlerResult.DoNotRetry()
} else {
OutcomeHandlerResult.Retry(null)
}
}
is OperationOutcome.Failure -> {
when(outcome.ex) {
is IllegalStateException -> {
log.debug(outcome.ex.message, outcome.ex)
OutcomeHandlerResult.Retry(null)
}
else -> {
OutcomeHandlerResult.DoNotRetry()
}
}
}
}
}
val random = Random(testArgs.seed)
val future =
executeWithRetry(executor, testArgs.maxAttempt, testArgs.initialDelay, testArgs.exp, outcomeHandler, null) {
val now = System.nanoTime()
val result = CompletableFuture<Int>()
executor.submit {
val n = random.nextInt(0, Integer.MAX_VALUE)
log.debug("Got new number: {}", n)
if(n % 3 == 0) {
val ex = IllegalStateException("Value $n can be divided by 3")
result.completeExceptionally(ex)
attempts += now to OperationOutcome.Failure(ex)
} else if(n % 7 == 0) {
val ex = RuntimeException("Value $n can be divided by 7")
result.completeExceptionally(ex)
attempts += now to OperationOutcome.Failure(ex)
} else {
result.complete(n)
attempts += now to OperationOutcome.Success(n)
}
}
result
}
Assertions.assertTrue(attempts.size <= testArgs.maxAttempt)
val result = future.handle { res, ex ->
if(ex != null) {
val err = ex.cause ?: ex
log.debug(err.message, err)
OperationOutcome.Failure(err)
} else {
OperationOutcome.Success(res)
}
}.get()
for ((index, attempt) in attempts.withIndex()) {
val (timestamp, value) = attempt
if (index > 0) {
/* Check the delay for subsequent attempts is correct */
val previousAttempt = attempts[index - 1]
val expectedTimestamp =
previousAttempt.first + testArgs.initialDelay * Math.pow(testArgs.exp, index.toDouble()) * 1e6
val actualTimestamp = timestamp
val err = Math.abs(expectedTimestamp - actualTimestamp) / expectedTimestamp
Assertions.assertTrue(err < 0.1)
}
if (index == attempts.size - 1 && index < testArgs.maxAttempt - 1) {
/*
* If the last attempt index is lower than the maximum number of attempts, then
* check the outcome handler returns DoNotRetry
*/
Assertions.assertTrue(outcomeHandler.shouldRetry(value) is OutcomeHandlerResult.DoNotRetry)
} else if (index < attempts.size - 1) {
/*
* If the attempt is not the last attempt check the outcome handler returns Retry
*/
Assertions.assertTrue(outcomeHandler.shouldRetry(value) is OutcomeHandlerResult.Retry)
}
}
}
}

View File

@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<rbcs-client:profiles xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs-client="urn:net.woggioni.rbcs.client"
xs:schemaLocation="urn:net.woggioni.rbcs.client jms://net.woggioni.rbcs.client/net/woggioni/rbcs/client/schema/rbcs-client.xsd"
>
<profile name="profile1" base-url="https://rbcs1.example.com/">
<tls-client-auth
key-store-file="keystore.pfx"
key-store-password="password"
key-alias="woggioni@c962475fa38"
key-password="key-password"/>
<connection write-idle-timeout="PT60S" read-idle-timeout="PT60S" write-timeout="PT0S" read-timeout="PT0S" idle-timeout="PT30S" />
<tls-trust-store file="file.pfx" password="password" check-certificate-status="false" verify-server-certificate="true"/>
</profile>
<profile name="profile2" base-url="https://rbcs2.example.com/">
<basic-auth user="user" password="password"/>
</profile>
</rbcs-client:profiles>

31
rbcs-common/.classpath Normal file
View File

@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="bin/main" path="src/main/java">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/main" path="src/main/kotlin">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/main" path="src/main/resources">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/test" path="src/test/kotlin">
<attributes>
<attribute name="gradle_scope" value="test"/>
<attribute name="gradle_used_by_scope" value="test"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-25/"/>
<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
<classpathentry kind="output" path="bin/default"/>
</classpath>

34
rbcs-common/.project Normal file
View File

@@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>rbcs-common</name>
<comment>Project rbcs-common created by Buildship.</comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
</natures>
<filteredResources>
<filter>
<id>1777370776954</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>

View File

@@ -0,0 +1,2 @@
connection.project.dir=..
eclipse.preferences.version=1

View File

@@ -0,0 +1,4 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=25
org.eclipse.jdt.core.compiler.compliance=25
org.eclipse.jdt.core.compiler.source=25

View File

@@ -0,0 +1 @@
net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory

View File

@@ -0,0 +1,15 @@
package net.woggioni.rbcs.common
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import io.netty.buffer.CompositeByteBuf
fun extractChunk(buf: CompositeByteBuf, alloc: ByteBufAllocator): ByteBuf {
val chunk = alloc.compositeBuffer()
for (component in buf.decompose(0, buf.readableBytes())) {
chunk.addComponent(true, component.retain())
}
buf.removeComponents(0, buf.numComponents())
buf.clear()
return chunk
}

View File

@@ -0,0 +1,25 @@
package net.woggioni.rbcs.common
import io.netty.buffer.ByteBuf
import java.io.InputStream
class ByteBufInputStream(private val buf : ByteBuf) : InputStream() {
override fun read(): Int {
return buf.takeIf {
it.readableBytes() > 0
}?.let(ByteBuf::readByte)
?.let(Byte::toInt) ?: -1
}
override fun read(b: ByteArray, off: Int, len: Int): Int {
val readableBytes = buf.readableBytes()
if(readableBytes == 0) return -1
val result = len.coerceAtMost(readableBytes)
buf.readBytes(b, off, result)
return result
}
override fun close() {
buf.release()
}
}

View File

@@ -0,0 +1,18 @@
package net.woggioni.rbcs.common
import io.netty.buffer.ByteBuf
import java.io.OutputStream
class ByteBufOutputStream(private val buf : ByteBuf) : OutputStream() {
override fun write(b: Int) {
buf.writeByte(b)
}
override fun write(b: ByteArray, off: Int, len: Int) {
buf.writeBytes(b, off, len)
}
override fun close() {
buf.release()
}
}

View File

@@ -0,0 +1,62 @@
package net.woggioni.rbcs.common
import java.net.InetAddress
data class Cidr private constructor(
val networkAddress: InetAddress,
val prefixLength: Int
) {
companion object {
fun from(cidr: String) : Cidr {
val separator = cidr.indexOf("/")
if(separator < 0) {
throw IllegalArgumentException("Invalid CIDR format: $cidr")
}
val networkAddress = InetAddress.getByName(cidr.substring(0, separator))
val prefixLength = cidr.substring(separator + 1, cidr.length).toInt()
// Validate prefix length
val maxPrefix = if (networkAddress.address.size == 4) 32 else 128
require(prefixLength in 0..maxPrefix) { "Invalid prefix length: $prefixLength" }
return Cidr(networkAddress, prefixLength)
}
}
fun contains(address: InetAddress): Boolean {
val networkBytes = networkAddress.address
val addressBytes = address.address
if (networkBytes.size != addressBytes.size) {
return false
}
// Calculate how many full bytes and remaining bits to check
val fullBytes = prefixLength / 8
val remainingBits = prefixLength % 8
// Check full bytes
for (i in 0..<fullBytes) {
if (networkBytes[i] != addressBytes[i]) {
return false
}
}
// Check remaining bits if any
if (remainingBits > 0 && fullBytes < networkBytes.size) {
val mask = (0xFF shl (8 - remainingBits)).toByte()
if ((networkBytes[fullBytes].toInt() and mask.toInt()) != (addressBytes[fullBytes].toInt() and mask.toInt())) {
return false
}
}
return true
}
override fun toString(): String {
return networkAddress.hostAddress + "/" + prefixLength
}
}

View File

@@ -0,0 +1,7 @@
package net.woggioni.rbcs.common
class ResourceNotFoundException(msg : String? = null, cause: Throwable? = null) : RuntimeException(msg, cause) {
}
class ModuleNotFoundException(msg : String? = null, cause: Throwable? = null) : RuntimeException(msg, cause) {
}

View File

@@ -0,0 +1,8 @@
package net.woggioni.rbcs.common
data class HostAndPort(val host: String, val port: Int = 0) {
override fun toString(): String {
return "$host:$port"
}
}

View File

@@ -0,0 +1,191 @@
package net.woggioni.rbcs.common
import io.netty.channel.Channel
import io.netty.channel.ChannelHandlerContext
import java.nio.file.Files
import java.nio.file.Path
import java.util.logging.LogManager
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.slf4j.MDC
import org.slf4j.event.Level
import org.slf4j.spi.LoggingEventBuilder
inline fun <reified T> T.contextLogger() = LoggerFactory.getLogger(T::class.java)
inline fun <reified T> createLogger() = LoggerFactory.getLogger(T::class.java)
inline fun Logger.traceParam(messageBuilder: () -> Pair<String, Array<Any>>) {
if (isTraceEnabled) {
val (format, params) = messageBuilder()
trace(format, params)
}
}
inline fun Logger.debugParam(messageBuilder: () -> Pair<String, Array<Any>>) {
if (isDebugEnabled) {
val (format, params) = messageBuilder()
info(format, params)
}
}
inline fun Logger.infoParam(messageBuilder: () -> Pair<String, Array<Any>>) {
if (isInfoEnabled) {
val (format, params) = messageBuilder()
info(format, params)
}
}
inline fun Logger.warnParam(messageBuilder: () -> Pair<String, Array<Any>>) {
if (isWarnEnabled) {
val (format, params) = messageBuilder()
warn(format, params)
}
}
inline fun Logger.errorParam(messageBuilder: () -> Pair<String, Array<Any>>) {
if (isErrorEnabled) {
val (format, params) = messageBuilder()
error(format, params)
}
}
inline fun log(
log: Logger,
filter: Logger.() -> Boolean,
loggerMethod: Logger.(String) -> Unit, messageBuilder: () -> String
) {
if (log.filter()) {
log.loggerMethod(messageBuilder())
}
}
fun withMDC(params: Array<Pair<String, String>>, cb: () -> Unit) {
object : AutoCloseable {
override fun close() {
for ((key, _) in params) MDC.remove(key)
}
}.use {
for ((key, value) in params) MDC.put(key, value)
cb()
}
}
inline fun Logger.log(level: Level, channel: Channel, crossinline messageBuilder: (LoggingEventBuilder) -> Unit ) {
if (isEnabledForLevel(level)) {
val params = arrayOf<Pair<String, String>>(
"channel-id-short" to channel.id().asShortText(),
"channel-id-long" to channel.id().asLongText(),
"remote-address" to channel.remoteAddress().toString(),
"local-address" to channel.localAddress().toString(),
)
withMDC(params) {
val builder = makeLoggingEventBuilder(level)
messageBuilder(builder)
builder.log()
}
}
}
inline fun Logger.log(level: Level, channel: Channel, crossinline messageBuilder: () -> String) {
log(level, channel) { builder ->
builder.setMessage(messageBuilder())
}
}
inline fun Logger.trace(ch: Channel, crossinline messageBuilder: () -> String) {
log(Level.TRACE, ch, messageBuilder)
}
inline fun Logger.debug(ch: Channel, crossinline messageBuilder: () -> String) {
log(Level.DEBUG, ch, messageBuilder)
}
inline fun Logger.info(ch: Channel, crossinline messageBuilder: () -> String) {
log(Level.INFO, ch, messageBuilder)
}
inline fun Logger.warn(ch: Channel, crossinline messageBuilder: () -> String) {
log(Level.WARN, ch, messageBuilder)
}
inline fun Logger.error(ch: Channel, crossinline messageBuilder: () -> String) {
log(Level.ERROR, ch, messageBuilder)
}
inline fun Logger.trace(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) {
log(Level.TRACE, ctx.channel(), messageBuilder)
}
inline fun Logger.debug(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) {
log(Level.DEBUG, ctx.channel(), messageBuilder)
}
inline fun Logger.info(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) {
log(Level.INFO, ctx.channel(), messageBuilder)
}
inline fun Logger.warn(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) {
log(Level.WARN, ctx.channel(), messageBuilder)
}
inline fun Logger.error(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) {
log(Level.ERROR, ctx.channel(), messageBuilder)
}
inline fun Logger.log(level: Level, messageBuilder: () -> String) {
if (isEnabledForLevel(level)) {
makeLoggingEventBuilder(level).log(messageBuilder())
}
}
inline fun Logger.trace(messageBuilder: () -> String) {
if (isTraceEnabled) {
trace(messageBuilder())
}
}
inline fun Logger.debug(messageBuilder: () -> String) {
if (isDebugEnabled) {
debug(messageBuilder())
}
}
inline fun Logger.info(messageBuilder: () -> String) {
if (isInfoEnabled) {
info(messageBuilder())
}
}
inline fun Logger.warn(messageBuilder: () -> String) {
if (isWarnEnabled) {
warn(messageBuilder())
}
}
inline fun Logger.error(messageBuilder: () -> String) {
if (isErrorEnabled) {
error(messageBuilder())
}
}
class LoggingConfig {
init {
val logManager = LogManager.getLogManager()
System.getProperty("log.config.source")?.let withSource@{ source ->
val urls = LoggingConfig::class.java.classLoader.getResources(source)
while (urls.hasMoreElements()) {
val url = urls.nextElement()
url.openStream().use { inputStream ->
logManager.readConfiguration(inputStream)
return@withSource
}
}
Path.of(source).takeIf(Files::exists)
?.let(Files::newInputStream)
?.use(logManager::readConfiguration)
}
}
}

View File

@@ -0,0 +1,57 @@
package net.woggioni.rbcs.common
import java.security.SecureRandom
import java.security.spec.KeySpec
import java.util.Base64
import javax.crypto.SecretKeyFactory
import javax.crypto.spec.PBEKeySpec
object PasswordSecurity {
enum class Algorithm(
val codeName : String,
val keyLength : Int,
val iterations : Int) {
PBEWithHmacSHA512_224AndAES_256("PBEWithHmacSHA512/224AndAES_256", 64, 1),
PBEWithHmacSHA1AndAES_256("PBEWithHmacSHA1AndAES_256",64, 1),
PBEWithHmacSHA384AndAES_128("PBEWithHmacSHA384AndAES_128", 64,1),
PBEWithHmacSHA384AndAES_256("PBEWithHmacSHA384AndAES_256",64,1),
PBKDF2WithHmacSHA512("PBKDF2WithHmacSHA512",512, 1),
PBKDF2WithHmacSHA384("PBKDF2WithHmacSHA384",384, 1);
}
private fun concat(arr1: ByteArray, arr2: ByteArray): ByteArray {
val result = ByteArray(arr1.size + arr2.size)
var j = 0
for(element in arr1) {
result[j] = element
j += 1
}
for(element in arr2) {
result[j] = element
j += 1
}
return result
}
fun hashPassword(password : String, salt : String? = null, algorithm : Algorithm = Algorithm.PBKDF2WithHmacSHA512) : String {
val actualSalt = salt?.let(Base64.getDecoder()::decode) ?: SecureRandom().run {
val result = ByteArray(16)
nextBytes(result)
result
}
val spec: KeySpec = PBEKeySpec(password.toCharArray(), actualSalt, algorithm.iterations, algorithm.keyLength)
val factory = SecretKeyFactory.getInstance(algorithm.codeName)
val hash = factory.generateSecret(spec).encoded
return String(Base64.getEncoder().encode(concat(hash, actualSalt)))
}
fun decodePasswordHash(encodedPasswordHash : String, algorithm: Algorithm = Algorithm.PBKDF2WithHmacSHA512) : Pair<ByteArray, ByteArray> {
val decoded = Base64.getDecoder().decode(encodedPasswordHash)
val hash = ByteArray(algorithm.keyLength / 8)
val salt = ByteArray(decoded.size - algorithm.keyLength / 8)
System.arraycopy(decoded, 0, hash, 0, hash.size)
System.arraycopy(decoded, hash.size, salt, 0, salt.size)
return hash to salt
}
}

View File

@@ -0,0 +1,167 @@
package net.woggioni.rbcs.common
import java.io.IOException
import java.net.InetAddress
import java.net.ServerSocket
import java.net.URI
import java.net.URL
import java.nio.file.Files
import java.nio.file.Path
import java.security.KeyStore
import java.security.MessageDigest
import java.security.cert.CertPathValidator
import java.security.cert.CertPathValidatorException
import java.security.cert.CertificateException
import java.security.cert.CertificateFactory
import java.security.cert.PKIXParameters
import java.security.cert.PKIXRevocationChecker
import java.security.cert.X509Certificate
import java.util.EnumSet
import javax.net.ssl.TrustManagerFactory
import javax.net.ssl.X509TrustManager
import net.woggioni.jwo.JWO
import net.woggioni.jwo.Tuple2
object RBCS {
fun String.toUrl(): URL = URL.of(URI(this), null)
const val RBCS_NAMESPACE_URI: String = "urn:net.woggioni.rbcs.server"
const val RBCS_PREFIX: String = "rbcs"
const val XML_SCHEMA_NAMESPACE_URI = "http://www.w3.org/2001/XMLSchema-instance"
fun ByteArray.toInt(index: Int = 0): Long {
if (index + 4 > size) throw IllegalArgumentException("Not enough bytes to decode a 32 bits integer")
var value: Long = 0
for (b in index until index + 4) {
value = (value shl 8) + (get(b).toInt() and 0xFF)
}
return value
}
fun ByteArray.toLong(index: Int = 0): Long {
if (index + 8 > size) throw IllegalArgumentException("Not enough bytes to decode a 64 bits long integer")
var value: Long = 0
for (b in index until index + 8) {
value = (value shl 8) + (get(b).toInt() and 0xFF)
}
return value
}
fun digest(
data: ByteArray,
md: MessageDigest
): ByteArray {
md.update(data)
return md.digest()
}
fun digestString(
data: ByteArray,
md: MessageDigest
): String {
return JWO.bytesToHex(digest(data, md))
}
fun processCacheKey(key: String, keyPrefix: String?, digestAlgorithm: String?) : ByteArray {
val prefixedKey = if (keyPrefix == null) {
key
} else {
key + keyPrefix
}.toByteArray(Charsets.UTF_8)
return digestAlgorithm
?.let(MessageDigest::getInstance)
?.let { md ->
digest(prefixedKey, md)
} ?: prefixedKey
}
fun Long.toIntOrNull(): Int? {
return if (this >= Int.MIN_VALUE && this <= Int.MAX_VALUE) {
toInt()
} else {
null
}
}
fun getFreePort(): Int {
var count = 0
while (count < 50) {
try {
ServerSocket(0, 50, InetAddress.getLocalHost()).use { serverSocket ->
val candidate = serverSocket.localPort
if (candidate > 0) {
return candidate
} else {
throw RuntimeException("Got invalid port number: $candidate")
}
}
} catch (ignored: IOException) {
++count
}
}
throw RuntimeException("Error trying to find an open port")
}
fun loadKeystore(file: Path, password: String?): KeyStore {
val ext = JWO.splitExtension(file)
.map(Tuple2<String, String>::get_2)
.orElseThrow {
IllegalArgumentException(
"Keystore file '${file}' must have .jks, .p12, .pfx extension"
)
}
val keystore = when (ext.substring(1).lowercase()) {
"jks" -> KeyStore.getInstance("JKS")
"p12", "pfx" -> KeyStore.getInstance("PKCS12")
else -> throw IllegalArgumentException(
"Keystore file '${file}' must have .jks, .p12, .pfx extension"
)
}
Files.newInputStream(file).use {
keystore.load(it, password?.let(String::toCharArray))
}
return keystore
}
fun getTrustManager(trustStore: KeyStore?, certificateRevocationEnabled: Boolean): X509TrustManager {
return if (trustStore != null) {
val certificateFactory = CertificateFactory.getInstance("X.509")
val validator = CertPathValidator.getInstance("PKIX").apply {
val rc = revocationChecker as PKIXRevocationChecker
rc.options = EnumSet.of(
PKIXRevocationChecker.Option.NO_FALLBACK
)
}
val params = PKIXParameters(trustStore).apply {
isRevocationEnabled = certificateRevocationEnabled
}
object : X509TrustManager {
override fun checkClientTrusted(chain: Array<out X509Certificate>, authType: String) {
val clientCertificateChain = certificateFactory.generateCertPath(chain.toList())
try {
validator.validate(clientCertificateChain, params)
} catch (ex: CertPathValidatorException) {
throw CertificateException(ex)
}
}
override fun checkServerTrusted(chain: Array<out X509Certificate>, authType: String) {
throw NotImplementedError()
}
private val acceptedIssuers = trustStore.aliases().asSequence()
.filter(trustStore::isCertificateEntry)
.map(trustStore::getCertificate)
.map { it as X509Certificate }
.toList()
.toTypedArray()
override fun getAcceptedIssuers() = acceptedIssuers
}
} else {
val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
trustManagerFactory.trustManagers.asSequence().filter { it is X509TrustManager }
.single() as X509TrustManager
}
}
}

View File

@@ -0,0 +1,113 @@
package net.woggioni.rbcs.common
import java.io.IOException
import java.io.InputStream
import java.net.URL
import java.net.URLConnection
import java.net.URLStreamHandler
import java.net.spi.URLStreamHandlerProvider
import java.util.concurrent.atomic.AtomicBoolean
import java.util.stream.Collectors
class RbcsUrlStreamHandlerFactory : URLStreamHandlerProvider() {
private class ClasspathHandler(private val classLoader: ClassLoader = RbcsUrlStreamHandlerFactory::class.java.classLoader) :
URLStreamHandler() {
override fun openConnection(u: URL): URLConnection? {
return javaClass.module
?.takeIf { m: Module -> m.layer != null }
?.let {
val path = u.path
val i = path.lastIndexOf('/')
val packageName = path.substring(0, i).replace('/', '.')
val modules = packageMap[packageName]!!
ClasspathResourceURLConnection(
u,
modules
)
}
?: classLoader.getResource(u.path)?.let(URL::openConnection)
}
}
private class JpmsHandler : URLStreamHandler() {
override fun openConnection(u: URL): URLConnection {
val moduleName = u.host
val thisModule = javaClass.module
val sourceModule =
thisModule
?.let(Module::getLayer)
?.let { layer: ModuleLayer ->
layer.findModule(moduleName).orElse(null)
} ?: if(thisModule.layer == null) {
thisModule
} else throw ModuleNotFoundException("Module '$moduleName' not found")
return JpmsResourceURLConnection(u, sourceModule)
}
}
private class JpmsResourceURLConnection(url: URL, private val module: Module) : URLConnection(url) {
override fun connect() {
}
@Throws(IOException::class)
override fun getInputStream(): InputStream {
val resource = getURL().path
return module.getResourceAsStream(resource)
?: throw ResourceNotFoundException("Resource '$resource' not found in module '${module.name}'")
}
}
override fun createURLStreamHandler(protocol: String): URLStreamHandler? {
return when (protocol) {
"classpath" -> ClasspathHandler()
"jpms" -> JpmsHandler()
else -> null
}
}
private class ClasspathResourceURLConnection(url: URL?, private val modules: List<Module>) :
URLConnection(url) {
override fun connect() {}
override fun getInputStream(): InputStream? {
for (module in modules) {
val result = module.getResourceAsStream(getURL().path)
if (result != null) return result
}
return null
}
}
companion object {
private val installed = AtomicBoolean(false)
fun install() {
if (!installed.getAndSet(true)) {
URL.setURLStreamHandlerFactory(RbcsUrlStreamHandlerFactory())
}
}
private val packageMap: Map<String, List<Module>> by lazy {
RbcsUrlStreamHandlerFactory::class.java.module.layer
.modules()
.stream()
.flatMap { m: Module ->
m.packages.stream()
.map { p: String -> p to m }
}
.collect(
Collectors.groupingBy(
Pair<String, Module>::first,
Collectors.mapping(
Pair<String, Module>::second,
Collectors.toUnmodifiableList<Module>()
)
)
)
}
}
}

View File

@@ -0,0 +1,243 @@
package net.woggioni.rbcs.common
import java.io.InputStream
import java.io.OutputStream
import java.net.URL
import javax.xml.XMLConstants.ACCESS_EXTERNAL_DTD
import javax.xml.XMLConstants.ACCESS_EXTERNAL_SCHEMA
import javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING
import javax.xml.XMLConstants.W3C_XML_SCHEMA_NS_URI
import javax.xml.parsers.DocumentBuilder
import javax.xml.parsers.DocumentBuilderFactory
import javax.xml.transform.OutputKeys
import javax.xml.transform.TransformerFactory
import javax.xml.transform.dom.DOMSource
import javax.xml.transform.stream.StreamResult
import javax.xml.transform.stream.StreamSource
import javax.xml.validation.Schema
import javax.xml.validation.SchemaFactory
import net.woggioni.jwo.JWO
import org.slf4j.event.Level
import org.w3c.dom.Document
import org.w3c.dom.Element
import org.w3c.dom.Node
import org.w3c.dom.NodeList
import org.xml.sax.ErrorHandler as ErrHandler
import org.xml.sax.SAXNotRecognizedException
import org.xml.sax.SAXNotSupportedException
import org.xml.sax.SAXParseException
class NodeListIterator(private val nodeList: NodeList) : Iterator<Node> {
private var cursor: Int = 0
override fun hasNext(): Boolean {
return cursor < nodeList.length
}
override fun next(): Node {
return if (hasNext()) nodeList.item(cursor++) else throw NoSuchElementException()
}
}
class ElementIterator(parent: Element, name: String? = null) : Iterator<Element> {
private val it: NodeListIterator
private val name: String?
private var next: Element?
init {
it = NodeListIterator(parent.childNodes)
this.name = name
next = getNext()
}
override fun hasNext(): Boolean {
return next != null
}
override fun next(): Element {
val result = next ?: throw NoSuchElementException()
next = getNext()
return result
}
private fun getNext(): Element? {
var result: Element? = null
while (it.hasNext()) {
val node: Node = it.next()
if (node is Element && (name == null || name == node.tagName)) {
result = node
break
}
}
return result
}
}
class Xml(val doc: Document, val element: Element) {
class ErrorHandler(private val fileURL: URL) : ErrHandler {
companion object {
private val log = createLogger<ErrorHandler>()
}
override fun warning(ex: SAXParseException)= err(ex, Level.WARN)
private fun err(ex: SAXParseException, level: Level) {
log.log(level) {
"Problem at ${fileURL}:${ex.lineNumber}:${ex.columnNumber} parsing deployment configuration: ${ex.message}"
}
throw ex
}
override fun error(ex: SAXParseException) = err(ex, Level.ERROR)
override fun fatalError(ex: SAXParseException) = err(ex, Level.ERROR)
}
companion object {
private val dictMap: Map<String, Map<String, Any>> = sequenceOf(
"env" to System.getenv().asSequence().map { (k, v) -> k to (v as Any) }.toMap(),
"sys" to System.getProperties().asSequence().map { (k, v) -> k as String to (v as Any) }.toMap()
).toMap()
private fun renderConfigurationTemplate(template: String): String {
return JWO.renderTemplate(template, emptyMap(), dictMap).replace("$$", "$")
}
fun Element.renderAttribute(name : String, namespaceURI: String? = null) = if(namespaceURI == null) {
getAttribute(name)
} else {
getAttributeNS(name, namespaceURI)
}.takeIf(String::isNotEmpty)?.let(Companion::renderConfigurationTemplate)
fun Element.asIterable() = Iterable { ElementIterator(this, null) }
fun NodeList.asIterable() = Iterable { NodeListIterator(this) }
private fun disableProperty(dbf: DocumentBuilderFactory, propertyName: String) {
try {
dbf.setAttribute(propertyName, "")
} catch (iae: IllegalArgumentException) {
// Property not supported.
}
}
private fun disableProperty(sf: SchemaFactory, propertyName: String) {
try {
sf.setProperty(propertyName, "")
} catch (ex: SAXNotRecognizedException) {
// Property not supported.
} catch (ex: SAXNotSupportedException) {
}
}
fun getSchema(schema: URL): Schema {
val sf = SchemaFactory.newInstance(W3C_XML_SCHEMA_NS_URI)
sf.setFeature(FEATURE_SECURE_PROCESSING, false)
sf.errorHandler = ErrorHandler(schema)
return sf.newSchema(schema)
}
fun getSchema(inputStream: InputStream): Schema {
val sf = SchemaFactory.newInstance(W3C_XML_SCHEMA_NS_URI)
sf.setFeature(FEATURE_SECURE_PROCESSING, true)
return sf.newSchema(StreamSource(inputStream))
}
fun newDocumentBuilderFactory(schemaResourceURL: URL?): DocumentBuilderFactory {
val dbf = DocumentBuilderFactory.newInstance()
dbf.setFeature(FEATURE_SECURE_PROCESSING, false)
dbf.setAttribute(ACCESS_EXTERNAL_SCHEMA, "all")
disableProperty(dbf, ACCESS_EXTERNAL_DTD)
dbf.isExpandEntityReferences = true
dbf.isIgnoringComments = true
dbf.isNamespaceAware = true
dbf.isValidating = schemaResourceURL == null
dbf.setFeature("http://apache.org/xml/features/validation/schema", true)
schemaResourceURL?.let {
dbf.schema = getSchema(it)
}
return dbf
}
fun newDocumentBuilder(resource: URL, schemaResourceURL: URL?): DocumentBuilder {
val db = newDocumentBuilderFactory(schemaResourceURL).newDocumentBuilder()
db.setErrorHandler(ErrorHandler(resource))
return db
}
fun parseXmlResource(resource: URL, schemaResourceURL: URL?): Document {
val db = newDocumentBuilder(resource, schemaResourceURL)
return resource.openStream().use(db::parse)
}
fun parseXml(sourceURL: URL, sourceStream: InputStream? = null, schemaResourceURL: URL? = null): Document {
val db = newDocumentBuilder(sourceURL, schemaResourceURL)
return sourceStream?.let(db::parse) ?: sourceURL.openStream().use(db::parse)
}
fun write(doc: Document, output: OutputStream) {
val transformerFactory = TransformerFactory.newInstance()
val transformer = transformerFactory.newTransformer()
transformer.setOutputProperty(OutputKeys.INDENT, "yes")
transformer.setOutputProperty(OutputKeys.INDENT, "yes")
transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "4")
transformer.setOutputProperty(OutputKeys.STANDALONE, "yes")
transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8")
val source = DOMSource(doc)
val result = StreamResult(output)
transformer.transform(source, result)
}
fun of(
namespaceURI: String,
qualifiedName: String,
schemaResourceURL: URL? = null,
cb: Xml.(el: Element) -> Unit
): Document {
val dbf = newDocumentBuilderFactory(schemaResourceURL)
val db = dbf.newDocumentBuilder()
val doc = db.newDocument()
val root = doc.createElementNS(namespaceURI, qualifiedName)
.also(doc::appendChild)
Xml(doc, root).cb(root)
return doc
}
fun of(doc: Document, el: Element, cb: Xml.(el: Element) -> Unit): Element {
Xml(doc, el).cb(el)
return el
}
fun Element.removeChildren() {
while (true) {
removeChild(firstChild ?: break)
}
}
}
fun node(
name: String,
namespaceURI: String? = null,
attrs: Map<String, String> = emptyMap(),
cb: Xml.(el: Element) -> Unit = {}
): Element {
val child = doc.createElementNS(namespaceURI, name)
for ((key, value) in attrs) {
child.setAttribute(key, value)
}
return child
.also {
element.appendChild(it)
Xml(doc, it).cb(it)
}
}
fun attr(key: String, value: String, namespaceURI: String? = null) {
element.setAttributeNS(namespaceURI, key, value)
}
fun text(txt: String) {
element.appendChild(doc.createTextNode(txt))
}
}

View File

@@ -0,0 +1,17 @@
package net.woggioni.rbcs.common
import java.net.InetAddress
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Test
class CidrTest {
class CidrTest {
@Test
fun test() {
val cidr = Cidr.from("2a02:4780:12:368b::1/128")
Assertions.assertTrue {
cidr.contains(InetAddress.ofLiteral("2a02:4780:12:368b::1"))
}
}
}
}

View File

@@ -0,0 +1,38 @@
package net.woggioni.rbcs.common
import java.security.Provider
import java.security.Security
import java.util.Base64
import net.woggioni.rbcs.common.PasswordSecurity.decodePasswordHash
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Test
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.EnumSource
class PasswordHashingTest {
@EnumSource(PasswordSecurity.Algorithm::class)
@ParameterizedTest
fun test(algo: PasswordSecurity.Algorithm) {
val password = "password"
val encoded = hashPassword(password, algorithm = algo)
val (_, salt) = decodePasswordHash(encoded, algo)
Assertions.assertEquals(encoded,
hashPassword(password, salt = salt.let(Base64.getEncoder()::encodeToString), algorithm = algo)
)
}
@Test
fun listAvailableAlgorithms() {
Security.getProviders().asSequence()
.flatMap { provider: Provider -> provider.services.asSequence() }
.filter { service: Provider.Service -> "SecretKeyFactory" == service.type }
.map(Provider.Service::getAlgorithm)
.forEach {
println(it)
}
}
}

View File

@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="bin/main" path="src/main/java">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/main" path="src/main/kotlin">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/main" path="src/main/resources">
<attributes>
<attribute name="gradle_scope" value="main"/>
<attribute name="gradle_used_by_scope" value="main,test"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="bin/test" path="src/test/kotlin">
<attributes>
<attribute name="gradle_scope" value="test"/>
<attribute name="gradle_used_by_scope" value="test"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-25/"/>
<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
<classpathentry kind="output" path="bin/default"/>
</classpath>

Some files were not shown because too many files have changed in this diff Show More