Compare commits

...

6 Commits
0.0.4 ... 0.0.6

Author SHA1 Message Date
3b7030c302 fixed mainClassName for native image build
All checks were successful
CI / build (push) Successful in 3m22s
2025-01-16 17:23:03 +08:00
a8670277e7 fixed XML error handler for server command 2025-01-16 17:01:14 +08:00
03ee75266d added average turnaround time calculation in benchmark 2025-01-16 14:54:21 +08:00
05a265e4b4 added connection pooling to gbcs-client
All checks were successful
CI / build (push) Successful in 3m55s
2025-01-16 13:37:14 +08:00
5af99330f8 server is now a subcommand 2025-01-16 11:35:05 +08:00
747168cda3 added client command 2025-01-16 11:16:01 +08:00
33 changed files with 706 additions and 411 deletions

View File

@@ -1,26 +0,0 @@
plugins {
alias catalog.plugins.gradle.jmh
alias catalog.plugins.lombok
}
import me.champeau.jmh.JMHTask
dependencies {
implementation rootProject
implementation catalog.jwo
implementation catalog.xz
implementation catalog.jackson.databind
jmhAnnotationProcessor catalog.lombok
}
jmh {
threads = 4
iterations = 2
fork = 1
warmupIterations = 1
warmupForks = 0
resultFormat = 'JSON'
}

View File

@@ -1,262 +0,0 @@
package net.woggioni.gbcs.benchmark;
import lombok.Getter;
import lombok.SneakyThrows;
import net.woggioni.jwo.Fun;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyStore;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
public class Main {
@SneakyThrows
private static Properties loadProperties() {
Properties properties = new Properties();
try (final var is = Main.class.getResourceAsStream("/benchmark.properties")) {
properties.load(is);
}
return properties;
}
private static final Properties properties = loadProperties();
@State(Scope.Thread)
public static class ExecutionPlan {
private final Random random = new Random(101325);
@Getter
private final HttpClient client = createHttpClient();
private final Map<String, byte[]> entries = new HashMap<>();
private HttpClient createHttpClient() {
final var clientBuilder = HttpClient.newBuilder();
getSslContext().ifPresent(clientBuilder::sslContext);
return clientBuilder.build();
}
public final Map<String, byte[]> getEntries() {
return Collections.unmodifiableMap(entries);
}
public Map.Entry<String, byte[]> newEntry() {
final var keyBuffer = new byte[0x20];
random.nextBytes(keyBuffer);
final var key = Base64.getUrlEncoder().encodeToString(keyBuffer);
final var value = new byte[0x1000];
random.nextBytes(value);
return Map.entry(key, value);
}
@SneakyThrows
public HttpRequest.Builder newRequestBuilder(String key) {
final var requestBuilder = HttpRequest.newBuilder()
.uri(getServerURI().resolve(key));
String user = getUser();
if (user != null) {
requestBuilder.header("Authorization", buildAuthorizationHeader(user, getPassword()));
}
return requestBuilder;
}
@SneakyThrows
public URI getServerURI() {
return new URI(properties.getProperty("gbcs.server.url"));
}
@SneakyThrows
public Optional<String> getClientTrustStorePassword() {
return Optional.ofNullable(properties.getProperty("gbcs.client.ssl.truststore.password"))
.filter(Predicate.not(String::isEmpty));
}
@SneakyThrows
public Optional<KeyStore> getClientTrustStore() {
return Optional.ofNullable(properties.getProperty("gbcs.client.ssl.truststore.file"))
.filter(Predicate.not(String::isEmpty))
.map(Path::of)
.map((Fun<Path, KeyStore>) keyStoreFile -> {
final var keyStore = KeyStore.getInstance("PKCS12");
try (final var is = Files.newInputStream(keyStoreFile)) {
keyStore.load(is, getClientTrustStorePassword().map(String::toCharArray).orElse(null));
}
return keyStore;
});
}
@SneakyThrows
public Optional<KeyStore> getClientKeyStore() {
return Optional.ofNullable(properties.getProperty("gbcs.client.ssl.keystore.file"))
.filter(Predicate.not(String::isEmpty))
.map(Path::of)
.map((Fun<Path, KeyStore>) keyStoreFile -> {
final var keyStore = KeyStore.getInstance("PKCS12");
try (final var is = Files.newInputStream(keyStoreFile)) {
keyStore.load(is, getClientKeyStorePassword().map(String::toCharArray).orElse(null));
}
return keyStore;
});
}
@SneakyThrows
public Optional<String> getClientKeyStorePassword() {
return Optional.ofNullable(properties.getProperty("gbcs.client.ssl.keystore.password"))
.filter(Predicate.not(String::isEmpty));
}
@SneakyThrows
public Optional<String> getClientKeyPassword() {
return Optional.ofNullable(properties.getProperty("gbcs.client.ssl.key.password"))
.filter(Predicate.not(String::isEmpty));
}
@SneakyThrows
public String getUser() {
return Optional.ofNullable(properties.getProperty("gbcs.server.username"))
.filter(Predicate.not(String::isEmpty))
.orElse(null);
}
@SneakyThrows
public String getPassword() {
return Optional.ofNullable(properties.getProperty("gbcs.server.password"))
.filter(Predicate.not(String::isEmpty))
.orElse(null);
}
private String buildAuthorizationHeader(String user, String password) {
final var b64 = Base64.getEncoder().encode(String.format("%s:%s", user, password).getBytes(StandardCharsets.UTF_8));
return "Basic " + new String(b64);
}
@SneakyThrows
private Optional<SSLContext> getSslContext() {
return getClientKeyStore().map((Fun<KeyStore, SSLContext>) clientKeyStore -> {
final var kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
kmf.init(clientKeyStore, getClientKeyStorePassword().map(String::toCharArray).orElse(null));
// Set up trust manager factory with the truststore
final var trustManagers = getClientTrustStore().map((Fun<KeyStore, TrustManager[]>) ts -> {
final var tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(ts);
return tmf.getTrustManagers();
}).orElse(new TrustManager[0]);
// Create SSL context with the key and trust managers
final var sslContext = SSLContext.getInstance("TLS");
sslContext.init(kmf.getKeyManagers(), trustManagers, null);
return sslContext;
});
}
@SneakyThrows
@Setup(Level.Trial)
public void setUp() {
final var client = getClient();
for (int i = 0; i < 1000; i++) {
final var pair = newEntry();
final var requestBuilder = newRequestBuilder(pair.getKey())
.header("Content-Type", "application/octet-stream")
.PUT(HttpRequest.BodyPublishers.ofByteArray(pair.getValue()));
final var response = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString());
if (201 != response.statusCode()) {
throw new IllegalStateException(Integer.toString(response.statusCode()));
} else {
entries.put(pair.getKey(), pair.getValue());
}
}
}
@TearDown
public void tearDown() {
client.close();
}
private Iterator<Map.Entry<String, byte[]>> it = null;
private Map.Entry<String, byte[]> nextEntry() {
if (it == null || !it.hasNext()) {
it = getEntries().entrySet().iterator();
}
return it.next();
}
}
@SneakyThrows
@Benchmark
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.SECONDS)
public void get(ExecutionPlan plan) {
final var client = plan.getClient();
final var entry = plan.nextEntry();
final var requestBuilder = plan.newRequestBuilder(entry.getKey())
.header("Accept", "application/octet-stream")
.GET();
final var response = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray());
if (200 != response.statusCode()) {
throw new IllegalStateException(Integer.toString(response.statusCode()));
} else {
if (!Arrays.equals(entry.getValue(), response.body())) {
throw new IllegalStateException("Retrieved unexpected value");
}
}
}
@SneakyThrows
@Benchmark
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.SECONDS)
public void put(Main.ExecutionPlan plan) {
final var client = plan.getClient();
final var entry = plan.nextEntry();
final var requestBuilder = plan.newRequestBuilder(entry.getKey())
.header("Content-Type", "application/octet-stream")
.PUT(HttpRequest.BodyPublishers.ofByteArray(entry.getValue()));
final var response = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray());
if (201 != response.statusCode()) {
throw new IllegalStateException(Integer.toString(response.statusCode()));
}
}
}

View File

@@ -1 +0,0 @@
gbcs.server.url= http://localhost:8080

View File

@@ -6,9 +6,9 @@ plugins {
id 'maven-publish'
}
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
allprojects { subproject ->
group = 'net.woggioni'

View File

@@ -7,7 +7,7 @@ WORKDIR /home/luser
FROM base-release AS release
ADD gbcs-cli-envelope-*.jar gbcs.jar
ENTRYPOINT ["java", "-jar", "/home/luser/gbcs.jar"]
ENTRYPOINT ["java", "-jar", "/home/luser/gbcs.jar", "server"]
FROM base-release AS release-memcached
ADD --chown=luser:luser gbcs-cli-envelope-*.jar gbcs.jar
@@ -15,7 +15,7 @@ RUN mkdir plugins
WORKDIR /home/luser/plugins
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/gbcs-memcached*.tar
WORKDIR /home/luser
ENTRYPOINT ["java", "-jar", "/home/luser/gbcs.jar"]
ENTRYPOINT ["java", "-jar", "/home/luser/gbcs.jar", "server"]
FROM release-memcached as compose
COPY --chown=luser:luser conf/gbcs-memcached.xml /home/luser/.config/gbcs/gbcs.xml

View File

@@ -2,6 +2,7 @@ package net.woggioni.gbcs.base
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.slf4j.event.Level
import java.nio.file.Files
import java.nio.file.Path
import java.util.logging.LogManager
@@ -52,6 +53,12 @@ inline fun log(log : Logger,
}
}
inline fun Logger.log(level : Level, messageBuilder : () -> String) {
if(isEnabledForLevel(level)) {
makeLoggingEventBuilder(level).log(messageBuilder())
}
}
inline fun Logger.trace(messageBuilder : () -> String) {
if(isTraceEnabled) {
trace(messageBuilder())

View File

@@ -1,6 +1,7 @@
package net.woggioni.gbcs.base
import org.slf4j.LoggerFactory
import org.slf4j.event.Level
import org.w3c.dom.Document
import org.w3c.dom.Element
import org.w3c.dom.Node
@@ -80,28 +81,17 @@ class Xml(val doc: Document, val element: Element) {
private val log = LoggerFactory.getLogger(ErrorHandler::class.java)
}
override fun warning(ex: SAXParseException) {
log.warn(
"Problem at {}:{}:{} parsing deployment configuration: {}",
fileURL, ex.lineNumber, ex.columnNumber, ex.message
)
}
override fun warning(ex: SAXParseException)= err(ex, Level.WARN)
override fun error(ex: SAXParseException) {
log.error(
"Problem at {}:{}:{} parsing deployment configuration: {}",
fileURL, ex.lineNumber, ex.columnNumber, ex.message
)
private fun err(ex: SAXParseException, level: Level) {
log.log(level) {
"Problem at ${fileURL}:${ex.lineNumber}:${ex.columnNumber} parsing deployment configuration: ${ex.message}"
}
throw ex
}
override fun fatalError(ex: SAXParseException) {
log.error(
"Problem at {}:{}:{} parsing deployment configuration: {}",
fileURL, ex.lineNumber, ex.columnNumber, ex.message
)
throw ex
}
override fun error(ex: SAXParseException) = err(ex, Level.ERROR)
override fun fatalError(ex: SAXParseException) = err(ex, Level.ERROR)
}
companion object {

View File

@@ -7,12 +7,11 @@ plugins {
id 'maven-publish'
}
import net.woggioni.gradle.envelope.EnvelopeJarTask
import net.woggioni.gradle.graalvm.NativeImageConfigurationTask
import net.woggioni.gradle.graalvm.NativeImagePlugin
import net.woggioni.gradle.graalvm.NativeImageTask
import net.woggioni.gradle.graalvm.NativeImageConfigurationTask
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
Property<String> mainClassName = objects.property(String.class)
mainClassName.set('net.woggioni.gbcs.cli.GradleBuildCacheServerCli')
@@ -43,6 +42,7 @@ dependencies {
implementation catalog.netty.codec.http
implementation catalog.picocli
implementation project(":gbcs-client")
implementation rootProject
// runtimeOnly catalog.slf4j.jdk14
@@ -56,11 +56,11 @@ Provider<EnvelopeJarTask> envelopeJarTaskProvider = tasks.named('envelopeJar', E
}
tasks.named(NativeImagePlugin.CONFIGURE_NATIVE_IMAGE_TASK_NAME, NativeImageConfigurationTask) {
mainClass = 'net.woggioni.gbcs.GraalNativeImageConfiguration'
mainClass = mainClassName
}
tasks.named(NativeImagePlugin.NATIVE_IMAGE_TASK_NAME, NativeImageTask) {
mainClass = 'net.woggioni.gbcs.GradleBuildCacheServer'
mainClass = mainClassName
useMusl = true
buildStaticImage = true
}

View File

@@ -3,8 +3,10 @@ module net.woggioni.gbcs.cli {
requires net.woggioni.gbcs;
requires info.picocli;
requires net.woggioni.gbcs.base;
requires net.woggioni.gbcs.client;
requires kotlin.stdlib;
requires net.woggioni.jwo;
requires net.woggioni.gbcs.api;
exports net.woggioni.gbcs.cli.impl.converters to info.picocli;
opens net.woggioni.gbcs.cli.impl.commands to info.picocli;

View File

@@ -1,28 +1,22 @@
package net.woggioni.gbcs.cli
import net.woggioni.gbcs.GradleBuildCacheServer
import net.woggioni.gbcs.GradleBuildCacheServer.Companion.DEFAULT_CONFIGURATION_URL
import net.woggioni.gbcs.base.GbcsUrlStreamHandlerFactory
import net.woggioni.gbcs.base.contextLogger
import net.woggioni.gbcs.base.debug
import net.woggioni.gbcs.base.info
import net.woggioni.gbcs.cli.impl.AbstractVersionProvider
import net.woggioni.gbcs.cli.impl.GbcsCommand
import net.woggioni.gbcs.cli.impl.commands.BenchmarkCommand
import net.woggioni.gbcs.cli.impl.commands.ClientCommand
import net.woggioni.gbcs.cli.impl.commands.PasswordHashCommand
import net.woggioni.gbcs.cli.impl.commands.ServerCommand
import net.woggioni.jwo.Application
import net.woggioni.jwo.JWO
import org.slf4j.Logger
import picocli.CommandLine
import picocli.CommandLine.Model.CommandSpec
import java.io.ByteArrayOutputStream
import java.nio.file.Files
import java.nio.file.Path
@CommandLine.Command(
name = "gbcs", versionProvider = GradleBuildCacheServerCli.VersionProvider::class
)
class GradleBuildCacheServerCli(application : Application, private val log : Logger) : GbcsCommand() {
class GradleBuildCacheServerCli : GbcsCommand() {
class VersionProvider : AbstractVersionProvider()
companion object {
@@ -35,24 +29,22 @@ class GradleBuildCacheServerCli(application : Application, private val log : Log
.configurationDirectoryEnvVar("GBCS_CONFIGURATION_DIR")
.configurationDirectoryPropertyKey("net.woggioni.gbcs.conf.dir")
.build()
val gbcsCli = GradleBuildCacheServerCli(app, log)
val gbcsCli = GradleBuildCacheServerCli()
val commandLine = CommandLine(gbcsCli)
commandLine.setExecutionExceptionHandler { ex, cl, parseResult ->
log.error(ex.message, ex)
CommandLine.ExitCode.SOFTWARE
}
commandLine.addSubcommand(ServerCommand(app))
commandLine.addSubcommand(PasswordHashCommand())
commandLine.addSubcommand(
CommandLine(ClientCommand(app)).apply {
addSubcommand(BenchmarkCommand())
})
System.exit(commandLine.execute(*args))
}
}
@CommandLine.Option(
names = ["-c", "--config-file"],
description = ["Read the application configuration from this file"],
paramLabel = "CONFIG_FILE"
)
private var configurationFile: Path = findConfigurationFile(application)
@CommandLine.Option(names = ["-V", "--version"], versionHelp = true)
var versionHelp = false
private set
@@ -60,40 +52,8 @@ class GradleBuildCacheServerCli(application : Application, private val log : Log
@CommandLine.Spec
private lateinit var spec: CommandSpec
private fun findConfigurationFile(app : Application): Path {
val confDir = app.computeConfigurationDirectory()
val configurationFile = confDir.resolve("gbcs.xml")
return configurationFile
}
private fun createDefaultConfigurationFile(configurationFile : Path) {
log.info {
"Creating default configuration file at '$configurationFile'"
}
val defaultConfigurationFileResource = DEFAULT_CONFIGURATION_URL
Files.newOutputStream(configurationFile).use { outputStream ->
defaultConfigurationFileResource.openStream().use { inputStream ->
JWO.copy(inputStream, outputStream)
}
}
}
override fun run() {
if (!Files.exists(configurationFile)) {
Files.createDirectories(configurationFile.parent)
createDefaultConfigurationFile(configurationFile)
}
val configuration = GradleBuildCacheServer.loadConfiguration(configurationFile)
log.debug {
ByteArrayOutputStream().also {
GradleBuildCacheServer.dumpConfiguration(configuration, it)
}.let {
"Server configuration:\n${String(it.toByteArray())}"
}
}
val server = GradleBuildCacheServer(configuration)
server.run().use {
}
spec.commandLine().usage(System.out);
}
}

View File

@@ -1,8 +1,6 @@
package net.woggioni.gbcs.cli.impl
import picocli.CommandLine
import java.net.URL
import java.util.Enumeration
import java.util.jar.Attributes
import java.util.jar.JarFile
import java.util.jar.Manifest

View File

@@ -1,6 +1,8 @@
package net.woggioni.gbcs.cli.impl
import net.woggioni.jwo.Application
import picocli.CommandLine
import java.nio.file.Path
abstract class GbcsCommand : Runnable {
@@ -8,4 +10,10 @@ abstract class GbcsCommand : Runnable {
@CommandLine.Option(names = ["-h", "--help"], usageHelp = true)
var usageHelp = false
private set
protected fun findConfigurationFile(app: Application, fileName : String): Path {
val confDir = app.computeConfigurationDirectory()
val configurationFile = confDir.resolve(fileName)
return configurationFile
}
}

View File

@@ -0,0 +1,132 @@
package net.woggioni.gbcs.cli.impl.commands
import net.woggioni.gbcs.base.contextLogger
import net.woggioni.gbcs.base.error
import net.woggioni.gbcs.base.info
import net.woggioni.gbcs.cli.impl.GbcsCommand
import net.woggioni.gbcs.client.GbcsClient
import picocli.CommandLine
import java.security.SecureRandom
import java.time.Duration
import java.time.Instant
import java.util.Base64
import java.util.concurrent.ExecutionException
import java.util.concurrent.Future
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.atomic.AtomicLong
import kotlin.random.Random
@CommandLine.Command(
name = "benchmark",
description = ["Run a load test against the server"],
showDefaultValues = true
)
class BenchmarkCommand : GbcsCommand() {
private val log = contextLogger()
@CommandLine.Spec
private lateinit var spec: CommandLine.Model.CommandSpec
@CommandLine.Option(
names = ["-e", "--entries"],
description = ["Total number of elements to be added to the cache"],
paramLabel = "NUMBER_OF_ENTRIES"
)
private var numberOfEntries = 1000
override fun run() {
val clientCommand = spec.parent().userObject() as ClientCommand
val profile = clientCommand.profileName.let { profileName ->
clientCommand.configuration.profiles[profileName]
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
}
val client = GbcsClient(profile)
val entryGenerator = sequence {
val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong())
while (true) {
val key = Base64.getUrlEncoder().encode(random.nextBytes(16)).toString(Charsets.UTF_8)
val value = random.nextBytes(0x1000)
yield(key to value)
}
}
val entries = let {
val completionQueue = LinkedBlockingQueue<Future<Pair<String, ByteArray>>>(numberOfEntries)
val start = Instant.now()
val totalElapsedTime = AtomicLong(0)
entryGenerator.take(numberOfEntries).forEach { entry ->
val requestStart = System.nanoTime()
val future = client.put(entry.first, entry.second).thenApply { entry }
future.whenComplete { _, _ ->
totalElapsedTime.addAndGet((System.nanoTime() - requestStart))
completionQueue.put(future)
}
}
val inserted = sequence<Pair<String, ByteArray>> {
var completionCounter = 0
while (completionCounter < numberOfEntries) {
val future = completionQueue.take()
try {
yield(future.get())
} catch (ee: ExecutionException) {
val cause = ee.cause ?: ee
log.error(cause.message, cause)
}
completionCounter += 1
}
}.toList()
val end = Instant.now()
log.info {
val elapsed = Duration.between(start, end).toMillis()
"Insertion rate: ${numberOfEntries.toDouble() / elapsed * 1000} ops/s"
}
log.info {
"Average time per insertion: ${totalElapsedTime.get() / numberOfEntries.toDouble() * 1000} ms"
}
inserted
}
log.info {
"Inserted ${entries.size} entries"
}
if (entries.isNotEmpty()) {
val completionQueue = LinkedBlockingQueue<Future<Unit>>(entries.size)
val start = Instant.now()
val totalElapsedTime = AtomicLong(0)
entries.forEach { entry ->
val requestStart = System.nanoTime()
val future = client.get(entry.first).thenApply {
totalElapsedTime.addAndGet((System.nanoTime() - requestStart))
if (it == null) {
log.error {
"Missing entry for key '${entry.first}'"
}
} else if (!entry.second.contentEquals(it)) {
log.error {
"Retrieved a value different from what was inserted for key '${entry.first}'"
}
}
}
future.whenComplete { _, _ ->
completionQueue.put(future)
}
}
var completionCounter = 0
while (completionCounter < entries.size) {
completionQueue.take()
completionCounter += 1
}
val end = Instant.now()
log.info {
val elapsed = Duration.between(start, end).toMillis()
"Retrieval rate: ${entries.size.toDouble() / elapsed * 1000} ops/s"
}
log.info {
"Average time per retrieval: ${totalElapsedTime.get() / numberOfEntries.toDouble() * 1e6} ms"
}
} else {
log.error("Skipping retrieval benchmark as it was not possible to insert any entry in the cache")
}
}
}

View File

@@ -0,0 +1,41 @@
package net.woggioni.gbcs.cli.impl.commands
import net.woggioni.gbcs.cli.impl.GbcsCommand
import net.woggioni.gbcs.client.GbcsClient
import net.woggioni.jwo.Application
import picocli.CommandLine
import java.nio.file.Path
@CommandLine.Command(
name = "client",
description = ["GBCS client"],
showDefaultValues = true
)
class ClientCommand(app : Application) : GbcsCommand() {
@CommandLine.Option(
names = ["-c", "--configuration"],
description = ["Path to the client configuration file"],
paramLabel = "CONFIGURATION_FILE"
)
private var configurationFile : Path = findConfigurationFile(app, "gbcs-client.xml")
@CommandLine.Option(
names = ["-p", "--profile"],
description = ["Name of the client profile to be used"],
paramLabel = "PROFILE",
required = true
)
var profileName : String? = null
val configuration : GbcsClient.Configuration by lazy {
GbcsClient.Configuration.parse(configurationFile)
}
override fun run() {
println("Available profiles:")
configuration.profiles.forEach { (profileName, _) ->
println(profileName)
}
}
}

View File

@@ -0,0 +1,67 @@
package net.woggioni.gbcs.cli.impl.commands
import net.woggioni.gbcs.GradleBuildCacheServer
import net.woggioni.gbcs.GradleBuildCacheServer.Companion.DEFAULT_CONFIGURATION_URL
import net.woggioni.gbcs.api.Configuration
import net.woggioni.gbcs.base.contextLogger
import net.woggioni.gbcs.base.debug
import net.woggioni.gbcs.base.info
import net.woggioni.gbcs.cli.impl.GbcsCommand
import net.woggioni.jwo.Application
import net.woggioni.jwo.JWO
import picocli.CommandLine
import java.io.ByteArrayOutputStream
import java.nio.file.Files
import java.nio.file.Path
@CommandLine.Command(
name = "server",
description = ["GBCS server"],
showDefaultValues = true
)
class ServerCommand(app : Application) : GbcsCommand() {
private val log = contextLogger()
private fun createDefaultConfigurationFile(configurationFile: Path) {
log.info {
"Creating default configuration file at '$configurationFile'"
}
val defaultConfigurationFileResource = DEFAULT_CONFIGURATION_URL
Files.newOutputStream(configurationFile).use { outputStream ->
defaultConfigurationFileResource.openStream().use { inputStream ->
JWO.copy(inputStream, outputStream)
}
}
}
@CommandLine.Option(
names = ["-c", "--config-file"],
description = ["Read the application configuration from this file"],
paramLabel = "CONFIG_FILE"
)
private var configurationFile: Path = findConfigurationFile(app, "gbcs-server.xml")
val configuration : Configuration by lazy {
GradleBuildCacheServer.loadConfiguration(configurationFile)
}
override fun run() {
if (!Files.exists(configurationFile)) {
Files.createDirectories(configurationFile.parent)
createDefaultConfigurationFile(configurationFile)
}
val configuration = GradleBuildCacheServer.loadConfiguration(configurationFile)
log.debug {
ByteArrayOutputStream().also {
GradleBuildCacheServer.dumpConfiguration(configuration, it)
}.let {
"Server configuration:\n${String(it.toByteArray())}"
}
}
val server = GradleBuildCacheServer(configuration)
server.run().use {
}
}
}

View File

@@ -15,8 +15,6 @@
<root level="info">
<appender-ref ref="console"/>
</root>
<logger name="io.netty" level="debug"/>
<logger name="io.netty.handler.ssl.BouncyCastlePemReader" level="info"/>
<logger name="com.google.code.yanf4j" level="warn"/>
<logger name="net.rubyeye.xmemcached" level="warn"/>
</configuration>

14
gbcs-client/build.gradle Normal file
View File

@@ -0,0 +1,14 @@
plugins {
id 'java-library'
alias catalog.plugins.kotlin.jvm
}
dependencies {
implementation project(':gbcs-base')
implementation catalog.picocli
implementation catalog.slf4j.api
implementation catalog.netty.buffer
implementation catalog.netty.codec.http
}

View File

@@ -0,0 +1,16 @@
module net.woggioni.gbcs.client {
requires io.netty.handler;
requires io.netty.codec.http;
requires io.netty.transport;
requires kotlin.stdlib;
requires io.netty.common;
requires io.netty.buffer;
requires java.xml;
requires net.woggioni.gbcs.base;
requires io.netty.codec;
requires org.slf4j;
exports net.woggioni.gbcs.client;
opens net.woggioni.gbcs.client.schema;
}

View File

@@ -0,0 +1,254 @@
package net.woggioni.gbcs.client
import io.netty.bootstrap.Bootstrap
import io.netty.buffer.ByteBuf
import io.netty.buffer.Unpooled
import io.netty.channel.Channel
import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelOption
import io.netty.channel.ChannelPipeline
import io.netty.channel.SimpleChannelInboundHandler
import io.netty.channel.nio.NioEventLoopGroup
import io.netty.channel.pool.AbstractChannelPoolHandler
import io.netty.channel.pool.ChannelPool
import io.netty.channel.pool.FixedChannelPool
import io.netty.channel.socket.nio.NioSocketChannel
import io.netty.handler.codec.DecoderException
import io.netty.handler.codec.http.DefaultFullHttpRequest
import io.netty.handler.codec.http.FullHttpRequest
import io.netty.handler.codec.http.FullHttpResponse
import io.netty.handler.codec.http.HttpClientCodec
import io.netty.handler.codec.http.HttpContentDecompressor
import io.netty.handler.codec.http.HttpHeaderNames
import io.netty.handler.codec.http.HttpHeaderValues
import io.netty.handler.codec.http.HttpMethod
import io.netty.handler.codec.http.HttpObjectAggregator
import io.netty.handler.codec.http.HttpResponseStatus
import io.netty.handler.codec.http.HttpVersion
import io.netty.handler.ssl.SslContext
import io.netty.handler.ssl.SslContextBuilder
import io.netty.handler.stream.ChunkedWriteHandler
import io.netty.util.concurrent.Future
import io.netty.util.concurrent.GenericFutureListener
import net.woggioni.gbcs.base.Xml
import net.woggioni.gbcs.base.contextLogger
import net.woggioni.gbcs.base.debug
import net.woggioni.gbcs.base.info
import net.woggioni.gbcs.client.impl.Parser
import java.net.InetSocketAddress
import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
import java.security.PrivateKey
import java.security.cert.X509Certificate
import java.util.Base64
import java.util.concurrent.CompletableFuture
import java.util.concurrent.atomic.AtomicInteger
import io.netty.util.concurrent.Future as NettyFuture
class GbcsClient(private val profile: Configuration.Profile) : AutoCloseable {
private val group: NioEventLoopGroup
private var sslContext: SslContext
private val log = contextLogger()
private val pool: ChannelPool
data class Configuration(
val profiles: Map<String, Profile>
) {
sealed class Authentication {
data class TlsClientAuthenticationCredentials(
val key: PrivateKey,
val certificateChain: Array<X509Certificate>
) : Authentication()
data class BasicAuthenticationCredentials(val username: String, val password: String) : Authentication()
}
data class Profile(
val serverURI: URI,
val authentication: Authentication?,
val maxConnections : Int
)
companion object {
fun parse(path: Path): Configuration {
return Files.newInputStream(path).use {
Xml.parseXml(path.toUri().toURL(), it)
}.let(Parser::parse)
}
}
}
init {
group = NioEventLoopGroup()
sslContext = SslContextBuilder.forClient().also { builder ->
(profile.authentication as? Configuration.Authentication.TlsClientAuthenticationCredentials)?.let { tlsClientAuthenticationCredentials ->
builder.keyManager(
tlsClientAuthenticationCredentials.key,
*tlsClientAuthenticationCredentials.certificateChain
)
}
}.build()
val (scheme, host, port) = profile.serverURI.run {
Triple(
if (scheme == null) "http" else profile.serverURI.scheme,
host,
port.takeIf { it > 0 } ?: if ("https" == scheme.lowercase()) 443 else 80
)
}
val bootstrap = Bootstrap().apply {
group(group)
channel(NioSocketChannel::class.java)
option(ChannelOption.TCP_NODELAY, true)
option(ChannelOption.SO_KEEPALIVE, true)
remoteAddress(InetSocketAddress(host, port))
}
val channelPoolHandler = object : AbstractChannelPoolHandler() {
@Volatile
private var connectionCount = AtomicInteger()
@Volatile
private var leaseCount = AtomicInteger()
override fun channelReleased(ch: Channel) {
log.debug {
"Released lease ${leaseCount.decrementAndGet()}"
}
}
override fun channelAcquired(ch: Channel?) {
log.debug {
"Acquired lease ${leaseCount.getAndIncrement()}"
}
}
override fun channelCreated(ch: Channel) {
log.debug {
"Created connection ${connectionCount.getAndIncrement()}"
}
val pipeline: ChannelPipeline = ch.pipeline()
// Add SSL handler if needed
if ("https".equals(scheme, ignoreCase = true)) {
pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port))
}
// HTTP handlers
pipeline.addLast("codec", HttpClientCodec())
pipeline.addLast("decompressor", HttpContentDecompressor())
pipeline.addLast("aggregator", HttpObjectAggregator(1048576))
pipeline.addLast("chunked", ChunkedWriteHandler())
}
}
pool = FixedChannelPool(bootstrap, channelPoolHandler, profile.maxConnections)
}
fun get(key: String): CompletableFuture<ByteArray?> {
return sendRequest(profile.serverURI.resolve(key), HttpMethod.GET, null)
.thenApply {
val status = it.status()
if (it.status() == HttpResponseStatus.NOT_FOUND) {
null
} else if (it.status() != HttpResponseStatus.OK) {
throw HttpException(status)
} else {
it.content()
}
}.thenApply { maybeByteBuf ->
maybeByteBuf?.let {
val result = ByteArray(it.readableBytes())
it.getBytes(0, result)
result
}
}
}
fun put(key: String, content: ByteArray): CompletableFuture<Unit> {
return sendRequest(profile.serverURI.resolve(key), HttpMethod.PUT, content).thenApply {
val status = it.status()
if (it.status() != HttpResponseStatus.CREATED) {
throw HttpException(status)
}
}
}
private fun sendRequest(uri: URI, method: HttpMethod, body: ByteArray?): CompletableFuture<FullHttpResponse> {
val responseFuture = CompletableFuture<FullHttpResponse>()
// Custom handler for processing responses
pool.acquire().addListener(object : GenericFutureListener<NettyFuture<Channel>> {
override fun operationComplete(channelFuture: Future<Channel>) {
if (channelFuture.isSuccess) {
val channel = channelFuture.now
val pipeline = channel.pipeline()
channel.pipeline().addLast("handler", object : SimpleChannelInboundHandler<FullHttpResponse>() {
override fun channelRead0(
ctx: ChannelHandlerContext,
response: FullHttpResponse
) {
responseFuture.complete(response)
pipeline.removeLast()
pool.release(channel)
}
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
val ex = when (cause) {
is DecoderException -> cause.cause
else -> cause
}
responseFuture.completeExceptionally(ex)
ctx.close()
pipeline.removeLast()
pool.release(channel)
}
})
// Prepare the HTTP request
val request: FullHttpRequest = let {
val content: ByteBuf? = body?.takeIf(ByteArray::isNotEmpty)?.let(Unpooled::wrappedBuffer)
DefaultFullHttpRequest(
HttpVersion.HTTP_1_1,
method,
uri.rawPath,
content ?: Unpooled.buffer(0)
).apply {
headers().apply {
if (content != null) {
set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.APPLICATION_OCTET_STREAM)
set(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes())
}
set(HttpHeaderNames.HOST, profile.serverURI.host)
set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE)
set(
HttpHeaderNames.ACCEPT_ENCODING,
HttpHeaderValues.GZIP.toString() + "," + HttpHeaderValues.DEFLATE.toString()
)
// Add basic auth if configured
(profile.authentication as? Configuration.Authentication.BasicAuthenticationCredentials)?.let { credentials ->
val auth = "${credentials.username}:${credentials.password}"
val encodedAuth = Base64.getEncoder().encodeToString(auth.toByteArray())
set(HttpHeaderNames.AUTHORIZATION, "Basic $encodedAuth")
}
}
}
}
// Set headers
// Send the request
channel.writeAndFlush(request)
}
}
})
return responseFuture
}
fun shutDown(): NettyFuture<*> {
return group.shutdownGracefully()
}
override fun close() {
shutDown().sync()
}
}

View File

@@ -0,0 +1,9 @@
package net.woggioni.gbcs.client
import io.netty.handler.codec.http.HttpResponseStatus
class HttpException(private val status : HttpResponseStatus) : RuntimeException(status.reasonPhrase()) {
override val message: String
get() = "Http status ${status.code()}: ${status.reasonPhrase()}"
}

View File

@@ -0,0 +1,68 @@
package net.woggioni.gbcs.client.impl
import net.woggioni.gbcs.base.Xml.Companion.asIterable
import net.woggioni.gbcs.client.GbcsClient
import org.w3c.dom.Document
import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
import java.security.KeyStore
import java.security.PrivateKey
import java.security.cert.X509Certificate
object Parser {
fun parse(document: Document): GbcsClient.Configuration {
val root = document.documentElement
val profiles = mutableMapOf<String, GbcsClient.Configuration.Profile>()
for (child in root.asIterable()) {
val tagName = child.localName
when (tagName) {
"profile" -> {
val name = child.getAttribute("name")
val uri = child.getAttribute("base-url").let(::URI)
var authentication: GbcsClient.Configuration.Authentication? = null
for (gchild in child.asIterable()) {
when (gchild.localName) {
"tls-client-auth" -> {
val keyStoreFile = gchild.getAttribute("key-store-file")
val keyStorePassword =
gchild.getAttribute("key-store-password").takeIf(String::isNotEmpty)
val keyAlias = gchild.getAttribute("key-alias")
val keyPassword = gchild.getAttribute("key-password").takeIf(String::isNotEmpty)
val keystore = KeyStore.getInstance("PKCS12").apply {
Files.newInputStream(Path.of(keyStoreFile)).use {
load(it, keyStorePassword?.toCharArray())
}
}
val key = keystore.getKey(keyAlias, keyPassword?.toCharArray()) as PrivateKey
val certChain = keystore.getCertificateChain(keyAlias).asSequence()
.map { it as X509Certificate }
.toList()
.toTypedArray()
authentication =
GbcsClient.Configuration.Authentication.TlsClientAuthenticationCredentials(key, certChain)
}
"basic-auth" -> {
val username = gchild.getAttribute("user")
val password = gchild.getAttribute("password")
authentication =
GbcsClient.Configuration.Authentication.BasicAuthenticationCredentials(username, password)
}
}
}
val maxConnections = child.getAttribute("max-connections")
.takeIf(String::isNotEmpty)
?.let(String::toInt)
?: 50
profiles[name] = GbcsClient.Configuration.Profile(uri, authentication, maxConnections)
}
}
}
return GbcsClient.Configuration(profiles)
}
}

View File

@@ -0,0 +1,37 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<xs:schema targetNamespace="urn:net.woggioni.gbcs.client"
xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns:gbcs-client="urn:net.woggioni.gbcs.client"
elementFormDefault="unqualified"
>
<xs:element name="profiles" type="gbcs-client:profilesType"/>
<xs:complexType name="profilesType">
<xs:sequence minOccurs="0">
<xs:element name="profile" type="gbcs-client:profileType" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="profileType">
<xs:choice>
<xs:element name="basic-auth" type="gbcs-client:basicAuthType"/>
<xs:element name="tls-client-auth" type="gbcs-client:tlsClientAuthType"/>
</xs:choice>
<xs:attribute name="name" type="xs:token" use="required"/>
<xs:attribute name="base-url" type="xs:anyURI" use="required"/>
<xs:attribute name="max-connections" type="xs:positiveInteger" default="50"/>
</xs:complexType>
<xs:complexType name="basicAuthType">
<xs:attribute name="user" type="xs:string" use="required"/>
<xs:attribute name="password" type="xs:string" use="required"/>
</xs:complexType>
<xs:complexType name="tlsClientAuthType">
<xs:attribute name="key-store-file" type="xs:anyURI" use="required"/>
<xs:attribute name="key-store-password" type="xs:string" use="required"/>
<xs:attribute name="key-alias" type="xs:string" use="required"/>
<xs:attribute name="key-password" type="xs:string" use="optional"/>
</xs:complexType>
</xs:schema>

View File

@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<gbcs-client:profiles xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:gbcs-client="urn:net.woggioni.gbcs.client"
xs:schemaLocation="urn:net.woggioni.gbcs.client jms://net.woggioni.gbcs.client/net/woggioni/gbcs/client/schema/gbcs-client.xsd"
>
<profile name="profile1" base-url="https://gbcs1.example.com/">
<tls-client-auth
key-store-file="keystore.pfx"
key-store-password="password"
key-alias="woggioni@c962475fa38"
key-password="key-password"/>
</profile>
<profile name="profile2" base-url="https://gbcs2.example.com/">
<basic-auth user="user" password="password"/>
</profile>
</gbcs-client:profiles>

View File

@@ -1,6 +1,3 @@
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
id 'java-library'
id 'maven-publish'

View File

@@ -1,6 +1,5 @@
package net.woggioni.gbcs.memcached
import net.rubyeye.xmemcached.MemcachedClient
import net.rubyeye.xmemcached.XMemcachedClientBuilder
import net.rubyeye.xmemcached.command.BinaryCommandFactory
import net.rubyeye.xmemcached.transcoders.CompressionMode

View File

@@ -31,5 +31,5 @@ include 'gbcs-base'
include 'gbcs-memcached'
include 'gbcs-cli'
include 'docker'
include 'benchmark'
include 'gbcs-client'

View File

@@ -507,9 +507,9 @@ class GradleBuildCacheServer(private val cfg: Configuration) {
val DEFAULT_CONFIGURATION_URL by lazy { "classpath:net/woggioni/gbcs/gbcs-default.xml".toUrl() }
fun loadConfiguration(configurationFile: Path): Configuration {
val dbf = Xml.newDocumentBuilderFactory(null)
val db = dbf.newDocumentBuilder()
val doc = Files.newInputStream(configurationFile).use(db::parse)
val doc = Files.newInputStream(configurationFile).use {
Xml.parseXml(configurationFile.toUri().toURL(), it)
}
return Parser.parse(doc)
}

View File

@@ -16,11 +16,9 @@ import net.woggioni.gbcs.base.Xml.Companion.asIterable
import org.w3c.dom.Document
import org.w3c.dom.Element
import org.w3c.dom.TypeInfo
import java.lang.IllegalArgumentException
import java.nio.file.Paths
object Parser {
fun parse(document: Document): Configuration {
val root = document.documentElement
val anonymousUser = User("", null, emptySet())

View File

@@ -48,8 +48,8 @@
<xs:complexType name="tlsCertificateAuthorizationType">
<xs:sequence>
<xs:element name="group-extractor" type="gbcs:X500NameExtractorType"/>
<xs:element name="user-extractor" type="gbcs:X500NameExtractorType"/>
<xs:element name="group-extractor" type="gbcs:X500NameExtractorType" minOccurs="0"/>
<xs:element name="user-extractor" type="gbcs:X500NameExtractorType" minOccurs="0"/>
</xs:sequence>
</xs:complexType>

View File

@@ -1,6 +1,5 @@
package net.woggioni.gbcs.test
import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.gbcs.api.Configuration
import net.woggioni.gbcs.api.Role
import net.woggioni.gbcs.base.Xml
@@ -10,13 +9,9 @@ import net.woggioni.gbcs.utils.CertificateUtils
import net.woggioni.gbcs.utils.CertificateUtils.X509Credentials
import net.woggioni.gbcs.utils.NetworkUtils
import org.bouncycastle.asn1.x500.X500Name
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Order
import org.junit.jupiter.api.Test
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Path

View File

@@ -2,7 +2,6 @@ package net.woggioni.gbcs.test
import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.gbcs.api.Configuration
import net.woggioni.gbcs.api.Role
import net.woggioni.gbcs.base.PasswordSecurity.hashPassword
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Order

View File

@@ -1,15 +1,14 @@
package net.woggioni.gbcs.test
import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.gbcs.base.Xml
import net.woggioni.gbcs.api.Configuration
import net.woggioni.gbcs.base.Xml
import net.woggioni.gbcs.cache.FileSystemCacheConfiguration
import net.woggioni.gbcs.configuration.Serializer
import net.woggioni.gbcs.utils.NetworkUtils
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Order
import org.junit.jupiter.api.Test
import java.net.ServerSocket
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest

View File

@@ -3,33 +3,13 @@ package net.woggioni.gbcs.test
import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.gbcs.api.Configuration
import net.woggioni.gbcs.api.Role
import net.woggioni.gbcs.base.Xml
import net.woggioni.gbcs.cache.FileSystemCacheConfiguration
import net.woggioni.gbcs.configuration.Serializer
import net.woggioni.gbcs.utils.CertificateUtils
import net.woggioni.gbcs.utils.CertificateUtils.X509Credentials
import net.woggioni.gbcs.utils.NetworkUtils
import org.bouncycastle.asn1.x500.X500Name
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Order
import org.junit.jupiter.api.Test
import java.net.ServerSocket
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Path
import java.security.KeyStore
import java.security.KeyStore.PasswordProtection
import java.time.Duration
import java.util.Base64
import java.util.zip.Deflater
import javax.net.ssl.KeyManagerFactory
import javax.net.ssl.SSLContext
import javax.net.ssl.TrustManagerFactory
import kotlin.random.Random
class TlsServerTest : AbstractTlsServerTest() {