Compare commits

..

4 Commits

Author SHA1 Message Date
3b7030c302 fixed mainClassName for native image build
All checks were successful
CI / build (push) Successful in 3m22s
2025-01-16 17:23:03 +08:00
a8670277e7 fixed XML error handler for server command 2025-01-16 17:01:14 +08:00
03ee75266d added average turnaround time calculation in benchmark 2025-01-16 14:54:21 +08:00
05a265e4b4 added connection pooling to gbcs-client
All checks were successful
CI / build (push) Successful in 3m55s
2025-01-16 13:37:14 +08:00
27 changed files with 183 additions and 464 deletions

View File

@@ -6,9 +6,9 @@ plugins {
id 'maven-publish' id 'maven-publish'
} }
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
allprojects { subproject -> allprojects { subproject ->
group = 'net.woggioni' group = 'net.woggioni'

View File

@@ -2,6 +2,7 @@ package net.woggioni.gbcs.base
import org.slf4j.Logger import org.slf4j.Logger
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.slf4j.event.Level
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
import java.util.logging.LogManager import java.util.logging.LogManager
@@ -52,6 +53,12 @@ inline fun log(log : Logger,
} }
} }
inline fun Logger.log(level : Level, messageBuilder : () -> String) {
if(isEnabledForLevel(level)) {
makeLoggingEventBuilder(level).log(messageBuilder())
}
}
inline fun Logger.trace(messageBuilder : () -> String) { inline fun Logger.trace(messageBuilder : () -> String) {
if(isTraceEnabled) { if(isTraceEnabled) {
trace(messageBuilder()) trace(messageBuilder())

View File

@@ -1,6 +1,7 @@
package net.woggioni.gbcs.base package net.woggioni.gbcs.base
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import org.slf4j.event.Level
import org.w3c.dom.Document import org.w3c.dom.Document
import org.w3c.dom.Element import org.w3c.dom.Element
import org.w3c.dom.Node import org.w3c.dom.Node
@@ -80,28 +81,17 @@ class Xml(val doc: Document, val element: Element) {
private val log = LoggerFactory.getLogger(ErrorHandler::class.java) private val log = LoggerFactory.getLogger(ErrorHandler::class.java)
} }
override fun warning(ex: SAXParseException) { override fun warning(ex: SAXParseException)= err(ex, Level.WARN)
log.warn(
"Problem at {}:{}:{} parsing deployment configuration: {}",
fileURL, ex.lineNumber, ex.columnNumber, ex.message
)
}
override fun error(ex: SAXParseException) { private fun err(ex: SAXParseException, level: Level) {
log.error( log.log(level) {
"Problem at {}:{}:{} parsing deployment configuration: {}", "Problem at ${fileURL}:${ex.lineNumber}:${ex.columnNumber} parsing deployment configuration: ${ex.message}"
fileURL, ex.lineNumber, ex.columnNumber, ex.message }
)
throw ex throw ex
} }
override fun fatalError(ex: SAXParseException) { override fun error(ex: SAXParseException) = err(ex, Level.ERROR)
log.error( override fun fatalError(ex: SAXParseException) = err(ex, Level.ERROR)
"Problem at {}:{}:{} parsing deployment configuration: {}",
fileURL, ex.lineNumber, ex.columnNumber, ex.message
)
throw ex
}
} }
companion object { companion object {

View File

@@ -56,11 +56,11 @@ Provider<EnvelopeJarTask> envelopeJarTaskProvider = tasks.named('envelopeJar', E
} }
tasks.named(NativeImagePlugin.CONFIGURE_NATIVE_IMAGE_TASK_NAME, NativeImageConfigurationTask) { tasks.named(NativeImagePlugin.CONFIGURE_NATIVE_IMAGE_TASK_NAME, NativeImageConfigurationTask) {
mainClass = 'net.woggioni.gbcs.GraalNativeImageConfiguration' mainClass = mainClassName
} }
tasks.named(NativeImagePlugin.NATIVE_IMAGE_TASK_NAME, NativeImageTask) { tasks.named(NativeImagePlugin.NATIVE_IMAGE_TASK_NAME, NativeImageTask) {
mainClass = 'net.woggioni.gbcs.GradleBuildCacheServer' mainClass = mainClassName
useMusl = true useMusl = true
buildStaticImage = true buildStaticImage = true
} }

View File

@@ -6,6 +6,7 @@ module net.woggioni.gbcs.cli {
requires net.woggioni.gbcs.client; requires net.woggioni.gbcs.client;
requires kotlin.stdlib; requires kotlin.stdlib;
requires net.woggioni.jwo; requires net.woggioni.jwo;
requires net.woggioni.gbcs.api;
exports net.woggioni.gbcs.cli.impl.converters to info.picocli; exports net.woggioni.gbcs.cli.impl.converters to info.picocli;
opens net.woggioni.gbcs.cli.impl.commands to info.picocli; opens net.woggioni.gbcs.cli.impl.commands to info.picocli;

View File

@@ -1,8 +1,6 @@
package net.woggioni.gbcs.cli.impl package net.woggioni.gbcs.cli.impl
import picocli.CommandLine import picocli.CommandLine
import java.net.URL
import java.util.Enumeration
import java.util.jar.Attributes import java.util.jar.Attributes
import java.util.jar.JarFile import java.util.jar.JarFile
import java.util.jar.Manifest import java.util.jar.Manifest

View File

@@ -13,6 +13,7 @@ import java.util.Base64
import java.util.concurrent.ExecutionException import java.util.concurrent.ExecutionException
import java.util.concurrent.Future import java.util.concurrent.Future
import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.atomic.AtomicLong
import kotlin.random.Random import kotlin.random.Random
@CommandLine.Command( @CommandLine.Command(
@@ -53,9 +54,12 @@ class BenchmarkCommand : GbcsCommand() {
val entries = let { val entries = let {
val completionQueue = LinkedBlockingQueue<Future<Pair<String, ByteArray>>>(numberOfEntries) val completionQueue = LinkedBlockingQueue<Future<Pair<String, ByteArray>>>(numberOfEntries)
val start = Instant.now() val start = Instant.now()
val totalElapsedTime = AtomicLong(0)
entryGenerator.take(numberOfEntries).forEach { entry -> entryGenerator.take(numberOfEntries).forEach { entry ->
val requestStart = System.nanoTime()
val future = client.put(entry.first, entry.second).thenApply { entry } val future = client.put(entry.first, entry.second).thenApply { entry }
future.whenComplete { _, _ -> future.whenComplete { _, _ ->
totalElapsedTime.addAndGet((System.nanoTime() - requestStart))
completionQueue.put(future) completionQueue.put(future)
} }
} }
@@ -78,6 +82,9 @@ class BenchmarkCommand : GbcsCommand() {
val elapsed = Duration.between(start, end).toMillis() val elapsed = Duration.between(start, end).toMillis()
"Insertion rate: ${numberOfEntries.toDouble() / elapsed * 1000} ops/s" "Insertion rate: ${numberOfEntries.toDouble() / elapsed * 1000} ops/s"
} }
log.info {
"Average time per insertion: ${totalElapsedTime.get() / numberOfEntries.toDouble() * 1000} ms"
}
inserted inserted
} }
log.info { log.info {
@@ -86,8 +93,11 @@ class BenchmarkCommand : GbcsCommand() {
if (entries.isNotEmpty()) { if (entries.isNotEmpty()) {
val completionQueue = LinkedBlockingQueue<Future<Unit>>(entries.size) val completionQueue = LinkedBlockingQueue<Future<Unit>>(entries.size)
val start = Instant.now() val start = Instant.now()
val totalElapsedTime = AtomicLong(0)
entries.forEach { entry -> entries.forEach { entry ->
val requestStart = System.nanoTime()
val future = client.get(entry.first).thenApply { val future = client.get(entry.first).thenApply {
totalElapsedTime.addAndGet((System.nanoTime() - requestStart))
if (it == null) { if (it == null) {
log.error { log.error {
"Missing entry for key '${entry.first}'" "Missing entry for key '${entry.first}'"
@@ -112,6 +122,9 @@ class BenchmarkCommand : GbcsCommand() {
val elapsed = Duration.between(start, end).toMillis() val elapsed = Duration.between(start, end).toMillis()
"Retrieval rate: ${entries.size.toDouble() / elapsed * 1000} ops/s" "Retrieval rate: ${entries.size.toDouble() / elapsed * 1000} ops/s"
} }
log.info {
"Average time per retrieval: ${totalElapsedTime.get() / numberOfEntries.toDouble() * 1e6} ms"
}
} else { } else {
log.error("Skipping retrieval benchmark as it was not possible to insert any entry in the cache") log.error("Skipping retrieval benchmark as it was not possible to insert any entry in the cache")
} }

View File

@@ -1,8 +1,7 @@
package net.woggioni.gbcs.cli.impl.commands package net.woggioni.gbcs.cli.impl.commands
import net.woggioni.gbcs.client.GbcsClient
import net.woggioni.gbcs.cli.impl.GbcsCommand import net.woggioni.gbcs.cli.impl.GbcsCommand
import net.woggioni.gbcs.client.GbcsClient
import net.woggioni.jwo.Application import net.woggioni.jwo.Application
import picocli.CommandLine import picocli.CommandLine
import java.nio.file.Path import java.nio.file.Path

View File

@@ -2,11 +2,11 @@ package net.woggioni.gbcs.cli.impl.commands
import net.woggioni.gbcs.GradleBuildCacheServer import net.woggioni.gbcs.GradleBuildCacheServer
import net.woggioni.gbcs.GradleBuildCacheServer.Companion.DEFAULT_CONFIGURATION_URL import net.woggioni.gbcs.GradleBuildCacheServer.Companion.DEFAULT_CONFIGURATION_URL
import net.woggioni.gbcs.api.Configuration
import net.woggioni.gbcs.base.contextLogger import net.woggioni.gbcs.base.contextLogger
import net.woggioni.gbcs.base.debug import net.woggioni.gbcs.base.debug
import net.woggioni.gbcs.base.info import net.woggioni.gbcs.base.info
import net.woggioni.gbcs.cli.impl.GbcsCommand import net.woggioni.gbcs.cli.impl.GbcsCommand
import net.woggioni.gbcs.client.GbcsClient
import net.woggioni.jwo.Application import net.woggioni.jwo.Application
import net.woggioni.jwo.JWO import net.woggioni.jwo.JWO
import picocli.CommandLine import picocli.CommandLine
@@ -42,8 +42,8 @@ class ServerCommand(app : Application) : GbcsCommand() {
) )
private var configurationFile: Path = findConfigurationFile(app, "gbcs-server.xml") private var configurationFile: Path = findConfigurationFile(app, "gbcs-server.xml")
val configuration : GbcsClient.Configuration by lazy { val configuration : Configuration by lazy {
GbcsClient.Configuration.parse(configurationFile) GradleBuildCacheServer.loadConfiguration(configurationFile)
} }
override fun run() { override fun run() {

View File

@@ -15,8 +15,6 @@
<root level="info"> <root level="info">
<appender-ref ref="console"/> <appender-ref ref="console"/>
</root> </root>
<logger name="io.netty" level="debug"/>
<logger name="io.netty.handler.ssl.BouncyCastlePemReader" level="info"/>
<logger name="com.google.code.yanf4j" level="warn"/> <logger name="com.google.code.yanf4j" level="warn"/>
<logger name="net.rubyeye.xmemcached" level="warn"/> <logger name="net.rubyeye.xmemcached" level="warn"/>
</configuration> </configuration>

View File

@@ -6,6 +6,7 @@ plugins {
dependencies { dependencies {
implementation project(':gbcs-base') implementation project(':gbcs-base')
implementation catalog.picocli implementation catalog.picocli
implementation catalog.slf4j.api
implementation catalog.netty.buffer implementation catalog.netty.buffer
implementation catalog.netty.codec.http implementation catalog.netty.codec.http
} }

View File

@@ -1,262 +0,0 @@
package net.woggioni.gbcs.benchmark;
import lombok.Getter;
import lombok.SneakyThrows;
import net.woggioni.jwo.Fun;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyStore;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
public class Main {
@SneakyThrows
private static Properties loadProperties() {
Properties properties = new Properties();
try (final var is = Main.class.getResourceAsStream("/benchmark.properties")) {
properties.load(is);
}
return properties;
}
private static final Properties properties = loadProperties();
@State(Scope.Thread)
public static class ExecutionPlan {
private final Random random = new Random(101325);
@Getter
private final HttpClient client = createHttpClient();
private final Map<String, byte[]> entries = new HashMap<>();
private HttpClient createHttpClient() {
final var clientBuilder = HttpClient.newBuilder();
getSslContext().ifPresent(clientBuilder::sslContext);
return clientBuilder.build();
}
public final Map<String, byte[]> getEntries() {
return Collections.unmodifiableMap(entries);
}
public Map.Entry<String, byte[]> newEntry() {
final var keyBuffer = new byte[0x20];
random.nextBytes(keyBuffer);
final var key = Base64.getUrlEncoder().encodeToString(keyBuffer);
final var value = new byte[0x1000];
random.nextBytes(value);
return Map.entry(key, value);
}
@SneakyThrows
public HttpRequest.Builder newRequestBuilder(String key) {
final var requestBuilder = HttpRequest.newBuilder()
.uri(getServerURI().resolve(key));
String user = getUser();
if (user != null) {
requestBuilder.header("Authorization", buildAuthorizationHeader(user, getPassword()));
}
return requestBuilder;
}
@SneakyThrows
public URI getServerURI() {
return new URI(properties.getProperty("gbcs.server.url"));
}
@SneakyThrows
public Optional<String> getClientTrustStorePassword() {
return Optional.ofNullable(properties.getProperty("gbcs.client.ssl.truststore.password"))
.filter(Predicate.not(String::isEmpty));
}
@SneakyThrows
public Optional<KeyStore> getClientTrustStore() {
return Optional.ofNullable(properties.getProperty("gbcs.client.ssl.truststore.file"))
.filter(Predicate.not(String::isEmpty))
.map(Path::of)
.map((Fun<Path, KeyStore>) keyStoreFile -> {
final var keyStore = KeyStore.getInstance("PKCS12");
try (final var is = Files.newInputStream(keyStoreFile)) {
keyStore.load(is, getClientTrustStorePassword().map(String::toCharArray).orElse(null));
}
return keyStore;
});
}
@SneakyThrows
public Optional<KeyStore> getClientKeyStore() {
return Optional.ofNullable(properties.getProperty("gbcs.client.ssl.keystore.file"))
.filter(Predicate.not(String::isEmpty))
.map(Path::of)
.map((Fun<Path, KeyStore>) keyStoreFile -> {
final var keyStore = KeyStore.getInstance("PKCS12");
try (final var is = Files.newInputStream(keyStoreFile)) {
keyStore.load(is, getClientKeyStorePassword().map(String::toCharArray).orElse(null));
}
return keyStore;
});
}
@SneakyThrows
public Optional<String> getClientKeyStorePassword() {
return Optional.ofNullable(properties.getProperty("gbcs.client.ssl.keystore.password"))
.filter(Predicate.not(String::isEmpty));
}
@SneakyThrows
public Optional<String> getClientKeyPassword() {
return Optional.ofNullable(properties.getProperty("gbcs.client.ssl.key.password"))
.filter(Predicate.not(String::isEmpty));
}
@SneakyThrows
public String getUser() {
return Optional.ofNullable(properties.getProperty("gbcs.server.username"))
.filter(Predicate.not(String::isEmpty))
.orElse(null);
}
@SneakyThrows
public String getPassword() {
return Optional.ofNullable(properties.getProperty("gbcs.server.password"))
.filter(Predicate.not(String::isEmpty))
.orElse(null);
}
private String buildAuthorizationHeader(String user, String password) {
final var b64 = Base64.getEncoder().encode(String.format("%s:%s", user, password).getBytes(StandardCharsets.UTF_8));
return "Basic " + new String(b64);
}
@SneakyThrows
private Optional<SSLContext> getSslContext() {
return getClientKeyStore().map((Fun<KeyStore, SSLContext>) clientKeyStore -> {
final var kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
kmf.init(clientKeyStore, getClientKeyStorePassword().map(String::toCharArray).orElse(null));
// Set up trust manager factory with the truststore
final var trustManagers = getClientTrustStore().map((Fun<KeyStore, TrustManager[]>) ts -> {
final var tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(ts);
return tmf.getTrustManagers();
}).orElse(new TrustManager[0]);
// Create SSL context with the key and trust managers
final var sslContext = SSLContext.getInstance("TLS");
sslContext.init(kmf.getKeyManagers(), trustManagers, null);
return sslContext;
});
}
@SneakyThrows
@Setup(Level.Trial)
public void setUp() {
final var client = getClient();
for (int i = 0; i < 1000; i++) {
final var pair = newEntry();
final var requestBuilder = newRequestBuilder(pair.getKey())
.header("Content-Type", "application/octet-stream")
.PUT(HttpRequest.BodyPublishers.ofByteArray(pair.getValue()));
final var response = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString());
if (201 != response.statusCode()) {
throw new IllegalStateException(Integer.toString(response.statusCode()));
} else {
entries.put(pair.getKey(), pair.getValue());
}
}
}
@TearDown
public void tearDown() {
client.close();
}
private Iterator<Map.Entry<String, byte[]>> it = null;
private Map.Entry<String, byte[]> nextEntry() {
if (it == null || !it.hasNext()) {
it = getEntries().entrySet().iterator();
}
return it.next();
}
}
@SneakyThrows
@Benchmark
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.SECONDS)
public void get(ExecutionPlan plan) {
final var client = plan.getClient();
final var entry = plan.nextEntry();
final var requestBuilder = plan.newRequestBuilder(entry.getKey())
.header("Accept", "application/octet-stream")
.GET();
final var response = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray());
if (200 != response.statusCode()) {
throw new IllegalStateException(Integer.toString(response.statusCode()));
} else {
if (!Arrays.equals(entry.getValue(), response.body())) {
throw new IllegalStateException("Retrieved unexpected value");
}
}
}
@SneakyThrows
@Benchmark
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.SECONDS)
public void put(Main.ExecutionPlan plan) {
final var client = plan.getClient();
final var entry = plan.nextEntry();
final var requestBuilder = plan.newRequestBuilder(entry.getKey())
.header("Content-Type", "application/octet-stream")
.PUT(HttpRequest.BodyPublishers.ofByteArray(entry.getValue()));
final var response = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray());
if (201 != response.statusCode()) {
throw new IllegalStateException(Integer.toString(response.statusCode()));
}
}
}

View File

@@ -1,6 +0,0 @@
gbcs.server.url= https://gbcs.woggioni.net:443
gbcs.client.ssl.keystore.file=conf/woggioni@c962475fa38.p12
gbcs.client.ssl.keystore.password=password
gbcs.client.ssl.key.password=password
gbcs.client.ssl.truststore.file=conf/truststore.pfx
gbcs.client.ssl.truststore.password=password

View File

@@ -8,6 +8,7 @@ module net.woggioni.gbcs.client {
requires java.xml; requires java.xml;
requires net.woggioni.gbcs.base; requires net.woggioni.gbcs.base;
requires io.netty.codec; requires io.netty.codec;
requires org.slf4j;
exports net.woggioni.gbcs.client; exports net.woggioni.gbcs.client;

View File

@@ -5,11 +5,13 @@ import io.netty.buffer.ByteBuf
import io.netty.buffer.Unpooled import io.netty.buffer.Unpooled
import io.netty.channel.Channel import io.netty.channel.Channel
import io.netty.channel.ChannelHandlerContext import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelInitializer import io.netty.channel.ChannelOption
import io.netty.channel.ChannelPipeline import io.netty.channel.ChannelPipeline
import io.netty.channel.SimpleChannelInboundHandler import io.netty.channel.SimpleChannelInboundHandler
import io.netty.channel.nio.NioEventLoopGroup import io.netty.channel.nio.NioEventLoopGroup
import io.netty.channel.socket.SocketChannel import io.netty.channel.pool.AbstractChannelPoolHandler
import io.netty.channel.pool.ChannelPool
import io.netty.channel.pool.FixedChannelPool
import io.netty.channel.socket.nio.NioSocketChannel import io.netty.channel.socket.nio.NioSocketChannel
import io.netty.handler.codec.DecoderException import io.netty.handler.codec.DecoderException
import io.netty.handler.codec.http.DefaultFullHttpRequest import io.netty.handler.codec.http.DefaultFullHttpRequest
@@ -26,8 +28,14 @@ import io.netty.handler.codec.http.HttpVersion
import io.netty.handler.ssl.SslContext import io.netty.handler.ssl.SslContext
import io.netty.handler.ssl.SslContextBuilder import io.netty.handler.ssl.SslContextBuilder
import io.netty.handler.stream.ChunkedWriteHandler import io.netty.handler.stream.ChunkedWriteHandler
import io.netty.util.concurrent.Future
import io.netty.util.concurrent.GenericFutureListener
import net.woggioni.gbcs.base.Xml import net.woggioni.gbcs.base.Xml
import net.woggioni.gbcs.base.contextLogger
import net.woggioni.gbcs.base.debug
import net.woggioni.gbcs.base.info
import net.woggioni.gbcs.client.impl.Parser import net.woggioni.gbcs.client.impl.Parser
import java.net.InetSocketAddress
import java.net.URI import java.net.URI
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
@@ -35,29 +43,36 @@ import java.security.PrivateKey
import java.security.cert.X509Certificate import java.security.cert.X509Certificate
import java.util.Base64 import java.util.Base64
import java.util.concurrent.CompletableFuture import java.util.concurrent.CompletableFuture
import java.util.concurrent.atomic.AtomicInteger
import io.netty.util.concurrent.Future as NettyFuture import io.netty.util.concurrent.Future as NettyFuture
class GbcsClient(private val profile: Configuration.Profile) : AutoCloseable { class GbcsClient(private val profile: Configuration.Profile) : AutoCloseable {
private val group: NioEventLoopGroup private val group: NioEventLoopGroup
private var sslContext: SslContext private var sslContext: SslContext
private val log = contextLogger()
private val pool: ChannelPool
data class Configuration( data class Configuration(
val profiles : Map<String, Profile> val profiles: Map<String, Profile>
) { ) {
sealed class Authentication { sealed class Authentication {
data class TlsClientAuthenticationCredentials(val key: PrivateKey, val certificateChain: Array<X509Certificate>) : Authentication() data class TlsClientAuthenticationCredentials(
val key: PrivateKey,
val certificateChain: Array<X509Certificate>
) : Authentication()
data class BasicAuthenticationCredentials(val username: String, val password: String) : Authentication() data class BasicAuthenticationCredentials(val username: String, val password: String) : Authentication()
} }
data class Profile( data class Profile(
val serverURI: URI, val serverURI: URI,
val authentication : Authentication? val authentication: Authentication?,
val maxConnections : Int
) )
companion object { companion object {
fun parse(path : Path) : Configuration { fun parse(path: Path): Configuration {
return Files.newInputStream(path).use { return Files.newInputStream(path).use {
Xml.parseXml(path.toUri().toURL(), it) Xml.parseXml(path.toUri().toURL(), it)
}.let(Parser::parse) }.let(Parser::parse)
@@ -67,9 +82,7 @@ class GbcsClient(private val profile: Configuration.Profile) : AutoCloseable {
init { init {
group = NioEventLoopGroup() group = NioEventLoopGroup()
sslContext = SslContextBuilder.forClient().also { builder ->
this.sslContext = SslContextBuilder.forClient().also { builder ->
(profile.authentication as? Configuration.Authentication.TlsClientAuthenticationCredentials)?.let { tlsClientAuthenticationCredentials -> (profile.authentication as? Configuration.Authentication.TlsClientAuthenticationCredentials)?.let { tlsClientAuthenticationCredentials ->
builder.keyManager( builder.keyManager(
tlsClientAuthenticationCredentials.key, tlsClientAuthenticationCredentials.key,
@@ -77,6 +90,61 @@ class GbcsClient(private val profile: Configuration.Profile) : AutoCloseable {
) )
} }
}.build() }.build()
val (scheme, host, port) = profile.serverURI.run {
Triple(
if (scheme == null) "http" else profile.serverURI.scheme,
host,
port.takeIf { it > 0 } ?: if ("https" == scheme.lowercase()) 443 else 80
)
}
val bootstrap = Bootstrap().apply {
group(group)
channel(NioSocketChannel::class.java)
option(ChannelOption.TCP_NODELAY, true)
option(ChannelOption.SO_KEEPALIVE, true)
remoteAddress(InetSocketAddress(host, port))
}
val channelPoolHandler = object : AbstractChannelPoolHandler() {
@Volatile
private var connectionCount = AtomicInteger()
@Volatile
private var leaseCount = AtomicInteger()
override fun channelReleased(ch: Channel) {
log.debug {
"Released lease ${leaseCount.decrementAndGet()}"
}
}
override fun channelAcquired(ch: Channel?) {
log.debug {
"Acquired lease ${leaseCount.getAndIncrement()}"
}
}
override fun channelCreated(ch: Channel) {
log.debug {
"Created connection ${connectionCount.getAndIncrement()}"
}
val pipeline: ChannelPipeline = ch.pipeline()
// Add SSL handler if needed
if ("https".equals(scheme, ignoreCase = true)) {
pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port))
}
// HTTP handlers
pipeline.addLast("codec", HttpClientCodec())
pipeline.addLast("decompressor", HttpContentDecompressor())
pipeline.addLast("aggregator", HttpObjectAggregator(1048576))
pipeline.addLast("chunked", ChunkedWriteHandler())
}
}
pool = FixedChannelPool(bootstrap, channelPoolHandler, profile.maxConnections)
} }
fun get(key: String): CompletableFuture<ByteArray?> { fun get(key: String): CompletableFuture<ByteArray?> {
@@ -110,43 +178,20 @@ class GbcsClient(private val profile: Configuration.Profile) : AutoCloseable {
private fun sendRequest(uri: URI, method: HttpMethod, body: ByteArray?): CompletableFuture<FullHttpResponse> { private fun sendRequest(uri: URI, method: HttpMethod, body: ByteArray?): CompletableFuture<FullHttpResponse> {
val responseFuture = CompletableFuture<FullHttpResponse>() val responseFuture = CompletableFuture<FullHttpResponse>()
try {
val scheme = if (uri.scheme == null) "http" else uri.scheme
val host = uri.host
var port = uri.port
if (port == -1) {
port = if ("https".equals(scheme, ignoreCase = true)) 443 else 80
}
val bootstrap = Bootstrap()
bootstrap.group(group)
.channel(NioSocketChannel::class.java)
.handler(object : ChannelInitializer<SocketChannel>() {
override fun initChannel(ch: SocketChannel) {
val pipeline: ChannelPipeline = ch.pipeline()
// Add SSL handler if needed
if ("https".equals(scheme, ignoreCase = true)) {
pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port))
}
// HTTP handlers
pipeline.addLast("codec", HttpClientCodec())
pipeline.addLast("decompressor", HttpContentDecompressor())
pipeline.addLast("aggregator", HttpObjectAggregator(1048576))
pipeline.addLast("chunked", ChunkedWriteHandler())
// Custom handler for processing responses // Custom handler for processing responses
pipeline.addLast("handler", object : SimpleChannelInboundHandler<FullHttpResponse>() { pool.acquire().addListener(object : GenericFutureListener<NettyFuture<Channel>> {
override fun operationComplete(channelFuture: Future<Channel>) {
if (channelFuture.isSuccess) {
val channel = channelFuture.now
val pipeline = channel.pipeline()
channel.pipeline().addLast("handler", object : SimpleChannelInboundHandler<FullHttpResponse>() {
override fun channelRead0( override fun channelRead0(
ctx: ChannelHandlerContext, ctx: ChannelHandlerContext,
response: FullHttpResponse response: FullHttpResponse
) { ) {
responseFuture.complete(response) responseFuture.complete(response)
ctx.close() pipeline.removeLast()
pool.release(channel)
} }
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
@@ -156,24 +201,25 @@ class GbcsClient(private val profile: Configuration.Profile) : AutoCloseable {
} }
responseFuture.completeExceptionally(ex) responseFuture.completeExceptionally(ex)
ctx.close() ctx.close()
pipeline.removeLast()
pool.release(channel)
} }
}) })
}
})
// Connect to host
val channel: Channel = bootstrap.connect(host, port).sync().channel()
// Prepare the HTTP request // Prepare the HTTP request
val request: FullHttpRequest = let { val request: FullHttpRequest = let {
val content: ByteBuf? = body?.takeIf(ByteArray::isNotEmpty)?.let(Unpooled::wrappedBuffer) val content: ByteBuf? = body?.takeIf(ByteArray::isNotEmpty)?.let(Unpooled::wrappedBuffer)
DefaultFullHttpRequest(HttpVersion.HTTP_1_1, method, uri.rawPath, content ?: Unpooled.buffer(0)).apply { DefaultFullHttpRequest(
HttpVersion.HTTP_1_1,
method,
uri.rawPath,
content ?: Unpooled.buffer(0)
).apply {
headers().apply { headers().apply {
if (content != null) { if (content != null) {
set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.APPLICATION_OCTET_STREAM) set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.APPLICATION_OCTET_STREAM)
set(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes()) set(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes())
} }
set(HttpHeaderNames.HOST, host) set(HttpHeaderNames.HOST, profile.serverURI.host)
set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE) set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE)
set( set(
HttpHeaderNames.ACCEPT_ENCODING, HttpHeaderNames.ACCEPT_ENCODING,
@@ -192,10 +238,9 @@ class GbcsClient(private val profile: Configuration.Profile) : AutoCloseable {
// Set headers // Set headers
// Send the request // Send the request
channel.writeAndFlush(request) channel.writeAndFlush(request)
} catch (e: Exception) {
responseFuture.completeExceptionally(e)
} }
}
})
return responseFuture return responseFuture
} }

View File

@@ -1,38 +0,0 @@
package net.woggioni.gbcs.client
import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
import java.security.KeyStore
import java.security.PrivateKey
import java.security.cert.X509Certificate
import kotlin.random.Random
//object Main {
// @JvmStatic
// fun main(vararg args : String) {
// val pwd = "PO%!*bW9p'Zp#=uu\$fl{Ij`Ad.8}x#ho".toCharArray()
// val keystore = KeyStore.getInstance("PKCS12").apply{
// Files.newInputStream(Path.of("/home/woggioni/ssl/woggioni@c962475fa38.pfx")).use {
// load(it, pwd)
// }
// }
// val key = keystore.getKey("woggioni@c962475fa38", pwd) as PrivateKey
// val certChain = keystore.getCertificateChain("woggioni@c962475fa38").asSequence()
// .map { it as X509Certificate }
// .toList()
// .toTypedArray()
// GbcsClient.Configuration(
// serverURI = URI("https://gbcs.woggioni.net/"),
// GbcsClient.TlsClientAuthenticationCredentials(
// key, certChain
// )
// ).let(::GbcsClient).use { client ->
// val random = Random(101325)
// val entry = "something" to ByteArray(0x1000).also(random::nextBytes)
// client.put(entry.first, entry.second)
// val retrieved = client.get(entry.first).get()
// println(retrieved.contentEquals(entry.second))
// }
// }
//}

View File

@@ -55,7 +55,11 @@ object Parser {
} }
} }
} }
profiles[name] = GbcsClient.Configuration.Profile(uri, authentication) val maxConnections = child.getAttribute("max-connections")
.takeIf(String::isNotEmpty)
?.let(String::toInt)
?: 50
profiles[name] = GbcsClient.Configuration.Profile(uri, authentication, maxConnections)
} }
} }
} }

View File

@@ -19,6 +19,7 @@
</xs:choice> </xs:choice>
<xs:attribute name="name" type="xs:token" use="required"/> <xs:attribute name="name" type="xs:token" use="required"/>
<xs:attribute name="base-url" type="xs:anyURI" use="required"/> <xs:attribute name="base-url" type="xs:anyURI" use="required"/>
<xs:attribute name="max-connections" type="xs:positiveInteger" default="50"/>
</xs:complexType> </xs:complexType>
<xs:complexType name="basicAuthType"> <xs:complexType name="basicAuthType">

View File

@@ -1,6 +1,3 @@
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins { plugins {
id 'java-library' id 'java-library'
id 'maven-publish' id 'maven-publish'

View File

@@ -1,6 +1,5 @@
package net.woggioni.gbcs.memcached package net.woggioni.gbcs.memcached
import net.rubyeye.xmemcached.MemcachedClient
import net.rubyeye.xmemcached.XMemcachedClientBuilder import net.rubyeye.xmemcached.XMemcachedClientBuilder
import net.rubyeye.xmemcached.command.BinaryCommandFactory import net.rubyeye.xmemcached.command.BinaryCommandFactory
import net.rubyeye.xmemcached.transcoders.CompressionMode import net.rubyeye.xmemcached.transcoders.CompressionMode

View File

@@ -507,9 +507,9 @@ class GradleBuildCacheServer(private val cfg: Configuration) {
val DEFAULT_CONFIGURATION_URL by lazy { "classpath:net/woggioni/gbcs/gbcs-default.xml".toUrl() } val DEFAULT_CONFIGURATION_URL by lazy { "classpath:net/woggioni/gbcs/gbcs-default.xml".toUrl() }
fun loadConfiguration(configurationFile: Path): Configuration { fun loadConfiguration(configurationFile: Path): Configuration {
val dbf = Xml.newDocumentBuilderFactory(null) val doc = Files.newInputStream(configurationFile).use {
val db = dbf.newDocumentBuilder() Xml.parseXml(configurationFile.toUri().toURL(), it)
val doc = Files.newInputStream(configurationFile).use(db::parse) }
return Parser.parse(doc) return Parser.parse(doc)
} }

View File

@@ -16,11 +16,9 @@ import net.woggioni.gbcs.base.Xml.Companion.asIterable
import org.w3c.dom.Document import org.w3c.dom.Document
import org.w3c.dom.Element import org.w3c.dom.Element
import org.w3c.dom.TypeInfo import org.w3c.dom.TypeInfo
import java.lang.IllegalArgumentException
import java.nio.file.Paths import java.nio.file.Paths
object Parser { object Parser {
fun parse(document: Document): Configuration { fun parse(document: Document): Configuration {
val root = document.documentElement val root = document.documentElement
val anonymousUser = User("", null, emptySet()) val anonymousUser = User("", null, emptySet())

View File

@@ -48,8 +48,8 @@
<xs:complexType name="tlsCertificateAuthorizationType"> <xs:complexType name="tlsCertificateAuthorizationType">
<xs:sequence> <xs:sequence>
<xs:element name="group-extractor" type="gbcs:X500NameExtractorType"/> <xs:element name="group-extractor" type="gbcs:X500NameExtractorType" minOccurs="0"/>
<xs:element name="user-extractor" type="gbcs:X500NameExtractorType"/> <xs:element name="user-extractor" type="gbcs:X500NameExtractorType" minOccurs="0"/>
</xs:sequence> </xs:sequence>
</xs:complexType> </xs:complexType>

View File

@@ -1,6 +1,5 @@
package net.woggioni.gbcs.test package net.woggioni.gbcs.test
import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.gbcs.api.Configuration import net.woggioni.gbcs.api.Configuration
import net.woggioni.gbcs.api.Role import net.woggioni.gbcs.api.Role
import net.woggioni.gbcs.base.Xml import net.woggioni.gbcs.base.Xml
@@ -10,13 +9,9 @@ import net.woggioni.gbcs.utils.CertificateUtils
import net.woggioni.gbcs.utils.CertificateUtils.X509Credentials import net.woggioni.gbcs.utils.CertificateUtils.X509Credentials
import net.woggioni.gbcs.utils.NetworkUtils import net.woggioni.gbcs.utils.NetworkUtils
import org.bouncycastle.asn1.x500.X500Name import org.bouncycastle.asn1.x500.X500Name
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Order
import org.junit.jupiter.api.Test
import java.net.URI import java.net.URI
import java.net.http.HttpClient import java.net.http.HttpClient
import java.net.http.HttpRequest import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.nio.charset.StandardCharsets import java.nio.charset.StandardCharsets
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path

View File

@@ -2,7 +2,6 @@ package net.woggioni.gbcs.test
import io.netty.handler.codec.http.HttpResponseStatus import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.gbcs.api.Configuration import net.woggioni.gbcs.api.Configuration
import net.woggioni.gbcs.api.Role
import net.woggioni.gbcs.base.PasswordSecurity.hashPassword import net.woggioni.gbcs.base.PasswordSecurity.hashPassword
import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Order import org.junit.jupiter.api.Order

View File

@@ -1,15 +1,14 @@
package net.woggioni.gbcs.test package net.woggioni.gbcs.test
import io.netty.handler.codec.http.HttpResponseStatus import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.gbcs.base.Xml
import net.woggioni.gbcs.api.Configuration import net.woggioni.gbcs.api.Configuration
import net.woggioni.gbcs.base.Xml
import net.woggioni.gbcs.cache.FileSystemCacheConfiguration import net.woggioni.gbcs.cache.FileSystemCacheConfiguration
import net.woggioni.gbcs.configuration.Serializer import net.woggioni.gbcs.configuration.Serializer
import net.woggioni.gbcs.utils.NetworkUtils import net.woggioni.gbcs.utils.NetworkUtils
import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Order import org.junit.jupiter.api.Order
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import java.net.ServerSocket
import java.net.URI import java.net.URI
import java.net.http.HttpClient import java.net.http.HttpClient
import java.net.http.HttpRequest import java.net.http.HttpRequest

View File

@@ -3,33 +3,13 @@ package net.woggioni.gbcs.test
import io.netty.handler.codec.http.HttpResponseStatus import io.netty.handler.codec.http.HttpResponseStatus
import net.woggioni.gbcs.api.Configuration import net.woggioni.gbcs.api.Configuration
import net.woggioni.gbcs.api.Role import net.woggioni.gbcs.api.Role
import net.woggioni.gbcs.base.Xml
import net.woggioni.gbcs.cache.FileSystemCacheConfiguration
import net.woggioni.gbcs.configuration.Serializer
import net.woggioni.gbcs.utils.CertificateUtils
import net.woggioni.gbcs.utils.CertificateUtils.X509Credentials
import net.woggioni.gbcs.utils.NetworkUtils
import org.bouncycastle.asn1.x500.X500Name import org.bouncycastle.asn1.x500.X500Name
import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Order import org.junit.jupiter.api.Order
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import java.net.ServerSocket
import java.net.URI
import java.net.http.HttpClient import java.net.http.HttpClient
import java.net.http.HttpRequest import java.net.http.HttpRequest
import java.net.http.HttpResponse import java.net.http.HttpResponse
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Path
import java.security.KeyStore
import java.security.KeyStore.PasswordProtection
import java.time.Duration
import java.util.Base64
import java.util.zip.Deflater
import javax.net.ssl.KeyManagerFactory
import javax.net.ssl.SSLContext
import javax.net.ssl.TrustManagerFactory
import kotlin.random.Random
class TlsServerTest : AbstractTlsServerTest() { class TlsServerTest : AbstractTlsServerTest() {