diff --git a/.project b/.project new file mode 100644 index 0000000..c130c71 --- /dev/null +++ b/.project @@ -0,0 +1,28 @@ + + + rbcs + Project rbcs created by Buildship. + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776929 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/.settings/org.eclipse.buildship.core.prefs b/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..5be7729 --- /dev/null +++ b/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,13 @@ +arguments=--init-script /var/lib/opencode/.eclipse/2139989682_linux_gtk_x86_64/configuration/org.eclipse.osgi/58/0/.cp/gradle/init/init.gradle +auto.sync=false +build.scans.enabled=false +connection.gradle.distribution=GRADLE_DISTRIBUTION(WRAPPER) +connection.project.dir= +eclipse.preferences.version=1 +gradle.user.home= +java.home=/usr/lib/jvm/java-26-openjdk +jvm.arguments= +offline.mode=false +override.workspace.settings=true +show.console.view=true +show.executions.view=true diff --git a/docker/.project b/docker/.project new file mode 100644 index 0000000..dc86921 --- /dev/null +++ b/docker/.project @@ -0,0 +1,28 @@ + + + docker + Project docker created by Buildship. + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776861 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/docker/.settings/org.eclipse.buildship.core.prefs b/docker/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/docker/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-api/.classpath b/rbcs-api/.classpath new file mode 100644 index 0000000..5f31ee4 --- /dev/null +++ b/rbcs-api/.classpath @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/rbcs-api/.project b/rbcs-api/.project new file mode 100644 index 0000000..f59a16d --- /dev/null +++ b/rbcs-api/.project @@ -0,0 +1,34 @@ + + + rbcs-api + Project rbcs-api created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776935 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-api/.settings/org.eclipse.buildship.core.prefs b/rbcs-api/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-api/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-api/.settings/org.eclipse.jdt.core.prefs b/rbcs-api/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-api/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-api/bin/main/module-info.class b/rbcs-api/bin/main/module-info.class new file mode 100644 index 0000000..219fd1e Binary files /dev/null and b/rbcs-api/bin/main/module-info.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/AsyncCloseable.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/AsyncCloseable.class new file mode 100644 index 0000000..45c46df Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/AsyncCloseable.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheHandler.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheHandler.class new file mode 100644 index 0000000..74e2877 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheHandler.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheHandlerFactory.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheHandlerFactory.class new file mode 100644 index 0000000..b3641db Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheHandlerFactory.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheProvider.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheProvider.class new file mode 100644 index 0000000..b652b8a Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheProvider.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheValueMetadata.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheValueMetadata.class new file mode 100644 index 0000000..86b0426 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheValueMetadata.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Authentication.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Authentication.class new file mode 100644 index 0000000..e4e68bf Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Authentication.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$BasicAuthentication.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$BasicAuthentication.class new file mode 100644 index 0000000..65f6c84 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$BasicAuthentication.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Cache.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Cache.class new file mode 100644 index 0000000..21e9538 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Cache.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$ClientCertificateAuthentication.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$ClientCertificateAuthentication.class new file mode 100644 index 0000000..97f8c73 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$ClientCertificateAuthentication.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Connection.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Connection.class new file mode 100644 index 0000000..d987d78 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Connection.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$EventExecutor.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$EventExecutor.class new file mode 100644 index 0000000..7916e05 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$EventExecutor.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$ForwardedClientCertificateAuthentication.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$ForwardedClientCertificateAuthentication.class new file mode 100644 index 0000000..1e46e6e Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$ForwardedClientCertificateAuthentication.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Group.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Group.class new file mode 100644 index 0000000..74acb29 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Group.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$GroupExtractor.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$GroupExtractor.class new file mode 100644 index 0000000..a89f937 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$GroupExtractor.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$KeyStore.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$KeyStore.class new file mode 100644 index 0000000..552dc62 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$KeyStore.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Quota.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Quota.class new file mode 100644 index 0000000..90390dd Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Quota.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$RateLimiter.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$RateLimiter.class new file mode 100644 index 0000000..cae1625 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$RateLimiter.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Tls.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Tls.class new file mode 100644 index 0000000..b66a53b Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Tls.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$TlsCertificateExtractor.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$TlsCertificateExtractor.class new file mode 100644 index 0000000..9e23557 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$TlsCertificateExtractor.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$TrustStore.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$TrustStore.class new file mode 100644 index 0000000..48fea41 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$TrustStore.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$User.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$User.class new file mode 100644 index 0000000..b6a1bb0 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$User.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$UserExtractor.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$UserExtractor.class new file mode 100644 index 0000000..bb1e735 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$UserExtractor.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration.class new file mode 100644 index 0000000..5db9937 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Role.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Role.class new file mode 100644 index 0000000..2c162d7 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/Role.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/CacheException.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/CacheException.class new file mode 100644 index 0000000..9d0ce34 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/CacheException.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/ConfigurationException.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/ConfigurationException.class new file mode 100644 index 0000000..20e5a85 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/ConfigurationException.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/ContentTooLargeException.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/ContentTooLargeException.class new file mode 100644 index 0000000..6504102 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/ContentTooLargeException.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/RbcsException.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/RbcsException.class new file mode 100644 index 0000000..f637a41 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/RbcsException.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheContent.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheContent.class new file mode 100644 index 0000000..a6be77d Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheContent.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetRequest.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetRequest.class new file mode 100644 index 0000000..6bf180b Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetRequest.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetResponse.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetResponse.class new file mode 100644 index 0000000..695bba9 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetResponse.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CachePutRequest.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CachePutRequest.class new file mode 100644 index 0000000..b2ac8f3 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CachePutRequest.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CachePutResponse.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CachePutResponse.class new file mode 100644 index 0000000..54968cf Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CachePutResponse.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheValueFoundResponse.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheValueFoundResponse.class new file mode 100644 index 0000000..20647d7 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheValueFoundResponse.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheValueNotFoundResponse.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheValueNotFoundResponse.class new file mode 100644 index 0000000..15589da Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheValueNotFoundResponse.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$LastCacheContent.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$LastCacheContent.class new file mode 100644 index 0000000..7a14653 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$LastCacheContent.class differ diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage.class new file mode 100644 index 0000000..2f2ffe9 Binary files /dev/null and b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage.class differ diff --git a/rbcs-api/src/main/java/net/woggioni/rbcs/api/Configuration.java b/rbcs-api/src/main/java/net/woggioni/rbcs/api/Configuration.java index 0fe1e73..11857f6 100644 --- a/rbcs-api/src/main/java/net/woggioni/rbcs/api/Configuration.java +++ b/rbcs-api/src/main/java/net/woggioni/rbcs/api/Configuration.java @@ -16,6 +16,7 @@ import java.util.stream.Collectors; @Value public class Configuration { + boolean enableTelemetry; String host; int port; boolean proxyProtocolEnabled; @@ -150,6 +151,7 @@ public class Configuration { } public static Configuration of( + boolean enableTelemetry, String host, int port, boolean proxyProtocolEnabled, @@ -166,6 +168,7 @@ public class Configuration { Tls tls ) { return new Configuration( + enableTelemetry, host, port, proxyProtocolEnabled, diff --git a/rbcs-cli/.classpath b/rbcs-cli/.classpath new file mode 100644 index 0000000..f11c7ec --- /dev/null +++ b/rbcs-cli/.classpath @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-cli/.project b/rbcs-cli/.project new file mode 100644 index 0000000..3a52806 --- /dev/null +++ b/rbcs-cli/.project @@ -0,0 +1,34 @@ + + + rbcs-cli + Project rbcs-cli created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776944 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-cli/.settings/org.eclipse.buildship.core.prefs b/rbcs-cli/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-cli/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-cli/.settings/org.eclipse.jdt.core.prefs b/rbcs-cli/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-cli/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-cli/bin/configureNativeImage/net/woggioni/rbcs/cli/graal/GraalNativeImageConfiguration.kt b/rbcs-cli/bin/configureNativeImage/net/woggioni/rbcs/cli/graal/GraalNativeImageConfiguration.kt new file mode 100644 index 0000000..fcc3343 --- /dev/null +++ b/rbcs-cli/bin/configureNativeImage/net/woggioni/rbcs/cli/graal/GraalNativeImageConfiguration.kt @@ -0,0 +1,200 @@ +package net.woggioni.rbcs.cli.graal + +import java.io.ByteArrayInputStream +import java.net.URI +import java.nio.file.Path +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.concurrent.ExecutionException +import java.util.zip.Deflater +import net.woggioni.jwo.NullOutputStream +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Configuration.User +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.cli.RemoteBuildCacheServerCli +import net.woggioni.rbcs.cli.impl.commands.BenchmarkCommand +import net.woggioni.rbcs.cli.impl.commands.GetCommand +import net.woggioni.rbcs.cli.impl.commands.HealthCheckCommand +import net.woggioni.rbcs.cli.impl.commands.PutCommand +import net.woggioni.rbcs.client.Configuration as ClientConfiguration +import net.woggioni.rbcs.client.impl.Parser as ClientConfigurationParser +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.server.RemoteBuildCacheServer +import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration +import net.woggioni.rbcs.server.cache.InMemoryCacheConfiguration +import net.woggioni.rbcs.server.configuration.Parser +import net.woggioni.rbcs.server.memcache.MemcacheCacheConfiguration +import net.woggioni.rbcs.server.redis.RedisCacheConfiguration + +object GraalNativeImageConfiguration { + @JvmStatic + fun main(vararg args : String) { + + let { + val serverURL = URI.create("file:conf/rbcs-server.xml").toURL() + val serverDoc = serverURL.openStream().use { + Xml.parseXml(serverURL, it) + } + Parser.parse(serverDoc) + } + + let { + val serverURL = URI.create("file:conf/rbcs-server-redis.xml").toURL() + val serverDoc = serverURL.openStream().use { + Xml.parseXml(serverURL, it) + } + Parser.parse(serverDoc) + } + + val url = URI.create("file:conf/rbcs-client.xml").toURL() + val clientDoc = url.openStream().use { + Xml.parseXml(url, it) + } + ClientConfigurationParser.parse(clientDoc) + + val PASSWORD = "password" + val readersGroup = Configuration.Group("readers", setOf(Role.Reader, Role.Healthcheck), null, null) + val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null, null) + + + val users = listOf( + User("user1", hashPassword(PASSWORD), setOf(readersGroup), null), + User("user2", hashPassword(PASSWORD), setOf(writersGroup), null), + User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup), null), + User("", null, setOf(readersGroup), null), + User("user4", hashPassword(PASSWORD), setOf(readersGroup), + Configuration.Quota(1, Duration.of(1, ChronoUnit.DAYS), 0, 1) + ), + User("user5", hashPassword(PASSWORD), setOf(readersGroup), + Configuration.Quota(1, Duration.of(5, ChronoUnit.SECONDS), 0, 1) + ) + ) + + val serverPort = RBCS.getFreePort() + + val caches = listOf( + InMemoryCacheConfiguration( + maxAge = Duration.ofSeconds(3600), + digestAlgorithm = "MD5", + compressionLevel = Deflater.DEFAULT_COMPRESSION, + compressionEnabled = false, + maxSize = 0x1000000, + ), + FileSystemCacheConfiguration( + Path.of(System.getProperty("java.io.tmpdir")).resolve("rbcs"), + maxAge = Duration.ofSeconds(3600), + digestAlgorithm = "MD5", + compressionLevel = Deflater.DEFAULT_COMPRESSION, + compressionEnabled = false, + ), + MemcacheCacheConfiguration( + listOf(MemcacheCacheConfiguration.Server( + HostAndPort("127.0.0.1", 11211), + 1000, + 4) + ), + Duration.ofSeconds(60), + "someCustomPrefix", + "MD5", + null, + 1, + ), + RedisCacheConfiguration( + listOf(RedisCacheConfiguration.Server( + HostAndPort("127.0.0.1", 6379), + 1000, + 4) + ), + Duration.ofSeconds(60), + "someCustomPrefix", + "MD5", + null, + 1, + ) + ) + + for (cache in caches) { + val serverConfiguration = Configuration( + "127.0.0.1", + serverPort, + false, + emptyList(), + 100, + null, + Configuration.EventExecutor(true), + Configuration.RateLimiter( + false, 0x100000, 10 + ), + Configuration.Connection( + Duration.ofSeconds(10), + Duration.ofSeconds(15), + Duration.ofSeconds(15), + 0x10000, + 0x1000 + ), + users.asSequence().map { it.name to it }.toMap(), + sequenceOf(writersGroup, readersGroup).map { it.name to it }.toMap(), + cache, + Configuration.BasicAuthentication(), + null, + ) + + val serverHandle = RemoteBuildCacheServer(serverConfiguration).run() + + val clientProfile = ClientConfiguration.Profile( + URI.create("http://127.0.0.1:$serverPort/"), + ClientConfiguration.Connection( + Duration.ofSeconds(5), + Duration.ofSeconds(5), + Duration.ofSeconds(7), + true, + ), + ClientConfiguration.Authentication.BasicAuthenticationCredentials("user3", PASSWORD), + Duration.ofSeconds(3), + 10, + true, + ClientConfiguration.RetryPolicy( + 3, + 1000, + 1.2 + ), + ClientConfiguration.TrustStore(null, null, false, false) + ) + + HealthCheckCommand.execute(clientProfile) + + BenchmarkCommand.execute( + clientProfile, + 1000, + 0x100, + true + ) + + PutCommand.execute( + clientProfile, + "some-file.bin", + ByteArrayInputStream(ByteArray(0x1000) { it.toByte() }), + "application/octet-setream", + "attachment; filename=\"some-file.bin\"" + ) + + GetCommand.execute( + clientProfile, + "some-file.bin", + NullOutputStream() + ) + + serverHandle.sendShutdownSignal() + try { + serverHandle.get() + } catch (ee : ExecutionException) { + } + } + System.setProperty("net.woggioni.rbcs.conf.dir", System.getProperty("gradle.tmp.dir")) + RemoteBuildCacheServerCli.createCommandLine().execute("--version") + RemoteBuildCacheServerCli.createCommandLine().execute("server", "-t", "PT10S") + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/module-info.class b/rbcs-cli/bin/main/module-info.class new file mode 100644 index 0000000..0c7d337 Binary files /dev/null and b/rbcs-cli/bin/main/module-info.class differ diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/RemoteBuildCacheServerCli.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/RemoteBuildCacheServerCli.kt new file mode 100644 index 0000000..0b0c273 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/RemoteBuildCacheServerCli.kt @@ -0,0 +1,79 @@ +package net.woggioni.rbcs.cli + +import net.woggioni.jwo.Application +import net.woggioni.rbcs.cli.impl.AbstractVersionProvider +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.cli.impl.commands.BenchmarkCommand +import net.woggioni.rbcs.cli.impl.commands.ClientCommand +import net.woggioni.rbcs.cli.impl.commands.GetCommand +import net.woggioni.rbcs.cli.impl.commands.HealthCheckCommand +import net.woggioni.rbcs.cli.impl.commands.PasswordHashCommand +import net.woggioni.rbcs.cli.impl.commands.PutCommand +import net.woggioni.rbcs.cli.impl.commands.ServerCommand +import net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory +import net.woggioni.rbcs.common.createLogger +import picocli.CommandLine +import picocli.CommandLine.Model.CommandSpec + + +@CommandLine.Command( + name = "rbcs", versionProvider = RemoteBuildCacheServerCli.VersionProvider::class +) +class RemoteBuildCacheServerCli : RbcsCommand() { + + class VersionProvider : AbstractVersionProvider() + companion object { + private fun setPropertyIfNotPresent(key: String, value: String) { + System.getProperty(key) ?: System.setProperty(key, value) + } + + fun createCommandLine() : CommandLine { + setPropertyIfNotPresent("logback.configurationFile", "net/woggioni/rbcs/cli/logback.xml") + setPropertyIfNotPresent("io.netty.leakDetectionLevel", "DISABLED") + val currentClassLoader = RemoteBuildCacheServerCli::class.java.classLoader + Thread.currentThread().contextClassLoader = currentClassLoader + if(currentClassLoader.javaClass.name == "net.woggioni.envelope.loader.ModuleClassLoader") { + //We're running in an envelope jar and custom URL protocols won't work + RbcsUrlStreamHandlerFactory.install() + } + val log = createLogger() + val app = Application.builder("rbcs") + .configurationDirectoryEnvVar("RBCS_CONFIGURATION_DIR") + .configurationDirectoryPropertyKey("net.woggioni.rbcs.conf.dir") + .build() + val rbcsCli = RemoteBuildCacheServerCli() + val commandLine = CommandLine(rbcsCli) + commandLine.setExecutionExceptionHandler { ex, cl, parseResult -> + log.error(ex.message, ex) + CommandLine.ExitCode.SOFTWARE + } + commandLine.addSubcommand(ServerCommand(app)) + commandLine.addSubcommand(PasswordHashCommand()) + commandLine.addSubcommand( + CommandLine(ClientCommand(app)).apply { + addSubcommand(BenchmarkCommand()) + addSubcommand(PutCommand()) + addSubcommand(GetCommand()) + addSubcommand(HealthCheckCommand()) + }) + return commandLine + } + + @JvmStatic + fun main(vararg args: String) { + System.exit(createCommandLine().execute(*args)) + } + } + + @CommandLine.Option(names = ["-V", "--version"], versionHelp = true) + var versionHelp = false + private set + + @CommandLine.Spec + private lateinit var spec: CommandSpec + + + override fun run() { + spec.commandLine().usage(System.out); + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/AbstractVersionProvider.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/AbstractVersionProvider.kt new file mode 100644 index 0000000..632689c --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/AbstractVersionProvider.kt @@ -0,0 +1,30 @@ +package net.woggioni.rbcs.cli.impl + +import java.util.jar.Attributes +import java.util.jar.JarFile +import java.util.jar.Manifest +import picocli.CommandLine + + +abstract class AbstractVersionProvider : CommandLine.IVersionProvider { + private val version: String + private val vcsHash: String + + init { + val mf = Manifest() + javaClass.module.getResourceAsStream(JarFile.MANIFEST_NAME).use { `is` -> + mf.read(`is`) + } + val mainAttributes = mf.mainAttributes + version = mainAttributes.getValue(Attributes.Name.SPECIFICATION_VERSION) ?: throw RuntimeException("Version information not found in manifest") + vcsHash = mainAttributes.getValue(Attributes.Name.IMPLEMENTATION_VERSION) ?: throw RuntimeException("Version information not found in manifest") + } + + override fun getVersion(): Array { + return if (version.endsWith("-SNAPSHOT")) { + arrayOf(version, vcsHash) + } else { + arrayOf(version) + } + } +} diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/RbcsCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/RbcsCommand.kt new file mode 100644 index 0000000..91d3159 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/RbcsCommand.kt @@ -0,0 +1,19 @@ +package net.woggioni.rbcs.cli.impl + +import java.nio.file.Path +import net.woggioni.jwo.Application +import picocli.CommandLine + + +abstract class RbcsCommand : Runnable { + + @CommandLine.Option(names = ["-h", "--help"], usageHelp = true) + var usageHelp = false + private set + + protected fun findConfigurationFile(app: Application, fileName : String): Path { + val confDir = app.computeConfigurationDirectory(false) + val configurationFile = confDir.resolve(fileName) + return configurationFile + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/BenchmarkCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/BenchmarkCommand.kt new file mode 100644 index 0000000..7f30b95 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/BenchmarkCommand.kt @@ -0,0 +1,199 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.security.SecureRandom +import java.time.Duration +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.Semaphore +import java.util.concurrent.atomic.AtomicLong +import kotlin.random.Random +import net.woggioni.jwo.JWO +import net.woggioni.jwo.LongMath +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.cli.impl.converters.ByteSizeConverter +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.client.RemoteBuildCacheClient +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.error +import net.woggioni.rbcs.common.info +import picocli.CommandLine + +@CommandLine.Command( + name = "benchmark", + description = ["Run a load test against the server"], + showDefaultValues = true +) +class BenchmarkCommand : RbcsCommand() { + companion object { + private val log = createLogger() + + fun execute(profile : Configuration.Profile, + numberOfEntries : Int, + entrySize : Int, + useRandomValue : Boolean, + ) { + val progressThreshold = LongMath.ceilDiv(numberOfEntries.toLong(), 20) + RemoteBuildCacheClient(profile).use { client -> + val entryGenerator = sequence { + val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong()) + while (true) { + val key = JWO.bytesToHex(random.nextBytes(16)) + val value = if (useRandomValue) { + random.nextBytes(entrySize) + } else { + val byteValue = random.nextInt().toByte() + ByteArray(entrySize) { _ -> byteValue } + } + yield(key to value) + } + } + + log.info { + "Starting insertion" + } + val entries = let { + val completionCounter = AtomicLong(0) + val completionQueue = LinkedBlockingQueue>(numberOfEntries) + val start = Instant.now() + val semaphore = Semaphore(profile.maxConnections * 5) + val iterator = entryGenerator.take(numberOfEntries).iterator() + while (completionCounter.get() < numberOfEntries) { + if (iterator.hasNext()) { + val entry = iterator.next() + semaphore.acquire() + val future = + client.put(entry.first, entry.second, CacheValueMetadata(null, null)).thenApply { entry } + future.whenComplete { result, ex -> + if (ex != null) { + log.error(ex.message, ex) + } else { + completionQueue.put(result) + } + semaphore.release() + val completed = completionCounter.incrementAndGet() + if (completed.mod(progressThreshold) == 0L) { + log.debug { + "Inserted $completed / $numberOfEntries" + } + } + } + } else { + Thread.sleep(Duration.of(500, ChronoUnit.MILLIS)) + } + } + + val inserted = completionQueue.toList() + val end = Instant.now() + log.info { + val elapsed = Duration.between(start, end).toMillis() + val opsPerSecond = String.format("%.2f", numberOfEntries.toDouble() / elapsed * 1000) + "Insertion rate: $opsPerSecond ops/s" + } + inserted + } + log.info { + "Inserted ${entries.size} entries" + } + log.info { + "Starting retrieval" + } + if (entries.isNotEmpty()) { + val errorCounter = AtomicLong(0) + val completionCounter = AtomicLong(0) + val semaphore = Semaphore(profile.maxConnections * 5) + val start = Instant.now() + val it = entries.iterator() + while (completionCounter.get() < entries.size) { + if (it.hasNext()) { + val entry = it.next() + semaphore.acquire() + val future = client.get(entry.first).handle { response, ex -> + if(ex != null) { + errorCounter.incrementAndGet() + log.error(ex.message, ex) + } else if (response == null) { + errorCounter.incrementAndGet() + log.error { + "Missing entry for key '${entry.first}'" + } + } else if (!entry.second.contentEquals(response)) { + errorCounter.incrementAndGet() + log.error { + "Retrieved a value different from what was inserted for key '${entry.first}': " + + "expected '${JWO.bytesToHex(entry.second)}', got '${JWO.bytesToHex(response)}' instead" + } + } + } + future.whenComplete { _, _ -> + val completed = completionCounter.incrementAndGet() + if (completed.mod(progressThreshold) == 0L) { + log.debug { + "Retrieved $completed / ${entries.size}" + } + } + semaphore.release() + } + } else { + Thread.sleep(Duration.of(500, ChronoUnit.MILLIS)) + } + } + val end = Instant.now() + val errors = errorCounter.get() + val successfulRetrievals = entries.size - errors + val successRate = successfulRetrievals.toDouble() / entries.size + log.info { + "Successfully retrieved ${entries.size - errors}/${entries.size} (${String.format("%.1f", successRate * 100)}%)" + } + log.info { + val elapsed = Duration.between(start, end).toMillis() + val opsPerSecond = String.format("%.2f", entries.size.toDouble() / elapsed * 1000) + "Retrieval rate: $opsPerSecond ops/s" + } + } else { + log.error("Skipping retrieval benchmark as it was not possible to insert any entry in the cache") + } + } + } + } + + @CommandLine.Spec + private lateinit var spec: CommandLine.Model.CommandSpec + + @CommandLine.Option( + names = ["-e", "--entries"], + description = ["Total number of elements to be added to the cache"], + paramLabel = "NUMBER_OF_ENTRIES" + ) + private var numberOfEntries = 1000 + + @CommandLine.Option( + names = ["-s", "--size"], + description = ["Size of a cache value in bytes"], + paramLabel = "SIZE", + converter = [ByteSizeConverter::class] + ) + private var size = 0x1000 + + @CommandLine.Option( + names = ["-r", "--random"], + description = ["Insert completely random byte values"] + ) + private var randomValues = false + + override fun run() { + val clientCommand = spec.parent().userObject() as ClientCommand + val profile = clientCommand.profileName.let { profileName -> + clientCommand.configuration.profiles[profileName] + ?: throw IllegalArgumentException("Profile $profileName does not exist in configuration") + } + execute( + profile, + numberOfEntries, + size, + randomValues + ) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ClientCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ClientCommand.kt new file mode 100644 index 0000000..4dcb8a9 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ClientCommand.kt @@ -0,0 +1,48 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.nio.file.Path +import net.woggioni.jwo.Application +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import picocli.CommandLine + +@CommandLine.Command( + name = "client", + description = ["RBCS client"], + showDefaultValues = true +) +class ClientCommand(app : Application) : RbcsCommand() { + + @CommandLine.Option( + names = ["-c", "--configuration"], + description = ["Path to the client configuration file"], + paramLabel = "CONFIGURATION_FILE" + ) + private var configurationFile : Path = findConfigurationFile(app, "rbcs-client.xml") + + @CommandLine.Option( + names = ["-p", "--profile"], + description = ["Name of the client profile to be used"], + paramLabel = "PROFILE", + required = false + ) + var profileName : String? = null + get() = field ?: throw IllegalArgumentException("A profile name must be specified using the '-p' command line parameter") + + val configuration : Configuration by lazy { + Configuration.parse(configurationFile) + } + + override fun run() { + val log = createLogger() + log.debug { + "Using configuration file '$configurationFile'" + } + println("Available profiles:") + configuration.profiles.forEach { (profileName, _) -> + println(profileName) + } + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/GetCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/GetCommand.kt new file mode 100644 index 0000000..63381ef --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/GetCommand.kt @@ -0,0 +1,59 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.io.OutputStream +import java.nio.file.Files +import java.nio.file.Path +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.client.RemoteBuildCacheClient +import net.woggioni.rbcs.common.createLogger +import picocli.CommandLine + +@CommandLine.Command( + name = "get", + description = ["Fetch a value from the cache with the specified key"], + showDefaultValues = true +) +class GetCommand : RbcsCommand() { + companion object { + private val log = createLogger() + + fun execute(profile : Configuration.Profile, key : String, outputStream: OutputStream) { + RemoteBuildCacheClient(profile).use { client -> + client.get(key).thenApply { value -> + value?.let { + outputStream.use { + it.write(value) + } + } ?: throw NoSuchElementException("No value found for key $key") + }.get() + } + } + } + + @CommandLine.Spec + private lateinit var spec: CommandLine.Model.CommandSpec + + @CommandLine.Option( + names = ["-k", "--key"], + description = ["The key for the new value"], + paramLabel = "KEY" + ) + private var key : String = "" + + @CommandLine.Option( + names = ["-v", "--value"], + description = ["Path to a file where the retrieved value will be written (defaults to stdout)"], + paramLabel = "VALUE_FILE", + ) + private var output : Path? = null + + override fun run() { + val clientCommand = spec.parent().userObject() as ClientCommand + val profile = clientCommand.profileName.let { profileName -> + clientCommand.configuration.profiles[profileName] + ?: throw IllegalArgumentException("Profile $profileName does not exist in configuration") + } + execute(profile, key, (output?.let(Files::newOutputStream) ?: System.out)) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/HealthCheckCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/HealthCheckCommand.kt new file mode 100644 index 0000000..224e63d --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/HealthCheckCommand.kt @@ -0,0 +1,53 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.security.SecureRandom +import kotlin.random.Random +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.client.RemoteBuildCacheClient +import net.woggioni.rbcs.common.createLogger +import picocli.CommandLine + +@CommandLine.Command( + name = "health", + description = ["Check server health"], + showDefaultValues = true +) +class HealthCheckCommand : RbcsCommand() { + companion object{ + private val log = createLogger() + + fun execute(profile : Configuration.Profile) { + RemoteBuildCacheClient(profile).use { client -> + val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong()) + val nonce = ByteArray(0xa0) + random.nextBytes(nonce) + client.healthCheck(nonce).thenApply { value -> + if(value == null) { + throw IllegalStateException("Empty response from server") + } + val offset = value.size - nonce.size + for(i in 0 until nonce.size) { + val a = nonce[i] + val b = value[offset + i] + if(a != b) { + throw IllegalStateException("Server nonce does not match") + } + } + }.get() + } + } + } + + @CommandLine.Spec + private lateinit var spec: CommandLine.Model.CommandSpec + + override fun run() { + val clientCommand = spec.parent().userObject() as ClientCommand + val profile = clientCommand.profileName.let { profileName -> + clientCommand.configuration.profiles[profileName] + ?: throw IllegalArgumentException("Profile $profileName does not exist in configuration") + } + execute(profile) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PasswordHashCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PasswordHashCommand.kt new file mode 100644 index 0000000..3862524 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PasswordHashCommand.kt @@ -0,0 +1,37 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.io.OutputStream +import java.io.OutputStreamWriter +import java.io.PrintWriter +import net.woggioni.jwo.UncloseableOutputStream +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.cli.impl.converters.OutputStreamConverter +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import picocli.CommandLine + + +@CommandLine.Command( + name = "password", + description = ["Generate a password hash to add to RBCS configuration file"], + showDefaultValues = true +) +class PasswordHashCommand : RbcsCommand() { + @CommandLine.Option( + names = ["-o", "--output-file"], + description = ["Write the output to a file instead of stdout"], + converter = [OutputStreamConverter::class], + showDefaultValue = CommandLine.Help.Visibility.NEVER, + paramLabel = "OUTPUT_FILE" + ) + private var outputStream: OutputStream = UncloseableOutputStream(System.out) + + override fun run() { + val password1 = String(System.console().readPassword("Type your password:")) + val password2 = String(System.console().readPassword("Type your password again for confirmation:")) + if(password1 != password2) throw IllegalArgumentException("Passwords do not match") + + PrintWriter(OutputStreamWriter(outputStream, Charsets.UTF_8)).use { + it.println(hashPassword(password1)) + } + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PutCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PutCommand.kt new file mode 100644 index 0000000..5e6d10f --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PutCommand.kt @@ -0,0 +1,113 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.io.InputStream +import java.nio.file.Files +import java.nio.file.Path +import java.util.UUID +import net.woggioni.jwo.Hash +import net.woggioni.jwo.JWO +import net.woggioni.jwo.NullOutputStream +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.client.RemoteBuildCacheClient +import net.woggioni.rbcs.common.createLogger +import picocli.CommandLine + +@CommandLine.Command( + name = "put", + description = ["Add or replace a value to the cache with the specified key"], + showDefaultValues = true +) +class PutCommand : RbcsCommand() { + companion object { + private val log = createLogger() + + fun execute(profile: Configuration.Profile, + actualKey : String, + inputStream: InputStream, + mimeType : String?, + contentDisposition: String? + ) { + RemoteBuildCacheClient(profile).use { client -> + inputStream.use { + client.put(actualKey, it.readAllBytes(), CacheValueMetadata(contentDisposition, mimeType)) + }.get() + println(profile.serverURI.resolve(actualKey)) + } + } + } + + + @CommandLine.Spec + private lateinit var spec: CommandLine.Model.CommandSpec + + @CommandLine.Option( + names = ["-k", "--key"], + description = ["The key for the new value, randomly generated if omitted"], + paramLabel = "KEY" + ) + private var key : String? = null + + @CommandLine.Option( + names = ["-i", "--inline"], + description = ["File is to be displayed in the browser"], + paramLabel = "INLINE", + ) + private var inline : Boolean = false + + @CommandLine.Option( + names = ["-t", "--type"], + description = ["File mime type"], + paramLabel = "MIME_TYPE", + ) + private var mimeType : String? = null + + @CommandLine.Option( + names = ["-v", "--value"], + description = ["Path to a file containing the value to be added (defaults to stdin)"], + paramLabel = "VALUE_FILE", + ) + private var value : Path? = null + + override fun run() { + val clientCommand = spec.parent().userObject() as ClientCommand + val profile = clientCommand.profileName.let { profileName -> + clientCommand.configuration.profiles[profileName] + ?: throw IllegalArgumentException("Profile $profileName does not exist in configuration") + } + RemoteBuildCacheClient(profile).use { client -> + val inputStream : InputStream + val mimeType : String? + val contentDisposition : String? + val valuePath = value + val actualKey : String? + if(valuePath != null) { + inputStream = Files.newInputStream(valuePath) + mimeType = this.mimeType ?: Files.probeContentType(valuePath) + contentDisposition = if(inline) { + "inline" + } else { + "attachment; filename=\"${valuePath.fileName}\"" + } + actualKey = key ?: let { + val md = Hash.Algorithm.SHA512.newInputStream(Files.newInputStream(valuePath)).use { + JWO.copy(it, NullOutputStream()) + it.messageDigest + } + UUID.nameUUIDFromBytes(md.digest()).toString() + } + } else { + inputStream = System.`in` + mimeType = this.mimeType + contentDisposition = if(inline) { + "inline" + } else { + null + } + actualKey = key ?: UUID.randomUUID().toString() + } + execute(profile, actualKey, inputStream, mimeType, contentDisposition) + } + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ServerCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ServerCommand.kt new file mode 100644 index 0000000..46af626 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ServerCommand.kt @@ -0,0 +1,90 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path +import java.time.Duration +import java.util.concurrent.TimeUnit +import net.woggioni.jwo.Application +import net.woggioni.jwo.JWO +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.cli.impl.converters.DurationConverter +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.info +import net.woggioni.rbcs.server.RemoteBuildCacheServer +import net.woggioni.rbcs.server.RemoteBuildCacheServer.Companion.DEFAULT_CONFIGURATION_URL +import picocli.CommandLine + +@CommandLine.Command( + name = "server", + description = ["RBCS server"], + showDefaultValues = true +) +class ServerCommand(app : Application) : RbcsCommand() { + companion object { + private val log = createLogger() + } + + private fun createDefaultConfigurationFile(configurationFile: Path) { + log.info { + "Creating default configuration file at '$configurationFile'" + } + val defaultConfigurationFileResource = DEFAULT_CONFIGURATION_URL + Files.newOutputStream(configurationFile).use { outputStream -> + defaultConfigurationFileResource.openStream().use { inputStream -> + JWO.copy(inputStream, outputStream) + } + } + } + + @CommandLine.Option( + names = ["-t", "--timeout"], + description = ["Exit after the specified time"], + paramLabel = "TIMEOUT", + converter = [DurationConverter::class] + ) + private var timeout: Duration? = null + + @CommandLine.Option( + names = ["-c", "--config-file"], + description = ["Read the application configuration from this file"], + paramLabel = "CONFIG_FILE" + ) + private var configurationFile: Path = findConfigurationFile(app, "rbcs-server.xml") + + override fun run() { + if (!Files.exists(configurationFile)) { + Files.createDirectories(configurationFile.parent) + createDefaultConfigurationFile(configurationFile) + } + + log.debug { + "Using configuration file '$configurationFile'" + } + val configuration = RemoteBuildCacheServer.loadConfiguration(configurationFile) + log.debug { + ByteArrayOutputStream().also { + RemoteBuildCacheServer.dumpConfiguration(configuration, it) + }.let { + "Server configuration:\n${String(it.toByteArray())}" + } + } + val server = RemoteBuildCacheServer(configuration) + val handle = server.run() + val shutdownHook = Thread.ofPlatform().unstarted { + handle.sendShutdownSignal() + try { + handle.get(60, TimeUnit.SECONDS) + } catch (ex : Throwable) { + log.warn(ex.message, ex) + } + } + Runtime.getRuntime().addShutdownHook(shutdownHook) + if(timeout != null) { + Thread.sleep(timeout) + handle.sendShutdownSignal() + } + handle.get() + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/ByteSizeConverter.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/ByteSizeConverter.kt new file mode 100644 index 0000000..e9d323e --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/ByteSizeConverter.kt @@ -0,0 +1,10 @@ +package net.woggioni.rbcs.cli.impl.converters + +import picocli.CommandLine + + +class ByteSizeConverter : CommandLine.ITypeConverter { + override fun convert(value: String): Int { + return Integer.decode(value) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/DurationConverter.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/DurationConverter.kt new file mode 100644 index 0000000..158a417 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/DurationConverter.kt @@ -0,0 +1,11 @@ +package net.woggioni.rbcs.cli.impl.converters + +import java.time.Duration +import picocli.CommandLine + + +class DurationConverter : CommandLine.ITypeConverter { + override fun convert(value: String): Duration { + return Duration.parse(value) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/InputStreamConverter.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/InputStreamConverter.kt new file mode 100644 index 0000000..81f5605 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/InputStreamConverter.kt @@ -0,0 +1,13 @@ +package net.woggioni.rbcs.cli.impl.converters + +import java.io.InputStream +import java.nio.file.Files +import java.nio.file.Paths +import picocli.CommandLine + + +class InputStreamConverter : CommandLine.ITypeConverter { + override fun convert(value: String): InputStream { + return Files.newInputStream(Paths.get(value)) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/OutputStreamConverter.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/OutputStreamConverter.kt new file mode 100644 index 0000000..8c16f40 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/OutputStreamConverter.kt @@ -0,0 +1,13 @@ +package net.woggioni.rbcs.cli.impl.converters + +import java.io.OutputStream +import java.nio.file.Files +import java.nio.file.Paths +import picocli.CommandLine + + +class OutputStreamConverter : CommandLine.ITypeConverter { + override fun convert(value: String): OutputStream { + return Files.newOutputStream(Paths.get(value)) + } +} \ No newline at end of file diff --git a/rbcs-client/.classpath b/rbcs-client/.classpath new file mode 100644 index 0000000..8bfaaaf --- /dev/null +++ b/rbcs-client/.classpath @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-client/.project b/rbcs-client/.project new file mode 100644 index 0000000..1af5415 --- /dev/null +++ b/rbcs-client/.project @@ -0,0 +1,34 @@ + + + rbcs-client + Project rbcs-client created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776950 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-client/.settings/org.eclipse.buildship.core.prefs b/rbcs-client/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-client/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-client/.settings/org.eclipse.jdt.core.prefs b/rbcs-client/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-client/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-client/bin/main/module-info.class b/rbcs-client/bin/main/module-info.class new file mode 100644 index 0000000..9e228c3 Binary files /dev/null and b/rbcs-client/bin/main/module-info.class differ diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/Configuration.kt b/rbcs-client/bin/main/net/woggioni/rbcs/client/Configuration.kt new file mode 100644 index 0000000..53e55a9 --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/Configuration.kt @@ -0,0 +1,62 @@ +package net.woggioni.rbcs.client + +import java.net.URI +import java.nio.file.Files +import java.nio.file.Path +import java.security.PrivateKey +import java.security.cert.X509Certificate +import java.time.Duration +import net.woggioni.rbcs.client.impl.Parser +import net.woggioni.rbcs.common.Xml + +data class Configuration( + val profiles: Map +) { + sealed class Authentication { + data class TlsClientAuthenticationCredentials( + val key: PrivateKey, + val certificateChain: Array + ) : Authentication() + + data class BasicAuthenticationCredentials(val username: String, val password: String) : Authentication() + } + + class TrustStore ( + var file: Path?, + var password: String?, + var checkCertificateStatus: Boolean = false, + var verifyServerCertificate: Boolean = true, + ) + + class RetryPolicy( + val maxAttempts: Int, + val initialDelayMillis: Long, + val exp: Double + ) + + class Connection( + val readIdleTimeout: Duration, + val writeIdleTimeout: Duration, + val idleTimeout: Duration, + val requestPipelining : Boolean, + ) + + data class Profile( + val serverURI: URI, + val connection: Connection, + val authentication: Authentication?, + val connectionTimeout: Duration?, + val maxConnections: Int, + val compressionEnabled: Boolean, + val retryPolicy: RetryPolicy?, + val tlsTruststore : TrustStore? + ) + + companion object { + fun parse(path: Path): Configuration { + return Files.newInputStream(path).use { + Xml.parseXml(path.toUri().toURL(), it) + }.let(Parser::parse) + } + } +} \ No newline at end of file diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/Exception.kt b/rbcs-client/bin/main/net/woggioni/rbcs/client/Exception.kt new file mode 100644 index 0000000..1d888bc --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/Exception.kt @@ -0,0 +1,9 @@ +package net.woggioni.rbcs.client + +import io.netty.handler.codec.http.HttpResponseStatus + +class HttpException(private val status : HttpResponseStatus) : RuntimeException(status.reasonPhrase()) { + + override val message: String + get() = "Http status ${status.code()}: ${status.reasonPhrase()}" +} \ No newline at end of file diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/RemoteBuildCacheClient.kt b/rbcs-client/bin/main/net/woggioni/rbcs/client/RemoteBuildCacheClient.kt new file mode 100644 index 0000000..cac8d8a --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/RemoteBuildCacheClient.kt @@ -0,0 +1,433 @@ +package net.woggioni.rbcs.client + +import io.netty.bootstrap.Bootstrap +import io.netty.buffer.ByteBuf +import io.netty.buffer.Unpooled +import io.netty.channel.Channel +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelOption +import io.netty.channel.ChannelPipeline +import io.netty.channel.IoEventLoopGroup +import io.netty.channel.MultiThreadIoEventLoopGroup +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.channel.nio.NioIoHandler +import io.netty.channel.pool.AbstractChannelPoolHandler +import io.netty.channel.pool.ChannelPool +import io.netty.channel.pool.FixedChannelPool +import io.netty.channel.socket.nio.NioSocketChannel +import io.netty.handler.codec.DecoderException +import io.netty.handler.codec.http.DefaultFullHttpRequest +import io.netty.handler.codec.http.FullHttpRequest +import io.netty.handler.codec.http.FullHttpResponse +import io.netty.handler.codec.http.HttpClientCodec +import io.netty.handler.codec.http.HttpContentDecompressor +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpHeaderValues +import io.netty.handler.codec.http.HttpMethod +import io.netty.handler.codec.http.HttpObjectAggregator +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import io.netty.handler.ssl.SslContext +import io.netty.handler.ssl.SslContextBuilder +import io.netty.handler.stream.ChunkedWriteHandler +import io.netty.handler.timeout.IdleState +import io.netty.handler.timeout.IdleStateEvent +import io.netty.handler.timeout.IdleStateHandler +import io.netty.util.concurrent.Future +import io.netty.util.concurrent.Future as NettyFuture +import io.netty.util.concurrent.GenericFutureListener +import java.io.IOException +import java.net.InetSocketAddress +import java.net.URI +import java.security.cert.X509Certificate +import java.util.Base64 +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException +import java.util.concurrent.atomic.AtomicInteger +import javax.net.ssl.TrustManagerFactory +import javax.net.ssl.X509TrustManager +import kotlin.random.Random +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.common.RBCS.loadKeystore +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.trace + +class RemoteBuildCacheClient(private val profile: Configuration.Profile) : AutoCloseable { + companion object { + private val log = createLogger() + } + + private val group: IoEventLoopGroup + private val sslContext: SslContext + private val pool: ChannelPool + + init { + group = MultiThreadIoEventLoopGroup(NioIoHandler.newFactory()) + sslContext = SslContextBuilder.forClient().also { builder -> + (profile.authentication as? Configuration.Authentication.TlsClientAuthenticationCredentials)?.let { tlsClientAuthenticationCredentials -> + builder.apply { + keyManager( + tlsClientAuthenticationCredentials.key, + *tlsClientAuthenticationCredentials.certificateChain + ) + profile.tlsTruststore?.let { trustStore -> + if (!trustStore.verifyServerCertificate) { + trustManager(object : X509TrustManager { + override fun checkClientTrusted(certChain: Array, p1: String?) { + } + + override fun checkServerTrusted(certChain: Array, p1: String?) { + } + + override fun getAcceptedIssuers() = null + }) + } else { + trustStore.file?.let { + val ts = loadKeystore(it, trustStore.password) + val trustManagerFactory: TrustManagerFactory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) + trustManagerFactory.init(ts) + trustManager(trustManagerFactory) + } + } + } + } + } + }.build() + + val (scheme, host, port) = profile.serverURI.run { + Triple( + if (scheme == null) "http" else profile.serverURI.scheme, + host, + port.takeIf { it > 0 } ?: if ("https" == scheme.lowercase()) 443 else 80 + ) + } + + val bootstrap = Bootstrap().apply { + group(group) + channel(NioSocketChannel::class.java) + option(ChannelOption.TCP_NODELAY, true) + option(ChannelOption.SO_KEEPALIVE, true) + remoteAddress(InetSocketAddress(host, port)) + profile.connectionTimeout?.let { + option(ChannelOption.CONNECT_TIMEOUT_MILLIS, it.toMillis().toInt()) + } + } + val channelPoolHandler = object : AbstractChannelPoolHandler() { + + @Volatile + private var connectionCount = AtomicInteger() + + @Volatile + private var leaseCount = AtomicInteger() + + override fun channelReleased(ch: Channel) { + val activeLeases = leaseCount.decrementAndGet() + log.trace { + "Released channel ${ch.id().asShortText()}, number of active leases: $activeLeases" + } + } + + override fun channelAcquired(ch: Channel) { + val activeLeases = leaseCount.getAndIncrement() + log.trace { + "Acquired channel ${ch.id().asShortText()}, number of active leases: $activeLeases" + } + } + + override fun channelCreated(ch: Channel) { + val connectionId = connectionCount.incrementAndGet() + log.debug { + "Created connection ${ch.id().asShortText()}, total number of active connections: $connectionId" + } + ch.closeFuture().addListener { + val activeConnections = connectionCount.decrementAndGet() + log.debug { + "Closed connection ${ + ch.id().asShortText() + }, total number of active connections: $activeConnections" + } + } + val pipeline: ChannelPipeline = ch.pipeline() + + profile.connection.also { conn -> + val readIdleTimeout = conn.readIdleTimeout.toMillis() + val writeIdleTimeout = conn.writeIdleTimeout.toMillis() + val idleTimeout = conn.idleTimeout.toMillis() + if (readIdleTimeout > 0 || writeIdleTimeout > 0 || idleTimeout > 0) { + pipeline.addLast( + IdleStateHandler( + true, + readIdleTimeout, + writeIdleTimeout, + idleTimeout, + TimeUnit.MILLISECONDS + ) + ) + } + } + + // Add SSL handler if needed + if ("https".equals(scheme, ignoreCase = true)) { + pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port)) + } + + // HTTP handlers + pipeline.addLast("codec", HttpClientCodec()) + if (profile.compressionEnabled) { + pipeline.addLast("decompressor", HttpContentDecompressor()) + } + pipeline.addLast("aggregator", HttpObjectAggregator(134217728)) + pipeline.addLast("chunked", ChunkedWriteHandler()) + } + } + pool = FixedChannelPool(bootstrap, channelPoolHandler, profile.maxConnections) + } + + private fun executeWithRetry(operation: () -> CompletableFuture): CompletableFuture { + val retryPolicy = profile.retryPolicy + return if (retryPolicy != null) { + val outcomeHandler = OutcomeHandler { outcome -> + when (outcome) { + is OperationOutcome.Success -> { + val response = outcome.result + val status = response.status() + when (status) { + HttpResponseStatus.TOO_MANY_REQUESTS -> { + val retryAfter = response.headers()[HttpHeaderNames.RETRY_AFTER]?.let { headerValue -> + try { + headerValue.toLong() * 1000 + } catch (nfe: NumberFormatException) { + null + } + } + OutcomeHandlerResult.Retry(retryAfter) + } + + HttpResponseStatus.INTERNAL_SERVER_ERROR, HttpResponseStatus.SERVICE_UNAVAILABLE -> + OutcomeHandlerResult.Retry() + + else -> OutcomeHandlerResult.DoNotRetry() + } + } + + is OperationOutcome.Failure -> { + OutcomeHandlerResult.Retry() + } + } + } + executeWithRetry( + group, + retryPolicy.maxAttempts, + retryPolicy.initialDelayMillis.toDouble(), + retryPolicy.exp, + outcomeHandler, + Random.Default, + operation + ) + } else { + operation() + } + } + + fun healthCheck(nonce: ByteArray): CompletableFuture { + return executeWithRetry { + sendRequest(profile.serverURI, HttpMethod.TRACE, nonce) + }.thenApply { + val status = it.status() + if (it.status() != HttpResponseStatus.OK) { + throw HttpException(status) + } else { + it.content() + } + }.thenApply { maybeByteBuf -> + maybeByteBuf?.let { + val result = ByteArray(it.readableBytes()) + it.getBytes(0, result) + result + } + } + } + + fun get(key: String): CompletableFuture { + return executeWithRetry { + sendRequest(profile.serverURI.resolve(key), HttpMethod.GET, null) + }.thenApply { response -> + val status = response.status() + if (response.status() == HttpResponseStatus.NOT_FOUND) { + response.release() + null + } else if (response.status() != HttpResponseStatus.OK) { + response.release() + throw HttpException(status) + } else { + response.content().also { + it.retain() + response.release() + } + } + }.thenApply { maybeByteBuf -> + maybeByteBuf?.let { buf -> + val result = ByteArray(buf.readableBytes()) + buf.getBytes(0, result) + buf.release() + result + } + } + } + + fun put(key: String, content: ByteArray, metadata: CacheValueMetadata): CompletableFuture { + return executeWithRetry { + val extraHeaders = sequenceOf( + metadata.mimeType?.let { HttpHeaderNames.CONTENT_TYPE to it }, + metadata.contentDisposition?.let { HttpHeaderNames.CONTENT_DISPOSITION to it } + ).filterNotNull() + sendRequest(profile.serverURI.resolve(key), HttpMethod.PUT, content, extraHeaders.asIterable()) + }.thenApply { + val status = it.status() + if (it.status() != HttpResponseStatus.CREATED && it.status() != HttpResponseStatus.OK) { + throw HttpException(status) + } + } + } + + private fun sendRequest( + uri: URI, + method: HttpMethod, + body: ByteArray?, + extraHeaders: Iterable>? = null + ): CompletableFuture { + val responseFuture = CompletableFuture() + // Custom handler for processing responses + pool.acquire().addListener(object : GenericFutureListener> { + + override fun operationComplete(channelFuture: Future) { + if (channelFuture.isSuccess) { + val channel = channelFuture.now + val pipeline = channel.pipeline() + + val closeListener = GenericFutureListener> { + responseFuture.completeExceptionally(IOException("The remote server closed the connection")) + } + channel.closeFuture().addListener(closeListener) + + val responseHandler = object : SimpleChannelInboundHandler() { + + override fun handlerAdded(ctx: ChannelHandlerContext) { + channel.closeFuture().removeListener(closeListener) + } + + override fun channelRead0( + ctx: ChannelHandlerContext, + response: FullHttpResponse + ) { + pipeline.remove(this) + responseFuture.complete(response.retainedDuplicate()) + if (!profile.connection.requestPipelining) { + pool.release(channel) + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + ctx.newPromise() + val ex = when (cause) { + is DecoderException -> cause.cause + else -> cause + } + responseFuture.completeExceptionally(ex) + ctx.close() + } + + override fun channelInactive(ctx: ChannelHandlerContext) { + responseFuture.completeExceptionally(IOException("The remote server closed the connection")) + super.channelInactive(ctx) + pool.release(channel) + } + + override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) { + if (evt is IdleStateEvent) { + val te = when (evt.state()) { + IdleState.READER_IDLE -> TimeoutException("Read timeout") + IdleState.WRITER_IDLE -> TimeoutException("Write timeout") + IdleState.ALL_IDLE -> TimeoutException("Idle timeout") + null -> throw IllegalStateException("This should never happen") + } + responseFuture.completeExceptionally(te) + super.userEventTriggered(ctx, evt) + if (this === pipeline.last()) { + ctx.close() + } + if (!profile.connection.requestPipelining) { + pool.release(channel) + } + } else { + super.userEventTriggered(ctx, evt) + } + } + } + pipeline.addLast(responseHandler) + + + // Prepare the HTTP request + val request: FullHttpRequest = let { + val content: ByteBuf? = body?.takeIf(ByteArray::isNotEmpty)?.let(Unpooled::wrappedBuffer) + DefaultFullHttpRequest( + HttpVersion.HTTP_1_1, + method, + uri.rawPath, + content ?: Unpooled.buffer(0) + ).apply { + // Set headers + headers().apply { + if (content != null) { + set(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes()) + } + set(HttpHeaderNames.HOST, profile.serverURI.host) + set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE) + if (profile.compressionEnabled) { + set( + HttpHeaderNames.ACCEPT_ENCODING, + HttpHeaderValues.GZIP.toString() + "," + HttpHeaderValues.DEFLATE.toString() + ) + } + extraHeaders?.forEach { (k, v) -> + add(k, v) + } + // Add basic auth if configured + (profile.authentication as? Configuration.Authentication.BasicAuthenticationCredentials)?.let { credentials -> + val auth = "${credentials.username}:${credentials.password}" + val encodedAuth = Base64.getEncoder().encodeToString(auth.toByteArray()) + set(HttpHeaderNames.AUTHORIZATION, "Basic $encodedAuth") + } + } + } + } + + // Send the request + channel.writeAndFlush(request).addListener { + if (!it.isSuccess) { + val ex = it.cause() + log.warn(ex.message, ex) + } + if (profile.connection.requestPipelining) { + pool.release(channel) + } + } + } else { + responseFuture.completeExceptionally(channelFuture.cause()) + } + } + }) + return responseFuture + } + + fun shutDown(): NettyFuture<*> { + return group.shutdownGracefully() + } + + override fun close() { + shutDown().sync() + } +} \ No newline at end of file diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/impl/Parser.kt b/rbcs-client/bin/main/net/woggioni/rbcs/client/impl/Parser.kt new file mode 100644 index 0000000..f41680a --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/impl/Parser.kt @@ -0,0 +1,151 @@ +package net.woggioni.rbcs.client.impl + +import java.net.URI +import java.nio.file.Files +import java.nio.file.Path +import java.security.KeyStore +import java.security.PrivateKey +import java.security.cert.X509Certificate +import java.time.Duration +import java.time.temporal.ChronoUnit +import net.woggioni.rbcs.api.exception.ConfigurationException +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.common.Xml.Companion.asIterable +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document + +object Parser { + + fun parse(document: Document): Configuration { + val root = document.documentElement + val profiles = mutableMapOf() + + for (child in root.asIterable()) { + val tagName = child.localName + when (tagName) { + "profile" -> { + val name = + child.renderAttribute("name") ?: throw ConfigurationException("name attribute is required") + val uri = child.renderAttribute("base-url")?.let(::URI) + ?: throw ConfigurationException("base-url attribute is required") + var authentication: Configuration.Authentication? = null + var retryPolicy: Configuration.RetryPolicy? = null + var connection : Configuration.Connection = Configuration.Connection( + Duration.ofSeconds(60), + Duration.ofSeconds(60), + Duration.ofSeconds(30), + false + ) + var trustStore : Configuration.TrustStore? = null + for (gchild in child.asIterable()) { + when (gchild.localName) { + "tls-client-auth" -> { + val keyStoreFile = gchild.renderAttribute("key-store-file") + val keyStorePassword = + gchild.renderAttribute("key-store-password") + val keyAlias = gchild.renderAttribute("key-alias") + val keyPassword = gchild.renderAttribute("key-password") + + val keystore = KeyStore.getInstance("PKCS12").apply { + Files.newInputStream(Path.of(keyStoreFile)).use { + load(it, keyStorePassword?.toCharArray()) + } + } + val key = keystore.getKey(keyAlias, keyPassword?.toCharArray()) as PrivateKey + val certChain = keystore.getCertificateChain(keyAlias).asSequence() + .map { it as X509Certificate } + .toList() + .toTypedArray() + authentication = + Configuration.Authentication.TlsClientAuthenticationCredentials( + key, + certChain + ) + } + + "basic-auth" -> { + val username = gchild.renderAttribute("user") + ?: throw ConfigurationException("username attribute is required") + val password = gchild.renderAttribute("password") + ?: throw ConfigurationException("password attribute is required") + authentication = + Configuration.Authentication.BasicAuthenticationCredentials( + username, + password + ) + } + + "retry-policy" -> { + val maxAttempts = + gchild.renderAttribute("max-attempts") + ?.let(String::toInt) + ?: throw ConfigurationException("max-attempts attribute is required") + val initialDelay = + gchild.renderAttribute("initial-delay") + ?.let(Duration::parse) + ?: Duration.ofSeconds(1) + val exp = + gchild.renderAttribute("exp") + ?.let(String::toDouble) + ?: 2.0f + retryPolicy = Configuration.RetryPolicy( + maxAttempts, + initialDelay.toMillis(), + exp.toDouble() + ) + } + + "connection" -> { + val idleTimeout = gchild.renderAttribute("idle-timeout") + ?.let(Duration::parse) ?: Duration.of(30, ChronoUnit.SECONDS) + val readIdleTimeout = gchild.renderAttribute("read-idle-timeout") + ?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS) + val writeIdleTimeout = gchild.renderAttribute("write-idle-timeout") + ?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS) + val requestPipelining = gchild.renderAttribute("request-pipelining") + ?.let(String::toBoolean) ?: false + connection = Configuration.Connection( + readIdleTimeout, + writeIdleTimeout, + idleTimeout, + requestPipelining + ) + } + + "tls-trust-store" -> { + val file = gchild.renderAttribute("file") + ?.let(Path::of) + val password = gchild.renderAttribute("password") + val checkCertificateStatus = gchild.renderAttribute("check-certificate-status") + ?.let(String::toBoolean) ?: false + val verifyServerCertificate = gchild.renderAttribute("verify-server-certificate") + ?.let(String::toBoolean) ?: true + trustStore = Configuration.TrustStore(file, password, checkCertificateStatus, verifyServerCertificate) + } + } + } + val maxConnections = child.renderAttribute("max-connections") + ?.let(String::toInt) + ?: 50 + val connectionTimeout = child.renderAttribute("connection-timeout") + ?.let(Duration::parse) + val compressionEnabled = child.renderAttribute("enable-compression") + ?.let(String::toBoolean) + ?: true + + profiles[name] = Configuration.Profile( + uri, + connection, + authentication, + connectionTimeout, + maxConnections, + compressionEnabled, + retryPolicy, + trustStore + ) + } + } + } + return Configuration(profiles) + } +} \ No newline at end of file diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/retry.kt b/rbcs-client/bin/main/net/woggioni/rbcs/client/retry.kt new file mode 100644 index 0000000..f90b4ab --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/retry.kt @@ -0,0 +1,79 @@ +package net.woggioni.rbcs.client + +import io.netty.util.concurrent.EventExecutorGroup +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit +import kotlin.math.pow +import kotlin.random.Random + +sealed class OperationOutcome { + class Success(val result: T) : OperationOutcome() + class Failure(val ex: Throwable) : OperationOutcome() +} + +sealed class OutcomeHandlerResult { + class Retry(val suggestedDelayMillis: Long? = null) : OutcomeHandlerResult() + class DoNotRetry : OutcomeHandlerResult() +} + +fun interface OutcomeHandler { + fun shouldRetry(result: OperationOutcome): OutcomeHandlerResult +} + +fun executeWithRetry( + eventExecutorGroup: EventExecutorGroup, + maxAttempts: Int, + initialDelay: Double, + exp: Double, + outcomeHandler: OutcomeHandler, + randomizer : Random?, + cb: () -> CompletableFuture +): CompletableFuture { + + val finalResult = cb() + var future = finalResult + var shortCircuit = false + for (i in 1 until maxAttempts) { + future = future.handle { result, ex -> + val operationOutcome = if (ex == null) { + OperationOutcome.Success(result) + } else { + OperationOutcome.Failure(ex.cause ?: ex) + } + if (shortCircuit) { + when(operationOutcome) { + is OperationOutcome.Failure -> throw operationOutcome.ex + is OperationOutcome.Success -> CompletableFuture.completedFuture(operationOutcome.result) + } + } else { + when(val outcomeHandlerResult = outcomeHandler.shouldRetry(operationOutcome)) { + is OutcomeHandlerResult.Retry -> { + val res = CompletableFuture() + val delay = run { + val scheduledDelay = (initialDelay * exp.pow(i.toDouble()) * (1.0 + (randomizer?.nextDouble(-0.5, 0.5) ?: 0.0))).toLong() + outcomeHandlerResult.suggestedDelayMillis?.coerceAtMost(scheduledDelay) ?: scheduledDelay + } + eventExecutorGroup.schedule({ + cb().handle { result, ex -> + if (ex == null) { + res.complete(result) + } else { + res.completeExceptionally(ex) + } + } + }, delay, TimeUnit.MILLISECONDS) + res + } + is OutcomeHandlerResult.DoNotRetry -> { + shortCircuit = true + when(operationOutcome) { + is OperationOutcome.Failure -> throw operationOutcome.ex + is OperationOutcome.Success -> CompletableFuture.completedFuture(operationOutcome.result) + } + } + } + } + }.thenCompose { it } + } + return future +} \ No newline at end of file diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/schema/rbcs-client.xsd b/rbcs-client/bin/main/net/woggioni/rbcs/client/schema/rbcs-client.xsd new file mode 100644 index 0000000..807cdea --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/schema/rbcs-client.xsd @@ -0,0 +1,260 @@ + + + + + + + + + + + + + + + + + Disable authentication. + + + + + + + Enable HTTP basic authentication. + + + + + + + Enable TLS certificate authentication. + + + + + + + + Set inactivity timeouts for connections to this server, + if not present, connections are only closed on network errors. + + + + + + + Set a retry policy for this server, if not present requests won't be retried + + + + + + + If set, specify an alternative truststore to validate the server certificate. + If not present the system truststore is used. + + + + + + + + Name of this server profile, to be referred to from rbcs-cli with the '-p' parameter + + + + + + + RBCs server URL + + + + + + + Maximum number of concurrent TCP connection to open with this server + + + + + + + Enable HTTP compression when communicating to this server + + + + + + + Enable HTTP compression when communicating to this server + + + + + + + + + + The client will close the connection with the server + when neither a read nor a write was performed for the specified period of time. + + + + + + + The client will close the connection with the server + when no read was performed for the specified period of time. + + + + + + + The client will close the connection with the server + when no write was performed for the specified period of time. + + + + + + + Enables HTTP/1.1 request pipelining + + + + + + + + + Add this tag to not use any type of authentication when talking to the RBCS server + + + + + + + + Add this tag to enable HTTP basic authentication for the communication to this server, + mind that HTTP basic authentication sends credentials directly over the network, so make sure + your communication is protected by TLS (i.e. your server's URL starts with "https") + + + + + + Username for HTTP basic authentication + + + + + + + Password used for HTTP basic authentication + + + + + + + + + + System path to the keystore file + + + + + + + Password to open they keystore file + + + + + + + Alias of the keystore entry containing the private key + + + + + + + Private key entry's encryption password + + + + + + + + + Retry policy to use in case of failures, based on exponential backoff + https://en.wikipedia.org/wiki/Exponential_backoff + + + + + + + Maximum number of attempts, after which the call will result in an error, + throwing an exception related to the last received failure + + + + + + + Delay to apply before retrying after the first failed call + + + + + + + Exponent to apply to compute the next delay + + + + + + + + + + Path to the truststore file + + + + + + + Truststore file password + + + + + + + Whether or not check the server certificate validity using CRL/OCSP + + + + + + + If false, the client will blindly trust the certificate provided by the server + + + + + diff --git a/rbcs-client/bin/test/logback.xml b/rbcs-client/bin/test/logback.xml new file mode 100644 index 0000000..400bcd5 --- /dev/null +++ b/rbcs-client/bin/test/logback.xml @@ -0,0 +1,21 @@ + + + + + + + + + System.err + + %d [%highlight(%-5level)] \(%thread\) %logger{36} -%kvp- %msg %n + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-client/bin/test/net/woggioni/rbcs/client/RetryTest.kt b/rbcs-client/bin/test/net/woggioni/rbcs/client/RetryTest.kt new file mode 100644 index 0000000..7295ed1 --- /dev/null +++ b/rbcs-client/bin/test/net/woggioni/rbcs/client/RetryTest.kt @@ -0,0 +1,152 @@ +package net.woggioni.rbcs.client + +import io.netty.util.concurrent.DefaultEventExecutorGroup +import io.netty.util.concurrent.EventExecutorGroup +import java.util.concurrent.CompletableFuture +import java.util.stream.Stream +import kotlin.random.Random +import net.woggioni.rbcs.common.contextLogger +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.extension.ExtensionContext +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.ArgumentsProvider +import org.junit.jupiter.params.provider.ArgumentsSource +import org.junit.jupiter.params.support.ParameterDeclarations + +class RetryTest { + + data class TestArgs( + val seed: Int, + val maxAttempt: Int, + val initialDelay: Double, + val exp: Double, + ) + + class TestArguments : ArgumentsProvider { + override fun provideArguments( + parameters: ParameterDeclarations, + context: ExtensionContext + ): Stream { + return Stream.of( + TestArgs( + seed = 101325, + maxAttempt = 5, + initialDelay = 50.0, + exp = 2.0, + ), + TestArgs( + seed = 101325, + maxAttempt = 20, + initialDelay = 100.0, + exp = 1.1, + ), + TestArgs( + seed = 123487, + maxAttempt = 20, + initialDelay = 100.0, + exp = 2.0, + ), + TestArgs( + seed = 20082024, + maxAttempt = 10, + initialDelay = 100.0, + exp = 2.0, + ) + ).map { + object: Arguments { + override fun get() = arrayOf(it) + } + } + } + } + + @ArgumentsSource(TestArguments::class) + @ParameterizedTest + fun test(testArgs: TestArgs) { + val log = contextLogger() + log.debug("Start") + val executor: EventExecutorGroup = DefaultEventExecutorGroup(1) + val attempts = mutableListOf>>() + val outcomeHandler = OutcomeHandler { outcome -> + when(outcome) { + is OperationOutcome.Success -> { + if(outcome.result % 10 == 0) { + OutcomeHandlerResult.DoNotRetry() + } else { + OutcomeHandlerResult.Retry(null) + } + } + is OperationOutcome.Failure -> { + when(outcome.ex) { + is IllegalStateException -> { + log.debug(outcome.ex.message, outcome.ex) + OutcomeHandlerResult.Retry(null) + } + else -> { + OutcomeHandlerResult.DoNotRetry() + } + } + } + } + } + val random = Random(testArgs.seed) + + val future = + executeWithRetry(executor, testArgs.maxAttempt, testArgs.initialDelay, testArgs.exp, outcomeHandler, null) { + val now = System.nanoTime() + val result = CompletableFuture() + executor.submit { + val n = random.nextInt(0, Integer.MAX_VALUE) + log.debug("Got new number: {}", n) + if(n % 3 == 0) { + val ex = IllegalStateException("Value $n can be divided by 3") + result.completeExceptionally(ex) + attempts += now to OperationOutcome.Failure(ex) + } else if(n % 7 == 0) { + val ex = RuntimeException("Value $n can be divided by 7") + result.completeExceptionally(ex) + attempts += now to OperationOutcome.Failure(ex) + } else { + result.complete(n) + attempts += now to OperationOutcome.Success(n) + } + } + result + } + Assertions.assertTrue(attempts.size <= testArgs.maxAttempt) + val result = future.handle { res, ex -> + if(ex != null) { + val err = ex.cause ?: ex + log.debug(err.message, err) + OperationOutcome.Failure(err) + } else { + OperationOutcome.Success(res) + } + }.get() + for ((index, attempt) in attempts.withIndex()) { + val (timestamp, value) = attempt + if (index > 0) { + /* Check the delay for subsequent attempts is correct */ + val previousAttempt = attempts[index - 1] + val expectedTimestamp = + previousAttempt.first + testArgs.initialDelay * Math.pow(testArgs.exp, index.toDouble()) * 1e6 + val actualTimestamp = timestamp + val err = Math.abs(expectedTimestamp - actualTimestamp) / expectedTimestamp + Assertions.assertTrue(err < 0.1) + } + if (index == attempts.size - 1 && index < testArgs.maxAttempt - 1) { + /* + * If the last attempt index is lower than the maximum number of attempts, then + * check the outcome handler returns DoNotRetry + */ + Assertions.assertTrue(outcomeHandler.shouldRetry(value) is OutcomeHandlerResult.DoNotRetry) + } else if (index < attempts.size - 1) { + /* + * If the attempt is not the last attempt check the outcome handler returns Retry + */ + Assertions.assertTrue(outcomeHandler.shouldRetry(value) is OutcomeHandlerResult.Retry) + } + } + } +} \ No newline at end of file diff --git a/rbcs-client/bin/test/net/woggioni/rbcs/client/test/rbcs-client.xml b/rbcs-client/bin/test/net/woggioni/rbcs/client/test/rbcs-client.xml new file mode 100644 index 0000000..5828d87 --- /dev/null +++ b/rbcs-client/bin/test/net/woggioni/rbcs/client/test/rbcs-client.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + diff --git a/rbcs-common/.classpath b/rbcs-common/.classpath new file mode 100644 index 0000000..df4ae25 --- /dev/null +++ b/rbcs-common/.classpath @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-common/.project b/rbcs-common/.project new file mode 100644 index 0000000..92bfd2b --- /dev/null +++ b/rbcs-common/.project @@ -0,0 +1,34 @@ + + + rbcs-common + Project rbcs-common created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776954 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-common/.settings/org.eclipse.buildship.core.prefs b/rbcs-common/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-common/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-common/.settings/org.eclipse.jdt.core.prefs b/rbcs-common/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-common/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-common/bin/main/META-INF/services/java.net.spi.URLStreamHandlerProvider b/rbcs-common/bin/main/META-INF/services/java.net.spi.URLStreamHandlerProvider new file mode 100644 index 0000000..738bdad --- /dev/null +++ b/rbcs-common/bin/main/META-INF/services/java.net.spi.URLStreamHandlerProvider @@ -0,0 +1 @@ +net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/BB.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/BB.kt new file mode 100644 index 0000000..65796c3 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/BB.kt @@ -0,0 +1,15 @@ +package net.woggioni.rbcs.common + +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.CompositeByteBuf + +fun extractChunk(buf: CompositeByteBuf, alloc: ByteBufAllocator): ByteBuf { + val chunk = alloc.compositeBuffer() + for (component in buf.decompose(0, buf.readableBytes())) { + chunk.addComponent(true, component.retain()) + } + buf.removeComponents(0, buf.numComponents()) + buf.clear() + return chunk +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufInputStream.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufInputStream.kt new file mode 100644 index 0000000..8d8ea77 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufInputStream.kt @@ -0,0 +1,25 @@ +package net.woggioni.rbcs.common + +import io.netty.buffer.ByteBuf +import java.io.InputStream + +class ByteBufInputStream(private val buf : ByteBuf) : InputStream() { + override fun read(): Int { + return buf.takeIf { + it.readableBytes() > 0 + }?.let(ByteBuf::readByte) + ?.let(Byte::toInt) ?: -1 + } + + override fun read(b: ByteArray, off: Int, len: Int): Int { + val readableBytes = buf.readableBytes() + if(readableBytes == 0) return -1 + val result = len.coerceAtMost(readableBytes) + buf.readBytes(b, off, result) + return result + } + + override fun close() { + buf.release() + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufOutputStream.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufOutputStream.kt new file mode 100644 index 0000000..8cc131e --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufOutputStream.kt @@ -0,0 +1,18 @@ +package net.woggioni.rbcs.common + +import io.netty.buffer.ByteBuf +import java.io.OutputStream + +class ByteBufOutputStream(private val buf : ByteBuf) : OutputStream() { + override fun write(b: Int) { + buf.writeByte(b) + } + + override fun write(b: ByteArray, off: Int, len: Int) { + buf.writeBytes(b, off, len) + } + + override fun close() { + buf.release() + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/Cidr.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/Cidr.kt new file mode 100644 index 0000000..c4a9ccf --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/Cidr.kt @@ -0,0 +1,62 @@ +package net.woggioni.rbcs.common + +import java.net.InetAddress + +data class Cidr private constructor( + val networkAddress: InetAddress, + val prefixLength: Int +) { + companion object { + fun from(cidr: String) : Cidr { + val separator = cidr.indexOf("/") + if(separator < 0) { + throw IllegalArgumentException("Invalid CIDR format: $cidr") + } + val networkAddress = InetAddress.getByName(cidr.substring(0, separator)) + val prefixLength = cidr.substring(separator + 1, cidr.length).toInt() + + + // Validate prefix length + val maxPrefix = if (networkAddress.address.size == 4) 32 else 128 + require(prefixLength in 0..maxPrefix) { "Invalid prefix length: $prefixLength" } + return Cidr(networkAddress, prefixLength) + } + } + + fun contains(address: InetAddress): Boolean { + val networkBytes = networkAddress.address + val addressBytes = address.address + + if (networkBytes.size != addressBytes.size) { + return false + } + + + // Calculate how many full bytes and remaining bits to check + val fullBytes = prefixLength / 8 + val remainingBits = prefixLength % 8 + + + // Check full bytes + for (i in 0.. 0 && fullBytes < networkBytes.size) { + val mask = (0xFF shl (8 - remainingBits)).toByte() + if ((networkBytes[fullBytes].toInt() and mask.toInt()) != (addressBytes[fullBytes].toInt() and mask.toInt())) { + return false + } + } + + return true + } + + override fun toString(): String { + return networkAddress.hostAddress + "/" + prefixLength + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/Exception.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/Exception.kt new file mode 100644 index 0000000..b4232da --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/Exception.kt @@ -0,0 +1,7 @@ +package net.woggioni.rbcs.common + +class ResourceNotFoundException(msg : String? = null, cause: Throwable? = null) : RuntimeException(msg, cause) { +} + +class ModuleNotFoundException(msg : String? = null, cause: Throwable? = null) : RuntimeException(msg, cause) { +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/HostAndPort.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/HostAndPort.kt new file mode 100644 index 0000000..59cfc47 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/HostAndPort.kt @@ -0,0 +1,8 @@ +package net.woggioni.rbcs.common + + +data class HostAndPort(val host: String, val port: Int = 0) { + override fun toString(): String { + return "$host:$port" + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/Logging.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/Logging.kt new file mode 100644 index 0000000..9247481 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/Logging.kt @@ -0,0 +1,191 @@ +package net.woggioni.rbcs.common + +import io.netty.channel.Channel +import io.netty.channel.ChannelHandlerContext +import java.nio.file.Files +import java.nio.file.Path +import java.util.logging.LogManager +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.slf4j.MDC +import org.slf4j.event.Level +import org.slf4j.spi.LoggingEventBuilder + +inline fun T.contextLogger() = LoggerFactory.getLogger(T::class.java) +inline fun createLogger() = LoggerFactory.getLogger(T::class.java) + +inline fun Logger.traceParam(messageBuilder: () -> Pair>) { + if (isTraceEnabled) { + val (format, params) = messageBuilder() + trace(format, params) + } +} + +inline fun Logger.debugParam(messageBuilder: () -> Pair>) { + if (isDebugEnabled) { + val (format, params) = messageBuilder() + info(format, params) + } +} + +inline fun Logger.infoParam(messageBuilder: () -> Pair>) { + if (isInfoEnabled) { + val (format, params) = messageBuilder() + info(format, params) + } +} + +inline fun Logger.warnParam(messageBuilder: () -> Pair>) { + if (isWarnEnabled) { + val (format, params) = messageBuilder() + warn(format, params) + } +} + +inline fun Logger.errorParam(messageBuilder: () -> Pair>) { + if (isErrorEnabled) { + val (format, params) = messageBuilder() + error(format, params) + } +} + + +inline fun log( + log: Logger, + filter: Logger.() -> Boolean, + loggerMethod: Logger.(String) -> Unit, messageBuilder: () -> String +) { + if (log.filter()) { + log.loggerMethod(messageBuilder()) + } +} + +fun withMDC(params: Array>, cb: () -> Unit) { + object : AutoCloseable { + override fun close() { + for ((key, _) in params) MDC.remove(key) + } + }.use { + for ((key, value) in params) MDC.put(key, value) + cb() + } +} + +inline fun Logger.log(level: Level, channel: Channel, crossinline messageBuilder: (LoggingEventBuilder) -> Unit ) { + if (isEnabledForLevel(level)) { + val params = arrayOf>( + "channel-id-short" to channel.id().asShortText(), + "channel-id-long" to channel.id().asLongText(), + "remote-address" to channel.remoteAddress().toString(), + "local-address" to channel.localAddress().toString(), + ) + withMDC(params) { + val builder = makeLoggingEventBuilder(level) + messageBuilder(builder) + builder.log() + } + } +} +inline fun Logger.log(level: Level, channel: Channel, crossinline messageBuilder: () -> String) { + log(level, channel) { builder -> + builder.setMessage(messageBuilder()) + } +} + +inline fun Logger.trace(ch: Channel, crossinline messageBuilder: () -> String) { + log(Level.TRACE, ch, messageBuilder) +} + +inline fun Logger.debug(ch: Channel, crossinline messageBuilder: () -> String) { + log(Level.DEBUG, ch, messageBuilder) +} + +inline fun Logger.info(ch: Channel, crossinline messageBuilder: () -> String) { + log(Level.INFO, ch, messageBuilder) +} + +inline fun Logger.warn(ch: Channel, crossinline messageBuilder: () -> String) { + log(Level.WARN, ch, messageBuilder) +} + +inline fun Logger.error(ch: Channel, crossinline messageBuilder: () -> String) { + log(Level.ERROR, ch, messageBuilder) +} + +inline fun Logger.trace(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) { + log(Level.TRACE, ctx.channel(), messageBuilder) +} + +inline fun Logger.debug(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) { + log(Level.DEBUG, ctx.channel(), messageBuilder) +} + +inline fun Logger.info(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) { + log(Level.INFO, ctx.channel(), messageBuilder) +} + +inline fun Logger.warn(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) { + log(Level.WARN, ctx.channel(), messageBuilder) +} + +inline fun Logger.error(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) { + log(Level.ERROR, ctx.channel(), messageBuilder) +} + + +inline fun Logger.log(level: Level, messageBuilder: () -> String) { + if (isEnabledForLevel(level)) { + makeLoggingEventBuilder(level).log(messageBuilder()) + } +} + +inline fun Logger.trace(messageBuilder: () -> String) { + if (isTraceEnabled) { + trace(messageBuilder()) + } +} + +inline fun Logger.debug(messageBuilder: () -> String) { + if (isDebugEnabled) { + debug(messageBuilder()) + } +} + +inline fun Logger.info(messageBuilder: () -> String) { + if (isInfoEnabled) { + info(messageBuilder()) + } +} + +inline fun Logger.warn(messageBuilder: () -> String) { + if (isWarnEnabled) { + warn(messageBuilder()) + } +} + +inline fun Logger.error(messageBuilder: () -> String) { + if (isErrorEnabled) { + error(messageBuilder()) + } +} + + +class LoggingConfig { + + init { + val logManager = LogManager.getLogManager() + System.getProperty("log.config.source")?.let withSource@{ source -> + val urls = LoggingConfig::class.java.classLoader.getResources(source) + while (urls.hasMoreElements()) { + val url = urls.nextElement() + url.openStream().use { inputStream -> + logManager.readConfiguration(inputStream) + return@withSource + } + } + Path.of(source).takeIf(Files::exists) + ?.let(Files::newInputStream) + ?.use(logManager::readConfiguration) + } + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/PasswordSecurity.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/PasswordSecurity.kt new file mode 100644 index 0000000..f7be95a --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/PasswordSecurity.kt @@ -0,0 +1,57 @@ +package net.woggioni.rbcs.common + +import java.security.SecureRandom +import java.security.spec.KeySpec +import java.util.Base64 +import javax.crypto.SecretKeyFactory +import javax.crypto.spec.PBEKeySpec + +object PasswordSecurity { + + enum class Algorithm( + val codeName : String, + val keyLength : Int, + val iterations : Int) { + PBEWithHmacSHA512_224AndAES_256("PBEWithHmacSHA512/224AndAES_256", 64, 1), + PBEWithHmacSHA1AndAES_256("PBEWithHmacSHA1AndAES_256",64, 1), + PBEWithHmacSHA384AndAES_128("PBEWithHmacSHA384AndAES_128", 64,1), + PBEWithHmacSHA384AndAES_256("PBEWithHmacSHA384AndAES_256",64,1), + PBKDF2WithHmacSHA512("PBKDF2WithHmacSHA512",512, 1), + PBKDF2WithHmacSHA384("PBKDF2WithHmacSHA384",384, 1); + } + + private fun concat(arr1: ByteArray, arr2: ByteArray): ByteArray { + val result = ByteArray(arr1.size + arr2.size) + var j = 0 + for(element in arr1) { + result[j] = element + j += 1 + } + for(element in arr2) { + result[j] = element + j += 1 + } + return result + } + + fun hashPassword(password : String, salt : String? = null, algorithm : Algorithm = Algorithm.PBKDF2WithHmacSHA512) : String { + val actualSalt = salt?.let(Base64.getDecoder()::decode) ?: SecureRandom().run { + val result = ByteArray(16) + nextBytes(result) + result + } + val spec: KeySpec = PBEKeySpec(password.toCharArray(), actualSalt, algorithm.iterations, algorithm.keyLength) + val factory = SecretKeyFactory.getInstance(algorithm.codeName) + val hash = factory.generateSecret(spec).encoded + return String(Base64.getEncoder().encode(concat(hash, actualSalt))) + } + + fun decodePasswordHash(encodedPasswordHash : String, algorithm: Algorithm = Algorithm.PBKDF2WithHmacSHA512) : Pair { + val decoded = Base64.getDecoder().decode(encodedPasswordHash) + val hash = ByteArray(algorithm.keyLength / 8) + val salt = ByteArray(decoded.size - algorithm.keyLength / 8) + System.arraycopy(decoded, 0, hash, 0, hash.size) + System.arraycopy(decoded, hash.size, salt, 0, salt.size) + return hash to salt + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/RBCS.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/RBCS.kt new file mode 100644 index 0000000..f228f6c --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/RBCS.kt @@ -0,0 +1,167 @@ +package net.woggioni.rbcs.common + +import java.io.IOException +import java.net.InetAddress +import java.net.ServerSocket +import java.net.URI +import java.net.URL +import java.nio.file.Files +import java.nio.file.Path +import java.security.KeyStore +import java.security.MessageDigest +import java.security.cert.CertPathValidator +import java.security.cert.CertPathValidatorException +import java.security.cert.CertificateException +import java.security.cert.CertificateFactory +import java.security.cert.PKIXParameters +import java.security.cert.PKIXRevocationChecker +import java.security.cert.X509Certificate +import java.util.EnumSet +import javax.net.ssl.TrustManagerFactory +import javax.net.ssl.X509TrustManager +import net.woggioni.jwo.JWO +import net.woggioni.jwo.Tuple2 + +object RBCS { + fun String.toUrl(): URL = URL.of(URI(this), null) + + const val RBCS_NAMESPACE_URI: String = "urn:net.woggioni.rbcs.server" + const val RBCS_PREFIX: String = "rbcs" + const val XML_SCHEMA_NAMESPACE_URI = "http://www.w3.org/2001/XMLSchema-instance" + + fun ByteArray.toInt(index: Int = 0): Long { + if (index + 4 > size) throw IllegalArgumentException("Not enough bytes to decode a 32 bits integer") + var value: Long = 0 + for (b in index until index + 4) { + value = (value shl 8) + (get(b).toInt() and 0xFF) + } + return value + } + + fun ByteArray.toLong(index: Int = 0): Long { + if (index + 8 > size) throw IllegalArgumentException("Not enough bytes to decode a 64 bits long integer") + var value: Long = 0 + for (b in index until index + 8) { + value = (value shl 8) + (get(b).toInt() and 0xFF) + } + return value + } + + fun digest( + data: ByteArray, + md: MessageDigest + ): ByteArray { + md.update(data) + return md.digest() + } + + fun digestString( + data: ByteArray, + md: MessageDigest + ): String { + return JWO.bytesToHex(digest(data, md)) + } + + fun processCacheKey(key: String, keyPrefix: String?, digestAlgorithm: String?) : ByteArray { + val prefixedKey = if (keyPrefix == null) { + key + } else { + key + keyPrefix + }.toByteArray(Charsets.UTF_8) + return digestAlgorithm + ?.let(MessageDigest::getInstance) + ?.let { md -> + digest(prefixedKey, md) + } ?: prefixedKey + } + + fun Long.toIntOrNull(): Int? { + return if (this >= Int.MIN_VALUE && this <= Int.MAX_VALUE) { + toInt() + } else { + null + } + } + + fun getFreePort(): Int { + var count = 0 + while (count < 50) { + try { + ServerSocket(0, 50, InetAddress.getLocalHost()).use { serverSocket -> + val candidate = serverSocket.localPort + if (candidate > 0) { + return candidate + } else { + throw RuntimeException("Got invalid port number: $candidate") + } + } + } catch (ignored: IOException) { + ++count + } + } + throw RuntimeException("Error trying to find an open port") + } + + fun loadKeystore(file: Path, password: String?): KeyStore { + val ext = JWO.splitExtension(file) + .map(Tuple2::get_2) + .orElseThrow { + IllegalArgumentException( + "Keystore file '${file}' must have .jks, .p12, .pfx extension" + ) + } + val keystore = when (ext.substring(1).lowercase()) { + "jks" -> KeyStore.getInstance("JKS") + "p12", "pfx" -> KeyStore.getInstance("PKCS12") + else -> throw IllegalArgumentException( + "Keystore file '${file}' must have .jks, .p12, .pfx extension" + ) + } + Files.newInputStream(file).use { + keystore.load(it, password?.let(String::toCharArray)) + } + return keystore + } + + fun getTrustManager(trustStore: KeyStore?, certificateRevocationEnabled: Boolean): X509TrustManager { + return if (trustStore != null) { + val certificateFactory = CertificateFactory.getInstance("X.509") + val validator = CertPathValidator.getInstance("PKIX").apply { + val rc = revocationChecker as PKIXRevocationChecker + rc.options = EnumSet.of( + PKIXRevocationChecker.Option.NO_FALLBACK + ) + } + val params = PKIXParameters(trustStore).apply { + isRevocationEnabled = certificateRevocationEnabled + } + object : X509TrustManager { + override fun checkClientTrusted(chain: Array, authType: String) { + val clientCertificateChain = certificateFactory.generateCertPath(chain.toList()) + try { + validator.validate(clientCertificateChain, params) + } catch (ex: CertPathValidatorException) { + throw CertificateException(ex) + } + } + + override fun checkServerTrusted(chain: Array, authType: String) { + throw NotImplementedError() + } + + private val acceptedIssuers = trustStore.aliases().asSequence() + .filter(trustStore::isCertificateEntry) + .map(trustStore::getCertificate) + .map { it as X509Certificate } + .toList() + .toTypedArray() + + override fun getAcceptedIssuers() = acceptedIssuers + } + } else { + val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) + trustManagerFactory.trustManagers.asSequence().filter { it is X509TrustManager } + .single() as X509TrustManager + } + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/RbcsUrlStreamHandlerFactory.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/RbcsUrlStreamHandlerFactory.kt new file mode 100644 index 0000000..71a3cd8 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/RbcsUrlStreamHandlerFactory.kt @@ -0,0 +1,113 @@ +package net.woggioni.rbcs.common + +import java.io.IOException +import java.io.InputStream +import java.net.URL +import java.net.URLConnection +import java.net.URLStreamHandler +import java.net.spi.URLStreamHandlerProvider +import java.util.concurrent.atomic.AtomicBoolean +import java.util.stream.Collectors + + +class RbcsUrlStreamHandlerFactory : URLStreamHandlerProvider() { + + private class ClasspathHandler(private val classLoader: ClassLoader = RbcsUrlStreamHandlerFactory::class.java.classLoader) : + URLStreamHandler() { + + override fun openConnection(u: URL): URLConnection? { + return javaClass.module + ?.takeIf { m: Module -> m.layer != null } + ?.let { + val path = u.path + val i = path.lastIndexOf('/') + val packageName = path.substring(0, i).replace('/', '.') + val modules = packageMap[packageName]!! + ClasspathResourceURLConnection( + u, + modules + ) + } + ?: classLoader.getResource(u.path)?.let(URL::openConnection) + } + } + + private class JpmsHandler : URLStreamHandler() { + + override fun openConnection(u: URL): URLConnection { + val moduleName = u.host + val thisModule = javaClass.module + val sourceModule = + thisModule + ?.let(Module::getLayer) + ?.let { layer: ModuleLayer -> + layer.findModule(moduleName).orElse(null) + } ?: if(thisModule.layer == null) { + thisModule + } else throw ModuleNotFoundException("Module '$moduleName' not found") + + return JpmsResourceURLConnection(u, sourceModule) + } + } + + private class JpmsResourceURLConnection(url: URL, private val module: Module) : URLConnection(url) { + override fun connect() { + } + + @Throws(IOException::class) + override fun getInputStream(): InputStream { + val resource = getURL().path + return module.getResourceAsStream(resource) + ?: throw ResourceNotFoundException("Resource '$resource' not found in module '${module.name}'") + } + } + + override fun createURLStreamHandler(protocol: String): URLStreamHandler? { + return when (protocol) { + "classpath" -> ClasspathHandler() + "jpms" -> JpmsHandler() + else -> null + } + } + + private class ClasspathResourceURLConnection(url: URL?, private val modules: List) : + URLConnection(url) { + override fun connect() {} + + override fun getInputStream(): InputStream? { + for (module in modules) { + val result = module.getResourceAsStream(getURL().path) + if (result != null) return result + } + return null + } + } + + companion object { + private val installed = AtomicBoolean(false) + fun install() { + if (!installed.getAndSet(true)) { + URL.setURLStreamHandlerFactory(RbcsUrlStreamHandlerFactory()) + } + } + + private val packageMap: Map> by lazy { + RbcsUrlStreamHandlerFactory::class.java.module.layer + .modules() + .stream() + .flatMap { m: Module -> + m.packages.stream() + .map { p: String -> p to m } + } + .collect( + Collectors.groupingBy( + Pair::first, + Collectors.mapping( + Pair::second, + Collectors.toUnmodifiableList() + ) + ) + ) + } + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/Xml.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/Xml.kt new file mode 100644 index 0000000..4963d60 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/Xml.kt @@ -0,0 +1,243 @@ +package net.woggioni.rbcs.common + +import java.io.InputStream +import java.io.OutputStream +import java.net.URL +import javax.xml.XMLConstants.ACCESS_EXTERNAL_DTD +import javax.xml.XMLConstants.ACCESS_EXTERNAL_SCHEMA +import javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING +import javax.xml.XMLConstants.W3C_XML_SCHEMA_NS_URI +import javax.xml.parsers.DocumentBuilder +import javax.xml.parsers.DocumentBuilderFactory +import javax.xml.transform.OutputKeys +import javax.xml.transform.TransformerFactory +import javax.xml.transform.dom.DOMSource +import javax.xml.transform.stream.StreamResult +import javax.xml.transform.stream.StreamSource +import javax.xml.validation.Schema +import javax.xml.validation.SchemaFactory +import net.woggioni.jwo.JWO +import org.slf4j.event.Level +import org.w3c.dom.Document +import org.w3c.dom.Element +import org.w3c.dom.Node +import org.w3c.dom.NodeList +import org.xml.sax.ErrorHandler as ErrHandler +import org.xml.sax.SAXNotRecognizedException +import org.xml.sax.SAXNotSupportedException +import org.xml.sax.SAXParseException + + +class NodeListIterator(private val nodeList: NodeList) : Iterator { + private var cursor: Int = 0 + override fun hasNext(): Boolean { + return cursor < nodeList.length + } + + override fun next(): Node { + return if (hasNext()) nodeList.item(cursor++) else throw NoSuchElementException() + } +} + +class ElementIterator(parent: Element, name: String? = null) : Iterator { + private val it: NodeListIterator + private val name: String? + private var next: Element? + + init { + it = NodeListIterator(parent.childNodes) + this.name = name + next = getNext() + } + + override fun hasNext(): Boolean { + return next != null + } + + override fun next(): Element { + val result = next ?: throw NoSuchElementException() + next = getNext() + return result + } + + private fun getNext(): Element? { + var result: Element? = null + while (it.hasNext()) { + val node: Node = it.next() + if (node is Element && (name == null || name == node.tagName)) { + result = node + break + } + } + return result + } +} + +class Xml(val doc: Document, val element: Element) { + + class ErrorHandler(private val fileURL: URL) : ErrHandler { + + companion object { + private val log = createLogger() + } + + override fun warning(ex: SAXParseException)= err(ex, Level.WARN) + + private fun err(ex: SAXParseException, level: Level) { + log.log(level) { + "Problem at ${fileURL}:${ex.lineNumber}:${ex.columnNumber} parsing deployment configuration: ${ex.message}" + } + throw ex + } + + override fun error(ex: SAXParseException) = err(ex, Level.ERROR) + override fun fatalError(ex: SAXParseException) = err(ex, Level.ERROR) + } + + companion object { + private val dictMap: Map> = sequenceOf( + "env" to System.getenv().asSequence().map { (k, v) -> k to (v as Any) }.toMap(), + "sys" to System.getProperties().asSequence().map { (k, v) -> k as String to (v as Any) }.toMap() + ).toMap() + + private fun renderConfigurationTemplate(template: String): String { + return JWO.renderTemplate(template, emptyMap(), dictMap).replace("$$", "$") + } + + fun Element.renderAttribute(name : String, namespaceURI: String? = null) = if(namespaceURI == null) { + getAttribute(name) + } else { + getAttributeNS(name, namespaceURI) + }.takeIf(String::isNotEmpty)?.let(Companion::renderConfigurationTemplate) + + + fun Element.asIterable() = Iterable { ElementIterator(this, null) } + fun NodeList.asIterable() = Iterable { NodeListIterator(this) } + + private fun disableProperty(dbf: DocumentBuilderFactory, propertyName: String) { + try { + dbf.setAttribute(propertyName, "") + } catch (iae: IllegalArgumentException) { + // Property not supported. + } + } + + private fun disableProperty(sf: SchemaFactory, propertyName: String) { + try { + sf.setProperty(propertyName, "") + } catch (ex: SAXNotRecognizedException) { + // Property not supported. + } catch (ex: SAXNotSupportedException) { + } + } + + fun getSchema(schema: URL): Schema { + val sf = SchemaFactory.newInstance(W3C_XML_SCHEMA_NS_URI) + sf.setFeature(FEATURE_SECURE_PROCESSING, false) + sf.errorHandler = ErrorHandler(schema) + return sf.newSchema(schema) + } + + fun getSchema(inputStream: InputStream): Schema { + val sf = SchemaFactory.newInstance(W3C_XML_SCHEMA_NS_URI) + sf.setFeature(FEATURE_SECURE_PROCESSING, true) + return sf.newSchema(StreamSource(inputStream)) + } + + fun newDocumentBuilderFactory(schemaResourceURL: URL?): DocumentBuilderFactory { + val dbf = DocumentBuilderFactory.newInstance() + dbf.setFeature(FEATURE_SECURE_PROCESSING, false) + dbf.setAttribute(ACCESS_EXTERNAL_SCHEMA, "all") + disableProperty(dbf, ACCESS_EXTERNAL_DTD) + dbf.isExpandEntityReferences = true + dbf.isIgnoringComments = true + dbf.isNamespaceAware = true + dbf.isValidating = schemaResourceURL == null + dbf.setFeature("http://apache.org/xml/features/validation/schema", true) + schemaResourceURL?.let { + dbf.schema = getSchema(it) + } + return dbf + } + + fun newDocumentBuilder(resource: URL, schemaResourceURL: URL?): DocumentBuilder { + val db = newDocumentBuilderFactory(schemaResourceURL).newDocumentBuilder() + db.setErrorHandler(ErrorHandler(resource)) + return db + } + + fun parseXmlResource(resource: URL, schemaResourceURL: URL?): Document { + val db = newDocumentBuilder(resource, schemaResourceURL) + return resource.openStream().use(db::parse) + } + + fun parseXml(sourceURL: URL, sourceStream: InputStream? = null, schemaResourceURL: URL? = null): Document { + val db = newDocumentBuilder(sourceURL, schemaResourceURL) + return sourceStream?.let(db::parse) ?: sourceURL.openStream().use(db::parse) + } + + fun write(doc: Document, output: OutputStream) { + val transformerFactory = TransformerFactory.newInstance() + val transformer = transformerFactory.newTransformer() + transformer.setOutputProperty(OutputKeys.INDENT, "yes") + transformer.setOutputProperty(OutputKeys.INDENT, "yes") + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "4") + transformer.setOutputProperty(OutputKeys.STANDALONE, "yes") + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8") + val source = DOMSource(doc) + val result = StreamResult(output) + transformer.transform(source, result) + } + + fun of( + namespaceURI: String, + qualifiedName: String, + schemaResourceURL: URL? = null, + cb: Xml.(el: Element) -> Unit + ): Document { + val dbf = newDocumentBuilderFactory(schemaResourceURL) + val db = dbf.newDocumentBuilder() + val doc = db.newDocument() + val root = doc.createElementNS(namespaceURI, qualifiedName) + .also(doc::appendChild) + Xml(doc, root).cb(root) + return doc + } + + fun of(doc: Document, el: Element, cb: Xml.(el: Element) -> Unit): Element { + Xml(doc, el).cb(el) + return el + } + + fun Element.removeChildren() { + while (true) { + removeChild(firstChild ?: break) + } + } + } + + fun node( + name: String, + namespaceURI: String? = null, + attrs: Map = emptyMap(), + cb: Xml.(el: Element) -> Unit = {} + ): Element { + val child = doc.createElementNS(namespaceURI, name) + for ((key, value) in attrs) { + child.setAttribute(key, value) + } + return child + .also { + element.appendChild(it) + Xml(doc, it).cb(it) + } + } + + fun attr(key: String, value: String, namespaceURI: String? = null) { + element.setAttributeNS(namespaceURI, key, value) + } + + fun text(txt: String) { + element.appendChild(doc.createTextNode(txt)) + } +} diff --git a/rbcs-common/bin/test/net/woggioni/rbcs/common/CidrTest.kt b/rbcs-common/bin/test/net/woggioni/rbcs/common/CidrTest.kt new file mode 100644 index 0000000..5ab6d87 --- /dev/null +++ b/rbcs-common/bin/test/net/woggioni/rbcs/common/CidrTest.kt @@ -0,0 +1,17 @@ +package net.woggioni.rbcs.common + +import java.net.InetAddress +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class CidrTest { + class CidrTest { + @Test + fun test() { + val cidr = Cidr.from("2a02:4780:12:368b::1/128") + Assertions.assertTrue { + cidr.contains(InetAddress.ofLiteral("2a02:4780:12:368b::1")) + } + } + } +} \ No newline at end of file diff --git a/rbcs-common/bin/test/net/woggioni/rbcs/common/PasswordHashingTest.kt b/rbcs-common/bin/test/net/woggioni/rbcs/common/PasswordHashingTest.kt new file mode 100644 index 0000000..2b8f287 --- /dev/null +++ b/rbcs-common/bin/test/net/woggioni/rbcs/common/PasswordHashingTest.kt @@ -0,0 +1,38 @@ +package net.woggioni.rbcs.common + +import java.security.Provider +import java.security.Security +import java.util.Base64 +import net.woggioni.rbcs.common.PasswordSecurity.decodePasswordHash +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource + + +class PasswordHashingTest { + + @EnumSource(PasswordSecurity.Algorithm::class) + @ParameterizedTest + fun test(algo: PasswordSecurity.Algorithm) { + val password = "password" + val encoded = hashPassword(password, algorithm = algo) + val (_, salt) = decodePasswordHash(encoded, algo) + Assertions.assertEquals(encoded, + hashPassword(password, salt = salt.let(Base64.getEncoder()::encodeToString), algorithm = algo) + ) + } + + @Test + fun listAvailableAlgorithms() { + Security.getProviders().asSequence() + .flatMap { provider: Provider -> provider.services.asSequence() } + .filter { service: Provider.Service -> "SecretKeyFactory" == service.type } + .map(Provider.Service::getAlgorithm) + .forEach { + println(it) + } + + } +} \ No newline at end of file diff --git a/rbcs-server-memcache/.classpath b/rbcs-server-memcache/.classpath new file mode 100644 index 0000000..df4ae25 --- /dev/null +++ b/rbcs-server-memcache/.classpath @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-server-memcache/.project b/rbcs-server-memcache/.project new file mode 100644 index 0000000..19cdffb --- /dev/null +++ b/rbcs-server-memcache/.project @@ -0,0 +1,34 @@ + + + rbcs-server-memcache + Project rbcs-server-memcache created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776958 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-server-memcache/.settings/org.eclipse.buildship.core.prefs b/rbcs-server-memcache/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-server-memcache/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-server-memcache/.settings/org.eclipse.jdt.core.prefs b/rbcs-server-memcache/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-server-memcache/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-server-memcache/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider b/rbcs-server-memcache/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider new file mode 100644 index 0000000..5a04b38 --- /dev/null +++ b/rbcs-server-memcache/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider @@ -0,0 +1 @@ +net.woggioni.rbcs.server.memcache.MemcacheCacheProvider \ No newline at end of file diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/Exception.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/Exception.kt new file mode 100644 index 0000000..31ca929 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/Exception.kt @@ -0,0 +1,4 @@ +package net.woggioni.rbcs.server.memcache + +class MemcacheException(status : Short, msg : String? = null, cause : Throwable? = null) + : RuntimeException(msg ?: "Memcached status $status", cause) \ No newline at end of file diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheConfiguration.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheConfiguration.kt new file mode 100644 index 0000000..a789876 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheConfiguration.kt @@ -0,0 +1,105 @@ +package net.woggioni.rbcs.server.memcache + +import io.netty.channel.ChannelFactory +import io.netty.channel.EventLoopGroup +import io.netty.channel.pool.FixedChannelPool +import io.netty.channel.socket.DatagramChannel +import io.netty.channel.socket.SocketChannel +import java.time.Duration +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.CacheHandlerFactory +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.server.memcache.client.MemcacheClient + +data class MemcacheCacheConfiguration( + val servers: List, + val maxAge: Duration = Duration.ofDays(1), + val keyPrefix : String? = null, + val digestAlgorithm: String? = null, + val compressionMode: CompressionMode? = null, + val compressionLevel: Int, +) : Configuration.Cache { + + companion object { + private val log = createLogger() + } + + enum class CompressionMode { + /** + * Deflate mode + */ + DEFLATE + } + + data class Server( + val endpoint: HostAndPort, + val connectionTimeoutMillis: Int?, + val maxConnections: Int + ) + + override fun materialize() = object : CacheHandlerFactory { + + private val connectionPoolMap = ConcurrentHashMap() + + override fun newHandler( + cfg : Configuration, + eventLoop: EventLoopGroup, + socketChannelFactory: ChannelFactory, + datagramChannelFactory: ChannelFactory, + ): CacheHandler { + return MemcacheCacheHandler( + MemcacheClient( + this@MemcacheCacheConfiguration.servers, + cfg.connection.chunkSize, + eventLoop, + socketChannelFactory, + connectionPoolMap + ), + keyPrefix, + digestAlgorithm, + compressionMode != null, + compressionLevel, + cfg.connection.chunkSize, + maxAge + ) + } + + override fun asyncClose() = object : CompletableFuture() { + init { + val failure = AtomicReference(null) + val pools = connectionPoolMap.values.toList() + val npools = pools.size + val finished = AtomicInteger(0) + if (pools.isEmpty()) { + complete(null) + } else { + pools.forEach { pool -> + pool.closeAsync().addListener { + if (!it.isSuccess) { + failure.compareAndSet(null, it.cause()) + } + if (finished.incrementAndGet() == npools) { + when (val ex = failure.get()) { + null -> complete(null) + else -> completeExceptionally(ex) + } + } + } + } + } + } + } + + } + + override fun getNamespaceURI() = "urn:net.woggioni.rbcs.server.memcache" + + override fun getTypeName() = "memcacheCacheType" +} + diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheHandler.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheHandler.kt new file mode 100644 index 0000000..d488eb1 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheHandler.kt @@ -0,0 +1,442 @@ +package net.woggioni.rbcs.server.memcache + +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.CompositeByteBuf +import io.netty.channel.Channel as NettyChannel +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.memcache.DefaultLastMemcacheContent +import io.netty.handler.codec.memcache.DefaultMemcacheContent +import io.netty.handler.codec.memcache.LastMemcacheContent +import io.netty.handler.codec.memcache.MemcacheContent +import io.netty.handler.codec.memcache.binary.BinaryMemcacheOpcodes +import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponse +import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponseStatus +import io.netty.handler.codec.memcache.binary.DefaultBinaryMemcacheRequest +import java.io.ByteArrayOutputStream +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.nio.ByteBuffer +import java.nio.channels.Channels +import java.nio.channels.FileChannel +import java.nio.channels.ReadableByteChannel +import java.nio.file.Files +import java.nio.file.StandardOpenOption +import java.time.Duration +import java.time.Instant +import java.util.concurrent.CompletableFuture +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream +import java.util.zip.InflaterOutputStream +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.api.exception.ContentTooLargeException +import net.woggioni.rbcs.api.message.CacheMessage +import net.woggioni.rbcs.api.message.CacheMessage.CacheContent +import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent +import net.woggioni.rbcs.common.ByteBufInputStream +import net.woggioni.rbcs.common.ByteBufOutputStream +import net.woggioni.rbcs.common.RBCS.processCacheKey +import net.woggioni.rbcs.common.RBCS.toIntOrNull +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.extractChunk +import net.woggioni.rbcs.common.trace +import net.woggioni.rbcs.server.memcache.client.MemcacheClient +import net.woggioni.rbcs.server.memcache.client.MemcacheRequestController +import net.woggioni.rbcs.server.memcache.client.MemcacheResponseHandler + +class MemcacheCacheHandler( + private val client: MemcacheClient, + private val keyPrefix: String?, + private val digestAlgorithm: String?, + private val compressionEnabled: Boolean, + private val compressionLevel: Int, + private val chunkSize: Int, + private val maxAge: Duration +) : CacheHandler() { + companion object { + private val log = createLogger() + + private fun encodeExpiry(expiry: Duration): Int { + val expirySeconds = expiry.toSeconds() + return expirySeconds.toInt().takeIf { it.toLong() == expirySeconds } + ?: Instant.ofEpochSecond(expirySeconds).epochSecond.toInt() + } + } + + private interface InProgressRequest { + + } + + private inner class InProgressGetRequest( + val key: String, + private val ctx: ChannelHandlerContext + ) : InProgressRequest { + private val acc = ctx.alloc().compositeBuffer() + private val chunk = ctx.alloc().compositeBuffer() + private val outputStream = ByteBufOutputStream(chunk).let { + if (compressionEnabled) { + InflaterOutputStream(it) + } else { + it + } + } + private var responseSent = false + private var metadataSize: Int? = null + + fun write(buf: ByteBuf) { + acc.addComponent(true, buf.retain()) + if (metadataSize == null && acc.readableBytes() >= Int.SIZE_BYTES) { + metadataSize = acc.readInt() + } + metadataSize + ?.takeIf { !responseSent } + ?.takeIf { acc.readableBytes() >= it } + ?.let { mSize -> + val metadata = ObjectInputStream(ByteBufInputStream(acc)).use { + acc.retain() + it.readObject() as CacheValueMetadata + } + log.trace(ctx) { + "Sending response from cache" + } + sendMessageAndFlush(ctx, CacheValueFoundResponse(key, metadata)) + responseSent = true + acc.readerIndex(Int.SIZE_BYTES + mSize) + } + if (responseSent) { + acc.readBytes(outputStream, acc.readableBytes()) + if (acc.readableBytes() >= chunkSize) { + flush(false) + } + } + } + + private fun flush(last: Boolean) { + val toSend = extractChunk(chunk, ctx.alloc()) + val msg = if (last) { + log.trace(ctx) { + "Sending last chunk to client" + } + LastCacheContent(toSend) + } else { + log.trace(ctx) { + "Sending chunk to client" + } + CacheContent(toSend) + } + sendMessageAndFlush(ctx, msg) + } + + fun commit() { + acc.release() + chunk.retain() + outputStream.close() + flush(true) + chunk.release() + } + + fun rollback() { + acc.release() + outputStream.close() + } + } + + private inner class InProgressPutRequest( + private val ch: NettyChannel, + metadata: CacheValueMetadata, + val digest: ByteBuf, + val requestController: CompletableFuture, + private val alloc: ByteBufAllocator + ) : InProgressRequest { + private var totalSize = 0 + private var tmpFile: FileChannel? = null + private val accumulator = alloc.compositeBuffer() + private val stream = ByteBufOutputStream(accumulator).let { + if (compressionEnabled) { + DeflaterOutputStream(it, Deflater(compressionLevel)) + } else { + it + } + } + + init { + ByteArrayOutputStream().let { baos -> + ObjectOutputStream(baos).use { + it.writeObject(metadata) + } + val serializedBytes = baos.toByteArray() + accumulator.writeInt(serializedBytes.size) + accumulator.writeBytes(serializedBytes) + } + } + + fun write(buf: ByteBuf) { + totalSize += buf.readableBytes() + buf.readBytes(stream, buf.readableBytes()) + tmpFile?.let { + flushToDisk(it, accumulator) + } + if (accumulator.readableBytes() > 0x100000) { + log.debug(ch) { + "Entry is too big, buffering it into a file" + } + val opts = arrayOf( + StandardOpenOption.DELETE_ON_CLOSE, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING + ) + FileChannel.open(Files.createTempFile("rbcs-memcache", ".tmp"), *opts).let { fc -> + tmpFile = fc + flushToDisk(fc, accumulator) + } + } + } + + private fun flushToDisk(fc: FileChannel, buf: CompositeByteBuf) { + val chunk = extractChunk(buf, alloc) + fc.write(chunk.nioBuffer()) + chunk.release() + } + + fun commit(): Pair { + digest.release() + accumulator.retain() + stream.close() + val fileChannel = tmpFile + return if (fileChannel != null) { + flushToDisk(fileChannel, accumulator) + accumulator.release() + fileChannel.position(0) + val fileSize = fileChannel.size().toIntOrNull() ?: let { + fileChannel.close() + throw ContentTooLargeException("Request body is too large", null) + } + fileSize to fileChannel + } else { + accumulator.readableBytes() to Channels.newChannel(ByteBufInputStream(accumulator)) + } + } + + fun rollback() { + stream.close() + digest.release() + tmpFile?.close() + } + } + + private var inProgressRequest: InProgressRequest? = null + + override fun channelRead0(ctx: ChannelHandlerContext, msg: CacheMessage) { + when (msg) { + is CacheGetRequest -> handleGetRequest(ctx, msg) + is CachePutRequest -> handlePutRequest(ctx, msg) + is LastCacheContent -> handleLastCacheContent(ctx, msg) + is CacheContent -> handleCacheContent(ctx, msg) + else -> ctx.fireChannelRead(msg) + } + } + + private fun handleGetRequest(ctx: ChannelHandlerContext, msg: CacheGetRequest) { + log.debug(ctx) { + "Fetching ${msg.key} from memcache" + } + val key = ctx.alloc().buffer().also { + it.writeBytes(processCacheKey(msg.key, keyPrefix, digestAlgorithm)) + } + val responseHandler = object : MemcacheResponseHandler { + override fun responseReceived(response: BinaryMemcacheResponse) { + val status = response.status() + when (status) { + BinaryMemcacheResponseStatus.SUCCESS -> { + log.debug(ctx) { + "Cache hit for key ${msg.key} on memcache" + } + inProgressRequest = InProgressGetRequest(msg.key, ctx) + } + + BinaryMemcacheResponseStatus.KEY_ENOENT -> { + log.debug(ctx) { + "Cache miss for key ${msg.key} on memcache" + } + sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key)) + } + } + } + + override fun contentReceived(content: MemcacheContent) { + log.trace(ctx) { + "${if (content is LastMemcacheContent) "Last chunk" else "Chunk"} of ${ + content.content().readableBytes() + } bytes received from memcache for key ${msg.key}" + } + (inProgressRequest as? InProgressGetRequest)?.let { inProgressGetRequest -> + inProgressGetRequest.write(content.content()) + if (content is LastMemcacheContent) { + inProgressRequest = null + inProgressGetRequest.commit() + } + } + } + + override fun exceptionCaught(ex: Throwable) { + (inProgressRequest as? InProgressGetRequest).let { inProgressGetRequest -> + inProgressGetRequest?.let { + inProgressRequest = null + it.rollback() + } + } + this@MemcacheCacheHandler.exceptionCaught(ctx, ex) + } + } + client.sendRequest(key.retainedDuplicate(), responseHandler).thenAccept { requestHandle -> + log.trace(ctx) { + "Sending GET request for key ${msg.key} to memcache" + } + val request = DefaultBinaryMemcacheRequest(key).apply { + setOpcode(BinaryMemcacheOpcodes.GET) + } + requestHandle.sendRequest(request) + requestHandle.sendContent(LastMemcacheContent.EMPTY_LAST_CONTENT) + } + } + + private fun handlePutRequest(ctx: ChannelHandlerContext, msg: CachePutRequest) { + val key = ctx.alloc().buffer().also { + it.writeBytes(processCacheKey(msg.key, keyPrefix, digestAlgorithm)) + } + val responseHandler = object : MemcacheResponseHandler { + override fun responseReceived(response: BinaryMemcacheResponse) { + val status = response.status() + when (status) { + BinaryMemcacheResponseStatus.SUCCESS -> { + log.debug(ctx) { + "Inserted key ${msg.key} into memcache" + } + sendMessageAndFlush(ctx, CachePutResponse(msg.key)) + } + + else -> this@MemcacheCacheHandler.exceptionCaught(ctx, MemcacheException(status)) + } + } + + override fun contentReceived(content: MemcacheContent) {} + + override fun exceptionCaught(ex: Throwable) { + this@MemcacheCacheHandler.exceptionCaught(ctx, ex) + } + } + + val requestController = client.sendRequest(key.retainedDuplicate(), responseHandler).whenComplete { _, ex -> + ex?.let { + this@MemcacheCacheHandler.exceptionCaught(ctx, ex) + } + } + inProgressRequest = InProgressPutRequest(ctx.channel(), msg.metadata, key, requestController, ctx.alloc()) + } + + private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + log.trace(ctx) { + "Received chunk of ${msg.content().readableBytes()} bytes for memcache" + } + request.write(msg.content()) + } + + is InProgressGetRequest -> { + msg.release() + } + } + } + + private fun handleLastCacheContent(ctx: ChannelHandlerContext, msg: LastCacheContent) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + inProgressRequest = null + log.trace(ctx) { + "Received last chunk of ${msg.content().readableBytes()} bytes for memcache" + } + request.write(msg.content()) + val key = request.digest.retainedDuplicate() + val (payloadSize, payloadSource) = request.commit() + val extras = ctx.alloc().buffer(8, 8) + extras.writeInt(0) + extras.writeInt(encodeExpiry(maxAge)) + val totalBodyLength = request.digest.readableBytes() + extras.readableBytes() + payloadSize + log.trace(ctx) { + "Trying to send SET request to memcache" + } + request.requestController.whenComplete { requestController, ex -> + if (ex == null) { + log.trace(ctx) { + "Sending SET request to memcache" + } + requestController.sendRequest(DefaultBinaryMemcacheRequest().apply { + setOpcode(BinaryMemcacheOpcodes.SET) + setKey(key) + setExtras(extras) + setTotalBodyLength(totalBodyLength) + }) + log.trace(ctx) { + "Sending request payload to memcache" + } + payloadSource.use { source -> + val bb = ByteBuffer.allocate(chunkSize) + while (true) { + val read = source.read(bb) + bb.limit() + if (read >= 0 && bb.position() < chunkSize && bb.hasRemaining()) { + continue + } + val chunk = ctx.alloc().buffer(chunkSize) + bb.flip() + chunk.writeBytes(bb) + bb.clear() + log.trace(ctx) { + "Sending ${chunk.readableBytes()} bytes chunk to memcache" + } + if (read < 0) { + requestController.sendContent(DefaultLastMemcacheContent(chunk)) + break + } else { + requestController.sendContent(DefaultMemcacheContent(chunk)) + } + } + } + } else { + payloadSource.close() + } + } + } + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + inProgressRequest = null + request.requestController.thenAccept { controller -> + controller.exceptionCaught(cause) + } + request.rollback() + } + + is InProgressGetRequest -> { + inProgressRequest = null + request.rollback() + } + } + super.exceptionCaught(ctx, cause) + } +} \ No newline at end of file diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheProvider.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheProvider.kt new file mode 100644 index 0000000..0ff29a1 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheProvider.kt @@ -0,0 +1,102 @@ +package net.woggioni.rbcs.server.memcache + +import java.time.Duration +import java.time.temporal.ChronoUnit +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.api.exception.ConfigurationException +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.common.Xml.Companion.asIterable +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document +import org.w3c.dom.Element + + +class MemcacheCacheProvider : CacheProvider { + override fun getXmlSchemaLocation() = "jpms://net.woggioni.rbcs.server.memcache/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd" + + override fun getXmlType() = "memcacheCacheType" + + override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server.memcache" + + val xmlNamespacePrefix : String + get() = "rbcs-memcache" + + override fun deserialize(el: Element): MemcacheCacheConfiguration { + val servers = mutableListOf() + val maxAge = el.renderAttribute("max-age") + ?.let(Duration::parse) + ?: Duration.ofDays(1) + val compressionLevel = el.renderAttribute("compression-level") + ?.let(Integer::decode) + ?: -1 + val compressionMode = el.renderAttribute("compression-mode") + ?.let { + when (it) { + "deflate" -> MemcacheCacheConfiguration.CompressionMode.DEFLATE + else -> MemcacheCacheConfiguration.CompressionMode.DEFLATE + } + } + val keyPrefix = el.renderAttribute("key-prefix") + val digestAlgorithm = el.renderAttribute("digest") + for (child in el.asIterable()) { + when (child.nodeName) { + "server" -> { + val host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required") + val port = child.renderAttribute("port")?.toInt() ?: throw ConfigurationException("port attribute is required") + val maxConnections = child.renderAttribute("max-connections")?.toInt() ?: 1 + val connectionTimeout = child.renderAttribute("connection-timeout") + ?.let(Duration::parse) + ?.let(Duration::toMillis) + ?.let(Long::toInt) + ?: 10000 + servers.add(MemcacheCacheConfiguration.Server(HostAndPort(host, port), connectionTimeout, maxConnections)) + } + } + } + return MemcacheCacheConfiguration( + servers, + maxAge, + keyPrefix, + digestAlgorithm, + compressionMode, + compressionLevel + ) + } + + override fun serialize(doc: Document, cache: MemcacheCacheConfiguration) = cache.run { + val result = doc.createElement("cache") + Xml.of(doc, result) { + attr("xmlns:${xmlNamespacePrefix}", xmlNamespace, namespaceURI = "http://www.w3.org/2000/xmlns/") + attr("xs:type", "${xmlNamespacePrefix}:$xmlType", RBCS.XML_SCHEMA_NAMESPACE_URI) + for (server in servers) { + node("server") { + attr("host", server.endpoint.host) + attr("port", server.endpoint.port.toString()) + server.connectionTimeoutMillis?.let { connectionTimeoutMillis -> + attr("connection-timeout", Duration.of(connectionTimeoutMillis.toLong(), ChronoUnit.MILLIS).toString()) + } + attr("max-connections", server.maxConnections.toString()) + } + + } + attr("max-age", maxAge.toString()) + keyPrefix?.let { + attr("key-prefix", it) + } + digestAlgorithm?.let { digestAlgorithm -> + attr("digest", digestAlgorithm) + } + compressionMode?.let { compressionMode -> + attr( + "compression-mode", when (compressionMode) { + MemcacheCacheConfiguration.CompressionMode.DEFLATE -> "deflate" + } + ) + } + attr("compression-level", compressionLevel.toString()) + } + result + } +} diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheClient.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheClient.kt new file mode 100644 index 0000000..07e37ab --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheClient.kt @@ -0,0 +1,198 @@ +package net.woggioni.rbcs.server.memcache.client + + +import io.netty.bootstrap.Bootstrap +import io.netty.buffer.ByteBuf +import io.netty.channel.Channel +import io.netty.channel.ChannelFactory +import io.netty.channel.ChannelFutureListener +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelOption +import io.netty.channel.ChannelPipeline +import io.netty.channel.EventLoopGroup +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.channel.pool.AbstractChannelPoolHandler +import io.netty.channel.pool.FixedChannelPool +import io.netty.channel.socket.SocketChannel +import io.netty.handler.codec.memcache.LastMemcacheContent +import io.netty.handler.codec.memcache.MemcacheContent +import io.netty.handler.codec.memcache.MemcacheObject +import io.netty.handler.codec.memcache.binary.BinaryMemcacheClientCodec +import io.netty.handler.codec.memcache.binary.BinaryMemcacheRequest +import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponse +import io.netty.util.concurrent.Future as NettyFuture +import io.netty.util.concurrent.GenericFutureListener +import java.io.IOException +import java.net.InetSocketAddress +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.trace +import net.woggioni.rbcs.server.memcache.MemcacheCacheConfiguration +import net.woggioni.rbcs.server.memcache.MemcacheCacheHandler + + +class MemcacheClient( + private val servers: List, + private val chunkSize : Int, + private val group: EventLoopGroup, + private val channelFactory: ChannelFactory, + private val connectionPool: ConcurrentHashMap +) : AutoCloseable { + + private companion object { + private val log = createLogger() + } + + private fun newConnectionPool(server: MemcacheCacheConfiguration.Server): FixedChannelPool { + val bootstrap = Bootstrap().apply { + group(group) + channelFactory(channelFactory) + option(ChannelOption.SO_KEEPALIVE, true) + remoteAddress(InetSocketAddress(server.endpoint.host, server.endpoint.port)) + server.connectionTimeoutMillis?.let { + option(ChannelOption.CONNECT_TIMEOUT_MILLIS, it) + } + } + val channelPoolHandler = object : AbstractChannelPoolHandler() { + + override fun channelCreated(ch: Channel) { + val pipeline: ChannelPipeline = ch.pipeline() + pipeline.addLast(BinaryMemcacheClientCodec(chunkSize, true)) + } + } + return FixedChannelPool(bootstrap, channelPoolHandler, server.maxConnections) + } + + fun sendRequest( + key: ByteBuf, + responseHandler: MemcacheResponseHandler + ): CompletableFuture { + val server = if (servers.size > 1) { + var checksum = 0 + while (key.readableBytes() > 4) { + val byte = key.readInt() + checksum = checksum xor byte + } + while (key.readableBytes() > 0) { + val byte = key.readByte() + checksum = checksum xor byte.toInt() + } + servers[checksum % servers.size] + } else { + servers.first() + } + key.release() + + val response = CompletableFuture() + // Custom handler for processing responses + val pool = connectionPool.computeIfAbsent(server.endpoint) { + newConnectionPool(server) + } + pool.acquire().addListener(object : GenericFutureListener> { + override fun operationComplete(channelFuture: NettyFuture) { + if (channelFuture.isSuccess) { + val channel = channelFuture.now + var connectionClosedByTheRemoteServer = true + val closeCallback = { + if (connectionClosedByTheRemoteServer) { + val ex = IOException("The memcache server closed the connection") + val completed = response.completeExceptionally(ex) + if(!completed) responseHandler.exceptionCaught(ex) + } + } + val closeListener = ChannelFutureListener { + closeCallback() + } + channel.closeFuture().addListener(closeListener) + val pipeline = channel.pipeline() + val handler = object : SimpleChannelInboundHandler() { + + override fun handlerAdded(ctx: ChannelHandlerContext) { + channel.closeFuture().removeListener(closeListener) + } + + override fun channelRead0( + ctx: ChannelHandlerContext, + msg: MemcacheObject + ) { + when (msg) { + is BinaryMemcacheResponse -> { + responseHandler.responseReceived(msg) + } + + is LastMemcacheContent -> { + responseHandler.contentReceived(msg) + pipeline.remove(this) + } + + is MemcacheContent -> { + responseHandler.contentReceived(msg) + } + } + } + + override fun channelInactive(ctx: ChannelHandlerContext) { + closeCallback() + ctx.fireChannelInactive() + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + connectionClosedByTheRemoteServer = false + ctx.close() + responseHandler.exceptionCaught(cause) + } + } + + channel.pipeline().addLast(handler) + response.complete(object : MemcacheRequestController { + private var channelReleased = false + + override fun sendRequest(request: BinaryMemcacheRequest) { + channel.writeAndFlush(request) + } + + override fun sendContent(content: MemcacheContent) { + channel.writeAndFlush(content).addListener { + if(content is LastMemcacheContent) { + if(!channelReleased) { + pool.release(channel) + channelReleased = true + log.trace(channel) { + "Channel released" + } + } + } + } + } + + override fun exceptionCaught(ex: Throwable) { + log.warn(ex.message, ex) + connectionClosedByTheRemoteServer = false + channel.close() + if(!channelReleased) { + pool.release(channel) + channelReleased = true + log.trace(channel) { + "Channel released" + } + } + } + }) + } else { + response.completeExceptionally(channelFuture.cause()) + } + } + }) + return response + } + + fun shutDown(): NettyFuture<*> { + return group.shutdownGracefully() + } + + override fun close() { + shutDown().sync() + } +} \ No newline at end of file diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheRequestController.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheRequestController.kt new file mode 100644 index 0000000..06cc772 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheRequestController.kt @@ -0,0 +1,13 @@ +package net.woggioni.rbcs.server.memcache.client + +import io.netty.handler.codec.memcache.MemcacheContent +import io.netty.handler.codec.memcache.binary.BinaryMemcacheRequest + +interface MemcacheRequestController { + + fun sendRequest(request : BinaryMemcacheRequest) + + fun sendContent(content : MemcacheContent) + + fun exceptionCaught(ex : Throwable) +} diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheResponseHandler.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheResponseHandler.kt new file mode 100644 index 0000000..19bb930 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheResponseHandler.kt @@ -0,0 +1,14 @@ +package net.woggioni.rbcs.server.memcache.client + +import io.netty.handler.codec.memcache.MemcacheContent +import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponse + +interface MemcacheResponseHandler { + + + fun responseReceived(response : BinaryMemcacheResponse) + + fun contentReceived(content : MemcacheContent) + + fun exceptionCaught(ex : Throwable) +} diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd new file mode 100644 index 0000000..1083f44 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + Prepend this string to all the keys inserted in memcache, + useful in case the caching backend is shared with other applications + + + + + + + + + + + + + + + + + diff --git a/rbcs-server-memcache/bin/test/net/woggioni/rbcs/server/memcache/client/ByteBufferTest.kt b/rbcs-server-memcache/bin/test/net/woggioni/rbcs/server/memcache/client/ByteBufferTest.kt new file mode 100644 index 0000000..21df339 --- /dev/null +++ b/rbcs-server-memcache/bin/test/net/woggioni/rbcs/server/memcache/client/ByteBufferTest.kt @@ -0,0 +1,27 @@ +package net.woggioni.rbcs.server.memcache.client + +import io.netty.buffer.ByteBufUtil +import io.netty.buffer.Unpooled +import java.io.ByteArrayInputStream +import java.nio.ByteBuffer +import java.nio.channels.Channels +import kotlin.random.Random +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class ByteBufferTest { + + @Test + fun test() { + val byteBuffer = ByteBuffer.allocate(0x100) + val originalBytes = Random(101325).nextBytes(0x100) + Channels.newChannel(ByteArrayInputStream(originalBytes)).use { source -> + source.read(byteBuffer) + } + byteBuffer.flip() + val buf = Unpooled.buffer() + buf.writeBytes(byteBuffer) + val finalBytes = ByteBufUtil.getBytes(buf) + Assertions.assertArrayEquals(originalBytes, finalBytes) + } +} \ No newline at end of file diff --git a/rbcs-server-redis/.classpath b/rbcs-server-redis/.classpath new file mode 100644 index 0000000..f330fac --- /dev/null +++ b/rbcs-server-redis/.classpath @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-server-redis/.project b/rbcs-server-redis/.project new file mode 100644 index 0000000..7db4573 --- /dev/null +++ b/rbcs-server-redis/.project @@ -0,0 +1,34 @@ + + + rbcs-server-redis + Project rbcs-server-redis created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776964 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-server-redis/.settings/org.eclipse.buildship.core.prefs b/rbcs-server-redis/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-server-redis/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-server-redis/.settings/org.eclipse.jdt.core.prefs b/rbcs-server-redis/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-server-redis/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-server-redis/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider b/rbcs-server-redis/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider new file mode 100644 index 0000000..20dc9e6 --- /dev/null +++ b/rbcs-server-redis/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider @@ -0,0 +1 @@ +net.woggioni.rbcs.server.redis.RedisCacheProvider diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/Exception.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/Exception.kt new file mode 100644 index 0000000..9e55570 --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/Exception.kt @@ -0,0 +1,4 @@ +package net.woggioni.rbcs.server.redis + +class RedisException(msg: String, cause: Throwable? = null) + : RuntimeException(msg, cause) diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheConfiguration.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheConfiguration.kt new file mode 100644 index 0000000..3b25bde --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheConfiguration.kt @@ -0,0 +1,107 @@ +package net.woggioni.rbcs.server.redis + +import io.netty.channel.ChannelFactory +import io.netty.channel.EventLoopGroup +import io.netty.channel.pool.FixedChannelPool +import io.netty.channel.socket.DatagramChannel +import io.netty.channel.socket.SocketChannel + +import java.time.Duration +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference + +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.CacheHandlerFactory +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.server.redis.client.RedisClient + +data class RedisCacheConfiguration( + val servers: List, + val maxAge: Duration = Duration.ofDays(1), + val keyPrefix: String? = null, + val digestAlgorithm: String? = null, + val compressionMode: CompressionMode? = null, + val compressionLevel: Int, +) : Configuration.Cache { + + companion object { + private val log = createLogger() + } + + enum class CompressionMode { + /** + * Deflate mode + */ + DEFLATE + } + + data class Server( + val endpoint: HostAndPort, + val connectionTimeoutMillis: Int?, + val maxConnections: Int, + val password: String? = null, + ) + + override fun materialize() = object : CacheHandlerFactory { + + private val connectionPoolMap = ConcurrentHashMap() + + override fun newHandler( + cfg: Configuration, + eventLoop: EventLoopGroup, + socketChannelFactory: ChannelFactory, + datagramChannelFactory: ChannelFactory, + ): CacheHandler { + return RedisCacheHandler( + RedisClient( + this@RedisCacheConfiguration.servers, + cfg.connection.chunkSize, + eventLoop, + socketChannelFactory, + connectionPoolMap + ), + keyPrefix, + digestAlgorithm, + compressionMode != null, + compressionLevel, + cfg.connection.chunkSize, + maxAge + ) + } + + override fun asyncClose() = object : CompletableFuture() { + init { + val failure = AtomicReference(null) + val pools = connectionPoolMap.values.toList() + val npools = pools.size + val finished = AtomicInteger(0) + if (pools.isEmpty()) { + complete(null) + } else { + pools.forEach { pool -> + pool.closeAsync().addListener { + if (!it.isSuccess) { + failure.compareAndSet(null, it.cause()) + } + if (finished.incrementAndGet() == npools) { + when (val ex = failure.get()) { + null -> complete(null) + else -> completeExceptionally(ex) + } + } + } + } + } + } + } + + } + + override fun getNamespaceURI() = "urn:net.woggioni.rbcs.server.redis" + + override fun getTypeName() = "redisCacheType" +} diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheHandler.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheHandler.kt new file mode 100644 index 0000000..584dd7f --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheHandler.kt @@ -0,0 +1,438 @@ +package net.woggioni.rbcs.server.redis + +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.CompositeByteBuf +import io.netty.channel.Channel as NettyChannel +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.redis.ArrayRedisMessage +import io.netty.handler.codec.redis.ErrorRedisMessage +import io.netty.handler.codec.redis.FullBulkStringRedisMessage +import io.netty.handler.codec.redis.RedisMessage +import io.netty.handler.codec.redis.SimpleStringRedisMessage + +import java.io.ByteArrayOutputStream +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.nio.ByteBuffer +import java.nio.channels.Channels +import java.nio.channels.FileChannel +import java.nio.channels.ReadableByteChannel +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.StandardOpenOption +import java.time.Duration +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream +import java.util.zip.InflaterOutputStream + +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.api.exception.ContentTooLargeException +import net.woggioni.rbcs.api.message.CacheMessage +import net.woggioni.rbcs.api.message.CacheMessage.CacheContent +import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent +import net.woggioni.rbcs.common.ByteBufInputStream +import net.woggioni.rbcs.common.ByteBufOutputStream +import net.woggioni.rbcs.common.RBCS.processCacheKey +import net.woggioni.rbcs.common.RBCS.toIntOrNull +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.extractChunk +import net.woggioni.rbcs.common.trace +import net.woggioni.rbcs.common.warn +import net.woggioni.rbcs.server.redis.client.RedisClient +import net.woggioni.rbcs.server.redis.client.RedisResponseHandler + +class RedisCacheHandler( + private val client: RedisClient, + private val keyPrefix: String?, + private val digestAlgorithm: String?, + private val compressionEnabled: Boolean, + private val compressionLevel: Int, + private val chunkSize: Int, + private val maxAge: Duration, +) : CacheHandler() { + companion object { + private val log = createLogger() + } + + private interface InProgressRequest + + private inner class InProgressGetRequest( + val key: String, + private val ctx: ChannelHandlerContext, + ) : InProgressRequest { + private val chunk = ctx.alloc().compositeBuffer() + private val outputStream = ByteBufOutputStream(chunk).let { + if (compressionEnabled) { + InflaterOutputStream(it) + } else { + it + } + } + + fun processResponse(data: ByteBuf) { + if (data.readableBytes() < Int.SIZE_BYTES) { + log.debug(ctx) { + "Received empty or corrupt data from Redis for key $key" + } + sendMessageAndFlush(ctx, CacheValueNotFoundResponse(key)) + data.release() + return + } + + val metadataSize = data.readInt() + if (data.readableBytes() < metadataSize) { + log.debug(ctx) { + "Received incomplete metadata from Redis for key $key" + } + sendMessageAndFlush(ctx, CacheValueNotFoundResponse(key)) + data.release() + return + } + + val metadata = ObjectInputStream(ByteBufInputStream(data)).use { + data.retain() + it.readObject() as CacheValueMetadata + } + data.readerIndex(Int.SIZE_BYTES + metadataSize) + + log.trace(ctx) { + "Sending response from cache" + } + sendMessageAndFlush(ctx, CacheValueFoundResponse(key, metadata)) + + // Decompress and stream the remaining payload + data.readBytes(outputStream, data.readableBytes()) + data.release() + commit() + } + + private fun flush(last: Boolean) { + val toSend = extractChunk(chunk, ctx.alloc()) + val msg = if (last) { + log.trace(ctx) { + "Sending last chunk to client" + } + LastCacheContent(toSend) + } else { + log.trace(ctx) { + "Sending chunk to client" + } + CacheContent(toSend) + } + sendMessageAndFlush(ctx, msg) + } + + fun commit() { + chunk.retain() + outputStream.close() + flush(true) + chunk.release() + } + + fun rollback() { + outputStream.close() + } + } + + private inner class InProgressPutRequest( + private val ch: NettyChannel, + metadata: CacheValueMetadata, + val keyString: String, + val keyBytes: ByteBuf, + private val alloc: ByteBufAllocator, + ) : InProgressRequest { + private var totalSize = 0 + private var tmpFile: FileChannel? = null + private val accumulator = alloc.compositeBuffer() + private val stream = ByteBufOutputStream(accumulator).let { + if (compressionEnabled) { + DeflaterOutputStream(it, Deflater(compressionLevel)) + } else { + it + } + } + + init { + ByteArrayOutputStream().let { baos -> + ObjectOutputStream(baos).use { + it.writeObject(metadata) + } + val serializedBytes = baos.toByteArray() + accumulator.writeInt(serializedBytes.size) + accumulator.writeBytes(serializedBytes) + } + } + + fun write(buf: ByteBuf) { + totalSize += buf.readableBytes() + buf.readBytes(stream, buf.readableBytes()) + tmpFile?.let { + flushToDisk(it, accumulator) + } + if (accumulator.readableBytes() > 0x100000) { + log.debug(ch) { + "Entry is too big, buffering it into a file" + } + val opts = arrayOf( + StandardOpenOption.DELETE_ON_CLOSE, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING + ) + FileChannel.open(Files.createTempFile("rbcs-server-redis", ".tmp"), *opts).let { fc -> + tmpFile = fc + flushToDisk(fc, accumulator) + } + } + } + + private fun flushToDisk(fc: FileChannel, buf: CompositeByteBuf) { + val chunk = extractChunk(buf, alloc) + fc.write(chunk.nioBuffer()) + chunk.release() + } + + fun commit(): Pair { + keyBytes.release() + accumulator.retain() + stream.close() + val fileChannel = tmpFile + return if (fileChannel != null) { + flushToDisk(fileChannel, accumulator) + accumulator.release() + fileChannel.position(0) + val fileSize = fileChannel.size().toIntOrNull() ?: let { + fileChannel.close() + throw ContentTooLargeException("Request body is too large", null) + } + fileSize to fileChannel + } else { + accumulator.readableBytes() to Channels.newChannel(ByteBufInputStream(accumulator)) + } + } + + fun rollback() { + stream.close() + keyBytes.release() + tmpFile?.close() + } + } + + private var inProgressRequest: InProgressRequest? = null + + override fun channelRead0(ctx: ChannelHandlerContext, msg: CacheMessage) { + when (msg) { + is CacheGetRequest -> handleGetRequest(ctx, msg) + is CachePutRequest -> handlePutRequest(ctx, msg) + is LastCacheContent -> handleLastCacheContent(ctx, msg) + is CacheContent -> handleCacheContent(ctx, msg) + else -> ctx.fireChannelRead(msg) + } + } + + private fun handleGetRequest(ctx: ChannelHandlerContext, msg: CacheGetRequest) { + log.debug(ctx) { + "Fetching ${msg.key} from Redis" + } + val keyBytes = processCacheKey(msg.key, keyPrefix, digestAlgorithm) + val keyString = String(keyBytes, StandardCharsets.UTF_8) + val responseHandler = object : RedisResponseHandler { + override fun responseReceived(response: RedisMessage) { + when (response) { + is FullBulkStringRedisMessage -> { + if (response === FullBulkStringRedisMessage.NULL_INSTANCE || response.content().readableBytes() == 0) { + log.debug(ctx) { + "Cache miss for key ${msg.key} on Redis" + } + sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key)) + } else { + log.debug(ctx) { + "Cache hit for key ${msg.key} on Redis" + } + val getRequest = InProgressGetRequest(msg.key, ctx) + inProgressRequest = getRequest + getRequest.processResponse(response.content()) + inProgressRequest = null + } + } + + is ErrorRedisMessage -> { + this@RedisCacheHandler.exceptionCaught( + ctx, RedisException("Redis error for GET ${msg.key}: ${response.content()}") + ) + } + + else -> { + log.warn(ctx) { + "Unexpected response type from Redis for key ${msg.key}: ${response.javaClass.name}" + } + sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key)) + } + } + } + + override fun exceptionCaught(ex: Throwable) { + this@RedisCacheHandler.exceptionCaught(ctx, ex) + } + } + client.sendCommand(keyBytes, ctx.alloc(), responseHandler).thenAccept { channel -> + log.trace(ctx) { + "Sending GET request for key ${msg.key} to Redis" + } + val cmd = buildRedisCommand(ctx.alloc(), "GET", keyString) + channel.writeAndFlush(cmd) + } + } + + private fun handlePutRequest(ctx: ChannelHandlerContext, msg: CachePutRequest) { + val keyBytes = processCacheKey(msg.key, keyPrefix, digestAlgorithm) + val keyBuf = ctx.alloc().buffer().also { + it.writeBytes(keyBytes) + } + inProgressRequest = InProgressPutRequest(ctx.channel(), msg.metadata, msg.key, keyBuf, ctx.alloc()) + } + + private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + log.trace(ctx) { + "Received chunk of ${msg.content().readableBytes()} bytes for Redis" + } + request.write(msg.content()) + } + + is InProgressGetRequest -> { + msg.release() + } + } + } + + private fun handleLastCacheContent(ctx: ChannelHandlerContext, msg: LastCacheContent) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + inProgressRequest = null + log.trace(ctx) { + "Received last chunk of ${msg.content().readableBytes()} bytes for Redis" + } + request.write(msg.content()) + val keyBytes = processCacheKey(request.keyString, keyPrefix, digestAlgorithm) + val keyString = String(keyBytes, StandardCharsets.UTF_8) + val (payloadSize, payloadSource) = request.commit() + + // Read the entire payload into a single ByteBuf for the SET command + val valueBuf = ctx.alloc().buffer(payloadSize) + payloadSource.use { source -> + val bb = ByteBuffer.allocate(chunkSize) + while (true) { + val read = source.read(bb) + if (read < 0) break + bb.flip() + valueBuf.writeBytes(bb) + bb.clear() + } + } + + val expirySeconds = maxAge.toSeconds().toString() + + val responseHandler = object : RedisResponseHandler { + override fun responseReceived(response: RedisMessage) { + when (response) { + is SimpleStringRedisMessage -> { + log.debug(ctx) { + "Inserted key ${request.keyString} into Redis" + } + sendMessageAndFlush(ctx, CachePutResponse(request.keyString)) + } + + is ErrorRedisMessage -> { + this@RedisCacheHandler.exceptionCaught( + ctx, RedisException("Redis error for SET ${request.keyString}: ${response.content()}") + ) + } + + else -> { + this@RedisCacheHandler.exceptionCaught( + ctx, RedisException("Unexpected response for SET ${request.keyString}: ${response.javaClass.name}") + ) + } + } + } + + override fun exceptionCaught(ex: Throwable) { + this@RedisCacheHandler.exceptionCaught(ctx, ex) + } + } + + // Use a ByteBuf key for server selection + client.sendCommand(keyBytes, ctx.alloc(), responseHandler).thenAccept { channel -> + log.trace(ctx) { + "Sending SET request to Redis" + } + // Build SET key value EX seconds + val cmd = buildRedisSetCommand(ctx.alloc(), keyString, valueBuf, expirySeconds) + channel.writeAndFlush(cmd) + }.whenComplete { _, ex -> + if (ex != null) { + valueBuf.release() + this@RedisCacheHandler.exceptionCaught(ctx, ex) + } + } + } + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + inProgressRequest = null + request.rollback() + } + + is InProgressGetRequest -> { + inProgressRequest = null + request.rollback() + } + } + super.exceptionCaught(ctx, cause) + } + + private fun buildRedisCommand(alloc: ByteBufAllocator, vararg args: String): ArrayRedisMessage { + val children = args.map { arg -> + FullBulkStringRedisMessage( + alloc.buffer(arg.toByteArray(StandardCharsets.UTF_8)) + ) + } + return ArrayRedisMessage(children) + } + + private fun ByteBufAllocator.buffer(bytes : ByteArray) = buffer().apply { + writeBytes(bytes) + } + + private fun buildRedisSetCommand( + alloc: ByteBufAllocator, + key: String, + value: ByteBuf, + expirySeconds: String, + ): ArrayRedisMessage { + val children = listOf( + FullBulkStringRedisMessage(alloc.buffer("SET".toByteArray(StandardCharsets.UTF_8))), + FullBulkStringRedisMessage(alloc.buffer(key.toByteArray(StandardCharsets.UTF_8))), + FullBulkStringRedisMessage(value), + FullBulkStringRedisMessage(alloc.buffer("EX".toByteArray(StandardCharsets.UTF_8))), + FullBulkStringRedisMessage(alloc.buffer(expirySeconds.toByteArray(StandardCharsets.UTF_8))), + ) + return ArrayRedisMessage(children) + } +} diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheProvider.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheProvider.kt new file mode 100644 index 0000000..bc4a5b8 --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheProvider.kt @@ -0,0 +1,108 @@ +package net.woggioni.rbcs.server.redis + +import java.time.Duration +import java.time.temporal.ChronoUnit + +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.api.exception.ConfigurationException +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.common.Xml.Companion.asIterable +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute + +import org.w3c.dom.Document +import org.w3c.dom.Element + + +class RedisCacheProvider : CacheProvider { + override fun getXmlSchemaLocation() = "jpms://net.woggioni.rbcs.server.redis/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd" + + override fun getXmlType() = "redisCacheType" + + override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server.redis" + + val xmlNamespacePrefix: String + get() = "rbcs-redis" + + override fun deserialize(el: Element): RedisCacheConfiguration { + val servers = mutableListOf() + val maxAge = el.renderAttribute("max-age") + ?.let(Duration::parse) + ?: Duration.ofDays(1) + val compressionLevel = el.renderAttribute("compression-level") + ?.let(Integer::decode) + ?: -1 + val compressionMode = el.renderAttribute("compression-mode") + ?.let { + when (it) { + "deflate" -> RedisCacheConfiguration.CompressionMode.DEFLATE + else -> RedisCacheConfiguration.CompressionMode.DEFLATE + } + } + val keyPrefix = el.renderAttribute("key-prefix") + val digestAlgorithm = el.renderAttribute("digest") + for (child in el.asIterable()) { + when (child.nodeName) { + "server" -> { + val host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required") + val port = child.renderAttribute("port")?.toInt() ?: throw ConfigurationException("port attribute is required") + val maxConnections = child.renderAttribute("max-connections")?.toInt() ?: 1 + val connectionTimeout = child.renderAttribute("connection-timeout") + ?.let(Duration::parse) + ?.let(Duration::toMillis) + ?.let(Long::toInt) + ?: 10000 + val password = child.renderAttribute("password") + servers.add(RedisCacheConfiguration.Server(HostAndPort(host, port), connectionTimeout, maxConnections, password)) + } + } + } + return RedisCacheConfiguration( + servers, + maxAge, + keyPrefix, + digestAlgorithm, + compressionMode, + compressionLevel + ) + } + + override fun serialize(doc: Document, cache: RedisCacheConfiguration) = cache.run { + val result = doc.createElement("cache") + Xml.of(doc, result) { + attr("xmlns:${xmlNamespacePrefix}", xmlNamespace, namespaceURI = "http://www.w3.org/2000/xmlns/") + attr("xs:type", "${xmlNamespacePrefix}:$xmlType", RBCS.XML_SCHEMA_NAMESPACE_URI) + for (server in servers) { + node("server") { + attr("host", server.endpoint.host) + attr("port", server.endpoint.port.toString()) + server.connectionTimeoutMillis?.let { connectionTimeoutMillis -> + attr("connection-timeout", Duration.of(connectionTimeoutMillis.toLong(), ChronoUnit.MILLIS).toString()) + } + attr("max-connections", server.maxConnections.toString()) + server.password?.let { password -> + attr("password", password) + } + } + + } + attr("max-age", maxAge.toString()) + keyPrefix?.let { + attr("key-prefix", it) + } + digestAlgorithm?.let { digestAlgorithm -> + attr("digest", digestAlgorithm) + } + compressionMode?.let { compressionMode -> + attr( + "compression-mode", when (compressionMode) { + RedisCacheConfiguration.CompressionMode.DEFLATE -> "deflate" + } + ) + } + attr("compression-level", compressionLevel.toString()) + } + result + } +} diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisClient.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisClient.kt new file mode 100644 index 0000000..d683377 --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisClient.kt @@ -0,0 +1,204 @@ +package net.woggioni.rbcs.server.redis.client + +import io.netty.bootstrap.Bootstrap +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.Unpooled +import io.netty.channel.Channel +import io.netty.channel.ChannelFactory +import io.netty.channel.ChannelFutureListener +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelOption +import io.netty.channel.ChannelPipeline +import io.netty.channel.EventLoopGroup +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.channel.pool.AbstractChannelPoolHandler +import io.netty.channel.pool.FixedChannelPool +import io.netty.channel.socket.SocketChannel +import io.netty.handler.codec.redis.ArrayRedisMessage +import io.netty.handler.codec.redis.ErrorRedisMessage +import io.netty.handler.codec.redis.FullBulkStringRedisMessage +import io.netty.handler.codec.redis.RedisArrayAggregator +import io.netty.handler.codec.redis.RedisBulkStringAggregator +import io.netty.handler.codec.redis.RedisDecoder +import io.netty.handler.codec.redis.RedisEncoder +import io.netty.handler.codec.redis.RedisMessage +import io.netty.util.concurrent.Future as NettyFuture +import io.netty.util.concurrent.GenericFutureListener + +import java.io.IOException +import java.net.InetSocketAddress +import java.nio.charset.StandardCharsets +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap + +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.trace +import net.woggioni.rbcs.server.redis.RedisCacheConfiguration +import net.woggioni.rbcs.server.redis.RedisCacheHandler + + +class RedisClient( + private val servers: List, + private val chunkSize: Int, + private val group: EventLoopGroup, + private val channelFactory: ChannelFactory, + private val connectionPool: ConcurrentHashMap, +) : AutoCloseable { + + private companion object { + private val log = createLogger() + } + + private fun newConnectionPool(server: RedisCacheConfiguration.Server): FixedChannelPool { + val bootstrap = Bootstrap().apply { + group(group) + channelFactory(channelFactory) + option(ChannelOption.SO_KEEPALIVE, true) + remoteAddress(InetSocketAddress(server.endpoint.host, server.endpoint.port)) + server.connectionTimeoutMillis?.let { + option(ChannelOption.CONNECT_TIMEOUT_MILLIS, it) + } + } + val channelPoolHandler = object : AbstractChannelPoolHandler() { + + override fun channelCreated(ch: Channel) { + val pipeline: ChannelPipeline = ch.pipeline() + pipeline.addLast(RedisEncoder()) + pipeline.addLast(RedisDecoder()) + pipeline.addLast(RedisBulkStringAggregator()) + pipeline.addLast(RedisArrayAggregator()) + server.password?.let { password -> + // Send AUTH command synchronously on new connections + val authCmd = buildCommand("AUTH", password) + ch.writeAndFlush(authCmd).addListener(ChannelFutureListener { future -> + if (!future.isSuccess) { + ch.close() + } + }) + // Install a one-shot handler to consume the AUTH response + pipeline.addLast(object : SimpleChannelInboundHandler() { + override fun channelRead0(ctx: ChannelHandlerContext, msg: RedisMessage) { + when (msg) { + is ErrorRedisMessage -> { + ctx.close() + } + else -> { + // AUTH succeeded, remove this one-shot handler + ctx.pipeline().remove(this) + } + } + } + }) + } + } + } + return FixedChannelPool(bootstrap, channelPoolHandler, server.maxConnections) + } + + private fun buildCommand(vararg args: String): ArrayRedisMessage { + val children = args.map { arg -> + FullBulkStringRedisMessage( + Unpooled.wrappedBuffer(arg.toByteArray(StandardCharsets.UTF_8)) + ) + } + return ArrayRedisMessage(children) + } + + fun sendCommand( + key: ByteArray, + alloc: ByteBufAllocator, + responseHandler: RedisResponseHandler, + ): CompletableFuture { + val server = if (servers.size > 1) { + val keyBuffer = alloc.buffer(key.size) + keyBuffer.writeBytes(key) + var checksum = 0 + while (keyBuffer.readableBytes() > 4) { + val byte = keyBuffer.readInt() + checksum = checksum xor byte + } + while (keyBuffer.readableBytes() > 0) { + val byte = keyBuffer.readByte() + checksum = checksum xor byte.toInt() + } + keyBuffer.release() + servers[Math.floorMod(checksum, servers.size)] + } else { + servers.first() + } + + val response = CompletableFuture() + val pool = connectionPool.computeIfAbsent(server.endpoint) { + newConnectionPool(server) + } + pool.acquire().addListener(object : GenericFutureListener> { + override fun operationComplete(channelFuture: NettyFuture) { + if (channelFuture.isSuccess) { + val channel = channelFuture.now + var connectionClosedByTheRemoteServer = true + val closeCallback = { + if (connectionClosedByTheRemoteServer) { + val ex = IOException("The Redis server closed the connection") + val completed = response.completeExceptionally(ex) + if (!completed) responseHandler.exceptionCaught(ex) + } + } + val closeListener = ChannelFutureListener { + closeCallback() + } + channel.closeFuture().addListener(closeListener) + val pipeline = channel.pipeline() + val handler = object : SimpleChannelInboundHandler(false) { + + override fun handlerAdded(ctx: ChannelHandlerContext) { + channel.closeFuture().removeListener(closeListener) + } + + override fun channelRead0( + ctx: ChannelHandlerContext, + msg: RedisMessage, + ) { + pipeline.remove(this) + pool.release(channel) + log.trace(channel) { + "Channel released" + } + responseHandler.responseReceived(msg) + } + + override fun channelInactive(ctx: ChannelHandlerContext) { + closeCallback() + ctx.fireChannelInactive() + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + connectionClosedByTheRemoteServer = false + pipeline.remove(this) + ctx.close() + pool.release(channel) + log.trace(channel) { + "Channel released after exception" + } + responseHandler.exceptionCaught(cause) + } + } + + channel.pipeline().addLast(handler) + response.complete(channel) + } else { + response.completeExceptionally(channelFuture.cause()) + } + } + }) + return response + } + + fun shutDown(): NettyFuture<*> { + return group.shutdownGracefully() + } + + override fun close() { + shutDown().sync() + } +} diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisResponseHandler.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisResponseHandler.kt new file mode 100644 index 0000000..0aa6fb0 --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisResponseHandler.kt @@ -0,0 +1,10 @@ +package net.woggioni.rbcs.server.redis.client + +import io.netty.handler.codec.redis.RedisMessage + +interface RedisResponseHandler { + + fun responseReceived(response: RedisMessage) + + fun exceptionCaught(ex: Throwable) +} diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd new file mode 100644 index 0000000..3ee4dc8 --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + Password for Redis AUTH command, used when the Redis server requires authentication + + + + + + + + + + + + + + + + + Prepend this string to all the keys inserted in Redis, + useful in case the caching backend is shared with other applications + + + + + + + + + + + + + + + + + diff --git a/rbcs-server/.classpath b/rbcs-server/.classpath new file mode 100644 index 0000000..da0cd61 --- /dev/null +++ b/rbcs-server/.classpath @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-server/.project b/rbcs-server/.project new file mode 100644 index 0000000..d562929 --- /dev/null +++ b/rbcs-server/.project @@ -0,0 +1,34 @@ + + + rbcs-server + Project rbcs-server created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776957 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-server/.settings/org.eclipse.buildship.core.prefs b/rbcs-server/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-server/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-server/.settings/org.eclipse.jdt.core.prefs b/rbcs-server/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-server/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-server/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider b/rbcs-server/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider new file mode 100644 index 0000000..46a7419 --- /dev/null +++ b/rbcs-server/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider @@ -0,0 +1,2 @@ +net.woggioni.rbcs.server.cache.FileSystemCacheProvider +net.woggioni.rbcs.server.cache.InMemoryCacheProvider \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt new file mode 100644 index 0000000..708d2ba --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt @@ -0,0 +1,562 @@ +package net.woggioni.rbcs.server + +import io.netty.bootstrap.ServerBootstrap +import io.netty.buffer.ByteBuf +import io.netty.channel.Channel +import io.netty.channel.ChannelFactory +import io.netty.channel.ChannelFuture +import io.netty.channel.ChannelHandler.Sharable +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.channel.ChannelInitializer +import io.netty.channel.ChannelOption +import io.netty.channel.ChannelPromise +import io.netty.channel.MultiThreadIoEventLoopGroup +import io.netty.channel.nio.NioIoHandler +import io.netty.channel.socket.DatagramChannel +import io.netty.channel.socket.ServerSocketChannel +import io.netty.channel.socket.SocketChannel +import io.netty.channel.socket.nio.NioDatagramChannel +import io.netty.channel.socket.nio.NioServerSocketChannel +import io.netty.channel.socket.nio.NioSocketChannel +import io.netty.handler.codec.compression.CompressionOptions +import io.netty.handler.codec.haproxy.HAProxyMessageDecoder +import io.netty.handler.codec.http.DefaultHttpContent +import io.netty.handler.codec.http.HttpContentCompressor +import io.netty.handler.codec.http.HttpDecoderConfig +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpRequest +import io.netty.handler.codec.http.HttpServerCodec +import io.netty.handler.ssl.ClientAuth +import io.netty.handler.ssl.SslContext +import io.netty.handler.ssl.SslContextBuilder +import io.netty.handler.ssl.SslHandler +import io.netty.handler.stream.ChunkedWriteHandler +import io.netty.handler.timeout.IdleState +import io.netty.handler.timeout.IdleStateEvent +import io.netty.handler.timeout.IdleStateHandler +import io.netty.util.AttributeKey +import io.netty.util.concurrent.EventExecutorGroup +import java.io.OutputStream +import java.net.InetSocketAddress +import java.nio.file.Files +import java.nio.file.Path +import java.security.PrivateKey +import java.security.cert.X509Certificate +import java.time.Duration +import java.time.Instant +import java.util.Arrays +import java.util.Base64 +import java.util.concurrent.CompletableFuture +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException +import java.util.regex.Matcher +import java.util.regex.Pattern +import javax.naming.ldap.LdapName +import javax.net.ssl.SSLPeerUnverifiedException +import net.woggioni.rbcs.api.AsyncCloseable +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.exception.ConfigurationException +import net.woggioni.rbcs.common.Cidr +import net.woggioni.rbcs.common.PasswordSecurity.decodePasswordHash +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import net.woggioni.rbcs.common.RBCS.getTrustManager +import net.woggioni.rbcs.common.RBCS.loadKeystore +import net.woggioni.rbcs.common.RBCS.toUrl +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.info +import net.woggioni.rbcs.server.otel.OtelIntegration +import net.woggioni.rbcs.server.auth.AbstractNettyHttpAuthenticator +import net.woggioni.rbcs.server.auth.Authorizer +import net.woggioni.rbcs.server.auth.RoleAuthorizer +import net.woggioni.rbcs.server.configuration.Parser +import net.woggioni.rbcs.server.configuration.Serializer +import net.woggioni.rbcs.server.exception.ExceptionHandler +import net.woggioni.rbcs.server.handler.MaxRequestSizeHandler +import net.woggioni.rbcs.server.handler.ProxyProtocolHandler +import net.woggioni.rbcs.server.handler.ReadTriggerDuplexHandler +import net.woggioni.rbcs.server.handler.ServerHandler +import net.woggioni.rbcs.server.throttling.BucketManager +import net.woggioni.rbcs.server.throttling.ThrottlingHandler + +class RemoteBuildCacheServer(private val cfg: Configuration) { + + companion object { + private val log = createLogger() + + val userAttribute: AttributeKey = AttributeKey.valueOf("user") + val groupAttribute: AttributeKey> = AttributeKey.valueOf("group") + val clientIp: AttributeKey = AttributeKey.valueOf("client-ip") + + val DEFAULT_CONFIGURATION_URL by lazy { "jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/rbcs-default.xml".toUrl() } + private const val SSL_HANDLER_NAME = "sslHandler" + + fun loadConfiguration(configurationFile: Path): Configuration { + val doc = Files.newInputStream(configurationFile).use { + Xml.parseXml(configurationFile.toUri().toURL(), it) + } + return Parser.parse(doc) + } + + fun dumpConfiguration(conf: Configuration, outputStream: OutputStream) { + Xml.write(Serializer.serialize(conf), outputStream) + } + } + + private class HttpChunkContentCompressor( + threshold: Int, + vararg compressionOptions: CompressionOptions = emptyArray() + ) : HttpContentCompressor(threshold, *compressionOptions) { + override fun write(ctx: ChannelHandlerContext, msg: Any, promise: ChannelPromise) { + var message: Any? = msg + if (message is ByteBuf) { + // convert ByteBuf to HttpContent to make it work with compression. This is needed as we use the + // ChunkedWriteHandler to send files when compression is enabled. + val buff = message + if (buff.isReadable) { + // We only encode non empty buffers, as empty buffers can be used for determining when + // the content has been flushed and it confuses the HttpContentCompressor + // if we let it go + message = DefaultHttpContent(buff) + } + } + super.write(ctx, message, promise) + } + } + + @Sharable + private class ClientCertificateAuthenticator( + authorizer: Authorizer, + private val anonymousUserGroups: Set?, + private val userExtractor: Configuration.UserExtractor?, + private val groupExtractor: Configuration.GroupExtractor?, + ) : AbstractNettyHttpAuthenticator(authorizer) { + + override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? { + return try { + val sslHandler = (ctx.pipeline().get(SSL_HANDLER_NAME) as? SslHandler) + ?: throw ConfigurationException("Client certificate authentication cannot be used when TLS is disabled") + val sslEngine = sslHandler.engine() + sslEngine.session.peerCertificates.takeIf { + it.isNotEmpty() + }?.let { peerCertificates -> + val clientCertificate = peerCertificates.first() as X509Certificate + val user = userExtractor?.extract(clientCertificate) + val group = groupExtractor?.extract(clientCertificate) + val allGroups = + ((user?.groups ?: emptySet()).asSequence() + sequenceOf(group).filterNotNull()).toSet() + AuthenticationResult(user, allGroups) + } ?: anonymousUserGroups?.let { AuthenticationResult(null, it) } + } catch (ex: SSLPeerUnverifiedException) { + log.debug(ctx) { + ex.message ?: "Error witch client certificate authentication" + } + anonymousUserGroups?.let { AuthenticationResult(null, it) } + } + } + } + + @Sharable + private class ForwardedClientCertificateAuthenticator( + authorizer: Authorizer, + private val anonymousUserGroups: Set?, + private val subjectDnUserExtractor: SubjectDnExtractor?, + private val subjectDnGroupExtractor: SubjectDnExtractor?, + private val headerName: String, + private val trustedProxyIPs: List, + private val users: Map, + private val groups: Map, + ) : AbstractNettyHttpAuthenticator(authorizer) { + + companion object { + private val log = createLogger() + } + + override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? { + val clientIp = ctx.channel().attr(clientIp).get() + if (clientIp == null || trustedProxyIPs.none { it.contains(clientIp.address) }) { + log.debug(ctx) { + "Rejecting forwarded client certificate authentication from untrusted address: $clientIp" + } + return null + } + val subjectDn = req.headers()[headerName] + ?: return anonymousUserGroups?.let { AuthenticationResult(null, it) } + val ldapName = try { + LdapName(subjectDn) + } catch (e: Exception) { + log.debug(ctx) { + "Invalid subject DN in header $headerName: $subjectDn" + } + return anonymousUserGroups?.let { AuthenticationResult(null, it) } + } + val user = subjectDnUserExtractor?.extract(ldapName)?.let { userName -> + users[userName] ?: throw RuntimeException("Failed to extract user '$userName'") + } + val group = subjectDnGroupExtractor?.extract(ldapName)?.let { groupName -> + groups[groupName] ?: throw RuntimeException("Failed to extract group '$groupName'") + } + val allGroups = ((user?.groups ?: emptySet()).asSequence() + sequenceOf(group).filterNotNull()).toSet() + return AuthenticationResult(user, allGroups) + } + } + + private data class SubjectDnExtractor(val rdnType: String, val pattern: Pattern) { + fun extract(ldapName: LdapName): String? { + return ldapName.rdns.find { it.type == rdnType } + ?.let { pattern.matcher(it.value.toString()) } + ?.takeIf(Matcher::matches)?.group(1) + } + } + + @Sharable + private class NettyHttpBasicAuthenticator( + private val users: Map, authorizer: Authorizer + ) : AbstractNettyHttpAuthenticator(authorizer) { + companion object { + private val log = createLogger() + } + + override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? { + val authorizationHeader = req.headers()[HttpHeaderNames.AUTHORIZATION] ?: let { + log.debug(ctx) { + "Missing Authorization header" + } + return users[""]?.let { AuthenticationResult(it, it.groups) } + } + val cursor = authorizationHeader.indexOf(' ') + if (cursor < 0) { + log.debug(ctx) { + "Invalid Authorization header: '$authorizationHeader'" + } + return users[""]?.let { AuthenticationResult(it, it.groups) } + } + val authenticationType = authorizationHeader.substring(0, cursor) + if ("Basic" != authenticationType) { + log.debug(ctx) { + "Invalid authentication type header: '$authenticationType'" + } + return users[""]?.let { AuthenticationResult(it, it.groups) } + } + val (username, password) = Base64.getDecoder().decode(authorizationHeader.substring(cursor + 1)) + .let(::String) + .let { + val colon = it.indexOf(':') + if (colon < 0) { + log.debug(ctx) { + "Missing colon from authentication" + } + return null + } + it.substring(0, colon) to it.substring(colon + 1) + } + + return username.let(users::get)?.takeIf { user -> + user.password?.let { passwordAndSalt -> + val (_, salt) = decodePasswordHash(passwordAndSalt) + hashPassword(password, Base64.getEncoder().encodeToString(salt)) == passwordAndSalt + } ?: false + }?.let { user -> + AuthenticationResult(user, user.groups) + } + } + } + + private class ServerInitializer( + private val cfg: Configuration, + private val channelFactory : ChannelFactory, + private val datagramChannelFactory : ChannelFactory, + ) : ChannelInitializer(), AsyncCloseable { + + companion object { + private fun createSslCtx(tls: Configuration.Tls): SslContext { + val keyStore = tls.keyStore + return if (keyStore == null) { + throw IllegalArgumentException("No keystore configured") + } else { + val javaKeyStore = loadKeystore(keyStore.file, keyStore.password) + val serverKey = javaKeyStore.getKey( + keyStore.keyAlias, (keyStore.keyPassword ?: "").let(String::toCharArray) + ) as PrivateKey + val serverCert: Array = + Arrays.stream(javaKeyStore.getCertificateChain(keyStore.keyAlias)) + .map { it as X509Certificate } + .toArray { size -> Array(size) { null } } + SslContextBuilder.forServer(serverKey, *serverCert).apply { + val clientAuth = tls.trustStore?.let { trustStore -> + val ts = loadKeystore(trustStore.file, trustStore.password) + trustManager( + getTrustManager(ts, trustStore.isCheckCertificateStatus) + ) + if (trustStore.isRequireClientCertificate) ClientAuth.REQUIRE + else ClientAuth.OPTIONAL + } ?: ClientAuth.NONE + clientAuth(clientAuth) + + }.build() + } + } + + private val log = createLogger() + } + + private val cacheHandlerFactory = cfg.cache.materialize() + + private val bucketManager = BucketManager.from(cfg) + + private val authenticator = when (val auth = cfg.authentication) { + is Configuration.BasicAuthentication -> NettyHttpBasicAuthenticator(cfg.users, RoleAuthorizer()) + is Configuration.ClientCertificateAuthentication -> { + ClientCertificateAuthenticator( + RoleAuthorizer(), + cfg.users[""]?.groups, + userExtractor(auth), + groupExtractor(auth) + ) + } + + is Configuration.ForwardedClientCertificateAuthentication -> { + ForwardedClientCertificateAuthenticator( + RoleAuthorizer(), + cfg.users[""]?.groups, + auth.userExtractor?.let { extractor -> + SubjectDnExtractor(extractor.rdnType, Pattern.compile(extractor.pattern)) + }, + auth.groupExtractor?.let { extractor -> + SubjectDnExtractor(extractor.rdnType, Pattern.compile(extractor.pattern)) + }, + auth.headerName, + cfg.trustedProxyIPs, + cfg.users, + cfg.groups, + ) + } + + else -> null + } + + private val proxyProtocolEnabled: Boolean = cfg.isProxyProtocolEnabled + private val trustedProxyIPs: List = cfg.trustedProxyIPs + + private val sslContext: SslContext? = cfg.tls?.let(Companion::createSslCtx) + + private fun userExtractor(authentication: Configuration.ClientCertificateAuthentication) = + authentication.userExtractor?.let { extractor -> + val pattern = Pattern.compile(extractor.pattern) + val rdnType = extractor.rdnType + Configuration.UserExtractor { cert: X509Certificate -> + val userName = LdapName(cert.subjectX500Principal.name).rdns.find { + it.type == rdnType + }?.let { + pattern.matcher(it.value.toString()) + }?.takeIf(Matcher::matches)?.group(1) + cfg.users[userName] ?: throw java.lang.RuntimeException("Failed to extract user") + } + } + + private fun groupExtractor(authentication: Configuration.ClientCertificateAuthentication) = + authentication.groupExtractor?.let { extractor -> + val pattern = Pattern.compile(extractor.pattern) + val rdnType = extractor.rdnType + Configuration.GroupExtractor { cert: X509Certificate -> + val groupName = LdapName(cert.subjectX500Principal.name).rdns.find { + it.type == rdnType + }?.let { + pattern.matcher(it.value.toString()) + }?.takeIf(Matcher::matches)?.group(1) + cfg.groups[groupName] ?: throw java.lang.RuntimeException("Failed to extract group") + } + } + + override fun initChannel(ch: Channel) { + ch.attr(clientIp).set(ch.remoteAddress() as InetSocketAddress) + log.debug { + "Created connection ${ch.id().asShortText()} with ${ch.remoteAddress()}" + } + ch.closeFuture().addListener { + log.debug { + "Closed connection ${ch.id().asShortText()} with ${ch.remoteAddress()}" + } + } + ch.config().isAutoRead = false + val pipeline = ch.pipeline() + cfg.connection.also { conn -> + val readIdleTimeout = conn.readIdleTimeout.toMillis() + val writeIdleTimeout = conn.writeIdleTimeout.toMillis() + val idleTimeout = conn.idleTimeout.toMillis() + if (readIdleTimeout > 0 || writeIdleTimeout > 0 || idleTimeout > 0) { + pipeline.addLast( + IdleStateHandler( + true, + readIdleTimeout, + writeIdleTimeout, + idleTimeout, + TimeUnit.MILLISECONDS + ) + ) + } + } + pipeline.addLast(object : ChannelInboundHandlerAdapter() { + override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) { + if (evt is IdleStateEvent) { + when (evt.state()) { + IdleState.READER_IDLE -> log.debug { + "Read timeout reached on channel ${ch.id().asShortText()}, closing the connection" + } + + IdleState.WRITER_IDLE -> log.debug { + "Write timeout reached on channel ${ch.id().asShortText()}, closing the connection" + } + + IdleState.ALL_IDLE -> log.debug { + "Idle timeout reached on channel ${ch.id().asShortText()}, closing the connection" + } + + null -> throw IllegalStateException("This should never happen") + } + ctx.close() + } + } + }) + if(proxyProtocolEnabled) { + pipeline.addLast(HAProxyMessageDecoder()) + pipeline.addLast(ProxyProtocolHandler(trustedProxyIPs)) + } + sslContext?.newHandler(ch.alloc())?.also { + pipeline.addLast(SSL_HANDLER_NAME, it) + } + val httpDecoderConfig = HttpDecoderConfig().apply { + maxChunkSize = cfg.connection.chunkSize + } + pipeline.addLast(HttpServerCodec(httpDecoderConfig)) + OtelIntegration.createHandler()?.let { pipeline.addLast(it) } + pipeline.addLast(ReadTriggerDuplexHandler.NAME, ReadTriggerDuplexHandler()) + pipeline.addLast(MaxRequestSizeHandler.NAME, MaxRequestSizeHandler(cfg.connection.maxRequestSize)) + pipeline.addLast(HttpChunkContentCompressor(1024)) + pipeline.addLast(ChunkedWriteHandler()) + authenticator?.let { + pipeline.addLast(it) + } + pipeline.addLast(ThrottlingHandler(bucketManager,cfg.rateLimiter, cfg.connection)) + + val serverHandler = let { + val prefix = Path.of("/").resolve(Path.of(cfg.serverPath ?: "/")) + ServerHandler(prefix) { + cacheHandlerFactory.newHandler(cfg, ch.eventLoop(), channelFactory, datagramChannelFactory) + } + } + pipeline.addLast(ServerHandler.NAME, serverHandler) + pipeline.addLast(ExceptionHandler.NAME, ExceptionHandler) + } + + override fun asyncClose() = cacheHandlerFactory.asyncClose() + } + + class ServerHandle( + closeFuture: ChannelFuture, + private val bossGroup: EventExecutorGroup, + private val executorGroups: Iterable, + private val serverInitializer: AsyncCloseable, + ) : Future by from(closeFuture, bossGroup, executorGroups, serverInitializer) { + + companion object { + private val log = createLogger() + + private fun from( + closeFuture: ChannelFuture, + bossGroup: EventExecutorGroup, + executorGroups: Iterable, + serverInitializer: AsyncCloseable + ): CompletableFuture { + val result = CompletableFuture() + closeFuture.addListener { + val errors = mutableListOf() + val deadline = Instant.now().plusSeconds(20) + + serverInitializer.asyncClose().whenCompleteAsync { _, ex -> + if(ex != null) { + log.error(ex.message, ex) + errors.addLast(ex) + } + + executorGroups.forEach(EventExecutorGroup::shutdownGracefully) + bossGroup.terminationFuture().sync() + + for (executorGroup in executorGroups) { + val future = executorGroup.terminationFuture() + try { + val now = Instant.now() + if (now > deadline) { + future.get(0, TimeUnit.SECONDS) + } else { + future.get(Duration.between(now, deadline).toMillis(), TimeUnit.MILLISECONDS) + } + } + catch (te: TimeoutException) { + errors.addLast(te) + log.warn("Timeout while waiting for shutdown of $executorGroup", te) + } catch (ex: Throwable) { + log.warn(ex.message, ex) + errors.addLast(ex) + } + } + + if(errors.isEmpty()) { + result.complete(null) + } else { + result.completeExceptionally(errors.first()) + } + } + } + + return result.thenAccept { + log.info { + "RemoteBuildCacheServer has been gracefully shut down" + } + } + } + } + + + fun sendShutdownSignal() { + bossGroup.shutdownGracefully() + } + } + + fun run(): ServerHandle { + // Create the multithreaded event loops for the server + val bossGroup = MultiThreadIoEventLoopGroup(1, NioIoHandler.newFactory()) + val channelFactory = ChannelFactory { NioSocketChannel() } + val datagramChannelFactory = ChannelFactory { NioDatagramChannel() } + val serverChannelFactory = ChannelFactory { NioServerSocketChannel() } + val workerGroup = MultiThreadIoEventLoopGroup(0, NioIoHandler.newFactory()) + + val serverInitializer = ServerInitializer(cfg, channelFactory, datagramChannelFactory) + val bootstrap = ServerBootstrap().apply { + // Configure the server + group(bossGroup, workerGroup) + channelFactory(serverChannelFactory) + childHandler(serverInitializer) + option(ChannelOption.SO_BACKLOG, cfg.incomingConnectionsBacklogSize) + childOption(ChannelOption.SO_KEEPALIVE, true) + } + + + // Bind and start to accept incoming connections. + val bindAddress = InetSocketAddress(cfg.host, cfg.port) + val httpChannel = bootstrap.bind(bindAddress).sync().channel() + log.info { + "RemoteBuildCacheServer is listening on ${cfg.host}:${cfg.port}" + } + + return ServerHandle( + httpChannel.closeFuture(), + bossGroup, + setOf(workerGroup), + serverInitializer + ) + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authenticator.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authenticator.kt new file mode 100644 index 0000000..202551e --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authenticator.kt @@ -0,0 +1,88 @@ +package net.woggioni.rbcs.server.auth + +import io.netty.buffer.Unpooled +import io.netty.channel.ChannelFutureListener +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.handler.codec.http.DefaultFullHttpResponse +import io.netty.handler.codec.http.FullHttpResponse +import io.netty.handler.codec.http.HttpContent +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpRequest +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import io.netty.util.ReferenceCountUtil +import java.net.InetSocketAddress +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Configuration.Group +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.server.RemoteBuildCacheServer + +abstract class AbstractNettyHttpAuthenticator(private val authorizer: Authorizer) : ChannelInboundHandlerAdapter() { + + companion object { + private val log = createLogger() + + private val AUTHENTICATION_FAILED: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.UNAUTHORIZED, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val NOT_AUTHORIZED: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + } + + class AuthenticationResult(val user: Configuration.User?, val groups: Set) + + abstract fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + if (msg is HttpRequest) { + val result = authenticate(ctx, msg) ?: return authenticationFailure(ctx, msg) + ctx.channel().attr(RemoteBuildCacheServer.userAttribute).set(result.user) + ctx.channel().attr(RemoteBuildCacheServer.groupAttribute).set(result.groups) + + val roles = ( + (result.user?.let { user -> + user.groups.asSequence().flatMap { group -> + group.roles.asSequence() + } + } ?: emptySequence()) + + result.groups.asSequence().flatMap { it.roles.asSequence() } + ).toSet() + val authorized = authorizer.authorize(roles, msg) + if(log.isDebugEnabled) { + val authorizedMessage = if(authorized) { "Authorized" } else { "Forbidden" } + val clientAddress = ctx.channel().attr(RemoteBuildCacheServer.clientIp).get() + val roleString = "[" + roles.asSequence().map { "\"" + it + "\""}.joinToString(", ") + "]" + result.user?.name?.takeUnless(String::isEmpty)?.let { username -> + log.debug("$authorizedMessage ${msg.method()} request from user $username with address $clientAddress, granted roles $roleString") + } ?: { + log.debug("$authorizedMessage anonymous ${msg.method()} request with address $clientAddress, granted roles $roleString") + } + } + if (authorized) { + super.channelRead(ctx, msg) + } else { + authorizationFailure(ctx, msg) + } + } else if(msg is HttpContent) { + ctx.fireChannelRead(msg) + } + } + + private fun authenticationFailure(ctx: ChannelHandlerContext, msg: Any) { + ReferenceCountUtil.release(msg) + ctx.writeAndFlush(AUTHENTICATION_FAILED.retainedDuplicate()).addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + + private fun authorizationFailure(ctx: ChannelHandlerContext, msg: Any) { + ReferenceCountUtil.release(msg) + ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate()).addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authorizer.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authorizer.kt new file mode 100644 index 0000000..1e777db --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authorizer.kt @@ -0,0 +1,8 @@ +package net.woggioni.rbcs.server.auth + +import io.netty.handler.codec.http.HttpRequest +import net.woggioni.rbcs.api.Role + +fun interface Authorizer { + fun authorize(roles : Set, request: HttpRequest) : Boolean +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/UserAuthorizer.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/UserAuthorizer.kt new file mode 100644 index 0000000..541e528 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/UserAuthorizer.kt @@ -0,0 +1,24 @@ +package net.woggioni.rbcs.server.auth + +import io.netty.handler.codec.http.HttpMethod +import io.netty.handler.codec.http.HttpRequest +import net.woggioni.rbcs.api.Role + +class RoleAuthorizer : Authorizer { + + companion object { + private val METHOD_MAP = mapOf( + Role.Reader to setOf(HttpMethod.GET, HttpMethod.HEAD), + Role.Writer to setOf(HttpMethod.PUT, HttpMethod.POST), + Role.Healthcheck to setOf(HttpMethod.TRACE) + ) + } + + override fun authorize(roles: Set, request: HttpRequest) : Boolean { + val allowedMethods = roles.asSequence() + .mapNotNull(METHOD_MAP::get) + .flatten() + .toSet() + return request.method() in allowedMethods + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCache.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCache.kt new file mode 100644 index 0000000..cefab14 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCache.kt @@ -0,0 +1,169 @@ +package net.woggioni.rbcs.server.cache + +import java.io.ByteArrayOutputStream +import java.io.InputStream +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.io.Serializable +import java.nio.ByteBuffer +import java.nio.channels.Channels +import java.nio.channels.FileChannel +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.nio.file.StandardOpenOption +import java.nio.file.attribute.BasicFileAttributes +import java.time.Duration +import java.time.Instant +import java.util.concurrent.CompletableFuture +import net.woggioni.jwo.JWO +import net.woggioni.rbcs.api.AsyncCloseable +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.common.createLogger + +class FileSystemCache( + val root: Path, + val maxAge: Duration +) : AsyncCloseable { + + class EntryValue(val metadata: CacheValueMetadata, val channel : FileChannel, val offset : Long, val size : Long) : Serializable + + private companion object { + private val log = createLogger() + } + + init { + Files.createDirectories(root) + } + + @Volatile + private var running = true + + private var nextGc = Instant.now() + + fun get(key: String): EntryValue? = + root.resolve(key).takeIf(Files::exists) + ?.let { file -> + val size = Files.size(file) + val channel = FileChannel.open(file, StandardOpenOption.READ) + val source = Channels.newInputStream(channel) + val tmp = ByteArray(Integer.BYTES) + val buffer = ByteBuffer.wrap(tmp) + source.read(tmp) + buffer.rewind() + val offset = (Integer.BYTES + buffer.getInt()).toLong() + var count = 0 + val wrapper = object : InputStream() { + override fun read(): Int { + return source.read().also { + if (it > 0) count += it + } + } + + override fun read(b: ByteArray, off: Int, len: Int): Int { + return source.read(b, off, len).also { + if (it > 0) count += it + } + } + + override fun close() { + } + } + val metadata = ObjectInputStream(wrapper).use { ois -> + ois.readObject() as CacheValueMetadata + } + EntryValue(metadata, channel, offset, size) + } + + class FileSink(metadata: CacheValueMetadata, private val path: Path, private val tmpFile: Path) { + val channel: FileChannel + + init { + val baos = ByteArrayOutputStream() + ObjectOutputStream(baos).use { + it.writeObject(metadata) + } + Files.newOutputStream(tmpFile).use { + val bytes = baos.toByteArray() + val buffer = ByteBuffer.allocate(Integer.BYTES) + buffer.putInt(bytes.size) + buffer.rewind() + it.write(buffer.array()) + it.write(bytes) + } + channel = FileChannel.open(tmpFile, StandardOpenOption.APPEND) + } + + fun commit() { + channel.close() + Files.move(tmpFile, path, StandardCopyOption.ATOMIC_MOVE) + } + + fun rollback() { + channel.close() + Files.delete(path) + } + } + + fun put( + key: String, + metadata: CacheValueMetadata, + ): FileSink { + val file = root.resolve(key) + val tmpFile = Files.createTempFile(root, null, ".tmp") + return FileSink(metadata, file, tmpFile) + } + + private val closeFuture = object : CompletableFuture() { + init { + Thread.ofVirtual().name("file-system-cache-gc").start { + try { + while (running) { + gc() + } + complete(null) + } catch (ex : Throwable) { + completeExceptionally(ex) + } + } + } + } + + private fun gc() { + val now = Instant.now() + if (nextGc < now) { + val oldestEntry = actualGc(now) + nextGc = (oldestEntry ?: now).plus(maxAge) + } + Thread.sleep(minOf(Duration.between(now, nextGc), Duration.ofSeconds(1))) + } + + /** + * Returns the creation timestamp of the oldest cache entry (if any) + */ + private fun actualGc(now: Instant): Instant? { + var result: Instant? = null + Files.list(root) + .filter { path -> + JWO.splitExtension(path) + .map { it._2 } + .map { it != ".tmp" } + .orElse(true) + } + .filter { + val creationTimeStamp = Files.readAttributes(it, BasicFileAttributes::class.java) + .creationTime() + .toInstant() + if (result == null || creationTimeStamp < result) { + result = creationTimeStamp + } + now > creationTimeStamp.plus(maxAge) + }.forEach(Files::delete) + return result + } + + override fun asyncClose() : CompletableFuture { + running = false + return closeFuture + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheConfiguration.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheConfiguration.kt new file mode 100644 index 0000000..9a867c8 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheConfiguration.kt @@ -0,0 +1,38 @@ +package net.woggioni.rbcs.server.cache + +import io.netty.channel.ChannelFactory +import io.netty.channel.EventLoopGroup +import io.netty.channel.socket.DatagramChannel +import io.netty.channel.socket.SocketChannel +import java.nio.file.Path +import java.time.Duration +import net.woggioni.jwo.Application +import net.woggioni.rbcs.api.CacheHandlerFactory +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.RBCS + +data class FileSystemCacheConfiguration( + val root: Path?, + val maxAge: Duration, + val digestAlgorithm : String?, + val compressionEnabled: Boolean, + val compressionLevel: Int, +) : Configuration.Cache { + + override fun materialize() = object : CacheHandlerFactory { + private val cache = FileSystemCache(root ?: Application.builder("rbcs").build().computeCacheDirectory(), maxAge) + + override fun asyncClose() = cache.asyncClose() + + override fun newHandler( + cfg : Configuration, + eventLoop: EventLoopGroup, + socketChannelFactory: ChannelFactory, + datagramChannelFactory: ChannelFactory + ) = FileSystemCacheHandler(cache, digestAlgorithm, compressionEnabled, compressionLevel, cfg.connection.chunkSize) + } + + override fun getNamespaceURI() = RBCS.RBCS_NAMESPACE_URI + + override fun getTypeName() = "fileSystemCacheType" +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheHandler.kt new file mode 100644 index 0000000..ea6ee09 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheHandler.kt @@ -0,0 +1,137 @@ +package net.woggioni.rbcs.server.cache + +import io.netty.buffer.ByteBuf +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.http.LastHttpContent +import io.netty.handler.stream.ChunkedNioFile +import java.nio.channels.Channels +import java.util.Base64 +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream +import java.util.zip.InflaterInputStream +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.message.CacheMessage +import net.woggioni.rbcs.api.message.CacheMessage.CacheContent +import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent +import net.woggioni.rbcs.common.RBCS.processCacheKey + +class FileSystemCacheHandler( + private val cache: FileSystemCache, + private val digestAlgorithm: String?, + private val compressionEnabled: Boolean, + private val compressionLevel: Int, + private val chunkSize: Int +) : CacheHandler() { + + private interface InProgressRequest{ + + } + + private class InProgressGetRequest(val request : CacheGetRequest) : InProgressRequest + + private inner class InProgressPutRequest( + val key : String, + private val fileSink : FileSystemCache.FileSink + ) : InProgressRequest { + + private val stream = Channels.newOutputStream(fileSink.channel).let { + if (compressionEnabled) { + DeflaterOutputStream(it, Deflater(compressionLevel)) + } else { + it + } + } + + fun write(buf: ByteBuf) { + buf.readBytes(stream, buf.readableBytes()) + } + + fun commit() { + stream.close() + fileSink.commit() + } + + fun rollback() { + fileSink.rollback() + } + } + + private var inProgressRequest: InProgressRequest? = null + + override fun channelRead0(ctx: ChannelHandlerContext, msg: CacheMessage) { + when (msg) { + is CacheGetRequest -> handleGetRequest(ctx, msg) + is CachePutRequest -> handlePutRequest(ctx, msg) + is LastCacheContent -> handleLastCacheContent(ctx, msg) + is CacheContent -> handleCacheContent(ctx, msg) + else -> ctx.fireChannelRead(msg) + } + } + + private fun handleGetRequest(ctx: ChannelHandlerContext, msg: CacheGetRequest) { + inProgressRequest = InProgressGetRequest(msg) + + } + + private fun handlePutRequest(ctx: ChannelHandlerContext, msg: CachePutRequest) { + val key = String(Base64.getUrlEncoder().encode(processCacheKey(msg.key, null, digestAlgorithm))) + val sink = cache.put(key, msg.metadata) + inProgressRequest = InProgressPutRequest(msg.key, sink) + } + + private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) { + val request = inProgressRequest + if(request is InProgressPutRequest) { + request.write(msg.content()) + } + } + + private fun handleLastCacheContent(ctx: ChannelHandlerContext, msg: LastCacheContent) { + when(val request = inProgressRequest) { + is InProgressPutRequest -> { + inProgressRequest = null + request.write(msg.content()) + request.commit() + sendMessageAndFlush(ctx, CachePutResponse(request.key)) + } + is InProgressGetRequest -> { + val key = String(Base64.getUrlEncoder().encode(processCacheKey(request.request.key, null, digestAlgorithm))) + cache.get(key)?.also { entryValue -> + sendMessageAndFlush(ctx, CacheValueFoundResponse(request.request.key, entryValue.metadata)) + entryValue.channel.let { channel -> + if(compressionEnabled) { + InflaterInputStream(Channels.newInputStream(channel)).use { stream -> + + outerLoop@ + while (true) { + val buf = ctx.alloc().heapBuffer(chunkSize) + while(buf.readableBytes() < chunkSize) { + val read = buf.writeBytes(stream, chunkSize) + if(read < 0) { + sendMessageAndFlush(ctx, LastCacheContent(buf)) + break@outerLoop + } + } + sendMessageAndFlush(ctx, CacheContent(buf)) + } + } + } else { + sendMessage(ctx, ChunkedNioFile(channel, entryValue.offset, entryValue.size - entryValue.offset, chunkSize)) + sendMessageAndFlush(ctx, LastHttpContent.EMPTY_LAST_CONTENT) + } + } + } ?: sendMessageAndFlush(ctx, CacheValueNotFoundResponse(key)) + } + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + (inProgressRequest as? InProgressPutRequest)?.rollback() + super.exceptionCaught(ctx, cause) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheProvider.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheProvider.kt new file mode 100644 index 0000000..d5112ce --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheProvider.kt @@ -0,0 +1,65 @@ +package net.woggioni.rbcs.server.cache + +import java.nio.file.Path +import java.time.Duration +import java.util.zip.Deflater +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document +import org.w3c.dom.Element + +class FileSystemCacheProvider : CacheProvider { + + override fun getXmlSchemaLocation() = "classpath:net/woggioni/rbcs/server/schema/rbcs-server.xsd" + + override fun getXmlType() = "fileSystemCacheType" + + override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server" + + override fun deserialize(el: Element): FileSystemCacheConfiguration { + val path = el.renderAttribute("path") + ?.let(Path::of) + val maxAge = el.renderAttribute("max-age") + ?.let(Duration::parse) + ?: Duration.ofDays(1) + val enableCompression = el.renderAttribute("enable-compression") + ?.let(String::toBoolean) + ?: true + val compressionLevel = el.renderAttribute("compression-level") + ?.let(String::toInt) + ?: Deflater.DEFAULT_COMPRESSION + val digestAlgorithm = el.renderAttribute("digest") + + return FileSystemCacheConfiguration( + path, + maxAge, + digestAlgorithm, + enableCompression, + compressionLevel, + ) + } + + override fun serialize(doc: Document, cache : FileSystemCacheConfiguration) = cache.run { + val result = doc.createElement("cache") + Xml.of(doc, result) { + val prefix = doc.lookupPrefix(RBCS.RBCS_NAMESPACE_URI) + attr("xs:type", "${prefix}:fileSystemCacheType", RBCS.XML_SCHEMA_NAMESPACE_URI) + root?.let { + attr("path", it.toString()) + } + attr("max-age", maxAge.toString()) + digestAlgorithm?.let { digestAlgorithm -> + attr("digest", digestAlgorithm) + } + attr("enable-compression", compressionEnabled.toString()) + compressionLevel.takeIf { + it != Deflater.DEFAULT_COMPRESSION + }?.let { + attr("compression-level", it.toString()) + } + } + result + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCache.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCache.kt new file mode 100644 index 0000000..4f3acbc --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCache.kt @@ -0,0 +1,131 @@ +package net.woggioni.rbcs.server.cache + +import java.time.Duration +import java.time.Instant +import java.util.PriorityQueue +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantReadWriteLock +import kotlin.concurrent.withLock +import net.woggioni.rbcs.api.AsyncCloseable +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.common.createLogger + +private class CacheKey(private val value: ByteArray) { + override fun equals(other: Any?) = if (other is CacheKey) { + value.contentEquals(other.value) + } else false + + override fun hashCode() = value.contentHashCode() +} + +class CacheEntry( + val metadata: CacheValueMetadata, + val content: ByteArray +) + +class InMemoryCache( + private val maxAge: Duration, + private val maxSize: Long +) : AsyncCloseable { + + companion object { + private val log = createLogger() + } + + private var mapSize : Long = 0 + private val map = HashMap() + private val lock = ReentrantReadWriteLock() + private val cond = lock.writeLock().newCondition() + + private class RemovalQueueElement(val key: CacheKey, val value: CacheEntry, val expiry: Instant) : + Comparable { + override fun compareTo(other: RemovalQueueElement) = expiry.compareTo(other.expiry) + } + + private val removalQueue = PriorityQueue() + + @Volatile + private var running = true + + private val closeFuture = object : CompletableFuture() { + init { + Thread.ofVirtual().name("in-memory-cache-gc").start { + try { + lock.writeLock().withLock { + while (running) { + val el = removalQueue.poll() + if(el == null) { + cond.await(1000, TimeUnit.MILLISECONDS) + continue + } + val value = el.value + val now = Instant.now() + if (now > el.expiry) { + val removed = map.remove(el.key, value) + if (removed) { + updateSizeAfterRemoval(value.content) + } + } else { + removalQueue.offer(el) + val interval = minOf(Duration.between(now, el.expiry), Duration.ofSeconds(1)) + cond.await(interval.toMillis(), TimeUnit.MILLISECONDS) + } + } + map.clear() + } + complete(null) + } catch (ex: Throwable) { + completeExceptionally(ex) + } + } + } + } + + fun removeEldest(): Long { + while (true) { + val el = removalQueue.poll() ?: return mapSize + val value = el.value + val removed = map.remove(el.key, value) + if (removed) { + val newSize = updateSizeAfterRemoval(value.content) + return newSize + } + } + } + + private fun updateSizeAfterRemoval(removed: ByteArray): Long { + mapSize -= removed.size + return mapSize + } + + override fun asyncClose() : CompletableFuture { + running = false + lock.writeLock().withLock { + cond.signal() + } + return closeFuture + } + + fun get(key: ByteArray) = lock.readLock().withLock { + map[CacheKey(key)]?.run { + CacheEntry(metadata, content) + } + } + + fun put( + key: ByteArray, + value: CacheEntry, + ) { + val cacheKey = CacheKey(key) + lock.writeLock().withLock { + val oldSize = map.put(cacheKey, value)?.content?.size ?: 0 + val delta = value.content.size - oldSize + mapSize += delta + removalQueue.offer(RemovalQueueElement(cacheKey, value, Instant.now().plus(maxAge))) + while (mapSize > maxSize) { + removeEldest() + } + } + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheConfiguration.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheConfiguration.kt new file mode 100644 index 0000000..99491f7 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheConfiguration.kt @@ -0,0 +1,35 @@ +package net.woggioni.rbcs.server.cache + +import io.netty.channel.ChannelFactory +import io.netty.channel.EventLoopGroup +import io.netty.channel.socket.DatagramChannel +import io.netty.channel.socket.SocketChannel +import java.time.Duration +import net.woggioni.rbcs.api.CacheHandlerFactory +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.RBCS + +data class InMemoryCacheConfiguration( + val maxAge: Duration, + val maxSize: Long, + val digestAlgorithm : String?, + val compressionEnabled: Boolean, + val compressionLevel: Int, +) : Configuration.Cache { + override fun materialize() = object : CacheHandlerFactory { + private val cache = InMemoryCache(maxAge, maxSize) + + override fun asyncClose() = cache.asyncClose() + + override fun newHandler( + cfg : Configuration, + eventLoop: EventLoopGroup, + socketChannelFactory: ChannelFactory, + datagramChannelFactory: ChannelFactory + ) = InMemoryCacheHandler(cache, digestAlgorithm, compressionEnabled, compressionLevel) + } + + override fun getNamespaceURI() = RBCS.RBCS_NAMESPACE_URI + + override fun getTypeName() = "inMemoryCacheType" +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheHandler.kt new file mode 100644 index 0000000..3a4d37d --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheHandler.kt @@ -0,0 +1,155 @@ +package net.woggioni.rbcs.server.cache + +import io.netty.buffer.ByteBuf +import io.netty.channel.ChannelHandlerContext +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream +import java.util.zip.InflaterOutputStream +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.message.CacheMessage +import net.woggioni.rbcs.api.message.CacheMessage.CacheContent +import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent +import net.woggioni.rbcs.common.ByteBufOutputStream +import net.woggioni.rbcs.common.RBCS.processCacheKey + +class InMemoryCacheHandler( + private val cache: InMemoryCache, + private val digestAlgorithm: String?, + private val compressionEnabled: Boolean, + private val compressionLevel: Int +) : CacheHandler() { + + private interface InProgressRequest : AutoCloseable { + } + + private class InProgressGetRequest(val request: CacheGetRequest) : InProgressRequest { + override fun close() { + } + } + + private interface InProgressPutRequest : InProgressRequest { + val request: CachePutRequest + val buf: ByteBuf + + fun append(buf: ByteBuf) + } + + private inner class InProgressPlainPutRequest(ctx: ChannelHandlerContext, override val request: CachePutRequest) : + InProgressPutRequest { + override val buf = ctx.alloc().compositeHeapBuffer() + + override fun append(buf: ByteBuf) { + if (buf.isDirect) { + this.buf.writeBytes(buf) + } else { + this.buf.addComponent(true, buf.retain()) + } + } + + override fun close() { + buf.release() + } + } + + private inner class InProgressCompressedPutRequest( + ctx: ChannelHandlerContext, + override val request: CachePutRequest + ) : InProgressPutRequest { + + override val buf = ctx.alloc().heapBuffer() + + private val stream = ByteBufOutputStream(buf).let { + DeflaterOutputStream(it, Deflater(compressionLevel)) + } + + override fun append(buf: ByteBuf) { + buf.readBytes(stream, buf.readableBytes()) + } + + override fun close() { + stream.close() + } + } + + private var inProgressRequest: InProgressRequest? = null + + override fun channelRead0(ctx: ChannelHandlerContext, msg: CacheMessage) { + when (msg) { + is CacheGetRequest -> handleGetRequest(ctx, msg) + is CachePutRequest -> handlePutRequest(ctx, msg) + is LastCacheContent -> handleLastCacheContent(ctx, msg) + is CacheContent -> handleCacheContent(ctx, msg) + else -> ctx.fireChannelRead(msg) + } + } + + private fun handleGetRequest(ctx: ChannelHandlerContext, msg: CacheGetRequest) { + inProgressRequest = InProgressGetRequest(msg) + } + + private fun handlePutRequest(ctx: ChannelHandlerContext, msg: CachePutRequest) { + inProgressRequest = if (compressionEnabled) { + InProgressCompressedPutRequest(ctx, msg) + } else { + InProgressPlainPutRequest(ctx, msg) + } + } + + private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) { + val req = inProgressRequest + if (req is InProgressPutRequest) { + req.append(msg.content()) + } + } + + private fun handleLastCacheContent(ctx: ChannelHandlerContext, msg: LastCacheContent) { + handleCacheContent(ctx, msg) + when (val req = inProgressRequest) { + is InProgressGetRequest -> { +// this.inProgressRequest = null + cache.get(processCacheKey(req.request.key, null, digestAlgorithm))?.let { value -> + sendMessageAndFlush(ctx, CacheValueFoundResponse(req.request.key, value.metadata)) + if (compressionEnabled) { + val buf = ctx.alloc().heapBuffer() + InflaterOutputStream(ByteBufOutputStream(buf)).use { + it.write(value.content) + buf.retain() + } + sendMessage(ctx, LastCacheContent(buf)) + } else { + val buf = ctx.alloc().heapBuffer() + ByteBufOutputStream(buf).use { + it.write(value.content) + buf.retain() + } + sendMessage(ctx, LastCacheContent(buf)) + } + } ?: sendMessage(ctx, CacheValueNotFoundResponse(req.request.key)) + } + + is InProgressPutRequest -> { + this.inProgressRequest = null + val buf = req.buf + buf.retain() + req.close() + + val bytes = ByteArray(buf.readableBytes()).also(buf::readBytes) + buf.release() + val cacheKey = processCacheKey(req.request.key, null, digestAlgorithm) + cache.put(cacheKey, CacheEntry(req.request.metadata, bytes)) + sendMessageAndFlush(ctx, CachePutResponse(req.request.key)) + } + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + inProgressRequest?.close() + inProgressRequest = null + super.exceptionCaught(ctx, cause) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheProvider.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheProvider.kt new file mode 100644 index 0000000..c231bc0 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheProvider.kt @@ -0,0 +1,62 @@ +package net.woggioni.rbcs.server.cache + +import java.time.Duration +import java.util.zip.Deflater +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document +import org.w3c.dom.Element + +class InMemoryCacheProvider : CacheProvider { + + override fun getXmlSchemaLocation() = "classpath:net/woggioni/rbcs/server/schema/rbcs-server.xsd" + + override fun getXmlType() = "inMemoryCacheType" + + override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server" + + override fun deserialize(el: Element): InMemoryCacheConfiguration { + val maxAge = el.renderAttribute("max-age") + ?.let(Duration::parse) + ?: Duration.ofDays(1) + val maxSize = el.renderAttribute("max-size") + ?.let(java.lang.Long::decode) + ?: 0x1000000 + val enableCompression = el.renderAttribute("enable-compression") + ?.let(String::toBoolean) + ?: true + val compressionLevel = el.renderAttribute("compression-level") + ?.let(String::toInt) + ?: Deflater.DEFAULT_COMPRESSION + val digestAlgorithm = el.renderAttribute("digest") + return InMemoryCacheConfiguration( + maxAge, + maxSize, + digestAlgorithm, + enableCompression, + compressionLevel, + ) + } + + override fun serialize(doc: Document, cache : InMemoryCacheConfiguration) = cache.run { + val result = doc.createElement("cache") + Xml.of(doc, result) { + val prefix = doc.lookupPrefix(RBCS.RBCS_NAMESPACE_URI) + attr("xs:type", "${prefix}:inMemoryCacheType", RBCS.XML_SCHEMA_NAMESPACE_URI) + attr("max-age", maxAge.toString()) + attr("max-size", maxSize.toString()) + digestAlgorithm?.let { digestAlgorithm -> + attr("digest", digestAlgorithm) + } + attr("enable-compression", compressionEnabled.toString()) + compressionLevel.takeIf { + it != Deflater.DEFAULT_COMPRESSION + }?.let { + attr("compression-level", it.toString()) + } + } + result + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/CacheSerializers.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/CacheSerializers.kt new file mode 100644 index 0000000..2cc708c --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/CacheSerializers.kt @@ -0,0 +1,15 @@ +package net.woggioni.rbcs.server.configuration + +import java.util.ServiceLoader +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.api.Configuration + +object CacheSerializers { + val index = (Configuration::class.java.module.layer?.let { layer -> + ServiceLoader.load(layer, CacheProvider::class.java) + } ?: ServiceLoader.load(CacheProvider::class.java)) + .asSequence() + .map { + (it.xmlNamespace to it.xmlType) to it + }.toMap() +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Parser.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Parser.kt new file mode 100644 index 0000000..f1ac5c6 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Parser.kt @@ -0,0 +1,361 @@ +package net.woggioni.rbcs.server.configuration + +import java.nio.file.Paths +import java.time.Duration +import java.time.temporal.ChronoUnit +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Configuration.Authentication +import net.woggioni.rbcs.api.Configuration.BasicAuthentication +import net.woggioni.rbcs.api.Configuration.Cache +import net.woggioni.rbcs.api.Configuration.ClientCertificateAuthentication +import net.woggioni.rbcs.api.Configuration.ForwardedClientCertificateAuthentication +import net.woggioni.rbcs.api.Configuration.Group +import net.woggioni.rbcs.api.Configuration.KeyStore +import net.woggioni.rbcs.api.Configuration.Tls +import net.woggioni.rbcs.api.Configuration.TlsCertificateExtractor +import net.woggioni.rbcs.api.Configuration.TrustStore +import net.woggioni.rbcs.api.Configuration.User +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.api.exception.ConfigurationException +import net.woggioni.rbcs.common.Cidr +import net.woggioni.rbcs.common.Xml.Companion.asIterable +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document +import org.w3c.dom.Element +import org.w3c.dom.TypeInfo + +object Parser { + fun parse(document: Document): Configuration { + val root = document.documentElement + val anonymousUser = User("", null, emptySet(), null) + var connection: Configuration.Connection = Configuration.Connection( + Duration.of(30, ChronoUnit.SECONDS), + Duration.of(60, ChronoUnit.SECONDS), + Duration.of(60, ChronoUnit.SECONDS), + 0x4000000, + 0x10000 + ) + var rateLimiter = Configuration.RateLimiter(false, 0x100000, 100) + var eventExecutor: Configuration.EventExecutor = Configuration.EventExecutor(true) + var cache: Cache? = null + var host = "127.0.0.1" + var port = 11080 + var proxyProtocolEnabled = false + var trustedProxies = emptyList() + var users: Map = mapOf(anonymousUser.name to anonymousUser) + var groups = emptyMap() + var tls: Tls? = null + val serverPath = root.renderAttribute("path") + var incomingConnectionsBacklogSize = 1024 + var authentication: Authentication? = null + for (child in root.asIterable()) { + val tagName = child.localName + when (tagName) { + "authentication" -> { + for (gchild in child.asIterable()) { + when (gchild.localName) { + "basic" -> { + authentication = BasicAuthentication() + } + + "client-certificate" -> { + var tlsExtractorUser: TlsCertificateExtractor? = null + var tlsExtractorGroup: TlsCertificateExtractor? = null + for (ggchild in gchild.asIterable()) { + when (ggchild.localName) { + "group-extractor" -> { + val attrName = ggchild.renderAttribute("attribute-name") + val pattern = ggchild.renderAttribute("pattern") + tlsExtractorGroup = TlsCertificateExtractor(attrName, pattern) + } + + "user-extractor" -> { + val attrName = ggchild.renderAttribute("attribute-name") + val pattern = ggchild.renderAttribute("pattern") + tlsExtractorUser = TlsCertificateExtractor(attrName, pattern) + } + } + } + authentication = ClientCertificateAuthentication(tlsExtractorUser, tlsExtractorGroup) + } + + "forwarded-client-certificate" -> { + val headerName = gchild.renderAttribute("header-name") ?: "X-Client-Cert-Subject-DN" + var tlsExtractorUser: TlsCertificateExtractor? = null + var tlsExtractorGroup: TlsCertificateExtractor? = null + for (ggchild in gchild.asIterable()) { + when (ggchild.localName) { + "group-extractor" -> { + val attrName = ggchild.renderAttribute("attribute-name") + val pattern = ggchild.renderAttribute("pattern") + tlsExtractorGroup = TlsCertificateExtractor(attrName, pattern) + } + + "user-extractor" -> { + val attrName = ggchild.renderAttribute("attribute-name") + val pattern = ggchild.renderAttribute("pattern") + tlsExtractorUser = TlsCertificateExtractor(attrName, pattern) + } + } + } + authentication = ForwardedClientCertificateAuthentication(headerName, tlsExtractorUser, tlsExtractorGroup) + } + } + } + } + + "authorization" -> { + var knownUsers = sequenceOf(anonymousUser) + for (gchild in child.asIterable()) { + when (gchild.localName) { + "users" -> { + knownUsers += parseUsers(gchild) + } + + "groups" -> { + val pair = parseGroups(gchild, knownUsers) + users = pair.first + groups = pair.second + } + } + } + } + + "bind" -> { + host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required") + port = Integer.parseInt(child.renderAttribute("port")) + proxyProtocolEnabled = child.renderAttribute("proxy-protocol") + ?.let(String::toBoolean) ?: false + incomingConnectionsBacklogSize = child.renderAttribute("incoming-connections-backlog-size") + ?.let(Integer::parseInt) + ?: 1024 + + for(grandChild in child.asIterable()) { + when(grandChild.localName) { + "trusted-proxies" -> { + trustedProxies = parseTrustedProxies(grandChild) + } + } + } + child.asIterable().filter { + it.localName == "trusted-proxies" + }.firstOrNull()?.let(::parseTrustedProxies) + + } + + "cache" -> { + cache = (child as TypeInfo).let { tf -> + val typeNamespace = tf.typeNamespace + val typeName = tf.typeName + CacheSerializers.index[typeNamespace to typeName] + ?: throw IllegalArgumentException("Cache provider for namespace '$typeNamespace' with name '$typeName' not found") + }.deserialize(child) + } + + "connection" -> { + val idleTimeout = child.renderAttribute("idle-timeout") + ?.let(Duration::parse) ?: Duration.of(30, ChronoUnit.SECONDS) + val readIdleTimeout = child.renderAttribute("read-idle-timeout") + ?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS) + val writeIdleTimeout = child.renderAttribute("write-idle-timeout") + ?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS) + val maxRequestSize = child.renderAttribute("max-request-size") + ?.let(Integer::decode) ?: 0x4000000 + val chunkSize = child.renderAttribute("chunk-size") + ?.let(Integer::decode) ?: 0x10000 + connection = Configuration.Connection( + idleTimeout, + readIdleTimeout, + writeIdleTimeout, + maxRequestSize, + chunkSize + ) + } + + "event-executor" -> { + val useVirtualThread = child.renderAttribute("use-virtual-threads") + ?.let(String::toBoolean) ?: true + eventExecutor = Configuration.EventExecutor(useVirtualThread) + } + + "rate-limiter" -> { + val delayResponse = child.renderAttribute("delay-response") + ?.let(String::toBoolean) + ?: false + val messageBufferSize = child.renderAttribute("message-buffer-size") + ?.let(Integer::decode) + ?: 0x100000 + val maxQueuedMessages = child.renderAttribute("max-queued-messages") + ?.let(Integer::decode) + ?: 100 + rateLimiter = Configuration.RateLimiter(delayResponse, messageBufferSize, maxQueuedMessages) + } + + "tls" -> { + var keyStore: KeyStore? = null + var trustStore: TrustStore? = null + + for (granChild in child.asIterable()) { + when (granChild.localName) { + "keystore" -> { + val keyStoreFile = Paths.get(granChild.renderAttribute("file")) + val keyStorePassword = granChild.renderAttribute("password") + val keyAlias = granChild.renderAttribute("key-alias") + val keyPassword = granChild.renderAttribute("key-password") + keyStore = KeyStore( + keyStoreFile, + keyStorePassword, + keyAlias, + keyPassword + ) + } + + "truststore" -> { + val trustStoreFile = Paths.get(granChild.renderAttribute("file")) + val trustStorePassword = granChild.renderAttribute("password") + val checkCertificateStatus = granChild.renderAttribute("check-certificate-status") + ?.let(String::toBoolean) + ?: false + val requireClientCertificate = child.renderAttribute("require-client-certificate") + ?.let(String::toBoolean) ?: false + + trustStore = TrustStore( + trustStoreFile, + trustStorePassword, + checkCertificateStatus, + requireClientCertificate + ) + } + } + } + tls = Tls(keyStore, trustStore) + } + } + } + return Configuration.of( + host, + port, + proxyProtocolEnabled, + trustedProxies, + incomingConnectionsBacklogSize, + serverPath, + eventExecutor, + rateLimiter, + connection, + users, + groups, + cache!!, + authentication, + tls, + ) + } + + private fun parseRoles(root: Element) = root.asIterable().asSequence().map { + when (it.localName) { + "reader" -> Role.Reader + "writer" -> Role.Writer + "healthcheck" -> Role.Healthcheck + else -> throw UnsupportedOperationException("Illegal node '${it.localName}'") + } + }.toSet() + + private fun parseTrustedProxies(root: Element) = root.asIterable().asSequence().map { + when (it.localName) { + "allow" -> it.renderAttribute("cidr") + ?.let(Cidr::from) + ?: throw ConfigurationException("Missing 'cidr' attribute") + else -> throw ConfigurationException("Unrecognized tag '${it.localName}'") + } + }.toList() + + private fun parseUserRefs(root: Element) = root.asIterable().asSequence().map { + when (it.localName) { + "user" -> it.renderAttribute("ref") + "anonymous" -> "" + else -> ConfigurationException("Unrecognized tag '${it.localName}'") + } + } + + private fun parseQuota(el: Element): Configuration.Quota { + val calls = el.renderAttribute("calls") + ?.let(String::toLong) + ?: throw ConfigurationException("Missing attribute 'calls'") + val maxAvailableCalls = el.renderAttribute("max-available-calls") + ?.let(String::toLong) + ?: calls + val initialAvailableCalls = el.renderAttribute("initial-available-calls") + ?.let(String::toLong) + ?: maxAvailableCalls + val period = el.renderAttribute("period") + ?.let(Duration::parse) + ?: throw ConfigurationException("Missing attribute 'period'") + return Configuration.Quota(calls, period, initialAvailableCalls, maxAvailableCalls) + } + + private fun parseUsers(root: Element): Sequence { + return root.asIterable().asSequence().mapNotNull { child -> + when (child.localName) { + "user" -> { + val username = child.renderAttribute("name") + val password = child.renderAttribute("password") + var quota: Configuration.Quota? = null + for (gchild in child.asIterable()) { + if (gchild.localName == "quota") { + quota = parseQuota(gchild) + } + } + User(username, password, emptySet(), quota) + } + "anonymous" -> { + var quota: Configuration.Quota? = null + for (gchild in child.asIterable()) { + if (gchild.localName == "quota") { + quota= parseQuota(gchild) + } + } + User("", null, emptySet(), quota) + } + else -> null + } + } + } + + private fun parseGroups(root: Element, knownUsers: Sequence): Pair, Map> { + val knownUsersMap = knownUsers.associateBy(User::getName) + val userGroups = mutableMapOf>() + val groups = root.asIterable().asSequence().filter { + it.localName == "group" + }.map { el -> + val groupName = el.renderAttribute("name") ?: throw ConfigurationException("Group name is required") + var roles = emptySet() + var userQuota: Configuration.Quota? = null + var groupQuota: Configuration.Quota? = null + for (child in el.asIterable()) { + when (child.localName) { + "users" -> { + parseUserRefs(child).mapNotNull(knownUsersMap::get).forEach { user -> + userGroups.computeIfAbsent(user.name) { + mutableSetOf() + }.add(groupName) + } + } + + "roles" -> { + roles = parseRoles(child) + } + "group-quota" -> { + userQuota = parseQuota(child) + } + "user-quota" -> { + groupQuota = parseQuota(child) + } + } + } + groupName to Group(groupName, roles, userQuota, groupQuota) + }.toMap() + val users = knownUsersMap.map { (name, user) -> + name to User(name, user.password, userGroups[name]?.mapNotNull { groups[it] }?.toSet() ?: emptySet(), user.quota) + }.toMap() + return users to groups + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Serializer.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Serializer.kt new file mode 100644 index 0000000..67e78d3 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Serializer.kt @@ -0,0 +1,218 @@ +package net.woggioni.rbcs.server.configuration + +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import org.w3c.dom.Document + +object Serializer { + + private fun Xml.serializeQuota(quota : Configuration.Quota) { + attr("calls", quota.calls.toString()) + attr("period", quota.period.toString()) + attr("max-available-calls", quota.maxAvailableCalls.toString()) + attr("initial-available-calls", quota.initialAvailableCalls.toString()) + } + + fun serialize(conf : Configuration) : Document { + val schemaLocations = CacheSerializers.index.values.asSequence().map { + it.xmlNamespace to it.xmlSchemaLocation + }.toMap() + return Xml.of(RBCS.RBCS_NAMESPACE_URI, RBCS.RBCS_PREFIX + ":server") { +// attr("xmlns:xs", GradleBuildCacheServer.XML_SCHEMA_NAMESPACE_URI) + val value = schemaLocations.asSequence().map { (k, v) -> "$k $v" }.joinToString(" ") + attr("xs:schemaLocation", value , namespaceURI = RBCS.XML_SCHEMA_NAMESPACE_URI) + + conf.serverPath + ?.takeIf(String::isNotEmpty) + ?.let { serverPath -> + attr("path", serverPath) + } + node("bind") { + attr("host", conf.host) + attr("port", conf.port.toString()) + attr("incoming-connections-backlog-size", conf.incomingConnectionsBacklogSize.toString()) + attr("proxy-protocol", conf.isProxyProtocolEnabled.toString()) + + if (conf.trustedProxyIPs.isNotEmpty()) { + node("trusted-proxies") { + for(trustedProxy in conf.trustedProxyIPs) { + node("allow") { + attr("cidr", trustedProxy.toString()) + } + } + } + } + } + node("connection") { + conf.connection.let { connection -> + attr("idle-timeout", connection.idleTimeout.toString()) + attr("read-idle-timeout", connection.readIdleTimeout.toString()) + attr("write-idle-timeout", connection.writeIdleTimeout.toString()) + attr("max-request-size", connection.maxRequestSize.toString()) + attr("chunk-size", connection.chunkSize.toString()) + } + } + node("event-executor") { + attr("use-virtual-threads", conf.eventExecutor.isUseVirtualThreads.toString()) + } + node("rate-limiter") { + attr("delay-response", conf.rateLimiter.isDelayRequest.toString()) + attr("max-queued-messages", conf.rateLimiter.maxQueuedMessages.toString()) + attr("message-buffer-size", conf.rateLimiter.messageBufferSize.toString()) + } + val cache = conf.cache + val serializer : CacheProvider = + (CacheSerializers.index[cache.namespaceURI to cache.typeName] as? CacheProvider) ?: throw NotImplementedError() + element.appendChild(serializer.serialize(doc, cache)) + node("authorization") { + node("users") { + for(user in conf.users.values) { + if(user.name.isNotEmpty()) { + node("user") { + attr("name", user.name) + user.password?.let { password -> + attr("password", password) + } + user.quota?.let { quota -> + node("quota") { + serializeQuota(quota) + } + } + } + } + } + conf.users[""] + ?.let { anonymousUser -> + anonymousUser.quota?.let { quota -> + node("anonymous") { + node("quota") { + serializeQuota(quota) + } + } + } + } + } + node("groups") { + val groups = conf.users.values.asSequence() + .flatMap { + user -> user.groups.map { it to user } + }.groupBy(Pair::first, Pair::second) + for(pair in groups) { + val group = pair.key + val users = pair.value + node("group") { + attr("name", group.name) + if(users.isNotEmpty()) { + node("users") { + var anonymousUser : Configuration.User? = null + for(user in users) { + if(user.name.isNotEmpty()) { + node("user") { + attr("ref", user.name) + } + } else { + anonymousUser = user + } + } + if(anonymousUser != null) { + node("anonymous") + } + } + } + if(group.roles.isNotEmpty()) { + node("roles") { + for(role in group.roles) { + node(role.toString().lowercase()) + } + } + } + group.userQuota?.let { quota -> + node("user-quota") { + serializeQuota(quota) + } + } + group.groupQuota?.let { quota -> + node("group-quota") { + serializeQuota(quota) + } + } + } + } + } + } + + conf.authentication?.let { authentication -> + node("authentication") { + when(authentication) { + is Configuration.BasicAuthentication -> { + node("basic") + } + is Configuration.ClientCertificateAuthentication -> { + node("client-certificate") { + authentication.groupExtractor?.let { extractor -> + node("group-extractor") { + attr("attribute-name", extractor.rdnType) + attr("pattern", extractor.pattern) + } + } + authentication.userExtractor?.let { extractor -> + node("user-extractor") { + attr("attribute-name", extractor.rdnType) + attr("pattern", extractor.pattern) + } + } + } + } + is Configuration.ForwardedClientCertificateAuthentication -> { + node("forwarded-client-certificate") { + attr("header-name", authentication.headerName) + authentication.groupExtractor?.let { extractor -> + node("group-extractor") { + attr("attribute-name", extractor.rdnType) + attr("pattern", extractor.pattern) + } + } + authentication.userExtractor?.let { extractor -> + node("user-extractor") { + attr("attribute-name", extractor.rdnType) + attr("pattern", extractor.pattern) + } + } + } + } + } + } + } + + conf.tls?.let { tlsConfiguration -> + node("tls") { + tlsConfiguration.keyStore?.let { keyStore -> + node("keystore") { + attr("file", keyStore.file.toString()) + keyStore.password?.let { keyStorePassword -> + attr("password", keyStorePassword) + } + attr("key-alias", keyStore.keyAlias) + keyStore.keyPassword?.let { keyPassword -> + attr("key-password", keyPassword) + } + } + } + + tlsConfiguration.trustStore?.let { trustStore -> + node("truststore") { + attr("file", trustStore.file.toString()) + trustStore.password?.let { password -> + attr("password", password) + } + attr("check-certificate-status", trustStore.isCheckCertificateStatus.toString()) + attr("require-client-certificate", trustStore.isRequireClientCertificate.toString()) + } + } + } + } + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/exception/ExceptionHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/exception/ExceptionHandler.kt new file mode 100644 index 0000000..0553952 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/exception/ExceptionHandler.kt @@ -0,0 +1,139 @@ +package net.woggioni.rbcs.server.exception + +import io.netty.buffer.Unpooled +import io.netty.channel.ChannelDuplexHandler +import io.netty.channel.ChannelFutureListener +import io.netty.channel.ChannelHandler.Sharable +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.DecoderException +import io.netty.handler.codec.http.DefaultFullHttpResponse +import io.netty.handler.codec.http.FullHttpResponse +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import io.netty.handler.timeout.ReadTimeoutException +import io.netty.handler.timeout.WriteTimeoutException +import java.net.ConnectException +import java.net.SocketException +import javax.net.ssl.SSLException +import javax.net.ssl.SSLPeerUnverifiedException +import net.woggioni.rbcs.api.exception.CacheException +import net.woggioni.rbcs.api.exception.ContentTooLargeException +import net.woggioni.rbcs.common.contextLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.log +import net.woggioni.rbcs.server.RemoteBuildCacheServer +import org.slf4j.event.Level +import org.slf4j.spi.LoggingEventBuilder + +@Sharable +object ExceptionHandler : ChannelDuplexHandler() { + + val NAME : String = this::class.java.name + + private val log = contextLogger() + + private val NOT_AUTHORIZED: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val NOT_AVAILABLE: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.SERVICE_UNAVAILABLE, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val SERVER_ERROR: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val TOO_BIG: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + when (cause) { + is DecoderException -> { + if(log.isDebugEnabled) { + log.debug(cause.message, cause) + } + ctx.close() + } + + is ConnectException -> { + if(log.isErrorEnabled) { + log.error(cause.message, cause) + } + ctx.writeAndFlush(SERVER_ERROR.retainedDuplicate()) + } + + is SocketException -> { + if(log.isDebugEnabled) { + log.debug(cause.message, cause) + } + ctx.close() + } + + is SSLPeerUnverifiedException -> { + if(log.isDebugEnabled) { + log.debug(cause.message, cause) + } + ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + + is SSLException -> { + if(log.isDebugEnabled) { + log.debug(cause.message, cause) + } + ctx.close() + } + + is ContentTooLargeException -> { + log.log(Level.DEBUG, ctx.channel()) { builder : LoggingEventBuilder -> + builder.setMessage("Request body is too large") + } + ctx.writeAndFlush(TOO_BIG.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + + is ReadTimeoutException -> { + log.debug { + val channelId = ctx.channel().id().asShortText() + "Read timeout on channel $channelId, closing the connection" + } + ctx.close() + } + + is WriteTimeoutException -> { + log.debug { + val channelId = ctx.channel().id().asShortText() + "Write timeout on channel $channelId, closing the connection" + } + ctx.close() + } + + is CacheException -> { + if(log.isErrorEnabled) { + log.error(cause.message, cause) + } + ctx.writeAndFlush(NOT_AVAILABLE.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + + else -> { + if(log.isErrorEnabled) { + log.error(cause.message, cause) + } + ctx.writeAndFlush(SERVER_ERROR.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/BlackHoleRequestHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/BlackHoleRequestHandler.kt new file mode 100644 index 0000000..d912c40 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/BlackHoleRequestHandler.kt @@ -0,0 +1,13 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.handler.codec.http.HttpContent + +class BlackHoleRequestHandler : SimpleChannelInboundHandler() { + companion object { + val NAME = BlackHoleRequestHandler::class.java.name + } + override fun channelRead0(ctx: ChannelHandlerContext, msg: HttpContent) { + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/MaxRequestSizeHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/MaxRequestSizeHandler.kt new file mode 100644 index 0000000..e5babeb --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/MaxRequestSizeHandler.kt @@ -0,0 +1,40 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.handler.codec.http.HttpContent +import io.netty.handler.codec.http.HttpRequest +import net.woggioni.rbcs.api.exception.ContentTooLargeException + + +class MaxRequestSizeHandler(private val maxRequestSize : Int) : ChannelInboundHandlerAdapter() { + companion object { + val NAME = MaxRequestSizeHandler::class.java.name + } + + private var cumulativeSize = 0 + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + when(msg) { + is HttpRequest -> { + cumulativeSize = 0 + ctx.fireChannelRead(msg) + } + is HttpContent -> { + val exceeded = cumulativeSize > maxRequestSize + if(!exceeded) { + cumulativeSize += msg.content().readableBytes() + } + if(cumulativeSize > maxRequestSize) { + msg.release() + if(!exceeded) { + ctx.fireExceptionCaught(ContentTooLargeException("Request body is too large", null)) + } + } else { + ctx.fireChannelRead(msg) + } + } + else -> ctx.fireChannelRead(msg) + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ProxyProtocolHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ProxyProtocolHandler.kt new file mode 100644 index 0000000..0a9860f --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ProxyProtocolHandler.kt @@ -0,0 +1,47 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.handler.codec.haproxy.HAProxyMessage +import java.net.InetAddress +import java.net.InetSocketAddress +import net.woggioni.rbcs.common.Cidr +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.trace +import net.woggioni.rbcs.server.RemoteBuildCacheServer + + +class ProxyProtocolHandler(private val trustedProxyIPs : List) : SimpleChannelInboundHandler() { + + companion object { + private val log = createLogger() + } + + override fun channelRead0( + ctx: ChannelHandlerContext, + msg: HAProxyMessage + ) { + val sourceAddress = ctx.channel().remoteAddress() + if (sourceAddress is InetSocketAddress && + trustedProxyIPs.isEmpty() || + trustedProxyIPs.any { it.contains((sourceAddress as InetSocketAddress).address) }.also { + if(!it && log.isTraceEnabled) { + log.trace { + "Received a proxied connection request from $sourceAddress which is not a trusted proxy address, " + + "the proxy server address will be used instead" + } + } + }) { + val proxiedClientAddress = InetSocketAddress( + InetAddress.ofLiteral(msg.sourceAddress()), + msg.sourcePort() + ) + if(log.isTraceEnabled) { + log.trace { + "Received proxied connection request from $sourceAddress forwarded for $proxiedClientAddress" + } + } + ctx.channel().attr(RemoteBuildCacheServer.clientIp).set(proxiedClientAddress) + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ReadTriggerDuplexHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ReadTriggerDuplexHandler.kt new file mode 100644 index 0000000..dc53ae2 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ReadTriggerDuplexHandler.kt @@ -0,0 +1,65 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.buffer.ByteBufHolder +import io.netty.channel.ChannelDuplexHandler +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelPromise +import io.netty.handler.codec.http.LastHttpContent +import net.woggioni.rbcs.common.createLogger + +class ReadTriggerDuplexHandler : ChannelDuplexHandler() { + companion object { + val NAME = ReadTriggerDuplexHandler::class.java.name + private val log = createLogger() + } + + private var inFlight = 0 + private val messageBuffer = ArrayDeque() + + override fun handlerAdded(ctx: ChannelHandlerContext) { + ctx.read() + } + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + if(inFlight > 0) { + messageBuffer.addLast(msg) + } else { + super.channelRead(ctx, msg) + if(msg !is LastHttpContent) { + invokeRead(ctx) + } else { + inFlight += 1 + } + } + } + + private fun invokeRead(ctx : ChannelHandlerContext) { + if(messageBuffer.isEmpty()) { + ctx.read() + } else { + this.channelRead(ctx, messageBuffer.removeFirst()) + } + } + + override fun write( + ctx: ChannelHandlerContext, + msg: Any, + promise: ChannelPromise + ) { + super.write(ctx, msg, promise) + if(msg is LastHttpContent) { + inFlight -= 1 + invokeRead(ctx) + } + } + + override fun channelInactive(ctx: ChannelHandlerContext) { + while(messageBuffer.isNotEmpty()) { + val msg = messageBuffer.removeFirst() + if(msg is ByteBufHolder) { + msg.release() + } + } + super.channelInactive(ctx) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ServerHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ServerHandler.kt new file mode 100644 index 0000000..48ca18c --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ServerHandler.kt @@ -0,0 +1,227 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.channel.ChannelDuplexHandler +import io.netty.channel.ChannelHandler +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelPromise +import io.netty.handler.codec.http.DefaultFullHttpResponse +import io.netty.handler.codec.http.DefaultHttpContent +import io.netty.handler.codec.http.DefaultHttpResponse +import io.netty.handler.codec.http.DefaultLastHttpContent +import io.netty.handler.codec.http.HttpContent +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpHeaderValues +import io.netty.handler.codec.http.HttpHeaders +import io.netty.handler.codec.http.HttpMethod +import io.netty.handler.codec.http.HttpRequest +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpUtil +import io.netty.handler.codec.http.HttpVersion +import io.netty.handler.codec.http.LastHttpContent +import java.nio.file.Path +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.api.message.CacheMessage +import net.woggioni.rbcs.api.message.CacheMessage.CacheContent +import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.warn +import net.woggioni.rbcs.server.exception.ExceptionHandler + +class ServerHandler(private val serverPrefix: Path, private val cacheHandlerSupplier : () -> ChannelHandler) : + ChannelDuplexHandler() { + + companion object { + private val log = createLogger() + val NAME = ServerHandler::class.java.name + } + + private var httpVersion = HttpVersion.HTTP_1_1 + private var keepAlive = true + + private fun resetRequestMetadata() { + httpVersion = HttpVersion.HTTP_1_1 + keepAlive = true + } + + private fun setRequestMetadata(req: HttpRequest) { + httpVersion = req.protocolVersion() + keepAlive = HttpUtil.isKeepAlive(req) + } + + private fun setKeepAliveHeader(headers: HttpHeaders) { + if (!keepAlive) { + headers.set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE) + } else { + headers.set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE) + } + } + + private var cacheRequestInProgress : Boolean = false + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + when (msg) { + is HttpRequest -> handleRequest(ctx, msg) + is HttpContent -> { + if(cacheRequestInProgress) { + if(msg is LastHttpContent) { + super.channelRead(ctx, LastCacheContent(msg.content().retain())) + cacheRequestInProgress = false + } else { + super.channelRead(ctx, CacheContent(msg.content().retain())) + } + msg.release() + } else { + super.channelRead(ctx, msg) + } + } + else -> super.channelRead(ctx, msg) + } + } + + override fun write(ctx: ChannelHandlerContext, msg: Any, promise: ChannelPromise?) { + if (msg is CacheMessage) { + try { + when (msg) { + is CachePutResponse -> { + log.debug(ctx) { + "Added value for key '${msg.key}' to build cache" + } + val response = DefaultFullHttpResponse(httpVersion, HttpResponseStatus.CREATED) + val keyBytes = msg.key.toByteArray(Charsets.UTF_8) + response.headers().apply { + set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.TEXT_PLAIN) + set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED) + } + setKeepAliveHeader(response.headers()) + ctx.write(response) + val buf = ctx.alloc().buffer(keyBytes.size).apply { + writeBytes(keyBytes) + } + ctx.writeAndFlush(DefaultLastHttpContent(buf)) + } + + is CacheValueNotFoundResponse -> { + log.debug(ctx) { + "Value not found for key '${msg.key}'" + } + val response = DefaultFullHttpResponse(httpVersion, HttpResponseStatus.NOT_FOUND) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0 + setKeepAliveHeader(response.headers()) + ctx.writeAndFlush(response) + } + + is CacheValueFoundResponse -> { + log.debug(ctx) { + "Retrieved value for key '${msg.key}'" + } + val response = DefaultHttpResponse(httpVersion, HttpResponseStatus.OK) + response.headers().apply { + set(HttpHeaderNames.CONTENT_TYPE, msg.metadata.mimeType ?: HttpHeaderValues.APPLICATION_OCTET_STREAM) + msg.metadata.contentDisposition?.let { contentDisposition -> + set(HttpHeaderNames.CONTENT_DISPOSITION, contentDisposition) + } + } + setKeepAliveHeader(response.headers()) + response.headers().set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED) + ctx.writeAndFlush(response) + } + + is LastCacheContent -> { + ctx.writeAndFlush(DefaultLastHttpContent(msg.content())) + } + + is CacheContent -> { + ctx.writeAndFlush(DefaultHttpContent(msg.content())) + } + + else -> throw UnsupportedOperationException("This should never happen") + }.let { channelFuture -> + if (promise != null) { + channelFuture.addListener { + if (it.isSuccess) promise.setSuccess() + else promise.setFailure(it.cause()) + } + } + } + } finally { + resetRequestMetadata() + } + } else if(msg is LastHttpContent) { + ctx.write(msg, promise) + } else super.write(ctx, msg, promise) + } + + + private fun handleRequest(ctx: ChannelHandlerContext, msg: HttpRequest) { + setRequestMetadata(msg) + val method = msg.method() + if (method === HttpMethod.GET) { + val path = Path.of(msg.uri()).normalize() + if (path.startsWith(serverPrefix)) { + cacheRequestInProgress = true + val relativePath = serverPrefix.relativize(path) + val key : String = relativePath.toString() + val cacheHandler = cacheHandlerSupplier() + ctx.pipeline().addBefore(ExceptionHandler.NAME, null, cacheHandler) + key.let(::CacheGetRequest) + .let(ctx::fireChannelRead) + ?: ctx.channel().write(CacheValueNotFoundResponse(key)) + } else { + cacheRequestInProgress = false + log.warn(ctx) { + "Got request for unhandled path '${msg.uri()}'" + } + val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.BAD_REQUEST) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0 + ctx.writeAndFlush(response) + } + } else if (method === HttpMethod.PUT) { + val path = Path.of(msg.uri()).normalize() + if (path.startsWith(serverPrefix)) { + cacheRequestInProgress = true + val relativePath = serverPrefix.relativize(path) + val key = relativePath.toString() + val cacheHandler = cacheHandlerSupplier() + ctx.pipeline().addAfter(NAME, null, cacheHandler) + + path.fileName?.toString() + ?.let { + val mimeType = HttpUtil.getMimeType(msg)?.toString() + CachePutRequest(key, CacheValueMetadata(msg.headers().get(HttpHeaderNames.CONTENT_DISPOSITION), mimeType)) + } + ?.let(ctx::fireChannelRead) + ?: ctx.channel().write(CacheValueNotFoundResponse(key)) + } else { + cacheRequestInProgress = false + log.warn(ctx) { + "Got request for unhandled path '${msg.uri()}'" + } + val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.BAD_REQUEST) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + ctx.writeAndFlush(response) + } + } else if (method == HttpMethod.TRACE) { + cacheRequestInProgress = false + ctx.pipeline().addAfter(NAME, null, TraceHandler) + super.channelRead(ctx, msg) + } else { + cacheRequestInProgress = false + log.warn(ctx) { + "Got request with unhandled method '${msg.method().name()}'" + } + val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.METHOD_NOT_ALLOWED) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + ctx.writeAndFlush(response) + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + super.exceptionCaught(ctx, cause) + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/TraceHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/TraceHandler.kt new file mode 100644 index 0000000..35f197f --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/TraceHandler.kt @@ -0,0 +1,55 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.channel.ChannelHandler.Sharable +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.handler.codec.http.DefaultHttpResponse +import io.netty.handler.codec.http.HttpContent +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpHeaderValues +import io.netty.handler.codec.http.HttpRequest +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.LastHttpContent +import java.nio.file.Path + +@Sharable +object TraceHandler : ChannelInboundHandlerAdapter() { + val NAME = this::class.java.name + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + when(msg) { + is HttpRequest -> { + val response = DefaultHttpResponse(msg.protocolVersion(), HttpResponseStatus.OK) + response.headers().apply { + set(HttpHeaderNames.CONTENT_TYPE, "message/http") + set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED) + } + ctx.write(response) + val replayedRequestHead = ctx.alloc().buffer() + replayedRequestHead.writeCharSequence( + "TRACE ${Path.of(msg.uri())} ${msg.protocolVersion().text()}\r\n", + Charsets.US_ASCII + ) + msg.headers().forEach { (key, value) -> + replayedRequestHead.apply { + writeCharSequence(key, Charsets.US_ASCII) + writeCharSequence(": ", Charsets.US_ASCII) + writeCharSequence(value, Charsets.UTF_8) + writeCharSequence("\r\n", Charsets.US_ASCII) + } + } + replayedRequestHead.writeCharSequence("\r\n", Charsets.US_ASCII) + ctx.writeAndFlush(replayedRequestHead) + } + is LastHttpContent -> { + ctx.writeAndFlush(msg) + ctx.pipeline().remove(this) + } + is HttpContent -> ctx.writeAndFlush(msg) + else -> super.channelRead(ctx, msg) + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext?, cause: Throwable?) { + super.exceptionCaught(ctx, cause) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/otel/OtelIntegration.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/otel/OtelIntegration.kt new file mode 100644 index 0000000..fcf9144 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/otel/OtelIntegration.kt @@ -0,0 +1,32 @@ +package net.woggioni.rbcs.server.otel + +import io.netty.channel.ChannelHandler +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.instrumentation.netty.v4_1.NettyServerTelemetry +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.warn + +object OtelIntegration { + + private val log = createLogger() + + val isAvailable: Boolean by lazy { + runCatching { + Class.forName("io.opentelemetry.api.OpenTelemetry") + }.fold( + onSuccess = { true }, + onFailure = { + log.warn { "OpenTelemetry classes not on classpath, instrumentation disabled" } + false + }, + ) + } + + fun createHandler(): ChannelHandler? { + return if (isAvailable) createHandlerInternal() else null + } + + private fun createHandlerInternal(): ChannelHandler { + return NettyServerTelemetry.create(GlobalOpenTelemetry.get()).createCombinedHandler() + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/rbcs-default.xml b/rbcs-server/bin/main/net/woggioni/rbcs/server/rbcs-default.xml new file mode 100644 index 0000000..6d45ea1 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/rbcs-default.xml @@ -0,0 +1,8 @@ + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/schema/rbcs-server.xsd b/rbcs-server/bin/main/net/woggioni/rbcs/server/schema/rbcs-server.xsd new file mode 100644 index 0000000..65aa616 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/schema/rbcs-server.xsd @@ -0,0 +1,767 @@ + + + + + + Root element containing the server configuration + + + + + + + + + + + + + + Cache storage backend implementation to use, more implementations can be added through + the use of plugins + + + + + + + + + + + + + + + + + Mechanism to use to assign a username to a specific client + + + + + + + Use TLS to encrypt all the communications + + + + + + + + URI path prefix, if your rbcs is hosted at "http://www.example.com" + and this parameter is set to "cache", then all the requests will need to be sent at + "http://www.example.com/cache/KEY", where "KEY" is the cache entry KEY + + + + + + + + + + + + Server bind address + + + + + Server port number + + + + + Enable proxy protocol + + + + + + + The maximum queue length for incoming connection indications (a request to connect) is set to + the backlog parameter. If a connection indication arrives when the queue is full, + the connection is refused. + + + + + + + + + + + + + + + + + + + + The server will close the connection with the client + when neither a read nor a write was performed for the specified period of time. + + + + + + + The server will close the connection with the client + when no read was performed for the specified period of time. + + + + + + + The server will close the connection with the client + when no write was performed for the specified period of time. + + + + + + + The maximum request body size the server will accept from a client + (if exceeded the server returns 413 HTTP status code) + + + + + + + Maximum byte size of socket write calls + (reduce it to reduce memory consumption, increase it for increased throughput) + + + + + + + + + + Whether or not to use virtual threads for the execution of the core server handler + (not for the I/O operations) + + + + + + + + + + If set to true, the server will delay responses to meet user quotas, otherwise it will simply + return an immediate 429 status code to all requests that exceed the configured quota + + + + + + + Only meaningful when "delay-response" is set to "true", + when a request is delayed, it and all the following messages are queued + as long as "max-queued-messages" is not crossed, all requests that would exceed the + max-queued-message limit are instead discarded and responded with a 429 status code + + + + + + + Only meaningful when "delay-response" is set to "true", + when a request is delayed, it and all the following requests are buffered + as long as "message-buffer-size" is not crossed, all requests that would exceed the buffer + size are instead discarded and responded with a 429 status code + + + + + + + + + + + A simple cache implementation that uses a java.util.ConcurrentHashMap as a storage backend + + + + + + + + Values will be removed from the cache after this amount of time + + + + + + + The maximum allowed total size of the cache in bytes, old values will be purged from the cache + when the insertion of a new value causes this limit to be exceeded + + + + + + + Hashing algorithm to apply to the key. If omitted, no hashing is performed. + + + + + + + Enable deflate compression for stored cache elements + + + + + + + Deflate compression level to use for cache compression, + use -1 to use the default compression level of java.util.zip.Deflater + + + + + + + + + + + A simple cache implementation that stores data in a folder on the filesystem + + + + + + + + File system path that will be used to store the cache data files + (it will be created if it doesn't already exist) + + + + + + + Values will be removed from the cache after this amount of time + + + + + + + Hashing algorithm to apply to the key. If omitted, no hashing is performed. + + + + + + + Enable deflate compression for stored cache elements + + + + + + + Deflate compression level to use for cache compression, + use -1 to use the default compression level of java.util.zip.Deflater + + + + + + + + + + + + + A regex based extractor that will be used to determine which group the client belongs to, + based on the X.500 name of the subject field in the client's TLS certificate. + When this is set RBAC works even if the user isn't listed in the <users/> section as + the client will be assigned role solely based on the group he is found to belong to. + Note that this does not allow for a client to be part of multiple groups. + + + + + + + A regex based extractor that will be used to assign a user to a connected client, + based on the X.500 name of the subject field in the client's TLS certificate. + + + + + + + + + + Authenticate clients based on a custom HTTP header containing the client TLS certificate + subject DN, forwarded by a reverse proxy that performs TLS termination. The proxy must be + listed in the trusted-proxies configuration for the header to be accepted. + + + + + + + A regex based extractor that will be used to determine which group the client belongs to, + based on the X.500 name of the subject DN forwarded by the reverse proxy. + When this is set RBAC works even if the user isn't listed in the <users/> section as + the client will be assigned role solely based on the group he is found to belong to. + Note that this does not allow for a client to be part of multiple groups. + + + + + + + A regex based extractor that will be used to assign a user to a connected client, + based on the X.500 name of the subject DN forwarded by the reverse proxy. + + + + + + + + Name of the HTTP header containing the client certificate subject DN + forwarded by the reverse proxy. Defaults to "X-Client-Cert-Subject-DN". + + + + + + + + + Extract informations from a client TLS certificates using + regular expressions applied to the X.500 name "Subject" field + + + + + + X.500 name attribute to apply the regex + + + + + + + Regex that wil be applied to the attribute value, + use regex groups to extract relevant data + (note that only the first group that appears in the regex is used) + + + + + + + + + + + List of users registered in the application + + + + + + + List of user groups registered in the application + + + + + + + + + + + + + + Authentication mechanism to assign usernames and groups to clients + + + + + + + Enable HTTP basic authentication + + + + + + + Enable TLS certificate authentication + + + + + + + Enable forwarded client certificate authentication. Authenticates clients based on + a custom HTTP header containing the client certificate subject DN, forwarded by a + reverse proxy that performs TLS termination. Requires trusted-proxies to be configured. + + + + + + + Disable authentication altogether + + + + + + + + + + Defines a quota for a user or a group + + + + + + Maximum number of allowed calls in a given period + + + + + + + The period length + + + + + + + Maximum number of available calls that can be accumulated + + + + + + + Number of available calls for users at their first call + + + + + + + + + Placeholder for a client that is not authenticated + + + + + + + Calls quota for the user + + + + + + + + + + An authenticated user + + + + + + + Calls quota for the user + + + + + + + + User's name + + + + + + + User's password hash used for HTTP basic authentication, this has to be generated with + the `password` subcommand of `rbcs-cli` + + + + + + + + + List of registered users, add an <anonymous> tag to enable authenticated user access + when authentication is enabled + + + + + + + + + + + + List of registered user groups + + + + + + + + + + + The definition of a user group, with the list of its member users + + + + + + + + + + + + + The list of application roles awarded to all the members of this group + + + + + + + The call quota for each user in this group + + + + + + + The cumulative call quota for all users in this group + + + + + + + + The group's name + + + + + + + + + + + + + + + + + + + A list of references to users in the <users> section + + + + + + + + + + + + A reference to a user in the <users> section + + + + + + Name of the referenced user + + + + + + + + + Enable TLS protocol + + + + + + + Path to the keystore file that contains the server's key and certificate + + + + + + + Path to the truststore file that contains the trusted CAs + for TLS client certificate verification + + + + + + + + + + + System path to the keystore file + + + + + + + Password to open they keystore file + + + + + + + Alias of the keystore entry containing the private key + + + + + + + Private key entry's encryption password + + + + + + + + + + Path to the trustore file + + + + + + + Trustore file password + + + + + + + Whether or not check the certificate validity using CRL/OCSP + + + + + + + If true, the server requires a TLS client certificate from the client and simply refuses to connect + when a client certificate isn't provided + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/BucketManager.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/BucketManager.kt new file mode 100644 index 0000000..24b0d60 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/BucketManager.kt @@ -0,0 +1,91 @@ +package net.woggioni.rbcs.server.throttling + +import java.net.InetSocketAddress +import java.util.Arrays +import java.util.concurrent.ConcurrentHashMap +import java.util.function.Function +import net.woggioni.jwo.Bucket +import net.woggioni.rbcs.api.Configuration + +class BucketManager private constructor( + private val bucketsByUser: Map> = HashMap(), + private val bucketsByGroup: Map = HashMap(), + loader: Function? +) { + + private class BucketsByAddress( + private val map: MutableMap, + private val loader: Function + ) { + fun getBucket(socketAddress : InetSocketAddress) = map.computeIfAbsent(ByteArrayKey(socketAddress.address.address)) { + loader.apply(socketAddress) + } + } + + private val bucketsByAddress: BucketsByAddress? = loader?.let { + BucketsByAddress(ConcurrentHashMap(), it) + } + + private class ByteArrayKey(val array: ByteArray) { + override fun equals(other: Any?) = (other as? ByteArrayKey)?.let { bak -> + array contentEquals bak.array + } ?: false + + override fun hashCode() = Arrays.hashCode(array) + } + + fun getBucketByAddress(address : InetSocketAddress) : Bucket? { + return bucketsByAddress?.getBucket(address) + } + + fun getBucketByUser(user : Configuration.User) = bucketsByUser[user] + fun getBucketByGroup(group : Configuration.Group) = bucketsByGroup[group] + + companion object { + fun from(cfg : Configuration) : BucketManager { + val bucketsByUser = cfg.users.values.asSequence().map { user -> + val buckets = ( + user.quota + ?.let { quota -> + sequenceOf(quota) + } ?: user.groups.asSequence() + .mapNotNull(Configuration.Group::getUserQuota) + ).map { quota -> + Bucket.local( + quota.maxAvailableCalls, + quota.calls, + quota.period, + quota.initialAvailableCalls + ) + }.toList() + user to buckets + }.toMap() + val bucketsByGroup = cfg.groups.values.asSequence().filter { + it.groupQuota != null + }.map { group -> + val quota = group.groupQuota + val bucket = Bucket.local( + quota.maxAvailableCalls, + quota.calls, + quota.period, + quota.initialAvailableCalls + ) + group to bucket + }.toMap() + return BucketManager( + bucketsByUser, + bucketsByGroup, + cfg.users[""]?.quota?.let { anonymousUserQuota -> + Function { + Bucket.local( + anonymousUserQuota.maxAvailableCalls, + anonymousUserQuota.calls, + anonymousUserQuota.period, + anonymousUserQuota.initialAvailableCalls + ) + } + } + ) + } + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/ThrottlingHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/ThrottlingHandler.kt new file mode 100644 index 0000000..1a695de --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/ThrottlingHandler.kt @@ -0,0 +1,228 @@ +package net.woggioni.rbcs.server.throttling + +import io.netty.buffer.ByteBufHolder +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.handler.codec.http.DefaultFullHttpResponse +import io.netty.handler.codec.http.FullHttpMessage +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpRequest +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import io.netty.handler.codec.http.LastHttpContent +import java.net.InetSocketAddress +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.ArrayDeque +import java.util.concurrent.TimeUnit +import net.woggioni.jwo.Bucket +import net.woggioni.jwo.LongMath +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.server.RemoteBuildCacheServer + +class ThrottlingHandler( + private val bucketManager: BucketManager, + rateLimiterConfiguration: Configuration.RateLimiter, + connectionConfiguration: Configuration.Connection +) : ChannelInboundHandlerAdapter() { + + private companion object { + private val log = createLogger() + + fun nextAttemptIsWithinThreshold(nextAttemptNanos : Long, waitThreshold : Duration) : Boolean { + val waitDuration = Duration.of(LongMath.ceilDiv(nextAttemptNanos, 100_000_000L) * 100L, ChronoUnit.MILLIS) + return waitDuration < waitThreshold + } + } + + private class RefusedRequest + + private val maxMessageBufferSize = rateLimiterConfiguration.messageBufferSize + private val maxQueuedMessages = rateLimiterConfiguration.maxQueuedMessages + private val delayRequests = rateLimiterConfiguration.isDelayRequest + private var requestBufferSize : Int = 0 + private var valveClosed = false + private var queuedContent = ArrayDeque() + + /** + * If the suggested waiting time from the bucket is lower than this + * amount, then the server will simply wait by itself before sending a response + * instead of replying with 429 + */ + private val waitThreshold = minOf( + connectionConfiguration.idleTimeout, + connectionConfiguration.readIdleTimeout, + connectionConfiguration.writeIdleTimeout + ).dividedBy(2) + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + if(valveClosed) { + if(msg !is HttpRequest && msg is ByteBufHolder) { + val newBufferSize = requestBufferSize + msg.content().readableBytes() + if(newBufferSize > maxMessageBufferSize || queuedContent.size + 1 > maxQueuedMessages) { + log.debug { + if (newBufferSize > maxMessageBufferSize) { + "New message part exceeds maxMessageBufferSize, removing previous chunks" + } else { + "New message part exceeds maxQueuedMessages, removing previous chunks" + } + } + // If this message overflows the maxMessageBufferSize, + // then remove the previously enqueued chunks of the request from the deque, + // then discard the message + while(true) { + val tail = queuedContent.last() + if(tail is ByteBufHolder) { + requestBufferSize -= tail.content().readableBytes() + tail.release() + } + queuedContent.removeLast() + if(tail is HttpRequest) { + break + } + } + msg.release() + //Add a placeholder to remember to return a 429 response corresponding to this request + queuedContent.addLast(RefusedRequest()) + } else { + //If the message does not overflow maxMessageBufferSize, just add it to the deque + queuedContent.addLast(msg) + requestBufferSize = newBufferSize + } + } else if(msg is HttpRequest && msg is FullHttpMessage){ + val newBufferSize = requestBufferSize + msg.content().readableBytes() + + // If this message overflows the maxMessageBufferSize, + // discard the message + if(newBufferSize > maxMessageBufferSize || queuedContent.size + 1 > maxQueuedMessages) { + log.debug { + if (newBufferSize > maxMessageBufferSize) { + "New message exceeds maxMessageBufferSize, discarding it" + } else { + "New message exceeds maxQueuedMessages, discarding it" + } + } + msg.release() + //Add a placeholder to remember to return a 429 response corresponding to this request + queuedContent.addLast(RefusedRequest()) + } else { + //If the message does not exceed maxMessageBufferSize or maxQueuedMessages, just add it to the deque + queuedContent.addLast(msg) + requestBufferSize = newBufferSize + } + } else { + queuedContent.addLast(msg) + } + } else { + entryPoint(ctx, msg) + } + } + + private fun entryPoint(ctx : ChannelHandlerContext, msg : Any) { + if(msg is RefusedRequest) { + sendThrottledResponse(ctx, null) + if(queuedContent.isEmpty()) { + valveClosed = false + } else { + val head = queuedContent.poll() + if(head is ByteBufHolder) { + requestBufferSize -= head.content().readableBytes() + } + entryPoint(ctx, head) + } + } else if(msg is HttpRequest) { + val nextAttempt = getNextAttempt(ctx) + if (nextAttempt < 0) { + super.channelRead(ctx, msg) + if(msg !is LastHttpContent) { + while (true) { + val head = queuedContent.poll() ?: break + if(head is ByteBufHolder) { + requestBufferSize -= head.content().readableBytes() + } + super.channelRead(ctx, head) + if (head is LastHttpContent) break + } + } + if(queuedContent.isEmpty()) { + valveClosed = false + } else { + val head = queuedContent.poll() + if(head is ByteBufHolder) { + requestBufferSize -= head.content().readableBytes() + } + entryPoint(ctx, head) + } + } else { + val waitDuration = Duration.of(LongMath.ceilDiv(nextAttempt, 100_000_000L) * 100L, ChronoUnit.MILLIS) + if (delayRequests && nextAttemptIsWithinThreshold(nextAttempt, waitThreshold)) { + valveClosed = true + ctx.executor().schedule({ + entryPoint(ctx, msg) + }, waitDuration.toMillis(), TimeUnit.MILLISECONDS) + } else { + sendThrottledResponse(ctx, waitDuration) + if(queuedContent.isEmpty()) { + valveClosed = false + } else { + val head = queuedContent.poll() + if(head is ByteBufHolder) { + requestBufferSize -= head.content().readableBytes() + } + entryPoint(ctx, head) + } + } + } + } else { + super.channelRead(ctx, msg) + } + } + + /** + * Returns the number amount of milliseconds to wait before the requests can be processed + * or -1 if the request can be performed immediately + */ + private fun getNextAttempt(ctx : ChannelHandlerContext) : Long { + val buckets = mutableListOf() + val user = ctx.channel().attr(RemoteBuildCacheServer.userAttribute).get() + if (user != null) { + bucketManager.getBucketByUser(user)?.let(buckets::addAll) + } + val groups = ctx.channel().attr(RemoteBuildCacheServer.groupAttribute).get() ?: emptySet() + if (groups.isNotEmpty()) { + groups.forEach { group -> + bucketManager.getBucketByGroup(group)?.let(buckets::add) + } + } + if (user == null && groups.isEmpty()) { + val clientAddress = ctx.channel().attr(RemoteBuildCacheServer.clientIp).get() + bucketManager.getBucketByAddress(clientAddress)?.let(buckets::add) + } + + var nextAttempt = -1L + for (bucket in buckets) { + val bucketNextAttempt = bucket.removeTokensWithEstimate(1) + if (bucketNextAttempt > nextAttempt) { + nextAttempt = bucketNextAttempt + } + } + return nextAttempt + } + + private fun sendThrottledResponse(ctx: ChannelHandlerContext, retryAfter: Duration?) { + val response = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, + HttpResponseStatus.TOO_MANY_REQUESTS + ) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0 + retryAfter?.seconds?.takeIf { + it > 0 + }?.let { + response.headers()[HttpHeaderNames.RETRY_AFTER] = it + } + + ctx.writeAndFlush(response) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/logback.xml b/rbcs-server/bin/test/logback.xml new file mode 100644 index 0000000..c6f9111 --- /dev/null +++ b/rbcs-server/bin/test/logback.xml @@ -0,0 +1,21 @@ + + + + + + + + + System.err + + %d [%highlight(%-5level)] \(%thread\) %logger{36} -%kvp- %msg %n + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt new file mode 100644 index 0000000..31d024c --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt @@ -0,0 +1,89 @@ +package net.woggioni.rbcs.server.test + +import java.net.URI +import java.net.http.HttpRequest +import java.nio.charset.StandardCharsets +import java.nio.file.Path +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.Base64 +import java.util.zip.Deflater +import kotlin.random.Random +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.common.RBCS.getFreePort +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration +import net.woggioni.rbcs.server.configuration.Serializer + + +abstract class AbstractBasicAuthServerTest : AbstractServerTest() { + + private lateinit var cacheDir : Path + + protected val random = Random(101325) + protected val keyValuePair = newEntry(random) + protected val serverPath = "rbcs" + protected val readersGroup = Configuration.Group("readers", setOf(Role.Reader), null, null) + protected val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null, null) + + abstract protected val users : List + + override fun setUp() { + this.cacheDir = testDir.resolve("cache") + cfg = Configuration.of( + "127.0.0.1", + getFreePort(), + false, + emptyList(), + 50, + serverPath, + Configuration.EventExecutor(false), + Configuration.RateLimiter(true, 0x100000, 50), + Configuration.Connection( + Duration.of(60, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + 0x1000, + 0x10000 + ), + users.asSequence().map { it.name to it}.toMap(), + sequenceOf(writersGroup, readersGroup).map { it.name to it}.toMap(), + FileSystemCacheConfiguration( + this.cacheDir, + maxAge = Duration.ofSeconds(3600 * 24), + digestAlgorithm = "MD5", + compressionLevel = Deflater.DEFAULT_COMPRESSION, + compressionEnabled = false + ), + Configuration.BasicAuthentication(), + null, + ) + Xml.write(Serializer.serialize(cfg), System.out) + } + + override fun tearDown() { + } + + protected fun buildAuthorizationHeader(user : Configuration.User, password : String) : String { + val b64 = Base64.getEncoder().encode("${user.name}:${password}".toByteArray(Charsets.UTF_8)).let{ + String(it, StandardCharsets.UTF_8) + } + return "Basic $b64" + } + + protected fun newRequestBuilder(key : String) = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/$serverPath/$key")) + + + protected fun newEntry(random : Random) : Pair { + val key = ByteArray(0x10).let { + random.nextBytes(it) + Base64.getUrlEncoder().encodeToString(it) + } + val value = ByteArray(0x1000).also { + random.nextBytes(it) + } + return key to value + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractServerTest.kt new file mode 100644 index 0000000..960fb4f --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractServerTest.kt @@ -0,0 +1,51 @@ +package net.woggioni.rbcs.server.test + +import java.nio.file.Path +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.server.RemoteBuildCacheServer +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.MethodOrderer +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.TestMethodOrder +import org.junit.jupiter.api.io.TempDir + + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation::class) +abstract class AbstractServerTest { + + protected lateinit var cfg : Configuration + + protected lateinit var testDir : Path + + private var serverHandle : RemoteBuildCacheServer.ServerHandle? = null + + @BeforeAll + fun setUp0(@TempDir tmpDir : Path) { + this.testDir = tmpDir + setUp() + startServer(cfg) + } + + @AfterAll + fun tearDown0() { + tearDown() + stopServer() + } + + abstract fun setUp() + + abstract fun tearDown() + + private fun startServer(cfg : Configuration) { + this.serverHandle = RemoteBuildCacheServer(cfg).run() + } + + private fun stopServer() { + this.serverHandle?.let { + it.sendShutdownSignal() + it.get() + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt new file mode 100644 index 0000000..ad8d660 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt @@ -0,0 +1,206 @@ +package net.woggioni.rbcs.server.test + +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.security.KeyStore +import java.security.KeyStore.PasswordProtection +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.Base64 +import java.util.zip.Deflater +import javax.net.ssl.KeyManagerFactory +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManagerFactory +import kotlin.random.Random +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.common.RBCS.getFreePort +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration +import net.woggioni.rbcs.server.configuration.Serializer +import net.woggioni.rbcs.server.test.utils.CertificateUtils +import net.woggioni.rbcs.server.test.utils.CertificateUtils.X509Credentials +import org.bouncycastle.asn1.x500.X500Name + + +abstract class AbstractTlsServerTest : AbstractServerTest() { + + companion object { + private const val CA_CERTIFICATE_ENTRY = "rbcs-ca" + private const val CLIENT_CERTIFICATE_ENTRY = "rbcs-client" + private const val SERVER_CERTIFICATE_ENTRY = "rbcs-server" + private const val PASSWORD = "password" + } + + private lateinit var cacheDir: Path + private lateinit var serverKeyStoreFile: Path + private lateinit var clientKeyStoreFile: Path + private lateinit var trustStoreFile: Path + private lateinit var serverKeyStore: KeyStore + private lateinit var clientKeyStore: KeyStore + private lateinit var trustStore: KeyStore + protected lateinit var ca: X509Credentials + + protected val readersGroup = Configuration.Group("readers", setOf(Role.Reader), null, null) + protected val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null, null) + protected val healthCheckGroup = Configuration.Group("healthcheckers", setOf(Role.Healthcheck), null, null) + protected val random = Random(101325) + protected val keyValuePair = newEntry(random) + private val serverPath : String? = null + + protected abstract val users : List + + protected fun createKeyStoreAndTrustStore() { + ca = CertificateUtils.createCertificateAuthority(CA_CERTIFICATE_ENTRY, 30) + val serverCert = CertificateUtils.createServerCertificate(ca, X500Name("CN=$SERVER_CERTIFICATE_ENTRY"), 30) + val clientCert = CertificateUtils.createClientCertificate(ca, X500Name("CN=$CLIENT_CERTIFICATE_ENTRY"), 30) + + serverKeyStore = KeyStore.getInstance("PKCS12").apply { + load(null, null) + setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null)) + setEntry( + SERVER_CERTIFICATE_ENTRY, + KeyStore.PrivateKeyEntry( + serverCert.keyPair().private, + arrayOf(serverCert.certificate(), ca.certificate) + ), + PasswordProtection(PASSWORD.toCharArray()) + ) + } + Files.newOutputStream(this.serverKeyStoreFile).use { + serverKeyStore.store(it, null) + } + + clientKeyStore = KeyStore.getInstance("PKCS12").apply { + load(null, null) + setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null)) + setEntry( + CLIENT_CERTIFICATE_ENTRY, + KeyStore.PrivateKeyEntry( + clientCert.keyPair().private, + arrayOf(clientCert.certificate(), ca.certificate) + ), + PasswordProtection(PASSWORD.toCharArray()) + ) + } + Files.newOutputStream(this.clientKeyStoreFile).use { + clientKeyStore.store(it, null) + } + + trustStore = KeyStore.getInstance("PKCS12").apply { + load(null, null) + setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null)) + } + Files.newOutputStream(this.trustStoreFile).use { + trustStore.store(it, null) + } + } + + protected fun getClientKeyStore(ca: X509Credentials, subject: X500Name) = KeyStore.getInstance("PKCS12").apply { + val clientCert = CertificateUtils.createClientCertificate(ca, subject, 30) + + load(null, null) + setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null)) + setEntry( + CLIENT_CERTIFICATE_ENTRY, + KeyStore.PrivateKeyEntry(clientCert.keyPair().private, arrayOf(clientCert.certificate(), ca.certificate)), + PasswordProtection(PASSWORD.toCharArray()) + ) + } + + protected fun getHttpClient(clientKeyStore: KeyStore?): HttpClient { + val kmf = clientKeyStore?.let { + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()).apply { + init(it, PASSWORD.toCharArray()) + } + } + + + // Set up trust manager factory with the truststore + val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) + tmf.init(trustStore) + + // Create SSL context with the key and trust managers + val sslContext = SSLContext.getInstance("TLS").apply { + init(kmf?.keyManagers ?: emptyArray(), tmf.trustManagers, null) + } + return HttpClient.newBuilder().sslContext(sslContext).build() + } + + override fun setUp() { + this.clientKeyStoreFile = testDir.resolve("client-keystore.p12") + this.serverKeyStoreFile = testDir.resolve("server-keystore.p12") + this.trustStoreFile = testDir.resolve("truststore.p12") + this.cacheDir = testDir.resolve("cache") + createKeyStoreAndTrustStore() + cfg = Configuration( + "127.0.0.1", + getFreePort(), + false, + emptyList(), + + 100, + serverPath, + Configuration.EventExecutor(false), + Configuration.RateLimiter(true, 0x100000, 50), + Configuration.Connection( + Duration.of(60, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + 0x1000, + 0x10000 + ), + users.asSequence().map { it.name to it }.toMap(), + sequenceOf(writersGroup, readersGroup).map { it.name to it }.toMap(), + FileSystemCacheConfiguration(this.cacheDir, + maxAge = Duration.ofSeconds(3600 * 24), + compressionEnabled = false, + compressionLevel = Deflater.DEFAULT_COMPRESSION, + digestAlgorithm = "MD5", + ), +// InMemoryCacheConfiguration( +// maxAge = Duration.ofSeconds(3600 * 24), +// compressionEnabled = true, +// compressionLevel = Deflater.DEFAULT_COMPRESSION, +// digestAlgorithm = "MD5" +// ), + Configuration.ClientCertificateAuthentication( + Configuration.TlsCertificateExtractor("CN", "(.*)"), + null + ), + Configuration.Tls( + Configuration.KeyStore(this.serverKeyStoreFile, null, SERVER_CERTIFICATE_ENTRY, PASSWORD), + Configuration.TrustStore(this.trustStoreFile, null, false, false), + ) + ) + Xml.write(Serializer.serialize(cfg), System.out) + } + + override fun tearDown() { + } + + protected fun newRequestBuilder(key: String) = HttpRequest.newBuilder() + .uri(URI.create("https://${cfg.host}:${cfg.port}/${serverPath ?: ""}/$key")) + + private fun buildAuthorizationHeader(user: Configuration.User, password: String): String { + val b64 = Base64.getEncoder().encode("${user.name}:${password}".toByteArray(Charsets.UTF_8)).let { + String(it, StandardCharsets.UTF_8) + } + return "Basic $b64" + } + + protected fun newEntry(random: Random): Pair { + val key = ByteArray(0x10).let { + random.nextBytes(it) + Base64.getUrlEncoder().encodeToString(it) + } + val value = ByteArray(0x1000).also { + random.nextBytes(it) + } + return key to value + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/BasicAuthServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/BasicAuthServerTest.kt new file mode 100644 index 0000000..316455c --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/BasicAuthServerTest.kt @@ -0,0 +1,192 @@ +package net.woggioni.rbcs.server.test + +import io.netty.handler.codec.http.HttpResponseStatus +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.time.Duration +import java.time.temporal.ChronoUnit +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test + + +class BasicAuthServerTest : AbstractBasicAuthServerTest() { + + companion object { + private const val PASSWORD = "password" + } + + override val users = listOf( + Configuration.User("user1", hashPassword(PASSWORD), setOf(readersGroup), null), + Configuration.User("user2", hashPassword(PASSWORD), setOf(writersGroup), null), + Configuration.User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup), null), + Configuration.User("", null, setOf(readersGroup), null), + Configuration.User("user4", hashPassword(PASSWORD), setOf(readersGroup), + Configuration.Quota(1, Duration.of(1, ChronoUnit.DAYS), 0, 1) + ), + Configuration.User("user5", hashPassword(PASSWORD), setOf(readersGroup), + Configuration.Quota(1, Duration.of(5, ChronoUnit.SECONDS), 0, 1) + ) + ) + + @Test + @Order(1) + fun putWithNoAuthorizationHeader() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(2) + fun putAsAReaderUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Reader in it.roles && Role.Writer !in it.roles + } ?: throw RuntimeException("Reader user not found") + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(3) + fun getAsAWriterUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, _) = keyValuePair + val user = cfg.users.values.find { + Role.Writer in it.roles + } ?: throw RuntimeException("Reader user not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(4) + fun putAsAWriterUser() { + val client: HttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Writer in it.roles + } ?: throw RuntimeException("Reader user not found") + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.CREATED.code(), response.statusCode()) + } + + @Test + @Order(5) + fun getAsAReaderUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Reader in it.roles + } ?: throw RuntimeException("Reader user not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(6) + fun getAsAnonymousUser() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + + val requestBuilder = newRequestBuilder(key) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(7) + fun getMissingKeyAsAReaderUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, _) = newEntry(random) + val user = cfg.users.values.find { + Role.Reader in it.roles + } ?: throw RuntimeException("Reader user not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.statusCode()) + } + + @Test + @Order(8) + fun getAsAThrottledUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + it.name == "user4" + } ?: throw RuntimeException("user4 not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.TOO_MANY_REQUESTS.code(), response.statusCode()) + } + + @Test + @Order(9) + fun getAsAThrottledUser2() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + it.name == "user5" + } ?: throw RuntimeException("user5 not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/ConfigurationTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/ConfigurationTest.kt new file mode 100644 index 0000000..17ab632 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/ConfigurationTest.kt @@ -0,0 +1,58 @@ +package net.woggioni.rbcs.server.test + +import java.nio.file.Files +import java.nio.file.Path +import net.woggioni.rbcs.common.RBCS.toUrl +import net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.server.configuration.Parser +import net.woggioni.rbcs.server.configuration.Serializer +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import org.xml.sax.SAXParseException + +class ConfigurationTest { + + @ValueSource( + strings = [ + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-default.xml", + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml", + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-tls.xml", + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml", + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-redis.xml", + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-redis-tls.xml", + ] + ) + @ParameterizedTest + fun test(configurationUrl: String, @TempDir testDir: Path) { + RbcsUrlStreamHandlerFactory.install() + val doc = Xml.parseXml(configurationUrl.toUrl()) + val cfg = Parser.parse(doc) + val configFile = testDir.resolve("rbcs.xml") + Files.newOutputStream(configFile).use { + Xml.write(Serializer.serialize(cfg), it) + } + Xml.write(Serializer.serialize(cfg), System.out) + + val parsed = Parser.parse(Xml.parseXml(configFile.toUri().toURL())) + Assertions.assertEquals(cfg, parsed) + } + + @ValueSource( + strings = [ + "classpath:net/woggioni/rbcs/server/test/invalid/invalid-user-ref.xml", + "classpath:net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user.xml", + "classpath:net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user2.xml", + "classpath:net/woggioni/rbcs/server/test/invalid/multiple-user-quota.xml", + ] + ) + @ParameterizedTest + fun invalidConfigurationTest(configurationUrl: String) { + RbcsUrlStreamHandlerFactory.install() + Assertions.assertThrows(SAXParseException::class.java) { + Xml.parseXml(configurationUrl.toUrl()) + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt new file mode 100644 index 0000000..f4e59f6 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt @@ -0,0 +1,52 @@ +package net.woggioni.rbcs.server.test + +import io.netty.handler.codec.http.HttpResponseStatus +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test + + +class NoAnonymousUserBasicAuthServerTest : AbstractBasicAuthServerTest() { + + companion object { + private const val PASSWORD = "anotherPassword" + } + + override val users = listOf( + Configuration.User("user1", hashPassword(PASSWORD), setOf(readersGroup), null), + Configuration.User("user2", hashPassword(PASSWORD), setOf(writersGroup), null), + Configuration.User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup), null), + ) + + @Test + @Order(1) + fun putWithNoAuthorizationHeader() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.UNAUTHORIZED.code(), response.statusCode()) + } + + @Test + @Order(2) + fun getWithNoAuthorizationHeader() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + + val requestBuilder = newRequestBuilder(key) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.UNAUTHORIZED.code(), response.statusCode()) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserTlsServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserTlsServerTest.kt new file mode 100644 index 0000000..a0fbd2f --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserTlsServerTest.kt @@ -0,0 +1,47 @@ +package net.woggioni.rbcs.server.test + +import io.netty.handler.codec.http.HttpResponseStatus +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import net.woggioni.rbcs.api.Configuration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test + +class NoAnonymousUserTlsServerTest : AbstractTlsServerTest() { + + override val users = listOf( + Configuration.User("user1", null, setOf(readersGroup), null), + Configuration.User("user2", null, setOf(writersGroup), null), + Configuration.User("user3", null, setOf(readersGroup, writersGroup), null), + ) + + @Test + @Order(1) + fun getAsAnonymousUser() { + val (key, _) = keyValuePair + val client: HttpClient = getHttpClient(null) + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.UNAUTHORIZED.code(), response.statusCode()) + } + + @Test + @Order(2) + fun putAsAnonymousUser() { + val (key, value) = keyValuePair + val client: HttpClient = getHttpClient(null) + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.UNAUTHORIZED.code(), response.statusCode()) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAuthServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAuthServerTest.kt new file mode 100644 index 0000000..d81f4d6 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAuthServerTest.kt @@ -0,0 +1,186 @@ +package net.woggioni.rbcs.server.test + +import io.netty.handler.codec.http.HttpResponseStatus +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.nio.file.Path +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.Base64 +import java.util.zip.Deflater +import kotlin.random.Random +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.RBCS.getFreePort +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.server.cache.InMemoryCacheConfiguration +import net.woggioni.rbcs.server.configuration.Serializer +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test + + +class NoAuthServerTest : AbstractServerTest() { + + private lateinit var cacheDir: Path + + private val random = Random(101325) + private val keyValuePair = newEntry(random) + private val serverPath = "/some/nested/path" + + override fun setUp() { + this.cacheDir = testDir.resolve("cache") + cfg = Configuration( + "127.0.0.1", + getFreePort(), + false, + emptyList(), + 100, + serverPath, + Configuration.EventExecutor(false), + Configuration.RateLimiter(true, 0x100000, 50), + Configuration.Connection( + Duration.of(60, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + 0x1000, + 0x10000 + ), + emptyMap(), + emptyMap(), + InMemoryCacheConfiguration( + maxAge = Duration.ofSeconds(3600 * 24), + compressionEnabled = true, + digestAlgorithm = "MD5", + compressionLevel = Deflater.DEFAULT_COMPRESSION, + maxSize = 0x1000000, + ), + null, + null, + ) + Xml.write(Serializer.serialize(cfg), System.out) + } + + override fun tearDown() { + } + + fun newRequestBuilder(key: String) = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/$serverPath/$key")) + + fun newEntry(random: Random): Pair { + val key = ByteArray(0x10).let { + random.nextBytes(it) + Base64.getUrlEncoder().encodeToString(it) + } + val value = ByteArray(0x1000).also { + random.nextBytes(it) + } + return key to value + } + + @Test + @Order(1) + fun putWithNoAuthorizationHeader() { + val client: HttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build() + val (key, value) = keyValuePair + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.CREATED.code(), response.statusCode()) + } + + @Test + @Order(2) + fun getWithNoAuthorizationHeader() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + val requestBuilder = newRequestBuilder(key) + .GET() + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(3) + fun getMissingKey() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, _) = newEntry(random) + val requestBuilder = newRequestBuilder(key).GET() + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.statusCode()) + } + + @Test + @Order(4) + fun getUnhandledPath() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, _) = newEntry(random) + val requestBuilder = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/some/other/path/$key")) + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.statusCode()) + } + + @Test + @Order(5) + fun putUnhandledPath() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = newEntry(random) + val requestBuilder = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/some/other/path/$key")) + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.statusCode()) + } + + @Test + @Order(6) + fun getRelativeUnhandledPath() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, _) = newEntry(random) + val requestBuilder = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/some/nested/path/../../../some/other/path/$key")) + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.statusCode()) + } + + @Test + @Order(7) + fun getRelativePath() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + val requestBuilder = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/some/other/path/../../nested/path/$key")) + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(10) + fun traceTest() { + val client: HttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build() + val requestBuilder = newRequestBuilder("").method( + "TRACE", + HttpRequest.BodyPublishers.ofByteArray("sfgsdgfaiousfiuhsd".toByteArray()) + ) + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + println(String(response.body())) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/TlsServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/TlsServerTest.kt new file mode 100644 index 0000000..2ac8315 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/TlsServerTest.kt @@ -0,0 +1,182 @@ +package net.woggioni.rbcs.server.test + +import io.netty.handler.codec.http.HttpResponseStatus +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Role +import org.bouncycastle.asn1.x500.X500Name +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test + + +class TlsServerTest : AbstractTlsServerTest() { + + override val users = listOf( + Configuration.User("user1", null, setOf(readersGroup), null), + Configuration.User("user2", null, setOf(writersGroup), null), + Configuration.User("user3", null, setOf(readersGroup, writersGroup), null), + Configuration.User("user4", null, setOf(healthCheckGroup), null), + Configuration.User("", null, setOf(readersGroup), null) + ) + + @Test + @Order(1) + fun putAsAReaderUser() { + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Reader in it.roles && Role.Writer !in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(2) + fun getAsAWriterUser() { + val (key, _) = keyValuePair + val user = cfg.users.values.find { + Role.Writer in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + + val requestBuilder = newRequestBuilder(key) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(3) + fun putAsAWriterUser() { + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Writer in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.CREATED.code(), response.statusCode()) + } + + @Test + @Order(4) + fun getAsAReaderUser() { + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Reader in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + + val requestBuilder = newRequestBuilder(key) + .GET() + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(5) + fun getMissingKeyAsAReaderUser() { + val (key, _) = newEntry(random) + val user = cfg.users.values.find { + Role.Reader in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + + val requestBuilder = newRequestBuilder(key) + .GET() + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.statusCode()) + } + + @Test + @Order(6) + fun getAsAnonymousUser() { + val (key, value) = keyValuePair + val client: HttpClient = getHttpClient(null) + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(7) + fun putAsAnonymousUser() { + val (key, value) = keyValuePair + val client: HttpClient = getHttpClient(null) + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(8) + fun traceAsAnonymousUser() { + val client: HttpClient = getHttpClient(null) + val requestBuilder = newRequestBuilder("").method( + "TRACE", + HttpRequest.BodyPublishers.ofByteArray("this is an healthcheck".toByteArray()) + ) + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(9) + fun traceAsHealthcheckUser() { + val user = cfg.users.values.find { + Role.Healthcheck in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + val requestBuilder = newRequestBuilder("").method( + "TRACE", + HttpRequest.BodyPublishers.ofByteArray("this is an healthcheck".toByteArray()) + ) + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + println(String(response.body())) + } + + @Test + @Order(10) + fun putAsUnknownUserUser() { + val (key, value) = keyValuePair + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=Unknown user"))) + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), response.statusCode()) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/X500NameTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/X500NameTest.kt new file mode 100644 index 0000000..ca2a1fe --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/X500NameTest.kt @@ -0,0 +1,19 @@ +package net.woggioni.rbcs.server.test + +import javax.naming.ldap.LdapName +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class X500NameTest { + + @Test + fun test() { + val name = + "C=SG, L=Bugis, CN=woggioni@f6aa5663ef26, emailAddress=oggioni.walter@gmail.com, street=1 Fraser Street\\, Duo Residences #23-05, postalCode=189350, GN=Walter, SN=Oggioni, pseudonym=woggioni" + val ldapName = LdapName(name) + val value = ldapName.rdns.asSequence().find { + it.type == "CN" + }!!.value + Assertions.assertEquals("woggioni@f6aa5663ef26", value) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user.xml new file mode 100644 index 0000000..d4f7f89 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user2.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user2.xml new file mode 100644 index 0000000..0c28def --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user2.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/invalid-user-ref.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/invalid-user-ref.xml new file mode 100644 index 0000000..3d0971e --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/invalid-user-ref.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/multiple-user-quota.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/multiple-user-quota.xml new file mode 100644 index 0000000..5384928 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/multiple-user-quota.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$CertificateAuthority.class b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$CertificateAuthority.class new file mode 100644 index 0000000..b2e9bd7 Binary files /dev/null and b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$CertificateAuthority.class differ diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$X509Credentials.class b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$X509Credentials.class new file mode 100644 index 0000000..843a126 Binary files /dev/null and b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$X509Credentials.class differ diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils.class b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils.class new file mode 100644 index 0000000..d03fe28 Binary files /dev/null and b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils.class differ diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-default.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-default.xml new file mode 100644 index 0000000..984260c --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-default.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml new file mode 100644 index 0000000..ef308b5 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml new file mode 100644 index 0000000..0f654b6 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis-tls.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis-tls.xml new file mode 100644 index 0000000..7cd5c41 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis-tls.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis.xml new file mode 100644 index 0000000..eb7995d --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-tls.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-tls.xml new file mode 100644 index 0000000..5001fdf --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-tls.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt index 708d2ba..6118875 100644 --- a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt +++ b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt @@ -432,7 +432,7 @@ class RemoteBuildCacheServer(private val cfg: Configuration) { maxChunkSize = cfg.connection.chunkSize } pipeline.addLast(HttpServerCodec(httpDecoderConfig)) - OtelIntegration.createHandler()?.let { pipeline.addLast(it) } + OtelIntegration.createHandler(cfg.isEnableTelemetry)?.let { pipeline.addLast(it) } pipeline.addLast(ReadTriggerDuplexHandler.NAME, ReadTriggerDuplexHandler()) pipeline.addLast(MaxRequestSizeHandler.NAME, MaxRequestSizeHandler(cfg.connection.maxRequestSize)) pipeline.addLast(HttpChunkContentCompressor(1024)) diff --git a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Parser.kt b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Parser.kt index f1ac5c6..355fc6e 100644 --- a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Parser.kt +++ b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Parser.kt @@ -46,6 +46,8 @@ object Parser { var groups = emptyMap() var tls: Tls? = null val serverPath = root.renderAttribute("path") + var enableTelemetry = root.renderAttribute("enable-telemetry") + ?.let(String::toBoolean) ?: false var incomingConnectionsBacklogSize = 1024 var authentication: Authentication? = null for (child in root.asIterable()) { @@ -233,6 +235,7 @@ object Parser { } } return Configuration.of( + enableTelemetry, host, port, proxyProtocolEnabled, diff --git a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Serializer.kt b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Serializer.kt index 67e78d3..f7506fb 100644 --- a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Serializer.kt +++ b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Serializer.kt @@ -29,6 +29,7 @@ object Serializer { ?.let { serverPath -> attr("path", serverPath) } + attr("enable-telemetry", conf.isEnableTelemetry.toString()) node("bind") { attr("host", conf.host) attr("port", conf.port.toString()) diff --git a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/otel/OtelIntegration.kt b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/otel/OtelIntegration.kt index fcf9144..fff39de 100644 --- a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/otel/OtelIntegration.kt +++ b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/otel/OtelIntegration.kt @@ -22,8 +22,8 @@ object OtelIntegration { ) } - fun createHandler(): ChannelHandler? { - return if (isAvailable) createHandlerInternal() else null + fun createHandler(enabled: Boolean): ChannelHandler? { + return if (enabled && isAvailable) createHandlerInternal() else null } private fun createHandlerInternal(): ChannelHandler { diff --git a/rbcs-server/src/main/resources/net/woggioni/rbcs/server/rbcs-default.xml b/rbcs-server/src/main/resources/net/woggioni/rbcs/server/rbcs-default.xml index 6d45ea1..55f3710 100644 --- a/rbcs-server/src/main/resources/net/woggioni/rbcs/server/rbcs-default.xml +++ b/rbcs-server/src/main/resources/net/woggioni/rbcs/server/rbcs-default.xml @@ -2,7 +2,8 @@ + xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd" + enable-telemetry="false"> \ No newline at end of file diff --git a/rbcs-server/src/main/resources/net/woggioni/rbcs/server/schema/rbcs-server.xsd b/rbcs-server/src/main/resources/net/woggioni/rbcs/server/schema/rbcs-server.xsd index 65aa616..b4c5399 100644 --- a/rbcs-server/src/main/resources/net/woggioni/rbcs/server/schema/rbcs-server.xsd +++ b/rbcs-server/src/main/resources/net/woggioni/rbcs/server/schema/rbcs-server.xsd @@ -59,6 +59,14 @@ + + + + Enable OpenTelemetry distributed tracing for the server. + Even when enabled, telemetry only activates if OpenTelemetry classes are present on the classpath. + + + diff --git a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt index 31d024c..acb5474 100644 --- a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt +++ b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt @@ -32,6 +32,7 @@ abstract class AbstractBasicAuthServerTest : AbstractServerTest() { override fun setUp() { this.cacheDir = testDir.resolve("cache") cfg = Configuration.of( + false, "127.0.0.1", getFreePort(), false, diff --git a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt index ad8d660..f160c50 100644 --- a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt +++ b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt @@ -138,6 +138,7 @@ abstract class AbstractTlsServerTest : AbstractServerTest() { this.cacheDir = testDir.resolve("cache") createKeyStoreAndTrustStore() cfg = Configuration( + false, "127.0.0.1", getFreePort(), false, diff --git a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/NoAuthServerTest.kt b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/NoAuthServerTest.kt index d81f4d6..63b2a61 100644 --- a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/NoAuthServerTest.kt +++ b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/NoAuthServerTest.kt @@ -32,6 +32,7 @@ class NoAuthServerTest : AbstractServerTest() { override fun setUp() { this.cacheDir = testDir.resolve("cache") cfg = Configuration( + false, "127.0.0.1", getFreePort(), false, diff --git a/rbcs-servlet/.classpath b/rbcs-servlet/.classpath new file mode 100644 index 0000000..86ae890 --- /dev/null +++ b/rbcs-servlet/.classpath @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-servlet/.project b/rbcs-servlet/.project new file mode 100644 index 0000000..b60a501 --- /dev/null +++ b/rbcs-servlet/.project @@ -0,0 +1,44 @@ + + + rbcs-servlet + Project rbcs-servlet created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.wst.common.project.facet.core.builder + + + + + org.eclipse.wst.validation.validationbuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370777023 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-servlet/.settings/org.eclipse.buildship.core.prefs b/rbcs-servlet/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-servlet/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-servlet/.settings/org.eclipse.jdt.core.prefs b/rbcs-servlet/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-servlet/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-servlet/bin/main/META-INF/beans.xml b/rbcs-servlet/bin/main/META-INF/beans.xml new file mode 100644 index 0000000..253b7a3 --- /dev/null +++ b/rbcs-servlet/bin/main/META-INF/beans.xml @@ -0,0 +1,5 @@ + + \ No newline at end of file diff --git a/rbcs-servlet/bin/main/META-INF/context.xml b/rbcs-servlet/bin/main/META-INF/context.xml new file mode 100644 index 0000000..85bd8de --- /dev/null +++ b/rbcs-servlet/bin/main/META-INF/context.xml @@ -0,0 +1,7 @@ + + + + \ No newline at end of file diff --git a/rbcs-servlet/bin/main/logging.properties b/rbcs-servlet/bin/main/logging.properties new file mode 100644 index 0000000..436e24a --- /dev/null +++ b/rbcs-servlet/bin/main/logging.properties @@ -0,0 +1,8 @@ +handlers = java.util.logging.ConsoleHandler +.level=INFO +net.woggioni.rbcs.servlet.level=FINEST +java.util.logging.ConsoleHandler.level=INFO +java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format = %1$tF %1$tT [%4$s] %2$s %5$s %6$s%n +org.apache.catalina.core.ContainerBase.[Catalina].level=ALL +org.apache.catalina.core.ContainerBase.[Catalina].handlers=java.util.logging.ConsoleHandler diff --git a/rbcs-servlet/bin/main/net/woggioni/rbcs/servlet/CacheServlet.kt b/rbcs-servlet/bin/main/net/woggioni/rbcs/servlet/CacheServlet.kt new file mode 100644 index 0000000..bd5713a --- /dev/null +++ b/rbcs-servlet/bin/main/net/woggioni/rbcs/servlet/CacheServlet.kt @@ -0,0 +1,169 @@ +package net.woggioni.rbcs.servlet + +import jakarta.annotation.PreDestroy +import jakarta.enterprise.context.ApplicationScoped +import jakarta.inject.Inject +import jakarta.servlet.annotation.WebServlet +import jakarta.servlet.http.HttpServlet +import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.nio.file.Path +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.PriorityBlockingQueue +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong +import java.util.logging.Logger +import net.woggioni.jwo.HttpClient.HttpStatus +import net.woggioni.jwo.JWO + + +private class CacheKey(private val value: ByteArray) { + override fun equals(other: Any?) = if (other is CacheKey) { + value.contentEquals(other.value) + } else false + + override fun hashCode() = value.contentHashCode() +} + + +@ApplicationScoped +open class InMemoryServletCache : AutoCloseable { + + private val maxAge= Duration.ofDays(7) + private val maxSize = 0x8000000 + + companion object { + @JvmStatic + private val log = Logger.getLogger(this::class.java.name) + } + + private val size = AtomicLong() + private val map = ConcurrentHashMap() + + private class RemovalQueueElement(val key: CacheKey, val value: ByteArray, val expiry: Instant) : + Comparable { + override fun compareTo(other: RemovalQueueElement) = expiry.compareTo(other.expiry) + } + + private val removalQueue = PriorityBlockingQueue() + + @Volatile + private var running = false + + private val garbageCollector = Thread.ofVirtual().name("in-memory-cache-gc").start { + while (running) { + val el = removalQueue.poll(1, TimeUnit.SECONDS) ?: continue + val value = el.value + val now = Instant.now() + if (now > el.expiry) { + val removed = map.remove(el.key, value) + if (removed) { + updateSizeAfterRemoval(value) + } + } else { + removalQueue.put(el) + Thread.sleep(minOf(Duration.between(now, el.expiry), Duration.ofSeconds(1))) + } + } + } + + private fun removeEldest(): Long { + while (true) { + val el = removalQueue.take() + val value = el.value + val removed = map.remove(el.key, value) + if (removed) { + val newSize = updateSizeAfterRemoval(value) + return newSize + } + } + } + + private fun updateSizeAfterRemoval(removed: ByteArray): Long { + return size.updateAndGet { currentSize: Long -> + currentSize - removed.size + } + } + + @PreDestroy + override fun close() { + running = false + garbageCollector.join() + } + + open fun get(key: ByteArray) = map[CacheKey(key)] + + open fun put( + key: ByteArray, + value: ByteArray, + ) { + val cacheKey = CacheKey(key) + val oldSize = map.put(cacheKey, value)?.let { old -> + val result = old.size + result + } ?: 0 + val delta = value.size - oldSize + var newSize = size.updateAndGet { currentSize: Long -> + currentSize + delta + } + removalQueue.put(RemovalQueueElement(cacheKey, value, Instant.now().plus(maxAge))) + while (newSize > maxSize) { + newSize = removeEldest() + } + } +} + + +@WebServlet(urlPatterns = ["/cache/*"]) +class CacheServlet : HttpServlet() { + companion object { + @JvmStatic + private val log = Logger.getLogger(this::class.java.name) + } + + @Inject + private lateinit var cache : InMemoryServletCache + + private fun getKey(req : HttpServletRequest) : String { + return Path.of(req.pathInfo).fileName.toString() + } + + override fun doPut(req: HttpServletRequest, resp: HttpServletResponse) { + val baos = ByteArrayOutputStream() + baos.use { + JWO.copy(req.inputStream, baos) + } + val key = getKey(req) + cache.put(key.toByteArray(Charsets.UTF_8), baos.toByteArray()) + resp.status = 201 + resp.setContentLength(0) + log.fine { + "[${Thread.currentThread().name}] Added value for key $key" + } + } + + override fun doGet(req: HttpServletRequest, resp: HttpServletResponse) { + val key = getKey(req) + val value = cache.get(key.toByteArray(Charsets.UTF_8)) + if (value == null) { + log.fine { + "[${Thread.currentThread().name}] Cache miss for key $key" + } + resp.status = HttpStatus.NOT_FOUND.code + resp.setContentLength(0) + } else { + log.fine { + "[${Thread.currentThread().name}] Cache hit for key $key" + } + resp.status = HttpStatus.OK.code + resp.setContentLength(value.size) + ByteArrayInputStream(value).use { + JWO.copy(it, resp.outputStream) + } + } + } +} \ No newline at end of file