From ee7bc7e8509e78cd94a8ab2ff5483d24a36a0ceb Mon Sep 17 00:00:00 2001 From: OpenCode Date: Tue, 28 Apr 2026 14:59:08 +0000 Subject: [PATCH] Add optional OpenTelemetry Netty server instrumentation - Update lys.version to 2026.04.14 - Add optional compileOnly dependency on opentelemetry-netty-4.1 in rbcs-server - Add runtime guard to only activate instrumentation when OTel classes are on classpath - Insert OTel combined handler after HttpServerCodec in the Netty pipeline - Add requires-static JPMS directives for optional module support - Add enableTelemetry config attribute to rbcs:server with default false - Update Configuration DTO, XSD schema, Parser, Serializer, and all tests --- .project | 28 + .settings/org.eclipse.buildship.core.prefs | 13 + docker/.project | 28 + .../org.eclipse.buildship.core.prefs | 2 + rbcs-api/.classpath | 12 + rbcs-api/.project | 34 + .../org.eclipse.buildship.core.prefs | 2 + rbcs-api/.settings/org.eclipse.jdt.core.prefs | 4 + rbcs-api/bin/main/module-info.class | Bin 0 -> 515 bytes .../woggioni/rbcs/api/AsyncCloseable.class | Bin 0 -> 631 bytes .../net/woggioni/rbcs/api/CacheHandler.class | Bin 0 -> 2960 bytes .../rbcs/api/CacheHandlerFactory.class | Bin 0 -> 664 bytes .../net/woggioni/rbcs/api/CacheProvider.class | Bin 0 -> 744 bytes .../rbcs/api/CacheValueMetadata.class | Bin 0 -> 782 bytes .../api/Configuration$Authentication.class | Bin 0 -> 252 bytes .../Configuration$BasicAuthentication.class | Bin 0 -> 1277 bytes .../rbcs/api/Configuration$Cache.class | Bin 0 -> 1241 bytes ...tion$ClientCertificateAuthentication.class | Bin 0 -> 2328 bytes .../rbcs/api/Configuration$Connection.class | Bin 0 -> 2306 bytes .../api/Configuration$EventExecutor.class | Bin 0 -> 1735 bytes ...ardedClientCertificateAuthentication.class | Bin 0 -> 2509 bytes .../rbcs/api/Configuration$Group.class | Bin 0 -> 2632 bytes .../api/Configuration$GroupExtractor.class | Bin 0 -> 1357 bytes .../rbcs/api/Configuration$KeyStore.class | Bin 0 -> 2166 bytes .../rbcs/api/Configuration$Quota.class | Bin 0 -> 2131 bytes .../rbcs/api/Configuration$RateLimiter.class | Bin 0 -> 1983 bytes .../woggioni/rbcs/api/Configuration$Tls.class | Bin 0 -> 2232 bytes ...onfiguration$TlsCertificateExtractor.class | Bin 0 -> 1901 bytes .../rbcs/api/Configuration$TrustStore.class | Bin 0 -> 2200 bytes .../rbcs/api/Configuration$User.class | Bin 0 -> 2812 bytes .../api/Configuration$UserExtractor.class | Bin 0 -> 1352 bytes .../net/woggioni/rbcs/api/Configuration.class | Bin 0 -> 8150 bytes .../bin/main/net/woggioni/rbcs/api/Role.class | Bin 0 -> 1048 bytes .../rbcs/api/exception/CacheException.class | Bin 0 -> 594 bytes .../exception/ConfigurationException.class | Bin 0 -> 618 bytes .../exception/ContentTooLargeException.class | Bin 0 -> 518 bytes .../rbcs/api/exception/RbcsException.class | Bin 0 -> 466 bytes .../message/CacheMessage$CacheContent.class | Bin 0 -> 3238 bytes .../CacheMessage$CacheGetRequest.class | Bin 0 -> 723 bytes .../CacheMessage$CacheGetResponse.class | Bin 0 -> 978 bytes .../CacheMessage$CachePutRequest.class | Bin 0 -> 988 bytes .../CacheMessage$CachePutResponse.class | Bin 0 -> 726 bytes ...CacheMessage$CacheValueFoundResponse.class | Bin 0 -> 967 bytes ...heMessage$CacheValueNotFoundResponse.class | Bin 0 -> 632 bytes .../CacheMessage$LastCacheContent.class | Bin 0 -> 3478 bytes .../rbcs/api/message/CacheMessage.class | Bin 0 -> 994 bytes .../net/woggioni/rbcs/api/Configuration.java | 3 + rbcs-cli/.classpath | 24 + rbcs-cli/.project | 34 + .../org.eclipse.buildship.core.prefs | 2 + rbcs-cli/.settings/org.eclipse.jdt.core.prefs | 4 + .../graal/GraalNativeImageConfiguration.kt | 200 +++++ rbcs-cli/bin/main/module-info.class | Bin 0 -> 582 bytes .../rbcs/cli/RemoteBuildCacheServerCli.kt | 79 ++ .../rbcs/cli/impl/AbstractVersionProvider.kt | 30 + .../net/woggioni/rbcs/cli/impl/RbcsCommand.kt | 19 + .../cli/impl/commands/BenchmarkCommand.kt | 199 +++++ .../rbcs/cli/impl/commands/ClientCommand.kt | 48 ++ .../rbcs/cli/impl/commands/GetCommand.kt | 59 ++ .../cli/impl/commands/HealthCheckCommand.kt | 53 ++ .../cli/impl/commands/PasswordHashCommand.kt | 37 + .../rbcs/cli/impl/commands/PutCommand.kt | 113 +++ .../rbcs/cli/impl/commands/ServerCommand.kt | 90 ++ .../cli/impl/converters/ByteSizeConverter.kt | 10 + .../cli/impl/converters/DurationConverter.kt | 11 + .../impl/converters/InputStreamConverter.kt | 13 + .../impl/converters/OutputStreamConverter.kt | 13 + rbcs-client/.classpath | 38 + rbcs-client/.project | 34 + .../org.eclipse.buildship.core.prefs | 2 + .../.settings/org.eclipse.jdt.core.prefs | 4 + rbcs-client/bin/main/module-info.class | Bin 0 -> 546 bytes .../net/woggioni/rbcs/client/Configuration.kt | 62 ++ .../net/woggioni/rbcs/client/Exception.kt | 9 + .../rbcs/client/RemoteBuildCacheClient.kt | 433 ++++++++++ .../net/woggioni/rbcs/client/impl/Parser.kt | 151 ++++ .../main/net/woggioni/rbcs/client/retry.kt | 79 ++ .../rbcs/client/schema/rbcs-client.xsd | 260 ++++++ rbcs-client/bin/test/logback.xml | 21 + .../net/woggioni/rbcs/client/RetryTest.kt | 152 ++++ .../woggioni/rbcs/client/test/rbcs-client.xml | 18 + rbcs-common/.classpath | 31 + rbcs-common/.project | 34 + .../org.eclipse.buildship.core.prefs | 2 + .../.settings/org.eclipse.jdt.core.prefs | 4 + .../java.net.spi.URLStreamHandlerProvider | 1 + .../bin/main/net/woggioni/rbcs/common/BB.kt | 15 + .../rbcs/common/ByteBufInputStream.kt | 25 + .../rbcs/common/ByteBufOutputStream.kt | 18 + .../bin/main/net/woggioni/rbcs/common/Cidr.kt | 62 ++ .../net/woggioni/rbcs/common/Exception.kt | 7 + .../net/woggioni/rbcs/common/HostAndPort.kt | 8 + .../main/net/woggioni/rbcs/common/Logging.kt | 191 +++++ .../woggioni/rbcs/common/PasswordSecurity.kt | 57 ++ .../bin/main/net/woggioni/rbcs/common/RBCS.kt | 167 ++++ .../common/RbcsUrlStreamHandlerFactory.kt | 113 +++ .../bin/main/net/woggioni/rbcs/common/Xml.kt | 243 ++++++ .../test/net/woggioni/rbcs/common/CidrTest.kt | 17 + .../rbcs/common/PasswordHashingTest.kt | 38 + rbcs-server-memcache/.classpath | 31 + rbcs-server-memcache/.project | 34 + .../org.eclipse.buildship.core.prefs | 2 + .../.settings/org.eclipse.jdt.core.prefs | 4 + .../net.woggioni.rbcs.api.CacheProvider | 1 + .../rbcs/server/memcache/Exception.kt | 4 + .../memcache/MemcacheCacheConfiguration.kt | 105 +++ .../server/memcache/MemcacheCacheHandler.kt | 442 ++++++++++ .../server/memcache/MemcacheCacheProvider.kt | 102 +++ .../server/memcache/client/MemcacheClient.kt | 198 +++++ .../client/MemcacheRequestController.kt | 13 + .../client/MemcacheResponseHandler.kt | 14 + .../server/memcache/schema/rbcs-memcache.xsd | 45 + .../server/memcache/client/ByteBufferTest.kt | 27 + rbcs-server-redis/.classpath | 24 + rbcs-server-redis/.project | 34 + .../org.eclipse.buildship.core.prefs | 2 + .../.settings/org.eclipse.jdt.core.prefs | 4 + .../net.woggioni.rbcs.api.CacheProvider | 1 + .../woggioni/rbcs/server/redis/Exception.kt | 4 + .../server/redis/RedisCacheConfiguration.kt | 107 +++ .../rbcs/server/redis/RedisCacheHandler.kt | 438 ++++++++++ .../rbcs/server/redis/RedisCacheProvider.kt | 108 +++ .../rbcs/server/redis/client/RedisClient.kt | 204 +++++ .../redis/client/RedisResponseHandler.kt | 10 + .../rbcs/server/redis/schema/rbcs-redis.xsd | 52 ++ rbcs-server/.classpath | 45 + rbcs-server/.project | 34 + .../org.eclipse.buildship.core.prefs | 2 + .../.settings/org.eclipse.jdt.core.prefs | 4 + .../net.woggioni.rbcs.api.CacheProvider | 2 + .../rbcs/server/RemoteBuildCacheServer.kt | 562 +++++++++++++ .../rbcs/server/auth/Authenticator.kt | 88 ++ .../woggioni/rbcs/server/auth/Authorizer.kt | 8 + .../rbcs/server/auth/UserAuthorizer.kt | 24 + .../rbcs/server/cache/FileSystemCache.kt | 169 ++++ .../cache/FileSystemCacheConfiguration.kt | 38 + .../server/cache/FileSystemCacheHandler.kt | 137 ++++ .../server/cache/FileSystemCacheProvider.kt | 65 ++ .../rbcs/server/cache/InMemoryCache.kt | 131 +++ .../cache/InMemoryCacheConfiguration.kt | 35 + .../rbcs/server/cache/InMemoryCacheHandler.kt | 155 ++++ .../server/cache/InMemoryCacheProvider.kt | 62 ++ .../server/configuration/CacheSerializers.kt | 15 + .../rbcs/server/configuration/Parser.kt | 361 +++++++++ .../rbcs/server/configuration/Serializer.kt | 218 +++++ .../rbcs/server/exception/ExceptionHandler.kt | 139 ++++ .../server/handler/BlackHoleRequestHandler.kt | 13 + .../server/handler/MaxRequestSizeHandler.kt | 40 + .../server/handler/ProxyProtocolHandler.kt | 47 ++ .../handler/ReadTriggerDuplexHandler.kt | 65 ++ .../rbcs/server/handler/ServerHandler.kt | 227 ++++++ .../rbcs/server/handler/TraceHandler.kt | 55 ++ .../rbcs/server/otel/OtelIntegration.kt | 32 + .../net/woggioni/rbcs/server/rbcs-default.xml | 8 + .../rbcs/server/schema/rbcs-server.xsd | 767 ++++++++++++++++++ .../rbcs/server/throttling/BucketManager.kt | 91 +++ .../server/throttling/ThrottlingHandler.kt | 228 ++++++ rbcs-server/bin/test/logback.xml | 21 + .../test/AbstractBasicAuthServerTest.kt | 89 ++ .../rbcs/server/test/AbstractServerTest.kt | 51 ++ .../rbcs/server/test/AbstractTlsServerTest.kt | 206 +++++ .../rbcs/server/test/BasicAuthServerTest.kt | 192 +++++ .../rbcs/server/test/ConfigurationTest.kt | 58 ++ .../NoAnonymousUserBasicAuthServerTest.kt | 52 ++ .../test/NoAnonymousUserTlsServerTest.kt | 47 ++ .../rbcs/server/test/NoAuthServerTest.kt | 186 +++++ .../rbcs/server/test/TlsServerTest.kt | 182 +++++ .../woggioni/rbcs/server/test/X500NameTest.kt | 19 + .../test/invalid/duplicate-anonymous-user.xml | 19 + .../invalid/duplicate-anonymous-user2.xml | 25 + .../server/test/invalid/invalid-user-ref.xml | 24 + .../test/invalid/multiple-user-quota.xml | 15 + ...ertificateUtils$CertificateAuthority.class | Bin 0 -> 986 bytes .../CertificateUtils$X509Credentials.class | Bin 0 -> 1795 bytes .../server/test/utils/CertificateUtils.class | Bin 0 -> 6730 bytes .../rbcs/server/test/valid/rbcs-default.xml | 24 + .../server/test/valid/rbcs-memcached-tls.xml | 53 ++ .../rbcs/server/test/valid/rbcs-memcached.xml | 21 + .../rbcs/server/test/valid/rbcs-redis-tls.xml | 53 ++ .../rbcs/server/test/valid/rbcs-redis.xml | 21 + .../rbcs/server/test/valid/rbcs-tls.xml | 67 ++ .../rbcs/server/RemoteBuildCacheServer.kt | 2 +- .../rbcs/server/configuration/Parser.kt | 3 + .../rbcs/server/configuration/Serializer.kt | 1 + .../rbcs/server/otel/OtelIntegration.kt | 4 +- .../net/woggioni/rbcs/server/rbcs-default.xml | 3 +- .../rbcs/server/schema/rbcs-server.xsd | 8 + .../test/AbstractBasicAuthServerTest.kt | 1 + .../rbcs/server/test/AbstractTlsServerTest.kt | 1 + .../rbcs/server/test/NoAuthServerTest.kt | 1 + rbcs-servlet/.classpath | 19 + rbcs-servlet/.project | 44 + .../org.eclipse.buildship.core.prefs | 2 + .../.settings/org.eclipse.jdt.core.prefs | 4 + rbcs-servlet/bin/main/META-INF/beans.xml | 5 + rbcs-servlet/bin/main/META-INF/context.xml | 7 + rbcs-servlet/bin/main/logging.properties | 8 + .../net/woggioni/rbcs/servlet/CacheServlet.kt | 169 ++++ 198 files changed, 11040 insertions(+), 4 deletions(-) create mode 100644 .project create mode 100644 .settings/org.eclipse.buildship.core.prefs create mode 100644 docker/.project create mode 100644 docker/.settings/org.eclipse.buildship.core.prefs create mode 100644 rbcs-api/.classpath create mode 100644 rbcs-api/.project create mode 100644 rbcs-api/.settings/org.eclipse.buildship.core.prefs create mode 100644 rbcs-api/.settings/org.eclipse.jdt.core.prefs create mode 100644 rbcs-api/bin/main/module-info.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/AsyncCloseable.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/CacheHandler.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/CacheHandlerFactory.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/CacheProvider.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/CacheValueMetadata.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Authentication.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$BasicAuthentication.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Cache.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$ClientCertificateAuthentication.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Connection.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$EventExecutor.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$ForwardedClientCertificateAuthentication.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Group.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$GroupExtractor.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$KeyStore.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Quota.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$RateLimiter.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Tls.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$TlsCertificateExtractor.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$TrustStore.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$User.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$UserExtractor.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/Role.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/exception/CacheException.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/exception/ConfigurationException.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/exception/ContentTooLargeException.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/exception/RbcsException.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheContent.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetRequest.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetResponse.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CachePutRequest.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CachePutResponse.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheValueFoundResponse.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheValueNotFoundResponse.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$LastCacheContent.class create mode 100644 rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage.class create mode 100644 rbcs-cli/.classpath create mode 100644 rbcs-cli/.project create mode 100644 rbcs-cli/.settings/org.eclipse.buildship.core.prefs create mode 100644 rbcs-cli/.settings/org.eclipse.jdt.core.prefs create mode 100644 rbcs-cli/bin/configureNativeImage/net/woggioni/rbcs/cli/graal/GraalNativeImageConfiguration.kt create mode 100644 rbcs-cli/bin/main/module-info.class create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/RemoteBuildCacheServerCli.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/AbstractVersionProvider.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/RbcsCommand.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/BenchmarkCommand.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ClientCommand.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/GetCommand.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/HealthCheckCommand.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PasswordHashCommand.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PutCommand.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ServerCommand.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/ByteSizeConverter.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/DurationConverter.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/InputStreamConverter.kt create mode 100644 rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/OutputStreamConverter.kt create mode 100644 rbcs-client/.classpath create mode 100644 rbcs-client/.project create mode 100644 rbcs-client/.settings/org.eclipse.buildship.core.prefs create mode 100644 rbcs-client/.settings/org.eclipse.jdt.core.prefs create mode 100644 rbcs-client/bin/main/module-info.class create mode 100644 rbcs-client/bin/main/net/woggioni/rbcs/client/Configuration.kt create mode 100644 rbcs-client/bin/main/net/woggioni/rbcs/client/Exception.kt create mode 100644 rbcs-client/bin/main/net/woggioni/rbcs/client/RemoteBuildCacheClient.kt create mode 100644 rbcs-client/bin/main/net/woggioni/rbcs/client/impl/Parser.kt create mode 100644 rbcs-client/bin/main/net/woggioni/rbcs/client/retry.kt create mode 100644 rbcs-client/bin/main/net/woggioni/rbcs/client/schema/rbcs-client.xsd create mode 100644 rbcs-client/bin/test/logback.xml create mode 100644 rbcs-client/bin/test/net/woggioni/rbcs/client/RetryTest.kt create mode 100644 rbcs-client/bin/test/net/woggioni/rbcs/client/test/rbcs-client.xml create mode 100644 rbcs-common/.classpath create mode 100644 rbcs-common/.project create mode 100644 rbcs-common/.settings/org.eclipse.buildship.core.prefs create mode 100644 rbcs-common/.settings/org.eclipse.jdt.core.prefs create mode 100644 rbcs-common/bin/main/META-INF/services/java.net.spi.URLStreamHandlerProvider create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/BB.kt create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufInputStream.kt create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufOutputStream.kt create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/Cidr.kt create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/Exception.kt create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/HostAndPort.kt create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/Logging.kt create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/PasswordSecurity.kt create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/RBCS.kt create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/RbcsUrlStreamHandlerFactory.kt create mode 100644 rbcs-common/bin/main/net/woggioni/rbcs/common/Xml.kt create mode 100644 rbcs-common/bin/test/net/woggioni/rbcs/common/CidrTest.kt create mode 100644 rbcs-common/bin/test/net/woggioni/rbcs/common/PasswordHashingTest.kt create mode 100644 rbcs-server-memcache/.classpath create mode 100644 rbcs-server-memcache/.project create mode 100644 rbcs-server-memcache/.settings/org.eclipse.buildship.core.prefs create mode 100644 rbcs-server-memcache/.settings/org.eclipse.jdt.core.prefs create mode 100644 rbcs-server-memcache/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider create mode 100644 rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/Exception.kt create mode 100644 rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheConfiguration.kt create mode 100644 rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheHandler.kt create mode 100644 rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheProvider.kt create mode 100644 rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheClient.kt create mode 100644 rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheRequestController.kt create mode 100644 rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheResponseHandler.kt create mode 100644 rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd create mode 100644 rbcs-server-memcache/bin/test/net/woggioni/rbcs/server/memcache/client/ByteBufferTest.kt create mode 100644 rbcs-server-redis/.classpath create mode 100644 rbcs-server-redis/.project create mode 100644 rbcs-server-redis/.settings/org.eclipse.buildship.core.prefs create mode 100644 rbcs-server-redis/.settings/org.eclipse.jdt.core.prefs create mode 100644 rbcs-server-redis/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider create mode 100644 rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/Exception.kt create mode 100644 rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheConfiguration.kt create mode 100644 rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheHandler.kt create mode 100644 rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheProvider.kt create mode 100644 rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisClient.kt create mode 100644 rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisResponseHandler.kt create mode 100644 rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd create mode 100644 rbcs-server/.classpath create mode 100644 rbcs-server/.project create mode 100644 rbcs-server/.settings/org.eclipse.buildship.core.prefs create mode 100644 rbcs-server/.settings/org.eclipse.jdt.core.prefs create mode 100644 rbcs-server/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authenticator.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authorizer.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/auth/UserAuthorizer.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCache.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheConfiguration.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheHandler.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheProvider.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCache.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheConfiguration.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheHandler.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheProvider.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/CacheSerializers.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Parser.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Serializer.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/exception/ExceptionHandler.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/handler/BlackHoleRequestHandler.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/handler/MaxRequestSizeHandler.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ProxyProtocolHandler.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ReadTriggerDuplexHandler.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ServerHandler.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/handler/TraceHandler.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/otel/OtelIntegration.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/rbcs-default.xml create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/schema/rbcs-server.xsd create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/BucketManager.kt create mode 100644 rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/ThrottlingHandler.kt create mode 100644 rbcs-server/bin/test/logback.xml create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractServerTest.kt create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/BasicAuthServerTest.kt create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/ConfigurationTest.kt create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserTlsServerTest.kt create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAuthServerTest.kt create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/TlsServerTest.kt create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/X500NameTest.kt create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user.xml create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user2.xml create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/invalid-user-ref.xml create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/multiple-user-quota.xml create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$CertificateAuthority.class create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$X509Credentials.class create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils.class create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-default.xml create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis-tls.xml create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis.xml create mode 100644 rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-tls.xml create mode 100644 rbcs-servlet/.classpath create mode 100644 rbcs-servlet/.project create mode 100644 rbcs-servlet/.settings/org.eclipse.buildship.core.prefs create mode 100644 rbcs-servlet/.settings/org.eclipse.jdt.core.prefs create mode 100644 rbcs-servlet/bin/main/META-INF/beans.xml create mode 100644 rbcs-servlet/bin/main/META-INF/context.xml create mode 100644 rbcs-servlet/bin/main/logging.properties create mode 100644 rbcs-servlet/bin/main/net/woggioni/rbcs/servlet/CacheServlet.kt diff --git a/.project b/.project new file mode 100644 index 0000000..c130c71 --- /dev/null +++ b/.project @@ -0,0 +1,28 @@ + + + rbcs + Project rbcs created by Buildship. + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776929 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/.settings/org.eclipse.buildship.core.prefs b/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..5be7729 --- /dev/null +++ b/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,13 @@ +arguments=--init-script /var/lib/opencode/.eclipse/2139989682_linux_gtk_x86_64/configuration/org.eclipse.osgi/58/0/.cp/gradle/init/init.gradle +auto.sync=false +build.scans.enabled=false +connection.gradle.distribution=GRADLE_DISTRIBUTION(WRAPPER) +connection.project.dir= +eclipse.preferences.version=1 +gradle.user.home= +java.home=/usr/lib/jvm/java-26-openjdk +jvm.arguments= +offline.mode=false +override.workspace.settings=true +show.console.view=true +show.executions.view=true diff --git a/docker/.project b/docker/.project new file mode 100644 index 0000000..dc86921 --- /dev/null +++ b/docker/.project @@ -0,0 +1,28 @@ + + + docker + Project docker created by Buildship. + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776861 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/docker/.settings/org.eclipse.buildship.core.prefs b/docker/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/docker/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-api/.classpath b/rbcs-api/.classpath new file mode 100644 index 0000000..5f31ee4 --- /dev/null +++ b/rbcs-api/.classpath @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/rbcs-api/.project b/rbcs-api/.project new file mode 100644 index 0000000..f59a16d --- /dev/null +++ b/rbcs-api/.project @@ -0,0 +1,34 @@ + + + rbcs-api + Project rbcs-api created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776935 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-api/.settings/org.eclipse.buildship.core.prefs b/rbcs-api/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-api/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-api/.settings/org.eclipse.jdt.core.prefs b/rbcs-api/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-api/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-api/bin/main/module-info.class b/rbcs-api/bin/main/module-info.class new file mode 100644 index 0000000000000000000000000000000000000000..219fd1e6e436ba0601a3ec1afb37075bf38f6b84 GIT binary patch literal 515 zcmZvZ%TmHH6o&tRaw#fEisA)P_of>kz>V%)xpyWlX(_f%rwxibp38*~;6pj4hk;=R zv-omuIVXSe^ZWG;;1Rb?)EGLcp3D<<7iXbna9;Gh2-H)YD2CovO3dWDWN187HAj;>`x(a>L#xsy z7HPt9VUy(fvQ#ccD}~-#p;syNtU?!KhOxcsZpu@OK;_0hh>IJB(f>uMDoYuuCBW=Q xtX2(mYt*oZeY?M+L170iJXp&i?|^tj>=FA!mv}-P63>^_R>viISHx=~;ScDVc)S1r literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/AsyncCloseable.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/AsyncCloseable.class new file mode 100644 index 0000000000000000000000000000000000000000..45c46df68548c762a30a6e61bdfe10f08a52f1be GIT binary patch literal 631 zcma)4O;5r=5PgearBp#d@!N|rz`>pk#DowyHG06{d094fP3e+$>&MUXVxk9sfIrGO z1qHov=+^ zWhmYA2kr(u^xf;>y%fg6DnoT~+D?p~moV6T-h-i0t9#ST#Hhd(IuuD1$OBc3g1XFVmXRIy8xzBCM#U;Tf69O){0DwJ2rcqpTr)H{)R zTJT`NBbDNx!>qYeF%f&qOo`gseVs%?_Eg%j{C|k&sd}0iDWfk(Zy-ZcC4+?WDS}N= z0S-m$gtLS;P@?F9B1U_m@rF%?SCnsGrUiSn+LTULv^&_R$eE?$0EdJaj%dx`_!|m7 Bq$L0V literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheHandler.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheHandler.class new file mode 100644 index 0000000000000000000000000000000000000000..74e28771b79662744ffd81d8dff4648fe7558b73 GIT binary patch literal 2960 zcmb7GT~`xV6x}xjCWH|cib_z6f=U89+R|zbYH3OkiD*OCY_o1@l)Ur z`0Q${r4P2g_W$&ytE;Q~PKG2R3M6ag%)N8&`PgUQbMn`JzrO-7g%u4kfs>95jK^-N zWV?=ScuSUVm{r@znO0d|H=SZddK%&aXKmM@YVg#s%BJJUijk|!8E46@ImJkKvS?NV z=?Sz|+>$^~!SzapUs)bsF$#1dy{y2%e9Z~$RXO84vVD80A}1Zk4a|TM`07Z-U0re? z8q?B|o*Bp@kL>s4Pc`WWdE2r5vMdUSdjg3GtHO=oD&38w7X{)uwpj)cgI1am>Ln_#goU5%a zN$-xyB*J7Zv$AMV5!FU z1a7_-)I6EhaD8ti3ki0+7R*b($}S}}+z^=B14}~#i)N)J=iDH#8fMc}L145&F*a4z za}A4~%reSBP&E|8uk#@yxI$r0$1U6@3d;&`Ua*mLv#9T7X;vMd*Rg;*Y@k)UD%nDb zW$(b`jrbxJwQ-0^Z_nQ&-2gV2keXt16bv%C;MP1# z=4~|=4z|wHb1DS3@)?d$FPt-y1K_|`ehH-B%^2!$ZF=z&ymM_F>oKI%c;ZUqO36gH z%W*@Q;I|7pclU9Xrc|HK9`PHxf5r#TxrpHqf7LnKN+9GeH;x_8Lx*~PelccPw;7&-UD33 zB?2=}gg?V&0x;Wn%2Klj9jRFj6aS)*zyClS=D#ZX*o5E)SE_yqCK`W%YfVU#v9GQ9;4LGJ7a7;&VFi18_b$wMK+R-+ANKGhSWajV#x8hvg4{yW_ zl~ffpR^cUn+**W-ycd^Rz|tCCwT4#>OR#7-h9V8cAhArDqSN2Vg-r_y9pjD73JM#3Cn{7~{HxnaFWkEtx>7X~Yb!HUGku5jP-!f6wbkWjrW^EV96o<@{c#~l!L3tzYFD>C9Z1Z@Dl8!$0(QdN3#k(!LNOeK OX%wailF%KlyQ5FHo8U44 literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheProvider.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/CacheProvider.class new file mode 100644 index 0000000000000000000000000000000000000000..b652b8ab1d256e245ab97bee4fda1758be002833 GIT binary patch literal 744 zcmb7COHaZ;5S~S$@=y@*dDR2GxF+7fcn~o$Y1E`Wa9x(QE@^kkwuR_V^WYEgM;WJu zC?qzK(`078neUs&=hyoO0QBM5f&zh4C1N_$zAv?s)C_q<*+kMF<1gaL=&2kDW5FhY z#+XeR4Vd!j<8Ul^OrYh9cn}6ILPO?g&SGqW;kNDkg5B6i<#!3x=jaC(ifF>PAW)e{ zyw?dj)scvVkt~pJ=-sp(ZG1Yr;txWiU`p<1H^X-7W+v=TD*99k7Qu@? zfDa{}*#;C6uov@p<~!dxXXe-MpXUHx;eG=Zfu7Y~esXDQoHa6!lR~P6kwcXn>#53? zdaS*gDX$u+30yl-r%GnZrt*D!q7&ahU7%Z{#>t7!jmpdy6=zzYlQ`?O^)F4aaE0+y zAUOKlV&ZdS(`N$Pb2HaRpBE(EOtl|w;B|ZbP0WK2OY2E~YfnvKDEq=%=T*=uf>`F} zvHK|BXsdJO^^9V*p_}O@7$!mk+XC&8vHIO|9_##Q<$UB4l}#1Xu;mJ@`D4SLgVAMQ zj9NQPGhSs2l6w)FL9%{mtntqU9`)8!tt0(u3soFM*vB1#YOg;PXieNQPxNaOo_cTn zzn+G7#zTr$Ikr$iC&cB~cY+750dHZ4HxW;Dp004c$|1QIT(<__(f-O!1=l%;5|G;A z99!A0rH8%bsG P;JsZ&uh8$^l6mhBWtpP; literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Authentication.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Authentication.class new file mode 100644 index 0000000000000000000000000000000000000000..e4e68bfae7b9f629c6d1150910f5d3a403b0bd1d GIT binary patch literal 252 zcma)1OA5j;5Pf5{KZ}T0(6s??AVtxQE<8aRVofQDG^yUrg$M9ZV%lAI&f*R8=JDRl z=lgmC*ddEBB5bsD>}kqU8Le1T2+R0gvE1mZD%*xT&#$(vyGiYo=*I|S!o1=S&T6hp zb}lL@93j1!wh?l#YDriO9M+whkR7#_OS`joh}HHOw@jy-FQt>ZeYe0ry?bS68MBc7Oisv@_@B=+K&vXZwe)yT!-Wt+4 zxyq-;dQRQ;eh;u$46sKPn=lhWM(#zuT`03PVc>)MFkA?1L^!5 z<2)XM_MF~T`VldD0@TSi$lEF7K=>8m`*!m)*1l7;f*WLOgemZCvd!`wVgonHTfrvT a3U1*xecN}D6gB!j68G~x16Jj?RbgtM3 z#%R-k7OQ+ZRR*ojE->M-&I_59L3m;GG{fA!;z4sVN1mgb>x8H1CT3wBCdWp9SPqYE zug_;j=iCv=Eeqt=lH~?xXe@WSB;?XssnQiuCg#$)%nNNCr{V{{iy!#n2b58+)gU4! zkA+g&@hOte%Y=jGnjb!#D1-bf%Enu;(f)4C`Ty#F01dd=hFj2KP}^V4_fBtgHPI)| z&|vNPiSLZ0!to-XqL~O9Vg}ujjzxARjP&*jY~{isA({M0l$X8!XpQ*1+>V4wGBk%G zcG@iWC zgtchv@QA^)HF^EH6c}ukYk?NP7_B?f*3}r$4xr@%P^I>5L5=J>Sua*-@D&>Gy1{2? ze z!9){%_eUAeZV@a}DFJ-goZU0$eCNz}F8kx>mu~<{xEDi&q3Cgwe=nO&Aw7{-)tb)R zZILfZuOXVgvJL4MijLr(DRN~*L)2`;7k$$rP4G_VF$^+{ZQ5IQ-m$%A{$+KO*Gvpi zh6nw?ccU>3``1pN-{zON$B9;6r^E(}vd&`|V;J3MucVYzaikgM z);-0wbhdcis!6vkoDj`wD@pL~BP*H2C?-;v#07@vW8n$&lwmTnypOhKl<=B`>;}bH z79L;m-6~fVn*f5OWX*OqY$by4UFD!@2}-!s7ad1*2HkO-xL!KIf0V!TZHIz%rP^tm zLiPXH*VXE)+lV3AM%u9v2QB`Pg$ zx!km5-6CQt%GcBit4eO~lvi~oYc-T~EgCM%?6kSnHHMbr4X(6hWN6?Dt*NxThN7-I z;KFT7Wh|PQ-0!i@1!J9~4sxZJE0U#EvxDBWs+=%84VFywI0(YoVUW`@Idclml;1N0 z|6lwwB-f;`YW%qfcR*K<<$}p#NR>U>Qbotsn(G*{4D)@gVA`|?SNZG_3nGTQ423?h z7kUgX!&J|c_zKtNh13T3L5qZ*Aw-Zy3~~AcrjZC`l=MlQqmvYAgRfzdBlJ8BeSqXw zjJ{5ff5!Rmq>W%|*ADoIi}ao%AF{$FOp|1oIaRdFxY8rHND%dyxn0F<547+O$3(L* zcZz6}xYh%0VDF~$hg5+bQiTCrC!Yvz;3mmws^kzdxD`!MeKNRBwYozxM=~GUWqtvA C1&JE~ literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Connection.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Connection.class new file mode 100644 index 0000000000000000000000000000000000000000..d987d78300271172a7d5e74cf545395459a2e2f8 GIT binary patch literal 2306 zcmcgs+iuf95S>j+ZVkPpH=tZxS|A1*w}K~%kg8B1QYcj$ir{S%Ze`_# z?v$k1<&8j5gZ@Rd1gnRiDHtX&zDEzJ<5AIY-qiN+2Z0gpdaTNQCIhslDp7<%+j-sw zStT&07|@o>eHhw_Ay}EZ1Ycy04Wp5)SH3WkL}S0GaWl-3U>GN=C%S6 z=1>*~T=Sa8)`XBIDoGQ_S3K$0k&h!cz`+ct`e#;U!8XH>Xg z8-ZVAs!9V?Y48+G&v@`T>hbd&04$RmCl8JCNiIO)<^5(uodsp-^8w|;i>mCN={ z8f=pI{P)Ia>`bQGWR}lNQ@R#bR8@h-o^L97bX?xl1>4$H(zgr-nd26-IwrtUY?mo* z85srmTx;xMJCM672IqcDDq~@niYF6Q3@56HQt_oImN3$)(@=y~jUi^Nq->_gkMKAe z9Wtnp3#Z^h`Mogk|HVIn?6wS4oxR}Euy<&z7+yFyZDqV`Rmr1TGo6AWfrWnw3`D(y9L$&E1e%G+HF|q zky`?J?Y9ey1I_IY-0hKDsDplUn}vHl&_cz=XoW6%BVF`11ov@I0v^D_a4+hQz!Es9 J(&fm%^c&%=DYgIr literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$EventExecutor.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$EventExecutor.class new file mode 100644 index 0000000000000000000000000000000000000000..7916e05a7128ea8a0b812c382b63ac9f34790859 GIT binary patch literal 1735 zcmchXZBJ7%6vzJ^u#Gtc2Ka;zMNqdlyCM2YjEPyME?LmXSR(Jbt!%}6+u8O`_&|OJ z6HWBJAIf;{h9L{ihyh>R-k#Ibe@}ny>GvO>zXGVDm_dqRQ%fIwFx{>)S_QV%c7f=t zpknmC>c&=h@^4fJQv2$mY{%Z%45k?t4#YqNq0rsnRqH^uJ;P$`WK&rmi?GqNQgj>x z+hbT#Zuj)aRIx;3t8ZfMRU~VApqy%j@|o7g4;Q7`g?ebBmU$PvkXlkIva`i7T`?V* z!5qWm}P}AZ#^qcr(h8VYyg3ojFk|?vLRRmYCRRn#}ubD~rJ zb=S(tT`SWOMMjq^6v^8nIegxdq&Ygl-0XybWH~WhPh~uuN?d2b8fwZGIGmE4WG)!!=x= zpq8TXv#Q;|>IAia$?Ya?ougU-wlaI0@-3v=mw{E-jEyh-u-8a#0oa5~&ImO)J6M#@-}bY;V|I_wo-& zd3JGA(JppU}oEwObK_7Ii@YVCU+w=vnd6_dr!@D8phyC z1}5Pufy6WJbNzzAMDEY>@;jJggZr!;cvU7V6am7bgiYNYDtUN+tGC?lpzfCeEwyds!61Wl=zBdcrRhG19bY zD0;KX5VNJ+bgIjPFddDO^eg1TDY#I6FAV&D@sG=TO9awpt2|mRZ5k_t3ywghPBq40u3bwuc3POZn_~A=m5XG2jt_r5=*Zb=fimCc8dKmYLG)LTOk4TqgJ%#Q!W9 zki-vc7E+Lo?4`&W_&ftc5QAa#Ch%?q?I`}9L}vlgUt#P`_VQ<#_>SHfTsw*b_yALQ zoxw901=DaHErA1~+pf)+9Ik7#p!ox&vC=~7$#xYN5G pW=cBDbO3UACI)%9k9G=mF$nYUFpyBNjI{{pUJus{F+ literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Group.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Group.class new file mode 100644 index 0000000000000000000000000000000000000000..74acb297065996f9ba5b5069b9111e8b6f4285e7 GIT binary patch literal 2632 zcmcgu>rN9v6h2dWw+Iwa5ig6Nwjy0oyu=a`QxTgGL<%JOZ`%&r!QGiUGmGMr_yRtI zi6;8rhccesE@HJ6YKp&R=3KsW<~wK4?6>cqz5u{7JV-!A znOPEihuNOAG|FdIr0|*vh!Pmywe~F2vAAtMtM1a8CJ^P8O9^CG4^-rfol;=bq3Of!QwB?KcRF{ji>* zGW1N?RIdfOkLB{6vY*`WxMnUb^F5|m)uE3#$Cy4Pc&F^DBiyQZWv)<8ktkY6{Afwk zX#&RZ%7=6;ODSX$&JdW|;*u)i?9sYW6K<0^0h-a20^!|zqBM0>S zQa|%|*^yNnM8E(%q<;<(Iv2Wg=`!fQby&SPeeGEeX4ghet0{~5mw17Or5b)Y2n^)% zWfW-9YD@y}IuP34gvB0m>%r8o+WRAeAHii0@p>@LZzo}nL_YkzMHTjFYO_HNm+FS7 z8(5T0=_%~~bt&HNm)BJ;Z|q3n8XArCev=w)-&8u)Md63Df*}1eIdKY3l;0Br|6ly$ zBTx~ZtkEYd7<+9!R`5qJflQg>U@ke9QdA}2Hi7A5%o}{P!sqVpF<-}Up2fm7xkin8qcR76UYaH)z_VdA=C|{G>)f{Knt*dNCfxk z&oKHTJN^;Qe#KJ+CPO=b$8Z7nnU)n?giClLa2X8rY{i@E8|oF9?qiNKeMP$p*Sh2u zK}LGz?>gkVp#4iV@_nuC1l;&p+u4q_js3jw%?=$yI&{nc6woIECfvd`i#vjSd N7I3|X>-|7K{{!l0tg-+A literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$GroupExtractor.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$GroupExtractor.class new file mode 100644 index 0000000000000000000000000000000000000000..a89f93795ee3639e000e6881f2cd16a6819a103c GIT binary patch literal 1357 zcmcgsO>Yx15FM8^n}pJqK!H-Amf+BE$RZ>TkO~P!DMX4&p{0VO<4jUlyLL48mi7nn z8<0SPJ3k6By9p{iAmuiT7XH86f?PE1I zaogFU8D*pb`yad{myJ88fs_u8A}9%1Jys_wPLv(RulvWSgMiAmPK-6-xqwSsjY9#Y zwi{psRRJsK=nj#i3S1TopWazQ?~)T7NbSYvG4_239)is8r&ggGzFRgO~_3;b`&I6q4~QC z&AbXt&M0-Z*=D5F%32p>ADMHKaIN~64|zCM1o>BxxtCzB^_?5}|EhltDsW{9mSIIe zd3)O5Yg@hZj@}Q%jn&FKex+wD?q+Ep=|FK43#j*;R>`3vliO2ggp*kk@McQnQD
*3T}`wEp0_1b4x`d(mGdCxJx>uTZ2rK}Fu l_+EnRg=YpXP{QCXv2LNny|*eDFRMlwi0hiq5R0;SD9_E z5y*9zF$YRF2sjBMKD=eMkgaHckN4IC!3=>(oK}y}*%r4iV&?gxdph^RyRNh%N^~sP?4eA90!JVWVPE5^WOaNh!#{OdfVU6$ki>S) z{fGf-Gt-V#BQREU%V@AFi5`JBeZL2Teo~9GSlpc)JRqqre~*B{5KL^hISsjOs|H1P zwsqHFaI0E^2?qM;LV5q6Ze6vN>i zR)-f=hle>4D(RIFN$V`iHd^DznWRj9V#r2x9g8m+739JtxKMpBjQoG;Pawagy1LGj z4&@Bwm=|>x0)?`~zEcXYqj8gkRRT+AsBf&V!-k8~O>XQfWh^`fEI;hrX>ed1GB|S} z3%Pil=#Tk$%pr0T3Xp*;s5{u literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Quota.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$Quota.class new file mode 100644 index 0000000000000000000000000000000000000000..90390dd996719a43e9890ac9cbcd2bfc8e67e176 GIT binary patch literal 2131 zcmcgsT~8B16unbQzpMfk5W$Z{P`iR%O?(n!LMm#TidB$A-=^)*4(`s>nb`_I$lqY1 ziN5=zjCXcR6m1Qf;zM^jGiT16d+we4_1nkK0I&&n(vTqF3TC`_vfbuVa8EUw+M`|W zm8IC@?NCvJ>&wqWX=oaf1Sa=sk9s~8ZSPrQpEV7E@h0_sO@LGhq`FLT*&;Aqi-3j) z%zN0UStBsR1vi}f8$HT>+VEL9K06iAgJ0J&ZD#u0V?`Iq7hZ(I5>`ZyYmSFEgg{+Z zAzojo`7&t8H{N3=80uy%yf0anEtZBU0uw)kZz?5K2F?>$d?^&u((kdB)09CMHCZ;N zt0d9~_nll0CSfKIvv2`}Tes>TU=+nTy2yrYqqu0Vxx08rjSIFF1`VcmY;abTM0a+n z;&$F&Oqvcy!7H^BTaS@s(T#fF+}WZ#*xoq=RR%jejayHDH#x=LhSX;7SWIzh*WA|x zG99Wr)iwn%MRJF*U{*bQ9(}Kf>ZVP)X4vr{{(W|GE`0W zh(|B^SdEozHzAO(2<$awpK8r?8g3IDR9mKilvd`nwQn+85M&J@$9wFP6k!rg-QZ@$`heVn{C;ryAYp^tg?O(F64A)1#|Z`1a4JGeVjXJ!%nKmG<2 zP4wL#Wjwn@p@AA9#fRP5IeX@tZ@%x$`SJ71Hvm|KdpSrk@FZIAgYtc$r0{g3X*}K) zURBAq@FUGF#phPI#kvTEMV*5TgNaSP#l3(_-+R^A#HM9XY+=B6R`7j#8^PZzbqKd5r2AQgAVGbr3jPJ2r)LQ90lo-ssks6H(wy@KgKB2U1|HCceYAU4EyTL!JKS_)q3F}9 z-gyYE>>fNNiBR6C)!g+w++e9sf0xgYej{rgbV9Tp)pCgYnvM)Pv#izjPHowgU8k*8 z=vYd!J6&}8D$UVIrD7bbl7vtgLrU)^ik9wyi?FM-bx6y~@5-&jm0KZ0LnSL!%5<7M zE)}PNgxT}EFf!mryd6(A8CJ@POK_t4o*4N5(w{+LRYkgq&qQ)1`)aHbUu0@nHAzjQ z8gOIK$LmF#fhQ`5Df9r&$Wl(L2rqxCQ6wUF>DS1-M8vYL9Svh-g=a zigp#Q4d9j{?ngbTGgNLfaD4!`;cmk6QlY?MvYOQnXNX?D%4|vU~s-PwO4x7``is*w!PAY|qxSYl` zW*Fj+g%tJ)!%Vg`VB4`;$i_x)mu!@TiN5#7U8bD|Sh>lafZL|3IsIty+O}374~yF-n{(<~1&$>p zd)VU6v6gW(uXAl2t2zpSFots4QxrAbB^N!;mgY8czk^ zGltWc&M~Zaf||EKI;opGzXX?tegx#=p#8|9ql@GgH}-`x7Hc3<`Y*+h z!3D(WD}w}*B;#~4h7_HQ_f~0=6ZAYJ1S6UHii>YEQ=gIfPOBKM&^tr{bzG%)x~qk2 zxK1mESKSQ=@+!?|aX0Ge$!UXjUV7c?{;TrXU?26v){gd`T}4bl^ik*Yg+o?o#}Lx(JHXb&;_EW zf|}87)#+Q|$-dl(TuoZ9+A0)Yt{-?SLT_vi;|$Y#VqXN2(4F8#b5DkzA!l2n3b4jFl^OGI%r4fnjw>J;N%@W07S({F*aI zq)jxslE>0_O^XxAhV46&W6PR@!}^vhS9sf+n0s>a)EoR%hjA<0(mL)<>JTgEs8^00 zRm%>7tGH*Z=R{!jxMQnH$5u(j*yw7FJb5UR{@_hXF$W?SCPo}2=|r03tX$5Wf^+5f z+=Blv{%JgROy7p`sY>VYRF74Yw}7Eo*Yr}=;7(oAhi=bszr zaSt?S^tfhF!UbgMUkSN%&C*F86LeA_YqGmYYmt8U=?0^<@EMb@N>d+k`77-*n4@pv z2fW7>`W6RXn8#JxFwi?Ge}#e z!YwM<5Y3L}mh4MAx?^nBDve2qGZ^3JhurnKY`R;uee^7YctiNeV6q%7Nuk`(;8wV8 ztusiqxiLpd*BNBH9?|Bi)j~FDecEfGcThxaMMHSpV%2inF$~xqgISFqIznU77bxv- zP7IPww6CLsr*rwskDlM|NGk#?$wOg8&Bs?#Dnh5U2tQl)RZvq0?i-Xy$zq+Ni5FF! z63sFgJ5O{&Yo${#$zb7~)M%7{h;_$PL0k9{n$y;b;DhH*It>#roq-vcWsrCwq_8g; zOyw?Sp5LW2%0l91C#a#`;RFx{r9AHMaxKF7i7{?lf&yMGU#(u0&M1*oIpcrwl&Ibc z%nDQNQ-h&go~ke>%y!>8Bxr^E6kbn~U2Uv~yHhQ0S|P0r2(Cm1tGd-^uzW&HuV^lk zwYQhwO+lW;KK;1>hAJ?!(?Tae+fsFh(vS3u^@>&>k4qJk%R3FN0>_e*J#M4ZqjpDQ z1GRCiiW~xA45{?QQPg!8Tm)^UtwYr)oOVwk?4ANiB2cnWBu~fVq1c=n5@uI1>5)DQ z;c+}VWKb^GPQkVEdu`zVi+={`s_JMDyBAcpM`MNXVquUeNxF85K6NLWBor7d{zZEe z^7(~Kt+a{8zE#G;8c=)D-5!T53`2sxvXF#SG>$+z8b_lsMV=YB0Wr8q)&$MQXdI{K z86hwl(_dlYefHL8nEFn$7~H0J=m%h7j^3HB7tF&QnlZQw_Xdb|ADjWAEesUx0W9{3 zEk;aSmA55$*az)js_rN9v6h2c5r3D2lC>Jk_ptc}gFW?1`#1KRhMC6v}ziB(PgS#_jW)|cnd<&n! zL=*k*LmAI(TS%cb#T5VS**$ain={|{?b%#OQlyZVPu&G0>SJBr{znLS;R^wN3AcNEJMulmzRh) zSg_mCaFbS*TsQ?6%I}4N|1bUtBsOKBa_kikhhvS$GIl@{=w1;xp%y)yOH9XLnn1dl z=@E{e7(5{`)}~q8&Q$E(x(p z?1VV3CNLL)F3fksCA_z}%g}?l@=7oCVRjl(2*$*BNWM?@e}$_*!}o^q+7Yg7!wBAY zm#g49+`uOSH{li{IhCqWjLo6Fsf%{IrD$V#%C^cW$6MCp9k|=V6W@bW1GETo*;tMG z{H0-{0ovcSt&cX@QZ)wQ{;?WUb=7D;K5?dw|41Fjw7~;>CxTzWBaFj{{}^WADGb3g PjI$UC#yO1hVcpChPovV( literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$UserExtractor.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration$UserExtractor.class new file mode 100644 index 0000000000000000000000000000000000000000..bb1e7354a4fe9171346b4ef42355ba723a7e057b GIT binary patch literal 1352 zcmcgsO>Yx15FMA0O+rgbpg<{5OK?ayWZ}R8QX!!z6_uhA(GS7VaVDv&T{{|kOZ$WP z4M-rtogamm-2{~$ka{2-{4ujnZ^pAT@5j$C-vFQoGJ+)m&n$-cqnk{OvnHmY_OY6p zxa;iLOfph|{f`ekQtvd7(!o&#B>}6)>O{qfvXl7j@ECOvP}$Onu_n9{aHZKg6j18A z5k^oIP(Mfah#XbmnppaDU3ZejtAzSAl>RAh2{?u>9YO9BRSB9A&F6V76d z6yo>KpT6kw#f*(s0Xwb!WrwsQptk2S(ztDk@E1m+ohL0|{at2*N%7El!%4l)o4Xi2 zFJSYJhizw8=i{%E9UG9wN+a_x?^uh}O_cZOBe*A^dD-kDxG&(*CD5!Lx#S2~xu86( zF}vu)8|On=0Lu*P7{C($p{r14J7Vj}+*|t!l@E>DXITDTU{-n0F~I9a;mKvJ0XO+x jf?I`W4Q}(9qwer|msf+?*I@%5@Gkg#Sgyimf!+88DF~)8 literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/Configuration.class new file mode 100644 index 0000000000000000000000000000000000000000..5db99374321e9c83a1cd90d96af4cbe829f1a893 GIT binary patch literal 8150 zcmds5S#uLd5bp7{l5AsJ7;_~cf(bTSBL5=~seCglOS`tgQy{8vsKTw8>Hemtr)T=x`Rm_beLSm0 zjM})RPnmpzo4mk1XO0oOz^EZ_yBLfQR}IEJ$FOn(7%SQi##owBXVI}|=f)h{vop3i zB$2Yf^qi9G@hk>r=h9;?Cia0!o?#}34c7xqblk{Ux>s^Iqjd|02ZvKe)){yfYMTrz zV;3+pR%5|qEX6&lXD*p`Zrr#E+nCFp8SaefUY=1apW&7_G|Mw3&vqE?8&(KS_fXIX zb8b9avOJ@})7FgP8i>vj%d$NgM!6Q#E=<{%lB2dYS~5)-Y+h(EF;@&5HDed}uu(8P z4!8Q1+&Wt|!5pznB>;@}C;?PM2K&Ym0tNTm0Sq^t?bB1RVC{snllvhjwbrEHwa5xc*ib|Hq=%Ruorg);F?%4@F;h|2%Vzh&j zEwqRF88r5FUNtx zUfDJynSgB0P;kmq2`lLcwrdu!piD_97Ntx!}v^66l$iH&CXwshl*FC=KyF zDal@EPw(QCix6^g%e_+rnc?G%Nn(gJ12yb2TuJ{Hk;C}n4MkGU&t0vDj5aLltX#zK z2{7qps9`H~#sloI9qmxLWOM*?sMM^Gncx(F3Y1YWO+}!95nzOe)#A!%uOdp-1|SiD z2=awp?+As=r+23p_TBFtULkluM!QwxQKotw@GQ(37#&o^ z2-_j9O#lO#0NqqX@JU}2zg*IVS?CJ)zqba&4IIhqZeG+C{Pb*}Vl!4!q9Ow;l7c+4 zG;xx#f0W{0xp`J({2%bd|4%6(*i`qz5BD+M(Lp@86Zqth&>M{YzIP3j^yr}z3!@<3 zUJ@u+ee=&fed4*N58Sh0TYY!o2Pz~K4-DgW$;t5Jh9s3CPSq#4FQb+;o?x7msk<(B zHNyS1SEY7_FILpfaEC^CxH+T(bX0eZOxT#w>Lq$Lx`YpP707a(X_QhUi&09+%rx+z zkm8PKOp9j)zR;`9se5Z^beU0~(o}(GpIEi-YlJtreieWabphoFwKclJNQnU*x1B4x zljYfER>r7DRaK*_irXZ~0p75r)aY%+%#-Rvv!0`8BSuM;IRi|QQFEZB(Wi{It3XKhQjfnHea2`&(Ik-8f!M7D z%u*v7eW{qq@-wa+##ptX(bvcs72gB_#RrZM8hxvn{z5yjjMh+FqaOe$u1(x-BmSqN zc)JCAYaZpUH^Rk-woCnl=Rdr-w^5uLsR_U0q)|l5D8-~~F88-kE3{XkuTJW>m&ZHG z`f*x~kxq;^L2nIY7yd_xttQ!u(^^`$jK+G}u#Cn=daUBb)6&H4SOE1*%}~$Mcnwe11f8oP+9bVLL$sIZ{9VxML?+8SQvG~hsOkB9xu)lH550nF z=if-LhBDz3-TMt+mPSE{to{1hbX+t2SMq+`4m72z<=`yy9+g-BVV*0PN zJU}!OpZ!tBJ4=Oqq5H5i=gv8I&YgSz{QYqWppI86Vhnj#1jbjd)3H6*HvG2LH~5Kd z9C(gU5oee_=BM0nxZ5%6?x4qzJP`ay_zVk;+oe_Fee~@>@}}Mi?gU+{E37Ys>U)Q~ z%gwFEVcld{{4_a9s%AO1YX=()N}#9| zI1Fk~FiU=duH7dKr`#C`+AI}{6HC&tj29}NllzF(VLuQ(9V;j>q`B|&q2-+n8A{QD z_S?t83aUGi8vlo=NCq|Zu>YB1Jt`Z&LvhkW%VO4a6!DVbbyU3V@P5AE^LwnQixI7s66fZ|oNnLG{tBvYrh1#Ofw8l4cvVQ=r)Vgq` zt^u=o*)z$YE>1xw{vASsu>7`ug0(W!$~k7vFc*UR@epJ~@GudAT=Ez3lo964R(yo4 z*-DI%GtV&posz_`NPmr1dCdwLdK59SRmo_3Y_3p`uR6T=5jbz?Z$wR>jclrMi#{q z@+c6>j`r1KG8w9{Z(M|m6S#6bkfApHO3cM09TBdbU0beS)+SgJ6-ClZ!5e!L@~%uG z{#e_(GV5EeYYc1CC)&?dR9~2Y@ab4sr|^O2pnwhoRI;dS)0z9-m52yar+#6XtcadXiyc zxb3<>u!91_**fh5+5JiwN>@_J_?n@0>_77t?v1$$y{>bPVAC5)4IbbMs z^rLW*N4bI`N(@zBDsi7ohQjnMi=pQ0fR6{G+s+KMg1KFM>ae*@|olM(;` literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/ContentTooLargeException.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/ContentTooLargeException.class new file mode 100644 index 0000000000000000000000000000000000000000..650410252cab7eeceea2b05073662730100a7d9e GIT binary patch literal 518 zcmah_O-lnY5PehIuCCSkc@*@fRhz3=@lq-Xiw9eJPuH-Gv`NWkt$)jt;K3i@j}j;K z13^m@n9RI+GxH{I?=PtT~vL`4Sz}q)eCL3za%sclY99f#Ymb=biLCiYyCocLuaRQFjB3fKB3&U zcPhpbR%)oAN~kB=sGDLuP_7qPgssG;a@3bjhxiLD`=QPW=gA_K^LA_zqOr%LH+_i;E)s+q+_vX!MhR!$u@s8mTUtW}L#L+dQ0goETC(l@ZVz-2t kZOkub#41+V+yA7YhBc130PCo8li+P&liB=H9F?&B0f+*Ms{jB1 literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/RbcsException.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/exception/RbcsException.class new file mode 100644 index 0000000000000000000000000000000000000000..f637a410dd8d5e41d24e6e26087731cce29499c6 GIT binary patch literal 466 zcmb7A%TB{E5S$HZ0-;cz7fx`4^6&=)aY02wia4}Y?@q8%qr|TAp!^mmBo2H4ABEV2 zx8Q;;t!HM(tKEHmd3*vez?Kh(aBNg1<~B{WHChyLQVKcKLfs{5R75UjHiX2%Q#cv!Gz$R+iMkI0ZUzU z;1PO}HtMR*V^xeB7hxl^iOeRl&<%eV-D;{!f{6Y#MM&^-RhBYs>;7t!&`D%na&r66 zjtRlY)RR2 literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheContent.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheContent.class new file mode 100644 index 0000000000000000000000000000000000000000..a6be77d9a01335bb57522331f80246f540d0b83c GIT binary patch literal 3238 zcmbtVYg5}s6g>+zeyQQnkd#8wLPBax6P1J};hBO-f?J2QKuMvGM3&c}U?h($r^A2h zhjylQX40Adfc~gX@0Cl5ad8j_Eyp%ef4hR(9 z9WsTqz!k$%$;3HTE3H~h+UokLBiCECO{-p&wm?!pYMI9ZW95j8O9Cm~qa`q&pNYAb zC|XsSfxw`G46?MA4O4ElPAbyg)hcxT-TY0-0*7Q8#qW zk<9L61y?-+Qwpx(y1>A)VH(b=z(kDyJ%M3cIvVAox)H?tM8Px^Lh-(`i-LkYA1&ug z=jY8pHyP*_6Cq-H)zXg`;EsYhX7VIo?rwyo|3~dEBi|Fa z8HyauHF0Dx&5>R{u7tHBf9=oDl$idCf<tH< zos%s)T8wYEOvgBpCG(ZhG^kb9Ow)2ShXx6C%1F6xom8x2^|3UiP1R9d%Ay}7u8J>R zPn7e?=w9`jpKM58F6vQ%YiIJ*a*VpVEoojzQu&ZU_L&t(L4^gFy;mB!~=u|%j|z-dpC~f+*uSycE@Vj zx_o5t>>Ccw`&(Sp0!qoG3>NEJvneUDO7JfrklvC_XVYprm?Zdq^dX59yb|=NvR+U#&kAp2%DVPDNMkoZDBFpPy2VE+ z(oOc2W0dhWKJA6sC-(;dTDcCb9HsX&mj;zsIIY{gc+ZK}i%?or=wY;G+xWZ}MiQ-B z1g-m6aB0!B^8s(Wdm!zpwfqak94y7-0HJh8Cz^@j_Fr_{HzMrUeft$(=Zvp&u^XTj z2ql>KhrYt}YbA(BM4Nl(F|FJot3GVD@EV(i**|dMx30kE1GxVc02K|1M+G&sM&0(6 zPjQ-7uT|S9zpW*&RcTVVz#x-6wtSNtzWad!mCn<6n{Uoxv+dgisqMtO35zDy3Z3ALnNd-=D)KOtc^3-Gqr2Ycd@w+E0;QXzaak>qkR{ PR&SOip;H-F-SgzXRk=n= literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetRequest.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetRequest.class new file mode 100644 index 0000000000000000000000000000000000000000..6bf180be7c4a1ffa859fab1808d95d4b4cc2f750 GIT binary patch literal 723 zcma)4O-~y!5FO_e!tz<5P(Dr^k|5RUl_De*wG|1ll=T$}W( zpjyg)FWk;HTW{Sj9Sw}vecm+}+UYpsZDXug9%5Xe%xpil=W>S)=gRX5C@UR%!lNW; zGNL+-z<0IbgF!#${HFp%_|7IO>#1A^{G6=#Qw`3|?)#;^9#{~xsg3qqg65k4>uL2G zRHly+&C&wK(ronvHID5dPx!kIjZME?$LBCFLD8;(xK5^=WAr!qYDt1d4zQ~q)_M8@ z)n!-}HWS3-9h4A{QUiVrs0I|lkHaKl0bgz3(!@1_5>4R@C4}N>oYAlVWFLUAC7OGK zXbZU#?i=lUns2Y)(ZmhVOSY(Bn;%v^v!T9dF|@o`)PJI-z=Hn-3CqZ?6#433Th^s@ literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetResponse.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheGetResponse.class new file mode 100644 index 0000000000000000000000000000000000000000..695bba9f84c1ed94846c3c646436bdd4513748dd GIT binary patch literal 978 zcmbVK+iuf95Iviln-EHy-XMiSp-4%P@COtsLDZIt5Kx2izII0Lw$5(lT^I3JAc2s0 z-~;$5#Mp_HLi$jUwLWt>do;5%KYxAy0bn1GJro3XH97x-i6UjR@{=I6zC2g{n5>l% z`CS>F(3^>~lRJmxj>(=IZOKDPVChs|NI#Z3^4|uh6gm$TfzI`11&U|%QJ~SADGgkr zbhIl_iO9Xqva7B3Rr$^1R68}MBYmN)3SxSpwQstogim{nqAQEbHi*(#q`#thMGF zY7c8DVxx{#+~#*_wTA+=fk~5)4pdgz^5ua%%^pl(}|!@)e7p*}JUP$ko;-tf?CF^AdR{aVO_DvB_9r)?IAj0o%=K VJKLoy$I5uf%q_-8j63H?i(c`E%_>d1c`98<{Erhqp+szB|8J_vM&zmX12 zrgZd3U^XEm$I?iK98Sh8NsVM&FwLr$=b90jR~B`#*Pn#DdQfPi657$HDpx^FPqj{s zG;Bg{sl;g#q$mCh(v(R#-2MmsX^iIE=ULD%eF9soYHBwu=Vb?TXtOWdN-OhN;7RXqTc(HXkJ_l=rh^sS z;`i(IM*{7`w8%nwuB;Oa7yr$D`>+Dej%GIpvCMN~L)<@p6F>GIf2x;hZt#F9*LXEB zgRXU(KWx`3;ArdfaQNgBXHCv7b98z6ycZlB-e+_^^2QEVEx;zO@ZPB!u!yUi30%YV z{{Ssb2eg9K2|&ACtMNsy_XRh+duQl)TW47M#HEYedL_5Ds`Z?km1KomL2fDCLftMW T*Kmhpo6oF8cd?B1vflU!<}2dh literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CachePutResponse.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CachePutResponse.class new file mode 100644 index 0000000000000000000000000000000000000000..54968cf3e14084e8dd22eb3f4a2b635d17f7d59b GIT binary patch literal 726 zcma)4O-~y!5Pi-kge?seD22AC9+DtoapprpQ45F^5TM+>%V;rdY_xVj{3|L^RqCNX zfFFezZ;(KR1BdaQc!}nN;q_XO#FlY66|N(+ZT&=~`gA_r!GMb8XU9 zfm%v_H&ka@?bqsyhePA_fOgEKb~?^z(-`ZOXEiQRW_A$UbGb!^a^>lSDJxxjLgNs4 zAEAyh_C0NAZ#al4KUPc;zO#wS`YP7}KPD^wR5Ry#@9olF+*nYwrj7O+0<*1`g4+EC z%J>*z2J?I`t#)6aab$;iLO*nvZ0h+!mcxY!L_3Dnbu;B0@qo?$*Ca5uN6v3s=dr@N z%WPG)=^&q$@5HqL> zEcTS9qj5T*?7bW$B+%%Yp-lQRQ{JCGR@|ww0lTpD*Q*l<4>Y5@iL{n%fOVd3@o9*B+BNZrqnmlW`3K*t{BZyP literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheValueNotFoundResponse.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$CacheValueNotFoundResponse.class new file mode 100644 index 0000000000000000000000000000000000000000..15589dada3b1efcb186b05b860a34946199dce5f GIT binary patch literal 632 zcmb7BO-}+b5Pbz?0a3ohud5ss6E~hQTnL~EauCD4z+|Z|E!i#_f0ic`5B>mulyP>^ zM588>CNuM9UZ<}!uW!#U0NU7fkPv8V3f+6-dCF+z2BWccYK1*7J70;c5bvK2PuKZs;#;del`$DpDC@vra<+ucO&nl>r3sqgD_CqYa9&) zQY|x~9FoZ9v4*rjp{F$U7qbxs*K*{OK)Gke(jUq|Mff9`3a82nbbG7H`}L;{f#fYc zL~s32DUchOMKGq0iqw?84(}vl3*;}frl93ZYsos;5~%-QmOwFX`E73de{V$~(YBS#73*yiHw0W)h@XS}^+A&(74%*Q4QanB;7 Z1WG8gFLRVc1=UoRYZbn=hn=`yc?Wm;slEUJ literal 0 HcmV?d00001 diff --git a/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$LastCacheContent.class b/rbcs-api/bin/main/net/woggioni/rbcs/api/message/CacheMessage$LastCacheContent.class new file mode 100644 index 0000000000000000000000000000000000000000..7a1465355db5fc45e84a82649cf5ff2db17fc968 GIT binary patch literal 3478 zcmb_eO>^5+6g@9rvW*~dpdY~vC23O}1u9CvXq~2MTuK5?+O#3SQkGw06-UbG1119t zHf-3ifemb61KTpgBr|1(AHa`dxK9#}YCkDB?s&%blkYw6+;h*n&wu~(^e+JKVN<~X zL&fI4{)Mx>Z8^53yLHpkja^IM;htw~bG>SqO}-i}&esgj4Gfc1i@W;Lp3j$pt;*~s!%)>}@B#*rS22tchS8d3^R-~7 z&fN{8-r@}7HODksn}%zNb^LJ1Z(1J1ty&KZJ4>uE44cis{)}NFl_XI#o!vc#+vVB5 zHp_$IWfd=Bw15*hrQjsPsT65cjNvpxiMC9Ms1yk9RTU+iVNhJY+cHej>XQt$`(n?K zZv?w7%QSpy*9n|eF^SWHM4qAOa^Ik~HSQ)u&Z(FZRVZfUC^AJ9C))jniW!v2qP?k_ zG#RyTvAoh>Qv{VcLFG+``5wVKxa2L{bomY)9R{{S*7BXeY>EvosnA8O>*d;G<8woA z8TPh*zy6q;ex-lI1rHgt|38B%J*gYRSm#h?$gOeDUv@knmx$LO-DjZA(SgDU zy(_d@zyQ5~5pC`Ox!>vA01EV82%mmQkze5jTAz%fpyEYZQM{PGghAV)7-LZ8o|7FfY&==zDdB0wqZ#1 zbeL%g%(qe)x(H&JMPeh=QU7Bj!T2trkJ{Ep#raqtI#@BC7 zPb1Hl_EE{&W(K#8k+X1&9GL>Qv#H&YC~!oIOtj3DNoi}AvWQJWwo7J833t0Bo0ANW zqytLy*E||whwVHnhc0(n&b_R1bSxhuXXO|Hy3_@A7#82NeOO* zE_!nAuna{8H4*=IVR>%-e4Lk2GY`25Z1YXqit*85e*JEpSo4tzmpSe093Ryy_ zqhGbx)u!z|G&}4&LcS5kk(#iY%*?CSgyvz7iu!sS%`wjUbH3dJb~c#P8lmZumX=^qS0ShzEWqsm~L zLPz&QC?+)hV)=1tWGFExN~anYjrb^R6RsxR)RXSUd=ti|Rp+7EX5#-hqp(Z3p30#X zx-pY-YBHWwIGAkZ_u9Hd?)T8icK(J6H91aI?ln2Ga|FVS+^wG{i&=SRFb5^LAZ#9a z!6lTiB9SZM^HRTxoWu!h*p@6|J)4sn!iMmY;HKb~U|DcSa8Gbw@KEqbbXF3bq4Jj0 G?0*AKRVv~D literal 0 HcmV?d00001 diff --git a/rbcs-api/src/main/java/net/woggioni/rbcs/api/Configuration.java b/rbcs-api/src/main/java/net/woggioni/rbcs/api/Configuration.java index 0fe1e73..11857f6 100644 --- a/rbcs-api/src/main/java/net/woggioni/rbcs/api/Configuration.java +++ b/rbcs-api/src/main/java/net/woggioni/rbcs/api/Configuration.java @@ -16,6 +16,7 @@ import java.util.stream.Collectors; @Value public class Configuration { + boolean enableTelemetry; String host; int port; boolean proxyProtocolEnabled; @@ -150,6 +151,7 @@ public class Configuration { } public static Configuration of( + boolean enableTelemetry, String host, int port, boolean proxyProtocolEnabled, @@ -166,6 +168,7 @@ public class Configuration { Tls tls ) { return new Configuration( + enableTelemetry, host, port, proxyProtocolEnabled, diff --git a/rbcs-cli/.classpath b/rbcs-cli/.classpath new file mode 100644 index 0000000..f11c7ec --- /dev/null +++ b/rbcs-cli/.classpath @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-cli/.project b/rbcs-cli/.project new file mode 100644 index 0000000..3a52806 --- /dev/null +++ b/rbcs-cli/.project @@ -0,0 +1,34 @@ + + + rbcs-cli + Project rbcs-cli created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776944 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-cli/.settings/org.eclipse.buildship.core.prefs b/rbcs-cli/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-cli/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-cli/.settings/org.eclipse.jdt.core.prefs b/rbcs-cli/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-cli/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-cli/bin/configureNativeImage/net/woggioni/rbcs/cli/graal/GraalNativeImageConfiguration.kt b/rbcs-cli/bin/configureNativeImage/net/woggioni/rbcs/cli/graal/GraalNativeImageConfiguration.kt new file mode 100644 index 0000000..fcc3343 --- /dev/null +++ b/rbcs-cli/bin/configureNativeImage/net/woggioni/rbcs/cli/graal/GraalNativeImageConfiguration.kt @@ -0,0 +1,200 @@ +package net.woggioni.rbcs.cli.graal + +import java.io.ByteArrayInputStream +import java.net.URI +import java.nio.file.Path +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.concurrent.ExecutionException +import java.util.zip.Deflater +import net.woggioni.jwo.NullOutputStream +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Configuration.User +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.cli.RemoteBuildCacheServerCli +import net.woggioni.rbcs.cli.impl.commands.BenchmarkCommand +import net.woggioni.rbcs.cli.impl.commands.GetCommand +import net.woggioni.rbcs.cli.impl.commands.HealthCheckCommand +import net.woggioni.rbcs.cli.impl.commands.PutCommand +import net.woggioni.rbcs.client.Configuration as ClientConfiguration +import net.woggioni.rbcs.client.impl.Parser as ClientConfigurationParser +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.server.RemoteBuildCacheServer +import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration +import net.woggioni.rbcs.server.cache.InMemoryCacheConfiguration +import net.woggioni.rbcs.server.configuration.Parser +import net.woggioni.rbcs.server.memcache.MemcacheCacheConfiguration +import net.woggioni.rbcs.server.redis.RedisCacheConfiguration + +object GraalNativeImageConfiguration { + @JvmStatic + fun main(vararg args : String) { + + let { + val serverURL = URI.create("file:conf/rbcs-server.xml").toURL() + val serverDoc = serverURL.openStream().use { + Xml.parseXml(serverURL, it) + } + Parser.parse(serverDoc) + } + + let { + val serverURL = URI.create("file:conf/rbcs-server-redis.xml").toURL() + val serverDoc = serverURL.openStream().use { + Xml.parseXml(serverURL, it) + } + Parser.parse(serverDoc) + } + + val url = URI.create("file:conf/rbcs-client.xml").toURL() + val clientDoc = url.openStream().use { + Xml.parseXml(url, it) + } + ClientConfigurationParser.parse(clientDoc) + + val PASSWORD = "password" + val readersGroup = Configuration.Group("readers", setOf(Role.Reader, Role.Healthcheck), null, null) + val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null, null) + + + val users = listOf( + User("user1", hashPassword(PASSWORD), setOf(readersGroup), null), + User("user2", hashPassword(PASSWORD), setOf(writersGroup), null), + User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup), null), + User("", null, setOf(readersGroup), null), + User("user4", hashPassword(PASSWORD), setOf(readersGroup), + Configuration.Quota(1, Duration.of(1, ChronoUnit.DAYS), 0, 1) + ), + User("user5", hashPassword(PASSWORD), setOf(readersGroup), + Configuration.Quota(1, Duration.of(5, ChronoUnit.SECONDS), 0, 1) + ) + ) + + val serverPort = RBCS.getFreePort() + + val caches = listOf( + InMemoryCacheConfiguration( + maxAge = Duration.ofSeconds(3600), + digestAlgorithm = "MD5", + compressionLevel = Deflater.DEFAULT_COMPRESSION, + compressionEnabled = false, + maxSize = 0x1000000, + ), + FileSystemCacheConfiguration( + Path.of(System.getProperty("java.io.tmpdir")).resolve("rbcs"), + maxAge = Duration.ofSeconds(3600), + digestAlgorithm = "MD5", + compressionLevel = Deflater.DEFAULT_COMPRESSION, + compressionEnabled = false, + ), + MemcacheCacheConfiguration( + listOf(MemcacheCacheConfiguration.Server( + HostAndPort("127.0.0.1", 11211), + 1000, + 4) + ), + Duration.ofSeconds(60), + "someCustomPrefix", + "MD5", + null, + 1, + ), + RedisCacheConfiguration( + listOf(RedisCacheConfiguration.Server( + HostAndPort("127.0.0.1", 6379), + 1000, + 4) + ), + Duration.ofSeconds(60), + "someCustomPrefix", + "MD5", + null, + 1, + ) + ) + + for (cache in caches) { + val serverConfiguration = Configuration( + "127.0.0.1", + serverPort, + false, + emptyList(), + 100, + null, + Configuration.EventExecutor(true), + Configuration.RateLimiter( + false, 0x100000, 10 + ), + Configuration.Connection( + Duration.ofSeconds(10), + Duration.ofSeconds(15), + Duration.ofSeconds(15), + 0x10000, + 0x1000 + ), + users.asSequence().map { it.name to it }.toMap(), + sequenceOf(writersGroup, readersGroup).map { it.name to it }.toMap(), + cache, + Configuration.BasicAuthentication(), + null, + ) + + val serverHandle = RemoteBuildCacheServer(serverConfiguration).run() + + val clientProfile = ClientConfiguration.Profile( + URI.create("http://127.0.0.1:$serverPort/"), + ClientConfiguration.Connection( + Duration.ofSeconds(5), + Duration.ofSeconds(5), + Duration.ofSeconds(7), + true, + ), + ClientConfiguration.Authentication.BasicAuthenticationCredentials("user3", PASSWORD), + Duration.ofSeconds(3), + 10, + true, + ClientConfiguration.RetryPolicy( + 3, + 1000, + 1.2 + ), + ClientConfiguration.TrustStore(null, null, false, false) + ) + + HealthCheckCommand.execute(clientProfile) + + BenchmarkCommand.execute( + clientProfile, + 1000, + 0x100, + true + ) + + PutCommand.execute( + clientProfile, + "some-file.bin", + ByteArrayInputStream(ByteArray(0x1000) { it.toByte() }), + "application/octet-setream", + "attachment; filename=\"some-file.bin\"" + ) + + GetCommand.execute( + clientProfile, + "some-file.bin", + NullOutputStream() + ) + + serverHandle.sendShutdownSignal() + try { + serverHandle.get() + } catch (ee : ExecutionException) { + } + } + System.setProperty("net.woggioni.rbcs.conf.dir", System.getProperty("gradle.tmp.dir")) + RemoteBuildCacheServerCli.createCommandLine().execute("--version") + RemoteBuildCacheServerCli.createCommandLine().execute("server", "-t", "PT10S") + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/module-info.class b/rbcs-cli/bin/main/module-info.class new file mode 100644 index 0000000000000000000000000000000000000000..0c7d3374d09ab530c571e80a5660f847b00c5edf GIT binary patch literal 582 zcmaJ;%TfX{5bRkM{G<)$E^ybMgSe8HqvkA4kz?JXv;0O3oma_q?v_Ny{ z$z*zZrYj$xZ|?veaZ^P=Xryj1HR@j5p(E5^+%!wnlQxRb{!xz%C3LJR z(cznYqelDidfCn&oIY(TVfz`y2U*OH*eB z4mMYfR<;ljgx0GojJ7f_21fS() + val app = Application.builder("rbcs") + .configurationDirectoryEnvVar("RBCS_CONFIGURATION_DIR") + .configurationDirectoryPropertyKey("net.woggioni.rbcs.conf.dir") + .build() + val rbcsCli = RemoteBuildCacheServerCli() + val commandLine = CommandLine(rbcsCli) + commandLine.setExecutionExceptionHandler { ex, cl, parseResult -> + log.error(ex.message, ex) + CommandLine.ExitCode.SOFTWARE + } + commandLine.addSubcommand(ServerCommand(app)) + commandLine.addSubcommand(PasswordHashCommand()) + commandLine.addSubcommand( + CommandLine(ClientCommand(app)).apply { + addSubcommand(BenchmarkCommand()) + addSubcommand(PutCommand()) + addSubcommand(GetCommand()) + addSubcommand(HealthCheckCommand()) + }) + return commandLine + } + + @JvmStatic + fun main(vararg args: String) { + System.exit(createCommandLine().execute(*args)) + } + } + + @CommandLine.Option(names = ["-V", "--version"], versionHelp = true) + var versionHelp = false + private set + + @CommandLine.Spec + private lateinit var spec: CommandSpec + + + override fun run() { + spec.commandLine().usage(System.out); + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/AbstractVersionProvider.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/AbstractVersionProvider.kt new file mode 100644 index 0000000..632689c --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/AbstractVersionProvider.kt @@ -0,0 +1,30 @@ +package net.woggioni.rbcs.cli.impl + +import java.util.jar.Attributes +import java.util.jar.JarFile +import java.util.jar.Manifest +import picocli.CommandLine + + +abstract class AbstractVersionProvider : CommandLine.IVersionProvider { + private val version: String + private val vcsHash: String + + init { + val mf = Manifest() + javaClass.module.getResourceAsStream(JarFile.MANIFEST_NAME).use { `is` -> + mf.read(`is`) + } + val mainAttributes = mf.mainAttributes + version = mainAttributes.getValue(Attributes.Name.SPECIFICATION_VERSION) ?: throw RuntimeException("Version information not found in manifest") + vcsHash = mainAttributes.getValue(Attributes.Name.IMPLEMENTATION_VERSION) ?: throw RuntimeException("Version information not found in manifest") + } + + override fun getVersion(): Array { + return if (version.endsWith("-SNAPSHOT")) { + arrayOf(version, vcsHash) + } else { + arrayOf(version) + } + } +} diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/RbcsCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/RbcsCommand.kt new file mode 100644 index 0000000..91d3159 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/RbcsCommand.kt @@ -0,0 +1,19 @@ +package net.woggioni.rbcs.cli.impl + +import java.nio.file.Path +import net.woggioni.jwo.Application +import picocli.CommandLine + + +abstract class RbcsCommand : Runnable { + + @CommandLine.Option(names = ["-h", "--help"], usageHelp = true) + var usageHelp = false + private set + + protected fun findConfigurationFile(app: Application, fileName : String): Path { + val confDir = app.computeConfigurationDirectory(false) + val configurationFile = confDir.resolve(fileName) + return configurationFile + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/BenchmarkCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/BenchmarkCommand.kt new file mode 100644 index 0000000..7f30b95 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/BenchmarkCommand.kt @@ -0,0 +1,199 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.security.SecureRandom +import java.time.Duration +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.Semaphore +import java.util.concurrent.atomic.AtomicLong +import kotlin.random.Random +import net.woggioni.jwo.JWO +import net.woggioni.jwo.LongMath +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.cli.impl.converters.ByteSizeConverter +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.client.RemoteBuildCacheClient +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.error +import net.woggioni.rbcs.common.info +import picocli.CommandLine + +@CommandLine.Command( + name = "benchmark", + description = ["Run a load test against the server"], + showDefaultValues = true +) +class BenchmarkCommand : RbcsCommand() { + companion object { + private val log = createLogger() + + fun execute(profile : Configuration.Profile, + numberOfEntries : Int, + entrySize : Int, + useRandomValue : Boolean, + ) { + val progressThreshold = LongMath.ceilDiv(numberOfEntries.toLong(), 20) + RemoteBuildCacheClient(profile).use { client -> + val entryGenerator = sequence { + val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong()) + while (true) { + val key = JWO.bytesToHex(random.nextBytes(16)) + val value = if (useRandomValue) { + random.nextBytes(entrySize) + } else { + val byteValue = random.nextInt().toByte() + ByteArray(entrySize) { _ -> byteValue } + } + yield(key to value) + } + } + + log.info { + "Starting insertion" + } + val entries = let { + val completionCounter = AtomicLong(0) + val completionQueue = LinkedBlockingQueue>(numberOfEntries) + val start = Instant.now() + val semaphore = Semaphore(profile.maxConnections * 5) + val iterator = entryGenerator.take(numberOfEntries).iterator() + while (completionCounter.get() < numberOfEntries) { + if (iterator.hasNext()) { + val entry = iterator.next() + semaphore.acquire() + val future = + client.put(entry.first, entry.second, CacheValueMetadata(null, null)).thenApply { entry } + future.whenComplete { result, ex -> + if (ex != null) { + log.error(ex.message, ex) + } else { + completionQueue.put(result) + } + semaphore.release() + val completed = completionCounter.incrementAndGet() + if (completed.mod(progressThreshold) == 0L) { + log.debug { + "Inserted $completed / $numberOfEntries" + } + } + } + } else { + Thread.sleep(Duration.of(500, ChronoUnit.MILLIS)) + } + } + + val inserted = completionQueue.toList() + val end = Instant.now() + log.info { + val elapsed = Duration.between(start, end).toMillis() + val opsPerSecond = String.format("%.2f", numberOfEntries.toDouble() / elapsed * 1000) + "Insertion rate: $opsPerSecond ops/s" + } + inserted + } + log.info { + "Inserted ${entries.size} entries" + } + log.info { + "Starting retrieval" + } + if (entries.isNotEmpty()) { + val errorCounter = AtomicLong(0) + val completionCounter = AtomicLong(0) + val semaphore = Semaphore(profile.maxConnections * 5) + val start = Instant.now() + val it = entries.iterator() + while (completionCounter.get() < entries.size) { + if (it.hasNext()) { + val entry = it.next() + semaphore.acquire() + val future = client.get(entry.first).handle { response, ex -> + if(ex != null) { + errorCounter.incrementAndGet() + log.error(ex.message, ex) + } else if (response == null) { + errorCounter.incrementAndGet() + log.error { + "Missing entry for key '${entry.first}'" + } + } else if (!entry.second.contentEquals(response)) { + errorCounter.incrementAndGet() + log.error { + "Retrieved a value different from what was inserted for key '${entry.first}': " + + "expected '${JWO.bytesToHex(entry.second)}', got '${JWO.bytesToHex(response)}' instead" + } + } + } + future.whenComplete { _, _ -> + val completed = completionCounter.incrementAndGet() + if (completed.mod(progressThreshold) == 0L) { + log.debug { + "Retrieved $completed / ${entries.size}" + } + } + semaphore.release() + } + } else { + Thread.sleep(Duration.of(500, ChronoUnit.MILLIS)) + } + } + val end = Instant.now() + val errors = errorCounter.get() + val successfulRetrievals = entries.size - errors + val successRate = successfulRetrievals.toDouble() / entries.size + log.info { + "Successfully retrieved ${entries.size - errors}/${entries.size} (${String.format("%.1f", successRate * 100)}%)" + } + log.info { + val elapsed = Duration.between(start, end).toMillis() + val opsPerSecond = String.format("%.2f", entries.size.toDouble() / elapsed * 1000) + "Retrieval rate: $opsPerSecond ops/s" + } + } else { + log.error("Skipping retrieval benchmark as it was not possible to insert any entry in the cache") + } + } + } + } + + @CommandLine.Spec + private lateinit var spec: CommandLine.Model.CommandSpec + + @CommandLine.Option( + names = ["-e", "--entries"], + description = ["Total number of elements to be added to the cache"], + paramLabel = "NUMBER_OF_ENTRIES" + ) + private var numberOfEntries = 1000 + + @CommandLine.Option( + names = ["-s", "--size"], + description = ["Size of a cache value in bytes"], + paramLabel = "SIZE", + converter = [ByteSizeConverter::class] + ) + private var size = 0x1000 + + @CommandLine.Option( + names = ["-r", "--random"], + description = ["Insert completely random byte values"] + ) + private var randomValues = false + + override fun run() { + val clientCommand = spec.parent().userObject() as ClientCommand + val profile = clientCommand.profileName.let { profileName -> + clientCommand.configuration.profiles[profileName] + ?: throw IllegalArgumentException("Profile $profileName does not exist in configuration") + } + execute( + profile, + numberOfEntries, + size, + randomValues + ) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ClientCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ClientCommand.kt new file mode 100644 index 0000000..4dcb8a9 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ClientCommand.kt @@ -0,0 +1,48 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.nio.file.Path +import net.woggioni.jwo.Application +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import picocli.CommandLine + +@CommandLine.Command( + name = "client", + description = ["RBCS client"], + showDefaultValues = true +) +class ClientCommand(app : Application) : RbcsCommand() { + + @CommandLine.Option( + names = ["-c", "--configuration"], + description = ["Path to the client configuration file"], + paramLabel = "CONFIGURATION_FILE" + ) + private var configurationFile : Path = findConfigurationFile(app, "rbcs-client.xml") + + @CommandLine.Option( + names = ["-p", "--profile"], + description = ["Name of the client profile to be used"], + paramLabel = "PROFILE", + required = false + ) + var profileName : String? = null + get() = field ?: throw IllegalArgumentException("A profile name must be specified using the '-p' command line parameter") + + val configuration : Configuration by lazy { + Configuration.parse(configurationFile) + } + + override fun run() { + val log = createLogger() + log.debug { + "Using configuration file '$configurationFile'" + } + println("Available profiles:") + configuration.profiles.forEach { (profileName, _) -> + println(profileName) + } + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/GetCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/GetCommand.kt new file mode 100644 index 0000000..63381ef --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/GetCommand.kt @@ -0,0 +1,59 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.io.OutputStream +import java.nio.file.Files +import java.nio.file.Path +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.client.RemoteBuildCacheClient +import net.woggioni.rbcs.common.createLogger +import picocli.CommandLine + +@CommandLine.Command( + name = "get", + description = ["Fetch a value from the cache with the specified key"], + showDefaultValues = true +) +class GetCommand : RbcsCommand() { + companion object { + private val log = createLogger() + + fun execute(profile : Configuration.Profile, key : String, outputStream: OutputStream) { + RemoteBuildCacheClient(profile).use { client -> + client.get(key).thenApply { value -> + value?.let { + outputStream.use { + it.write(value) + } + } ?: throw NoSuchElementException("No value found for key $key") + }.get() + } + } + } + + @CommandLine.Spec + private lateinit var spec: CommandLine.Model.CommandSpec + + @CommandLine.Option( + names = ["-k", "--key"], + description = ["The key for the new value"], + paramLabel = "KEY" + ) + private var key : String = "" + + @CommandLine.Option( + names = ["-v", "--value"], + description = ["Path to a file where the retrieved value will be written (defaults to stdout)"], + paramLabel = "VALUE_FILE", + ) + private var output : Path? = null + + override fun run() { + val clientCommand = spec.parent().userObject() as ClientCommand + val profile = clientCommand.profileName.let { profileName -> + clientCommand.configuration.profiles[profileName] + ?: throw IllegalArgumentException("Profile $profileName does not exist in configuration") + } + execute(profile, key, (output?.let(Files::newOutputStream) ?: System.out)) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/HealthCheckCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/HealthCheckCommand.kt new file mode 100644 index 0000000..224e63d --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/HealthCheckCommand.kt @@ -0,0 +1,53 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.security.SecureRandom +import kotlin.random.Random +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.client.RemoteBuildCacheClient +import net.woggioni.rbcs.common.createLogger +import picocli.CommandLine + +@CommandLine.Command( + name = "health", + description = ["Check server health"], + showDefaultValues = true +) +class HealthCheckCommand : RbcsCommand() { + companion object{ + private val log = createLogger() + + fun execute(profile : Configuration.Profile) { + RemoteBuildCacheClient(profile).use { client -> + val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong()) + val nonce = ByteArray(0xa0) + random.nextBytes(nonce) + client.healthCheck(nonce).thenApply { value -> + if(value == null) { + throw IllegalStateException("Empty response from server") + } + val offset = value.size - nonce.size + for(i in 0 until nonce.size) { + val a = nonce[i] + val b = value[offset + i] + if(a != b) { + throw IllegalStateException("Server nonce does not match") + } + } + }.get() + } + } + } + + @CommandLine.Spec + private lateinit var spec: CommandLine.Model.CommandSpec + + override fun run() { + val clientCommand = spec.parent().userObject() as ClientCommand + val profile = clientCommand.profileName.let { profileName -> + clientCommand.configuration.profiles[profileName] + ?: throw IllegalArgumentException("Profile $profileName does not exist in configuration") + } + execute(profile) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PasswordHashCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PasswordHashCommand.kt new file mode 100644 index 0000000..3862524 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PasswordHashCommand.kt @@ -0,0 +1,37 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.io.OutputStream +import java.io.OutputStreamWriter +import java.io.PrintWriter +import net.woggioni.jwo.UncloseableOutputStream +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.cli.impl.converters.OutputStreamConverter +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import picocli.CommandLine + + +@CommandLine.Command( + name = "password", + description = ["Generate a password hash to add to RBCS configuration file"], + showDefaultValues = true +) +class PasswordHashCommand : RbcsCommand() { + @CommandLine.Option( + names = ["-o", "--output-file"], + description = ["Write the output to a file instead of stdout"], + converter = [OutputStreamConverter::class], + showDefaultValue = CommandLine.Help.Visibility.NEVER, + paramLabel = "OUTPUT_FILE" + ) + private var outputStream: OutputStream = UncloseableOutputStream(System.out) + + override fun run() { + val password1 = String(System.console().readPassword("Type your password:")) + val password2 = String(System.console().readPassword("Type your password again for confirmation:")) + if(password1 != password2) throw IllegalArgumentException("Passwords do not match") + + PrintWriter(OutputStreamWriter(outputStream, Charsets.UTF_8)).use { + it.println(hashPassword(password1)) + } + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PutCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PutCommand.kt new file mode 100644 index 0000000..5e6d10f --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/PutCommand.kt @@ -0,0 +1,113 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.io.InputStream +import java.nio.file.Files +import java.nio.file.Path +import java.util.UUID +import net.woggioni.jwo.Hash +import net.woggioni.jwo.JWO +import net.woggioni.jwo.NullOutputStream +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.client.RemoteBuildCacheClient +import net.woggioni.rbcs.common.createLogger +import picocli.CommandLine + +@CommandLine.Command( + name = "put", + description = ["Add or replace a value to the cache with the specified key"], + showDefaultValues = true +) +class PutCommand : RbcsCommand() { + companion object { + private val log = createLogger() + + fun execute(profile: Configuration.Profile, + actualKey : String, + inputStream: InputStream, + mimeType : String?, + contentDisposition: String? + ) { + RemoteBuildCacheClient(profile).use { client -> + inputStream.use { + client.put(actualKey, it.readAllBytes(), CacheValueMetadata(contentDisposition, mimeType)) + }.get() + println(profile.serverURI.resolve(actualKey)) + } + } + } + + + @CommandLine.Spec + private lateinit var spec: CommandLine.Model.CommandSpec + + @CommandLine.Option( + names = ["-k", "--key"], + description = ["The key for the new value, randomly generated if omitted"], + paramLabel = "KEY" + ) + private var key : String? = null + + @CommandLine.Option( + names = ["-i", "--inline"], + description = ["File is to be displayed in the browser"], + paramLabel = "INLINE", + ) + private var inline : Boolean = false + + @CommandLine.Option( + names = ["-t", "--type"], + description = ["File mime type"], + paramLabel = "MIME_TYPE", + ) + private var mimeType : String? = null + + @CommandLine.Option( + names = ["-v", "--value"], + description = ["Path to a file containing the value to be added (defaults to stdin)"], + paramLabel = "VALUE_FILE", + ) + private var value : Path? = null + + override fun run() { + val clientCommand = spec.parent().userObject() as ClientCommand + val profile = clientCommand.profileName.let { profileName -> + clientCommand.configuration.profiles[profileName] + ?: throw IllegalArgumentException("Profile $profileName does not exist in configuration") + } + RemoteBuildCacheClient(profile).use { client -> + val inputStream : InputStream + val mimeType : String? + val contentDisposition : String? + val valuePath = value + val actualKey : String? + if(valuePath != null) { + inputStream = Files.newInputStream(valuePath) + mimeType = this.mimeType ?: Files.probeContentType(valuePath) + contentDisposition = if(inline) { + "inline" + } else { + "attachment; filename=\"${valuePath.fileName}\"" + } + actualKey = key ?: let { + val md = Hash.Algorithm.SHA512.newInputStream(Files.newInputStream(valuePath)).use { + JWO.copy(it, NullOutputStream()) + it.messageDigest + } + UUID.nameUUIDFromBytes(md.digest()).toString() + } + } else { + inputStream = System.`in` + mimeType = this.mimeType + contentDisposition = if(inline) { + "inline" + } else { + null + } + actualKey = key ?: UUID.randomUUID().toString() + } + execute(profile, actualKey, inputStream, mimeType, contentDisposition) + } + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ServerCommand.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ServerCommand.kt new file mode 100644 index 0000000..46af626 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/commands/ServerCommand.kt @@ -0,0 +1,90 @@ +package net.woggioni.rbcs.cli.impl.commands + +import java.io.ByteArrayOutputStream +import java.nio.file.Files +import java.nio.file.Path +import java.time.Duration +import java.util.concurrent.TimeUnit +import net.woggioni.jwo.Application +import net.woggioni.jwo.JWO +import net.woggioni.rbcs.cli.impl.RbcsCommand +import net.woggioni.rbcs.cli.impl.converters.DurationConverter +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.info +import net.woggioni.rbcs.server.RemoteBuildCacheServer +import net.woggioni.rbcs.server.RemoteBuildCacheServer.Companion.DEFAULT_CONFIGURATION_URL +import picocli.CommandLine + +@CommandLine.Command( + name = "server", + description = ["RBCS server"], + showDefaultValues = true +) +class ServerCommand(app : Application) : RbcsCommand() { + companion object { + private val log = createLogger() + } + + private fun createDefaultConfigurationFile(configurationFile: Path) { + log.info { + "Creating default configuration file at '$configurationFile'" + } + val defaultConfigurationFileResource = DEFAULT_CONFIGURATION_URL + Files.newOutputStream(configurationFile).use { outputStream -> + defaultConfigurationFileResource.openStream().use { inputStream -> + JWO.copy(inputStream, outputStream) + } + } + } + + @CommandLine.Option( + names = ["-t", "--timeout"], + description = ["Exit after the specified time"], + paramLabel = "TIMEOUT", + converter = [DurationConverter::class] + ) + private var timeout: Duration? = null + + @CommandLine.Option( + names = ["-c", "--config-file"], + description = ["Read the application configuration from this file"], + paramLabel = "CONFIG_FILE" + ) + private var configurationFile: Path = findConfigurationFile(app, "rbcs-server.xml") + + override fun run() { + if (!Files.exists(configurationFile)) { + Files.createDirectories(configurationFile.parent) + createDefaultConfigurationFile(configurationFile) + } + + log.debug { + "Using configuration file '$configurationFile'" + } + val configuration = RemoteBuildCacheServer.loadConfiguration(configurationFile) + log.debug { + ByteArrayOutputStream().also { + RemoteBuildCacheServer.dumpConfiguration(configuration, it) + }.let { + "Server configuration:\n${String(it.toByteArray())}" + } + } + val server = RemoteBuildCacheServer(configuration) + val handle = server.run() + val shutdownHook = Thread.ofPlatform().unstarted { + handle.sendShutdownSignal() + try { + handle.get(60, TimeUnit.SECONDS) + } catch (ex : Throwable) { + log.warn(ex.message, ex) + } + } + Runtime.getRuntime().addShutdownHook(shutdownHook) + if(timeout != null) { + Thread.sleep(timeout) + handle.sendShutdownSignal() + } + handle.get() + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/ByteSizeConverter.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/ByteSizeConverter.kt new file mode 100644 index 0000000..e9d323e --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/ByteSizeConverter.kt @@ -0,0 +1,10 @@ +package net.woggioni.rbcs.cli.impl.converters + +import picocli.CommandLine + + +class ByteSizeConverter : CommandLine.ITypeConverter { + override fun convert(value: String): Int { + return Integer.decode(value) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/DurationConverter.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/DurationConverter.kt new file mode 100644 index 0000000..158a417 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/DurationConverter.kt @@ -0,0 +1,11 @@ +package net.woggioni.rbcs.cli.impl.converters + +import java.time.Duration +import picocli.CommandLine + + +class DurationConverter : CommandLine.ITypeConverter { + override fun convert(value: String): Duration { + return Duration.parse(value) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/InputStreamConverter.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/InputStreamConverter.kt new file mode 100644 index 0000000..81f5605 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/InputStreamConverter.kt @@ -0,0 +1,13 @@ +package net.woggioni.rbcs.cli.impl.converters + +import java.io.InputStream +import java.nio.file.Files +import java.nio.file.Paths +import picocli.CommandLine + + +class InputStreamConverter : CommandLine.ITypeConverter { + override fun convert(value: String): InputStream { + return Files.newInputStream(Paths.get(value)) + } +} \ No newline at end of file diff --git a/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/OutputStreamConverter.kt b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/OutputStreamConverter.kt new file mode 100644 index 0000000..8c16f40 --- /dev/null +++ b/rbcs-cli/bin/main/net/woggioni/rbcs/cli/impl/converters/OutputStreamConverter.kt @@ -0,0 +1,13 @@ +package net.woggioni.rbcs.cli.impl.converters + +import java.io.OutputStream +import java.nio.file.Files +import java.nio.file.Paths +import picocli.CommandLine + + +class OutputStreamConverter : CommandLine.ITypeConverter { + override fun convert(value: String): OutputStream { + return Files.newOutputStream(Paths.get(value)) + } +} \ No newline at end of file diff --git a/rbcs-client/.classpath b/rbcs-client/.classpath new file mode 100644 index 0000000..8bfaaaf --- /dev/null +++ b/rbcs-client/.classpath @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-client/.project b/rbcs-client/.project new file mode 100644 index 0000000..1af5415 --- /dev/null +++ b/rbcs-client/.project @@ -0,0 +1,34 @@ + + + rbcs-client + Project rbcs-client created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776950 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-client/.settings/org.eclipse.buildship.core.prefs b/rbcs-client/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-client/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-client/.settings/org.eclipse.jdt.core.prefs b/rbcs-client/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-client/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-client/bin/main/module-info.class b/rbcs-client/bin/main/module-info.class new file mode 100644 index 0000000000000000000000000000000000000000..9e228c3ebdb3b88b20f3c740b37c0bcb2f138f5f GIT binary patch literal 546 zcmZuuNm2qa6n!6+Q3eSpjyP>iZ@hx;TzP_~I}O2fLUjj+9p`f40X&ptIv_=rWbxjg z^54(T_s{1SfCt=GP$D!^6VH{nlR7bk`m>p5k$99!5IP$vJ`G<(Lix#8GgJrzEi8XC zjbC)UmwnuP8;VY5)@Gn2VC zZ9?n*!O=WP3bj6=S`5wKQ^jz!ecn}rV?zJ`Yr>gi zI3=_;1l`aK=Y*Qc#ynTa{gh$oT(2|=bJ4@qQY!dcO;D^RSgxjr8^X{@ZOsOGG!bdI z0LcB&@|3XSj1qRS=Qu{$hxbwSTwi$wwY=T&*!9@+c<6E9@x +) { + sealed class Authentication { + data class TlsClientAuthenticationCredentials( + val key: PrivateKey, + val certificateChain: Array + ) : Authentication() + + data class BasicAuthenticationCredentials(val username: String, val password: String) : Authentication() + } + + class TrustStore ( + var file: Path?, + var password: String?, + var checkCertificateStatus: Boolean = false, + var verifyServerCertificate: Boolean = true, + ) + + class RetryPolicy( + val maxAttempts: Int, + val initialDelayMillis: Long, + val exp: Double + ) + + class Connection( + val readIdleTimeout: Duration, + val writeIdleTimeout: Duration, + val idleTimeout: Duration, + val requestPipelining : Boolean, + ) + + data class Profile( + val serverURI: URI, + val connection: Connection, + val authentication: Authentication?, + val connectionTimeout: Duration?, + val maxConnections: Int, + val compressionEnabled: Boolean, + val retryPolicy: RetryPolicy?, + val tlsTruststore : TrustStore? + ) + + companion object { + fun parse(path: Path): Configuration { + return Files.newInputStream(path).use { + Xml.parseXml(path.toUri().toURL(), it) + }.let(Parser::parse) + } + } +} \ No newline at end of file diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/Exception.kt b/rbcs-client/bin/main/net/woggioni/rbcs/client/Exception.kt new file mode 100644 index 0000000..1d888bc --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/Exception.kt @@ -0,0 +1,9 @@ +package net.woggioni.rbcs.client + +import io.netty.handler.codec.http.HttpResponseStatus + +class HttpException(private val status : HttpResponseStatus) : RuntimeException(status.reasonPhrase()) { + + override val message: String + get() = "Http status ${status.code()}: ${status.reasonPhrase()}" +} \ No newline at end of file diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/RemoteBuildCacheClient.kt b/rbcs-client/bin/main/net/woggioni/rbcs/client/RemoteBuildCacheClient.kt new file mode 100644 index 0000000..cac8d8a --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/RemoteBuildCacheClient.kt @@ -0,0 +1,433 @@ +package net.woggioni.rbcs.client + +import io.netty.bootstrap.Bootstrap +import io.netty.buffer.ByteBuf +import io.netty.buffer.Unpooled +import io.netty.channel.Channel +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelOption +import io.netty.channel.ChannelPipeline +import io.netty.channel.IoEventLoopGroup +import io.netty.channel.MultiThreadIoEventLoopGroup +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.channel.nio.NioIoHandler +import io.netty.channel.pool.AbstractChannelPoolHandler +import io.netty.channel.pool.ChannelPool +import io.netty.channel.pool.FixedChannelPool +import io.netty.channel.socket.nio.NioSocketChannel +import io.netty.handler.codec.DecoderException +import io.netty.handler.codec.http.DefaultFullHttpRequest +import io.netty.handler.codec.http.FullHttpRequest +import io.netty.handler.codec.http.FullHttpResponse +import io.netty.handler.codec.http.HttpClientCodec +import io.netty.handler.codec.http.HttpContentDecompressor +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpHeaderValues +import io.netty.handler.codec.http.HttpMethod +import io.netty.handler.codec.http.HttpObjectAggregator +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import io.netty.handler.ssl.SslContext +import io.netty.handler.ssl.SslContextBuilder +import io.netty.handler.stream.ChunkedWriteHandler +import io.netty.handler.timeout.IdleState +import io.netty.handler.timeout.IdleStateEvent +import io.netty.handler.timeout.IdleStateHandler +import io.netty.util.concurrent.Future +import io.netty.util.concurrent.Future as NettyFuture +import io.netty.util.concurrent.GenericFutureListener +import java.io.IOException +import java.net.InetSocketAddress +import java.net.URI +import java.security.cert.X509Certificate +import java.util.Base64 +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException +import java.util.concurrent.atomic.AtomicInteger +import javax.net.ssl.TrustManagerFactory +import javax.net.ssl.X509TrustManager +import kotlin.random.Random +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.common.RBCS.loadKeystore +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.trace + +class RemoteBuildCacheClient(private val profile: Configuration.Profile) : AutoCloseable { + companion object { + private val log = createLogger() + } + + private val group: IoEventLoopGroup + private val sslContext: SslContext + private val pool: ChannelPool + + init { + group = MultiThreadIoEventLoopGroup(NioIoHandler.newFactory()) + sslContext = SslContextBuilder.forClient().also { builder -> + (profile.authentication as? Configuration.Authentication.TlsClientAuthenticationCredentials)?.let { tlsClientAuthenticationCredentials -> + builder.apply { + keyManager( + tlsClientAuthenticationCredentials.key, + *tlsClientAuthenticationCredentials.certificateChain + ) + profile.tlsTruststore?.let { trustStore -> + if (!trustStore.verifyServerCertificate) { + trustManager(object : X509TrustManager { + override fun checkClientTrusted(certChain: Array, p1: String?) { + } + + override fun checkServerTrusted(certChain: Array, p1: String?) { + } + + override fun getAcceptedIssuers() = null + }) + } else { + trustStore.file?.let { + val ts = loadKeystore(it, trustStore.password) + val trustManagerFactory: TrustManagerFactory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) + trustManagerFactory.init(ts) + trustManager(trustManagerFactory) + } + } + } + } + } + }.build() + + val (scheme, host, port) = profile.serverURI.run { + Triple( + if (scheme == null) "http" else profile.serverURI.scheme, + host, + port.takeIf { it > 0 } ?: if ("https" == scheme.lowercase()) 443 else 80 + ) + } + + val bootstrap = Bootstrap().apply { + group(group) + channel(NioSocketChannel::class.java) + option(ChannelOption.TCP_NODELAY, true) + option(ChannelOption.SO_KEEPALIVE, true) + remoteAddress(InetSocketAddress(host, port)) + profile.connectionTimeout?.let { + option(ChannelOption.CONNECT_TIMEOUT_MILLIS, it.toMillis().toInt()) + } + } + val channelPoolHandler = object : AbstractChannelPoolHandler() { + + @Volatile + private var connectionCount = AtomicInteger() + + @Volatile + private var leaseCount = AtomicInteger() + + override fun channelReleased(ch: Channel) { + val activeLeases = leaseCount.decrementAndGet() + log.trace { + "Released channel ${ch.id().asShortText()}, number of active leases: $activeLeases" + } + } + + override fun channelAcquired(ch: Channel) { + val activeLeases = leaseCount.getAndIncrement() + log.trace { + "Acquired channel ${ch.id().asShortText()}, number of active leases: $activeLeases" + } + } + + override fun channelCreated(ch: Channel) { + val connectionId = connectionCount.incrementAndGet() + log.debug { + "Created connection ${ch.id().asShortText()}, total number of active connections: $connectionId" + } + ch.closeFuture().addListener { + val activeConnections = connectionCount.decrementAndGet() + log.debug { + "Closed connection ${ + ch.id().asShortText() + }, total number of active connections: $activeConnections" + } + } + val pipeline: ChannelPipeline = ch.pipeline() + + profile.connection.also { conn -> + val readIdleTimeout = conn.readIdleTimeout.toMillis() + val writeIdleTimeout = conn.writeIdleTimeout.toMillis() + val idleTimeout = conn.idleTimeout.toMillis() + if (readIdleTimeout > 0 || writeIdleTimeout > 0 || idleTimeout > 0) { + pipeline.addLast( + IdleStateHandler( + true, + readIdleTimeout, + writeIdleTimeout, + idleTimeout, + TimeUnit.MILLISECONDS + ) + ) + } + } + + // Add SSL handler if needed + if ("https".equals(scheme, ignoreCase = true)) { + pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port)) + } + + // HTTP handlers + pipeline.addLast("codec", HttpClientCodec()) + if (profile.compressionEnabled) { + pipeline.addLast("decompressor", HttpContentDecompressor()) + } + pipeline.addLast("aggregator", HttpObjectAggregator(134217728)) + pipeline.addLast("chunked", ChunkedWriteHandler()) + } + } + pool = FixedChannelPool(bootstrap, channelPoolHandler, profile.maxConnections) + } + + private fun executeWithRetry(operation: () -> CompletableFuture): CompletableFuture { + val retryPolicy = profile.retryPolicy + return if (retryPolicy != null) { + val outcomeHandler = OutcomeHandler { outcome -> + when (outcome) { + is OperationOutcome.Success -> { + val response = outcome.result + val status = response.status() + when (status) { + HttpResponseStatus.TOO_MANY_REQUESTS -> { + val retryAfter = response.headers()[HttpHeaderNames.RETRY_AFTER]?.let { headerValue -> + try { + headerValue.toLong() * 1000 + } catch (nfe: NumberFormatException) { + null + } + } + OutcomeHandlerResult.Retry(retryAfter) + } + + HttpResponseStatus.INTERNAL_SERVER_ERROR, HttpResponseStatus.SERVICE_UNAVAILABLE -> + OutcomeHandlerResult.Retry() + + else -> OutcomeHandlerResult.DoNotRetry() + } + } + + is OperationOutcome.Failure -> { + OutcomeHandlerResult.Retry() + } + } + } + executeWithRetry( + group, + retryPolicy.maxAttempts, + retryPolicy.initialDelayMillis.toDouble(), + retryPolicy.exp, + outcomeHandler, + Random.Default, + operation + ) + } else { + operation() + } + } + + fun healthCheck(nonce: ByteArray): CompletableFuture { + return executeWithRetry { + sendRequest(profile.serverURI, HttpMethod.TRACE, nonce) + }.thenApply { + val status = it.status() + if (it.status() != HttpResponseStatus.OK) { + throw HttpException(status) + } else { + it.content() + } + }.thenApply { maybeByteBuf -> + maybeByteBuf?.let { + val result = ByteArray(it.readableBytes()) + it.getBytes(0, result) + result + } + } + } + + fun get(key: String): CompletableFuture { + return executeWithRetry { + sendRequest(profile.serverURI.resolve(key), HttpMethod.GET, null) + }.thenApply { response -> + val status = response.status() + if (response.status() == HttpResponseStatus.NOT_FOUND) { + response.release() + null + } else if (response.status() != HttpResponseStatus.OK) { + response.release() + throw HttpException(status) + } else { + response.content().also { + it.retain() + response.release() + } + } + }.thenApply { maybeByteBuf -> + maybeByteBuf?.let { buf -> + val result = ByteArray(buf.readableBytes()) + buf.getBytes(0, result) + buf.release() + result + } + } + } + + fun put(key: String, content: ByteArray, metadata: CacheValueMetadata): CompletableFuture { + return executeWithRetry { + val extraHeaders = sequenceOf( + metadata.mimeType?.let { HttpHeaderNames.CONTENT_TYPE to it }, + metadata.contentDisposition?.let { HttpHeaderNames.CONTENT_DISPOSITION to it } + ).filterNotNull() + sendRequest(profile.serverURI.resolve(key), HttpMethod.PUT, content, extraHeaders.asIterable()) + }.thenApply { + val status = it.status() + if (it.status() != HttpResponseStatus.CREATED && it.status() != HttpResponseStatus.OK) { + throw HttpException(status) + } + } + } + + private fun sendRequest( + uri: URI, + method: HttpMethod, + body: ByteArray?, + extraHeaders: Iterable>? = null + ): CompletableFuture { + val responseFuture = CompletableFuture() + // Custom handler for processing responses + pool.acquire().addListener(object : GenericFutureListener> { + + override fun operationComplete(channelFuture: Future) { + if (channelFuture.isSuccess) { + val channel = channelFuture.now + val pipeline = channel.pipeline() + + val closeListener = GenericFutureListener> { + responseFuture.completeExceptionally(IOException("The remote server closed the connection")) + } + channel.closeFuture().addListener(closeListener) + + val responseHandler = object : SimpleChannelInboundHandler() { + + override fun handlerAdded(ctx: ChannelHandlerContext) { + channel.closeFuture().removeListener(closeListener) + } + + override fun channelRead0( + ctx: ChannelHandlerContext, + response: FullHttpResponse + ) { + pipeline.remove(this) + responseFuture.complete(response.retainedDuplicate()) + if (!profile.connection.requestPipelining) { + pool.release(channel) + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + ctx.newPromise() + val ex = when (cause) { + is DecoderException -> cause.cause + else -> cause + } + responseFuture.completeExceptionally(ex) + ctx.close() + } + + override fun channelInactive(ctx: ChannelHandlerContext) { + responseFuture.completeExceptionally(IOException("The remote server closed the connection")) + super.channelInactive(ctx) + pool.release(channel) + } + + override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) { + if (evt is IdleStateEvent) { + val te = when (evt.state()) { + IdleState.READER_IDLE -> TimeoutException("Read timeout") + IdleState.WRITER_IDLE -> TimeoutException("Write timeout") + IdleState.ALL_IDLE -> TimeoutException("Idle timeout") + null -> throw IllegalStateException("This should never happen") + } + responseFuture.completeExceptionally(te) + super.userEventTriggered(ctx, evt) + if (this === pipeline.last()) { + ctx.close() + } + if (!profile.connection.requestPipelining) { + pool.release(channel) + } + } else { + super.userEventTriggered(ctx, evt) + } + } + } + pipeline.addLast(responseHandler) + + + // Prepare the HTTP request + val request: FullHttpRequest = let { + val content: ByteBuf? = body?.takeIf(ByteArray::isNotEmpty)?.let(Unpooled::wrappedBuffer) + DefaultFullHttpRequest( + HttpVersion.HTTP_1_1, + method, + uri.rawPath, + content ?: Unpooled.buffer(0) + ).apply { + // Set headers + headers().apply { + if (content != null) { + set(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes()) + } + set(HttpHeaderNames.HOST, profile.serverURI.host) + set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE) + if (profile.compressionEnabled) { + set( + HttpHeaderNames.ACCEPT_ENCODING, + HttpHeaderValues.GZIP.toString() + "," + HttpHeaderValues.DEFLATE.toString() + ) + } + extraHeaders?.forEach { (k, v) -> + add(k, v) + } + // Add basic auth if configured + (profile.authentication as? Configuration.Authentication.BasicAuthenticationCredentials)?.let { credentials -> + val auth = "${credentials.username}:${credentials.password}" + val encodedAuth = Base64.getEncoder().encodeToString(auth.toByteArray()) + set(HttpHeaderNames.AUTHORIZATION, "Basic $encodedAuth") + } + } + } + } + + // Send the request + channel.writeAndFlush(request).addListener { + if (!it.isSuccess) { + val ex = it.cause() + log.warn(ex.message, ex) + } + if (profile.connection.requestPipelining) { + pool.release(channel) + } + } + } else { + responseFuture.completeExceptionally(channelFuture.cause()) + } + } + }) + return responseFuture + } + + fun shutDown(): NettyFuture<*> { + return group.shutdownGracefully() + } + + override fun close() { + shutDown().sync() + } +} \ No newline at end of file diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/impl/Parser.kt b/rbcs-client/bin/main/net/woggioni/rbcs/client/impl/Parser.kt new file mode 100644 index 0000000..f41680a --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/impl/Parser.kt @@ -0,0 +1,151 @@ +package net.woggioni.rbcs.client.impl + +import java.net.URI +import java.nio.file.Files +import java.nio.file.Path +import java.security.KeyStore +import java.security.PrivateKey +import java.security.cert.X509Certificate +import java.time.Duration +import java.time.temporal.ChronoUnit +import net.woggioni.rbcs.api.exception.ConfigurationException +import net.woggioni.rbcs.client.Configuration +import net.woggioni.rbcs.common.Xml.Companion.asIterable +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document + +object Parser { + + fun parse(document: Document): Configuration { + val root = document.documentElement + val profiles = mutableMapOf() + + for (child in root.asIterable()) { + val tagName = child.localName + when (tagName) { + "profile" -> { + val name = + child.renderAttribute("name") ?: throw ConfigurationException("name attribute is required") + val uri = child.renderAttribute("base-url")?.let(::URI) + ?: throw ConfigurationException("base-url attribute is required") + var authentication: Configuration.Authentication? = null + var retryPolicy: Configuration.RetryPolicy? = null + var connection : Configuration.Connection = Configuration.Connection( + Duration.ofSeconds(60), + Duration.ofSeconds(60), + Duration.ofSeconds(30), + false + ) + var trustStore : Configuration.TrustStore? = null + for (gchild in child.asIterable()) { + when (gchild.localName) { + "tls-client-auth" -> { + val keyStoreFile = gchild.renderAttribute("key-store-file") + val keyStorePassword = + gchild.renderAttribute("key-store-password") + val keyAlias = gchild.renderAttribute("key-alias") + val keyPassword = gchild.renderAttribute("key-password") + + val keystore = KeyStore.getInstance("PKCS12").apply { + Files.newInputStream(Path.of(keyStoreFile)).use { + load(it, keyStorePassword?.toCharArray()) + } + } + val key = keystore.getKey(keyAlias, keyPassword?.toCharArray()) as PrivateKey + val certChain = keystore.getCertificateChain(keyAlias).asSequence() + .map { it as X509Certificate } + .toList() + .toTypedArray() + authentication = + Configuration.Authentication.TlsClientAuthenticationCredentials( + key, + certChain + ) + } + + "basic-auth" -> { + val username = gchild.renderAttribute("user") + ?: throw ConfigurationException("username attribute is required") + val password = gchild.renderAttribute("password") + ?: throw ConfigurationException("password attribute is required") + authentication = + Configuration.Authentication.BasicAuthenticationCredentials( + username, + password + ) + } + + "retry-policy" -> { + val maxAttempts = + gchild.renderAttribute("max-attempts") + ?.let(String::toInt) + ?: throw ConfigurationException("max-attempts attribute is required") + val initialDelay = + gchild.renderAttribute("initial-delay") + ?.let(Duration::parse) + ?: Duration.ofSeconds(1) + val exp = + gchild.renderAttribute("exp") + ?.let(String::toDouble) + ?: 2.0f + retryPolicy = Configuration.RetryPolicy( + maxAttempts, + initialDelay.toMillis(), + exp.toDouble() + ) + } + + "connection" -> { + val idleTimeout = gchild.renderAttribute("idle-timeout") + ?.let(Duration::parse) ?: Duration.of(30, ChronoUnit.SECONDS) + val readIdleTimeout = gchild.renderAttribute("read-idle-timeout") + ?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS) + val writeIdleTimeout = gchild.renderAttribute("write-idle-timeout") + ?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS) + val requestPipelining = gchild.renderAttribute("request-pipelining") + ?.let(String::toBoolean) ?: false + connection = Configuration.Connection( + readIdleTimeout, + writeIdleTimeout, + idleTimeout, + requestPipelining + ) + } + + "tls-trust-store" -> { + val file = gchild.renderAttribute("file") + ?.let(Path::of) + val password = gchild.renderAttribute("password") + val checkCertificateStatus = gchild.renderAttribute("check-certificate-status") + ?.let(String::toBoolean) ?: false + val verifyServerCertificate = gchild.renderAttribute("verify-server-certificate") + ?.let(String::toBoolean) ?: true + trustStore = Configuration.TrustStore(file, password, checkCertificateStatus, verifyServerCertificate) + } + } + } + val maxConnections = child.renderAttribute("max-connections") + ?.let(String::toInt) + ?: 50 + val connectionTimeout = child.renderAttribute("connection-timeout") + ?.let(Duration::parse) + val compressionEnabled = child.renderAttribute("enable-compression") + ?.let(String::toBoolean) + ?: true + + profiles[name] = Configuration.Profile( + uri, + connection, + authentication, + connectionTimeout, + maxConnections, + compressionEnabled, + retryPolicy, + trustStore + ) + } + } + } + return Configuration(profiles) + } +} \ No newline at end of file diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/retry.kt b/rbcs-client/bin/main/net/woggioni/rbcs/client/retry.kt new file mode 100644 index 0000000..f90b4ab --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/retry.kt @@ -0,0 +1,79 @@ +package net.woggioni.rbcs.client + +import io.netty.util.concurrent.EventExecutorGroup +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit +import kotlin.math.pow +import kotlin.random.Random + +sealed class OperationOutcome { + class Success(val result: T) : OperationOutcome() + class Failure(val ex: Throwable) : OperationOutcome() +} + +sealed class OutcomeHandlerResult { + class Retry(val suggestedDelayMillis: Long? = null) : OutcomeHandlerResult() + class DoNotRetry : OutcomeHandlerResult() +} + +fun interface OutcomeHandler { + fun shouldRetry(result: OperationOutcome): OutcomeHandlerResult +} + +fun executeWithRetry( + eventExecutorGroup: EventExecutorGroup, + maxAttempts: Int, + initialDelay: Double, + exp: Double, + outcomeHandler: OutcomeHandler, + randomizer : Random?, + cb: () -> CompletableFuture +): CompletableFuture { + + val finalResult = cb() + var future = finalResult + var shortCircuit = false + for (i in 1 until maxAttempts) { + future = future.handle { result, ex -> + val operationOutcome = if (ex == null) { + OperationOutcome.Success(result) + } else { + OperationOutcome.Failure(ex.cause ?: ex) + } + if (shortCircuit) { + when(operationOutcome) { + is OperationOutcome.Failure -> throw operationOutcome.ex + is OperationOutcome.Success -> CompletableFuture.completedFuture(operationOutcome.result) + } + } else { + when(val outcomeHandlerResult = outcomeHandler.shouldRetry(operationOutcome)) { + is OutcomeHandlerResult.Retry -> { + val res = CompletableFuture() + val delay = run { + val scheduledDelay = (initialDelay * exp.pow(i.toDouble()) * (1.0 + (randomizer?.nextDouble(-0.5, 0.5) ?: 0.0))).toLong() + outcomeHandlerResult.suggestedDelayMillis?.coerceAtMost(scheduledDelay) ?: scheduledDelay + } + eventExecutorGroup.schedule({ + cb().handle { result, ex -> + if (ex == null) { + res.complete(result) + } else { + res.completeExceptionally(ex) + } + } + }, delay, TimeUnit.MILLISECONDS) + res + } + is OutcomeHandlerResult.DoNotRetry -> { + shortCircuit = true + when(operationOutcome) { + is OperationOutcome.Failure -> throw operationOutcome.ex + is OperationOutcome.Success -> CompletableFuture.completedFuture(operationOutcome.result) + } + } + } + } + }.thenCompose { it } + } + return future +} \ No newline at end of file diff --git a/rbcs-client/bin/main/net/woggioni/rbcs/client/schema/rbcs-client.xsd b/rbcs-client/bin/main/net/woggioni/rbcs/client/schema/rbcs-client.xsd new file mode 100644 index 0000000..807cdea --- /dev/null +++ b/rbcs-client/bin/main/net/woggioni/rbcs/client/schema/rbcs-client.xsd @@ -0,0 +1,260 @@ + + + + + + + + + + + + + + + + + Disable authentication. + + + + + + + Enable HTTP basic authentication. + + + + + + + Enable TLS certificate authentication. + + + + + + + + Set inactivity timeouts for connections to this server, + if not present, connections are only closed on network errors. + + + + + + + Set a retry policy for this server, if not present requests won't be retried + + + + + + + If set, specify an alternative truststore to validate the server certificate. + If not present the system truststore is used. + + + + + + + + Name of this server profile, to be referred to from rbcs-cli with the '-p' parameter + + + + + + + RBCs server URL + + + + + + + Maximum number of concurrent TCP connection to open with this server + + + + + + + Enable HTTP compression when communicating to this server + + + + + + + Enable HTTP compression when communicating to this server + + + + + + + + + + The client will close the connection with the server + when neither a read nor a write was performed for the specified period of time. + + + + + + + The client will close the connection with the server + when no read was performed for the specified period of time. + + + + + + + The client will close the connection with the server + when no write was performed for the specified period of time. + + + + + + + Enables HTTP/1.1 request pipelining + + + + + + + + + Add this tag to not use any type of authentication when talking to the RBCS server + + + + + + + + Add this tag to enable HTTP basic authentication for the communication to this server, + mind that HTTP basic authentication sends credentials directly over the network, so make sure + your communication is protected by TLS (i.e. your server's URL starts with "https") + + + + + + Username for HTTP basic authentication + + + + + + + Password used for HTTP basic authentication + + + + + + + + + + System path to the keystore file + + + + + + + Password to open they keystore file + + + + + + + Alias of the keystore entry containing the private key + + + + + + + Private key entry's encryption password + + + + + + + + + Retry policy to use in case of failures, based on exponential backoff + https://en.wikipedia.org/wiki/Exponential_backoff + + + + + + + Maximum number of attempts, after which the call will result in an error, + throwing an exception related to the last received failure + + + + + + + Delay to apply before retrying after the first failed call + + + + + + + Exponent to apply to compute the next delay + + + + + + + + + + Path to the truststore file + + + + + + + Truststore file password + + + + + + + Whether or not check the server certificate validity using CRL/OCSP + + + + + + + If false, the client will blindly trust the certificate provided by the server + + + + + diff --git a/rbcs-client/bin/test/logback.xml b/rbcs-client/bin/test/logback.xml new file mode 100644 index 0000000..400bcd5 --- /dev/null +++ b/rbcs-client/bin/test/logback.xml @@ -0,0 +1,21 @@ + + + + + + + + + System.err + + %d [%highlight(%-5level)] \(%thread\) %logger{36} -%kvp- %msg %n + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-client/bin/test/net/woggioni/rbcs/client/RetryTest.kt b/rbcs-client/bin/test/net/woggioni/rbcs/client/RetryTest.kt new file mode 100644 index 0000000..7295ed1 --- /dev/null +++ b/rbcs-client/bin/test/net/woggioni/rbcs/client/RetryTest.kt @@ -0,0 +1,152 @@ +package net.woggioni.rbcs.client + +import io.netty.util.concurrent.DefaultEventExecutorGroup +import io.netty.util.concurrent.EventExecutorGroup +import java.util.concurrent.CompletableFuture +import java.util.stream.Stream +import kotlin.random.Random +import net.woggioni.rbcs.common.contextLogger +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.extension.ExtensionContext +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.ArgumentsProvider +import org.junit.jupiter.params.provider.ArgumentsSource +import org.junit.jupiter.params.support.ParameterDeclarations + +class RetryTest { + + data class TestArgs( + val seed: Int, + val maxAttempt: Int, + val initialDelay: Double, + val exp: Double, + ) + + class TestArguments : ArgumentsProvider { + override fun provideArguments( + parameters: ParameterDeclarations, + context: ExtensionContext + ): Stream { + return Stream.of( + TestArgs( + seed = 101325, + maxAttempt = 5, + initialDelay = 50.0, + exp = 2.0, + ), + TestArgs( + seed = 101325, + maxAttempt = 20, + initialDelay = 100.0, + exp = 1.1, + ), + TestArgs( + seed = 123487, + maxAttempt = 20, + initialDelay = 100.0, + exp = 2.0, + ), + TestArgs( + seed = 20082024, + maxAttempt = 10, + initialDelay = 100.0, + exp = 2.0, + ) + ).map { + object: Arguments { + override fun get() = arrayOf(it) + } + } + } + } + + @ArgumentsSource(TestArguments::class) + @ParameterizedTest + fun test(testArgs: TestArgs) { + val log = contextLogger() + log.debug("Start") + val executor: EventExecutorGroup = DefaultEventExecutorGroup(1) + val attempts = mutableListOf>>() + val outcomeHandler = OutcomeHandler { outcome -> + when(outcome) { + is OperationOutcome.Success -> { + if(outcome.result % 10 == 0) { + OutcomeHandlerResult.DoNotRetry() + } else { + OutcomeHandlerResult.Retry(null) + } + } + is OperationOutcome.Failure -> { + when(outcome.ex) { + is IllegalStateException -> { + log.debug(outcome.ex.message, outcome.ex) + OutcomeHandlerResult.Retry(null) + } + else -> { + OutcomeHandlerResult.DoNotRetry() + } + } + } + } + } + val random = Random(testArgs.seed) + + val future = + executeWithRetry(executor, testArgs.maxAttempt, testArgs.initialDelay, testArgs.exp, outcomeHandler, null) { + val now = System.nanoTime() + val result = CompletableFuture() + executor.submit { + val n = random.nextInt(0, Integer.MAX_VALUE) + log.debug("Got new number: {}", n) + if(n % 3 == 0) { + val ex = IllegalStateException("Value $n can be divided by 3") + result.completeExceptionally(ex) + attempts += now to OperationOutcome.Failure(ex) + } else if(n % 7 == 0) { + val ex = RuntimeException("Value $n can be divided by 7") + result.completeExceptionally(ex) + attempts += now to OperationOutcome.Failure(ex) + } else { + result.complete(n) + attempts += now to OperationOutcome.Success(n) + } + } + result + } + Assertions.assertTrue(attempts.size <= testArgs.maxAttempt) + val result = future.handle { res, ex -> + if(ex != null) { + val err = ex.cause ?: ex + log.debug(err.message, err) + OperationOutcome.Failure(err) + } else { + OperationOutcome.Success(res) + } + }.get() + for ((index, attempt) in attempts.withIndex()) { + val (timestamp, value) = attempt + if (index > 0) { + /* Check the delay for subsequent attempts is correct */ + val previousAttempt = attempts[index - 1] + val expectedTimestamp = + previousAttempt.first + testArgs.initialDelay * Math.pow(testArgs.exp, index.toDouble()) * 1e6 + val actualTimestamp = timestamp + val err = Math.abs(expectedTimestamp - actualTimestamp) / expectedTimestamp + Assertions.assertTrue(err < 0.1) + } + if (index == attempts.size - 1 && index < testArgs.maxAttempt - 1) { + /* + * If the last attempt index is lower than the maximum number of attempts, then + * check the outcome handler returns DoNotRetry + */ + Assertions.assertTrue(outcomeHandler.shouldRetry(value) is OutcomeHandlerResult.DoNotRetry) + } else if (index < attempts.size - 1) { + /* + * If the attempt is not the last attempt check the outcome handler returns Retry + */ + Assertions.assertTrue(outcomeHandler.shouldRetry(value) is OutcomeHandlerResult.Retry) + } + } + } +} \ No newline at end of file diff --git a/rbcs-client/bin/test/net/woggioni/rbcs/client/test/rbcs-client.xml b/rbcs-client/bin/test/net/woggioni/rbcs/client/test/rbcs-client.xml new file mode 100644 index 0000000..5828d87 --- /dev/null +++ b/rbcs-client/bin/test/net/woggioni/rbcs/client/test/rbcs-client.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + diff --git a/rbcs-common/.classpath b/rbcs-common/.classpath new file mode 100644 index 0000000..df4ae25 --- /dev/null +++ b/rbcs-common/.classpath @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-common/.project b/rbcs-common/.project new file mode 100644 index 0000000..92bfd2b --- /dev/null +++ b/rbcs-common/.project @@ -0,0 +1,34 @@ + + + rbcs-common + Project rbcs-common created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776954 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-common/.settings/org.eclipse.buildship.core.prefs b/rbcs-common/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-common/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-common/.settings/org.eclipse.jdt.core.prefs b/rbcs-common/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-common/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-common/bin/main/META-INF/services/java.net.spi.URLStreamHandlerProvider b/rbcs-common/bin/main/META-INF/services/java.net.spi.URLStreamHandlerProvider new file mode 100644 index 0000000..738bdad --- /dev/null +++ b/rbcs-common/bin/main/META-INF/services/java.net.spi.URLStreamHandlerProvider @@ -0,0 +1 @@ +net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/BB.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/BB.kt new file mode 100644 index 0000000..65796c3 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/BB.kt @@ -0,0 +1,15 @@ +package net.woggioni.rbcs.common + +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.CompositeByteBuf + +fun extractChunk(buf: CompositeByteBuf, alloc: ByteBufAllocator): ByteBuf { + val chunk = alloc.compositeBuffer() + for (component in buf.decompose(0, buf.readableBytes())) { + chunk.addComponent(true, component.retain()) + } + buf.removeComponents(0, buf.numComponents()) + buf.clear() + return chunk +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufInputStream.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufInputStream.kt new file mode 100644 index 0000000..8d8ea77 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufInputStream.kt @@ -0,0 +1,25 @@ +package net.woggioni.rbcs.common + +import io.netty.buffer.ByteBuf +import java.io.InputStream + +class ByteBufInputStream(private val buf : ByteBuf) : InputStream() { + override fun read(): Int { + return buf.takeIf { + it.readableBytes() > 0 + }?.let(ByteBuf::readByte) + ?.let(Byte::toInt) ?: -1 + } + + override fun read(b: ByteArray, off: Int, len: Int): Int { + val readableBytes = buf.readableBytes() + if(readableBytes == 0) return -1 + val result = len.coerceAtMost(readableBytes) + buf.readBytes(b, off, result) + return result + } + + override fun close() { + buf.release() + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufOutputStream.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufOutputStream.kt new file mode 100644 index 0000000..8cc131e --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/ByteBufOutputStream.kt @@ -0,0 +1,18 @@ +package net.woggioni.rbcs.common + +import io.netty.buffer.ByteBuf +import java.io.OutputStream + +class ByteBufOutputStream(private val buf : ByteBuf) : OutputStream() { + override fun write(b: Int) { + buf.writeByte(b) + } + + override fun write(b: ByteArray, off: Int, len: Int) { + buf.writeBytes(b, off, len) + } + + override fun close() { + buf.release() + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/Cidr.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/Cidr.kt new file mode 100644 index 0000000..c4a9ccf --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/Cidr.kt @@ -0,0 +1,62 @@ +package net.woggioni.rbcs.common + +import java.net.InetAddress + +data class Cidr private constructor( + val networkAddress: InetAddress, + val prefixLength: Int +) { + companion object { + fun from(cidr: String) : Cidr { + val separator = cidr.indexOf("/") + if(separator < 0) { + throw IllegalArgumentException("Invalid CIDR format: $cidr") + } + val networkAddress = InetAddress.getByName(cidr.substring(0, separator)) + val prefixLength = cidr.substring(separator + 1, cidr.length).toInt() + + + // Validate prefix length + val maxPrefix = if (networkAddress.address.size == 4) 32 else 128 + require(prefixLength in 0..maxPrefix) { "Invalid prefix length: $prefixLength" } + return Cidr(networkAddress, prefixLength) + } + } + + fun contains(address: InetAddress): Boolean { + val networkBytes = networkAddress.address + val addressBytes = address.address + + if (networkBytes.size != addressBytes.size) { + return false + } + + + // Calculate how many full bytes and remaining bits to check + val fullBytes = prefixLength / 8 + val remainingBits = prefixLength % 8 + + + // Check full bytes + for (i in 0.. 0 && fullBytes < networkBytes.size) { + val mask = (0xFF shl (8 - remainingBits)).toByte() + if ((networkBytes[fullBytes].toInt() and mask.toInt()) != (addressBytes[fullBytes].toInt() and mask.toInt())) { + return false + } + } + + return true + } + + override fun toString(): String { + return networkAddress.hostAddress + "/" + prefixLength + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/Exception.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/Exception.kt new file mode 100644 index 0000000..b4232da --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/Exception.kt @@ -0,0 +1,7 @@ +package net.woggioni.rbcs.common + +class ResourceNotFoundException(msg : String? = null, cause: Throwable? = null) : RuntimeException(msg, cause) { +} + +class ModuleNotFoundException(msg : String? = null, cause: Throwable? = null) : RuntimeException(msg, cause) { +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/HostAndPort.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/HostAndPort.kt new file mode 100644 index 0000000..59cfc47 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/HostAndPort.kt @@ -0,0 +1,8 @@ +package net.woggioni.rbcs.common + + +data class HostAndPort(val host: String, val port: Int = 0) { + override fun toString(): String { + return "$host:$port" + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/Logging.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/Logging.kt new file mode 100644 index 0000000..9247481 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/Logging.kt @@ -0,0 +1,191 @@ +package net.woggioni.rbcs.common + +import io.netty.channel.Channel +import io.netty.channel.ChannelHandlerContext +import java.nio.file.Files +import java.nio.file.Path +import java.util.logging.LogManager +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.slf4j.MDC +import org.slf4j.event.Level +import org.slf4j.spi.LoggingEventBuilder + +inline fun T.contextLogger() = LoggerFactory.getLogger(T::class.java) +inline fun createLogger() = LoggerFactory.getLogger(T::class.java) + +inline fun Logger.traceParam(messageBuilder: () -> Pair>) { + if (isTraceEnabled) { + val (format, params) = messageBuilder() + trace(format, params) + } +} + +inline fun Logger.debugParam(messageBuilder: () -> Pair>) { + if (isDebugEnabled) { + val (format, params) = messageBuilder() + info(format, params) + } +} + +inline fun Logger.infoParam(messageBuilder: () -> Pair>) { + if (isInfoEnabled) { + val (format, params) = messageBuilder() + info(format, params) + } +} + +inline fun Logger.warnParam(messageBuilder: () -> Pair>) { + if (isWarnEnabled) { + val (format, params) = messageBuilder() + warn(format, params) + } +} + +inline fun Logger.errorParam(messageBuilder: () -> Pair>) { + if (isErrorEnabled) { + val (format, params) = messageBuilder() + error(format, params) + } +} + + +inline fun log( + log: Logger, + filter: Logger.() -> Boolean, + loggerMethod: Logger.(String) -> Unit, messageBuilder: () -> String +) { + if (log.filter()) { + log.loggerMethod(messageBuilder()) + } +} + +fun withMDC(params: Array>, cb: () -> Unit) { + object : AutoCloseable { + override fun close() { + for ((key, _) in params) MDC.remove(key) + } + }.use { + for ((key, value) in params) MDC.put(key, value) + cb() + } +} + +inline fun Logger.log(level: Level, channel: Channel, crossinline messageBuilder: (LoggingEventBuilder) -> Unit ) { + if (isEnabledForLevel(level)) { + val params = arrayOf>( + "channel-id-short" to channel.id().asShortText(), + "channel-id-long" to channel.id().asLongText(), + "remote-address" to channel.remoteAddress().toString(), + "local-address" to channel.localAddress().toString(), + ) + withMDC(params) { + val builder = makeLoggingEventBuilder(level) + messageBuilder(builder) + builder.log() + } + } +} +inline fun Logger.log(level: Level, channel: Channel, crossinline messageBuilder: () -> String) { + log(level, channel) { builder -> + builder.setMessage(messageBuilder()) + } +} + +inline fun Logger.trace(ch: Channel, crossinline messageBuilder: () -> String) { + log(Level.TRACE, ch, messageBuilder) +} + +inline fun Logger.debug(ch: Channel, crossinline messageBuilder: () -> String) { + log(Level.DEBUG, ch, messageBuilder) +} + +inline fun Logger.info(ch: Channel, crossinline messageBuilder: () -> String) { + log(Level.INFO, ch, messageBuilder) +} + +inline fun Logger.warn(ch: Channel, crossinline messageBuilder: () -> String) { + log(Level.WARN, ch, messageBuilder) +} + +inline fun Logger.error(ch: Channel, crossinline messageBuilder: () -> String) { + log(Level.ERROR, ch, messageBuilder) +} + +inline fun Logger.trace(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) { + log(Level.TRACE, ctx.channel(), messageBuilder) +} + +inline fun Logger.debug(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) { + log(Level.DEBUG, ctx.channel(), messageBuilder) +} + +inline fun Logger.info(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) { + log(Level.INFO, ctx.channel(), messageBuilder) +} + +inline fun Logger.warn(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) { + log(Level.WARN, ctx.channel(), messageBuilder) +} + +inline fun Logger.error(ctx: ChannelHandlerContext, crossinline messageBuilder: () -> String) { + log(Level.ERROR, ctx.channel(), messageBuilder) +} + + +inline fun Logger.log(level: Level, messageBuilder: () -> String) { + if (isEnabledForLevel(level)) { + makeLoggingEventBuilder(level).log(messageBuilder()) + } +} + +inline fun Logger.trace(messageBuilder: () -> String) { + if (isTraceEnabled) { + trace(messageBuilder()) + } +} + +inline fun Logger.debug(messageBuilder: () -> String) { + if (isDebugEnabled) { + debug(messageBuilder()) + } +} + +inline fun Logger.info(messageBuilder: () -> String) { + if (isInfoEnabled) { + info(messageBuilder()) + } +} + +inline fun Logger.warn(messageBuilder: () -> String) { + if (isWarnEnabled) { + warn(messageBuilder()) + } +} + +inline fun Logger.error(messageBuilder: () -> String) { + if (isErrorEnabled) { + error(messageBuilder()) + } +} + + +class LoggingConfig { + + init { + val logManager = LogManager.getLogManager() + System.getProperty("log.config.source")?.let withSource@{ source -> + val urls = LoggingConfig::class.java.classLoader.getResources(source) + while (urls.hasMoreElements()) { + val url = urls.nextElement() + url.openStream().use { inputStream -> + logManager.readConfiguration(inputStream) + return@withSource + } + } + Path.of(source).takeIf(Files::exists) + ?.let(Files::newInputStream) + ?.use(logManager::readConfiguration) + } + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/PasswordSecurity.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/PasswordSecurity.kt new file mode 100644 index 0000000..f7be95a --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/PasswordSecurity.kt @@ -0,0 +1,57 @@ +package net.woggioni.rbcs.common + +import java.security.SecureRandom +import java.security.spec.KeySpec +import java.util.Base64 +import javax.crypto.SecretKeyFactory +import javax.crypto.spec.PBEKeySpec + +object PasswordSecurity { + + enum class Algorithm( + val codeName : String, + val keyLength : Int, + val iterations : Int) { + PBEWithHmacSHA512_224AndAES_256("PBEWithHmacSHA512/224AndAES_256", 64, 1), + PBEWithHmacSHA1AndAES_256("PBEWithHmacSHA1AndAES_256",64, 1), + PBEWithHmacSHA384AndAES_128("PBEWithHmacSHA384AndAES_128", 64,1), + PBEWithHmacSHA384AndAES_256("PBEWithHmacSHA384AndAES_256",64,1), + PBKDF2WithHmacSHA512("PBKDF2WithHmacSHA512",512, 1), + PBKDF2WithHmacSHA384("PBKDF2WithHmacSHA384",384, 1); + } + + private fun concat(arr1: ByteArray, arr2: ByteArray): ByteArray { + val result = ByteArray(arr1.size + arr2.size) + var j = 0 + for(element in arr1) { + result[j] = element + j += 1 + } + for(element in arr2) { + result[j] = element + j += 1 + } + return result + } + + fun hashPassword(password : String, salt : String? = null, algorithm : Algorithm = Algorithm.PBKDF2WithHmacSHA512) : String { + val actualSalt = salt?.let(Base64.getDecoder()::decode) ?: SecureRandom().run { + val result = ByteArray(16) + nextBytes(result) + result + } + val spec: KeySpec = PBEKeySpec(password.toCharArray(), actualSalt, algorithm.iterations, algorithm.keyLength) + val factory = SecretKeyFactory.getInstance(algorithm.codeName) + val hash = factory.generateSecret(spec).encoded + return String(Base64.getEncoder().encode(concat(hash, actualSalt))) + } + + fun decodePasswordHash(encodedPasswordHash : String, algorithm: Algorithm = Algorithm.PBKDF2WithHmacSHA512) : Pair { + val decoded = Base64.getDecoder().decode(encodedPasswordHash) + val hash = ByteArray(algorithm.keyLength / 8) + val salt = ByteArray(decoded.size - algorithm.keyLength / 8) + System.arraycopy(decoded, 0, hash, 0, hash.size) + System.arraycopy(decoded, hash.size, salt, 0, salt.size) + return hash to salt + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/RBCS.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/RBCS.kt new file mode 100644 index 0000000..f228f6c --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/RBCS.kt @@ -0,0 +1,167 @@ +package net.woggioni.rbcs.common + +import java.io.IOException +import java.net.InetAddress +import java.net.ServerSocket +import java.net.URI +import java.net.URL +import java.nio.file.Files +import java.nio.file.Path +import java.security.KeyStore +import java.security.MessageDigest +import java.security.cert.CertPathValidator +import java.security.cert.CertPathValidatorException +import java.security.cert.CertificateException +import java.security.cert.CertificateFactory +import java.security.cert.PKIXParameters +import java.security.cert.PKIXRevocationChecker +import java.security.cert.X509Certificate +import java.util.EnumSet +import javax.net.ssl.TrustManagerFactory +import javax.net.ssl.X509TrustManager +import net.woggioni.jwo.JWO +import net.woggioni.jwo.Tuple2 + +object RBCS { + fun String.toUrl(): URL = URL.of(URI(this), null) + + const val RBCS_NAMESPACE_URI: String = "urn:net.woggioni.rbcs.server" + const val RBCS_PREFIX: String = "rbcs" + const val XML_SCHEMA_NAMESPACE_URI = "http://www.w3.org/2001/XMLSchema-instance" + + fun ByteArray.toInt(index: Int = 0): Long { + if (index + 4 > size) throw IllegalArgumentException("Not enough bytes to decode a 32 bits integer") + var value: Long = 0 + for (b in index until index + 4) { + value = (value shl 8) + (get(b).toInt() and 0xFF) + } + return value + } + + fun ByteArray.toLong(index: Int = 0): Long { + if (index + 8 > size) throw IllegalArgumentException("Not enough bytes to decode a 64 bits long integer") + var value: Long = 0 + for (b in index until index + 8) { + value = (value shl 8) + (get(b).toInt() and 0xFF) + } + return value + } + + fun digest( + data: ByteArray, + md: MessageDigest + ): ByteArray { + md.update(data) + return md.digest() + } + + fun digestString( + data: ByteArray, + md: MessageDigest + ): String { + return JWO.bytesToHex(digest(data, md)) + } + + fun processCacheKey(key: String, keyPrefix: String?, digestAlgorithm: String?) : ByteArray { + val prefixedKey = if (keyPrefix == null) { + key + } else { + key + keyPrefix + }.toByteArray(Charsets.UTF_8) + return digestAlgorithm + ?.let(MessageDigest::getInstance) + ?.let { md -> + digest(prefixedKey, md) + } ?: prefixedKey + } + + fun Long.toIntOrNull(): Int? { + return if (this >= Int.MIN_VALUE && this <= Int.MAX_VALUE) { + toInt() + } else { + null + } + } + + fun getFreePort(): Int { + var count = 0 + while (count < 50) { + try { + ServerSocket(0, 50, InetAddress.getLocalHost()).use { serverSocket -> + val candidate = serverSocket.localPort + if (candidate > 0) { + return candidate + } else { + throw RuntimeException("Got invalid port number: $candidate") + } + } + } catch (ignored: IOException) { + ++count + } + } + throw RuntimeException("Error trying to find an open port") + } + + fun loadKeystore(file: Path, password: String?): KeyStore { + val ext = JWO.splitExtension(file) + .map(Tuple2::get_2) + .orElseThrow { + IllegalArgumentException( + "Keystore file '${file}' must have .jks, .p12, .pfx extension" + ) + } + val keystore = when (ext.substring(1).lowercase()) { + "jks" -> KeyStore.getInstance("JKS") + "p12", "pfx" -> KeyStore.getInstance("PKCS12") + else -> throw IllegalArgumentException( + "Keystore file '${file}' must have .jks, .p12, .pfx extension" + ) + } + Files.newInputStream(file).use { + keystore.load(it, password?.let(String::toCharArray)) + } + return keystore + } + + fun getTrustManager(trustStore: KeyStore?, certificateRevocationEnabled: Boolean): X509TrustManager { + return if (trustStore != null) { + val certificateFactory = CertificateFactory.getInstance("X.509") + val validator = CertPathValidator.getInstance("PKIX").apply { + val rc = revocationChecker as PKIXRevocationChecker + rc.options = EnumSet.of( + PKIXRevocationChecker.Option.NO_FALLBACK + ) + } + val params = PKIXParameters(trustStore).apply { + isRevocationEnabled = certificateRevocationEnabled + } + object : X509TrustManager { + override fun checkClientTrusted(chain: Array, authType: String) { + val clientCertificateChain = certificateFactory.generateCertPath(chain.toList()) + try { + validator.validate(clientCertificateChain, params) + } catch (ex: CertPathValidatorException) { + throw CertificateException(ex) + } + } + + override fun checkServerTrusted(chain: Array, authType: String) { + throw NotImplementedError() + } + + private val acceptedIssuers = trustStore.aliases().asSequence() + .filter(trustStore::isCertificateEntry) + .map(trustStore::getCertificate) + .map { it as X509Certificate } + .toList() + .toTypedArray() + + override fun getAcceptedIssuers() = acceptedIssuers + } + } else { + val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) + trustManagerFactory.trustManagers.asSequence().filter { it is X509TrustManager } + .single() as X509TrustManager + } + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/RbcsUrlStreamHandlerFactory.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/RbcsUrlStreamHandlerFactory.kt new file mode 100644 index 0000000..71a3cd8 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/RbcsUrlStreamHandlerFactory.kt @@ -0,0 +1,113 @@ +package net.woggioni.rbcs.common + +import java.io.IOException +import java.io.InputStream +import java.net.URL +import java.net.URLConnection +import java.net.URLStreamHandler +import java.net.spi.URLStreamHandlerProvider +import java.util.concurrent.atomic.AtomicBoolean +import java.util.stream.Collectors + + +class RbcsUrlStreamHandlerFactory : URLStreamHandlerProvider() { + + private class ClasspathHandler(private val classLoader: ClassLoader = RbcsUrlStreamHandlerFactory::class.java.classLoader) : + URLStreamHandler() { + + override fun openConnection(u: URL): URLConnection? { + return javaClass.module + ?.takeIf { m: Module -> m.layer != null } + ?.let { + val path = u.path + val i = path.lastIndexOf('/') + val packageName = path.substring(0, i).replace('/', '.') + val modules = packageMap[packageName]!! + ClasspathResourceURLConnection( + u, + modules + ) + } + ?: classLoader.getResource(u.path)?.let(URL::openConnection) + } + } + + private class JpmsHandler : URLStreamHandler() { + + override fun openConnection(u: URL): URLConnection { + val moduleName = u.host + val thisModule = javaClass.module + val sourceModule = + thisModule + ?.let(Module::getLayer) + ?.let { layer: ModuleLayer -> + layer.findModule(moduleName).orElse(null) + } ?: if(thisModule.layer == null) { + thisModule + } else throw ModuleNotFoundException("Module '$moduleName' not found") + + return JpmsResourceURLConnection(u, sourceModule) + } + } + + private class JpmsResourceURLConnection(url: URL, private val module: Module) : URLConnection(url) { + override fun connect() { + } + + @Throws(IOException::class) + override fun getInputStream(): InputStream { + val resource = getURL().path + return module.getResourceAsStream(resource) + ?: throw ResourceNotFoundException("Resource '$resource' not found in module '${module.name}'") + } + } + + override fun createURLStreamHandler(protocol: String): URLStreamHandler? { + return when (protocol) { + "classpath" -> ClasspathHandler() + "jpms" -> JpmsHandler() + else -> null + } + } + + private class ClasspathResourceURLConnection(url: URL?, private val modules: List) : + URLConnection(url) { + override fun connect() {} + + override fun getInputStream(): InputStream? { + for (module in modules) { + val result = module.getResourceAsStream(getURL().path) + if (result != null) return result + } + return null + } + } + + companion object { + private val installed = AtomicBoolean(false) + fun install() { + if (!installed.getAndSet(true)) { + URL.setURLStreamHandlerFactory(RbcsUrlStreamHandlerFactory()) + } + } + + private val packageMap: Map> by lazy { + RbcsUrlStreamHandlerFactory::class.java.module.layer + .modules() + .stream() + .flatMap { m: Module -> + m.packages.stream() + .map { p: String -> p to m } + } + .collect( + Collectors.groupingBy( + Pair::first, + Collectors.mapping( + Pair::second, + Collectors.toUnmodifiableList() + ) + ) + ) + } + } +} \ No newline at end of file diff --git a/rbcs-common/bin/main/net/woggioni/rbcs/common/Xml.kt b/rbcs-common/bin/main/net/woggioni/rbcs/common/Xml.kt new file mode 100644 index 0000000..4963d60 --- /dev/null +++ b/rbcs-common/bin/main/net/woggioni/rbcs/common/Xml.kt @@ -0,0 +1,243 @@ +package net.woggioni.rbcs.common + +import java.io.InputStream +import java.io.OutputStream +import java.net.URL +import javax.xml.XMLConstants.ACCESS_EXTERNAL_DTD +import javax.xml.XMLConstants.ACCESS_EXTERNAL_SCHEMA +import javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING +import javax.xml.XMLConstants.W3C_XML_SCHEMA_NS_URI +import javax.xml.parsers.DocumentBuilder +import javax.xml.parsers.DocumentBuilderFactory +import javax.xml.transform.OutputKeys +import javax.xml.transform.TransformerFactory +import javax.xml.transform.dom.DOMSource +import javax.xml.transform.stream.StreamResult +import javax.xml.transform.stream.StreamSource +import javax.xml.validation.Schema +import javax.xml.validation.SchemaFactory +import net.woggioni.jwo.JWO +import org.slf4j.event.Level +import org.w3c.dom.Document +import org.w3c.dom.Element +import org.w3c.dom.Node +import org.w3c.dom.NodeList +import org.xml.sax.ErrorHandler as ErrHandler +import org.xml.sax.SAXNotRecognizedException +import org.xml.sax.SAXNotSupportedException +import org.xml.sax.SAXParseException + + +class NodeListIterator(private val nodeList: NodeList) : Iterator { + private var cursor: Int = 0 + override fun hasNext(): Boolean { + return cursor < nodeList.length + } + + override fun next(): Node { + return if (hasNext()) nodeList.item(cursor++) else throw NoSuchElementException() + } +} + +class ElementIterator(parent: Element, name: String? = null) : Iterator { + private val it: NodeListIterator + private val name: String? + private var next: Element? + + init { + it = NodeListIterator(parent.childNodes) + this.name = name + next = getNext() + } + + override fun hasNext(): Boolean { + return next != null + } + + override fun next(): Element { + val result = next ?: throw NoSuchElementException() + next = getNext() + return result + } + + private fun getNext(): Element? { + var result: Element? = null + while (it.hasNext()) { + val node: Node = it.next() + if (node is Element && (name == null || name == node.tagName)) { + result = node + break + } + } + return result + } +} + +class Xml(val doc: Document, val element: Element) { + + class ErrorHandler(private val fileURL: URL) : ErrHandler { + + companion object { + private val log = createLogger() + } + + override fun warning(ex: SAXParseException)= err(ex, Level.WARN) + + private fun err(ex: SAXParseException, level: Level) { + log.log(level) { + "Problem at ${fileURL}:${ex.lineNumber}:${ex.columnNumber} parsing deployment configuration: ${ex.message}" + } + throw ex + } + + override fun error(ex: SAXParseException) = err(ex, Level.ERROR) + override fun fatalError(ex: SAXParseException) = err(ex, Level.ERROR) + } + + companion object { + private val dictMap: Map> = sequenceOf( + "env" to System.getenv().asSequence().map { (k, v) -> k to (v as Any) }.toMap(), + "sys" to System.getProperties().asSequence().map { (k, v) -> k as String to (v as Any) }.toMap() + ).toMap() + + private fun renderConfigurationTemplate(template: String): String { + return JWO.renderTemplate(template, emptyMap(), dictMap).replace("$$", "$") + } + + fun Element.renderAttribute(name : String, namespaceURI: String? = null) = if(namespaceURI == null) { + getAttribute(name) + } else { + getAttributeNS(name, namespaceURI) + }.takeIf(String::isNotEmpty)?.let(Companion::renderConfigurationTemplate) + + + fun Element.asIterable() = Iterable { ElementIterator(this, null) } + fun NodeList.asIterable() = Iterable { NodeListIterator(this) } + + private fun disableProperty(dbf: DocumentBuilderFactory, propertyName: String) { + try { + dbf.setAttribute(propertyName, "") + } catch (iae: IllegalArgumentException) { + // Property not supported. + } + } + + private fun disableProperty(sf: SchemaFactory, propertyName: String) { + try { + sf.setProperty(propertyName, "") + } catch (ex: SAXNotRecognizedException) { + // Property not supported. + } catch (ex: SAXNotSupportedException) { + } + } + + fun getSchema(schema: URL): Schema { + val sf = SchemaFactory.newInstance(W3C_XML_SCHEMA_NS_URI) + sf.setFeature(FEATURE_SECURE_PROCESSING, false) + sf.errorHandler = ErrorHandler(schema) + return sf.newSchema(schema) + } + + fun getSchema(inputStream: InputStream): Schema { + val sf = SchemaFactory.newInstance(W3C_XML_SCHEMA_NS_URI) + sf.setFeature(FEATURE_SECURE_PROCESSING, true) + return sf.newSchema(StreamSource(inputStream)) + } + + fun newDocumentBuilderFactory(schemaResourceURL: URL?): DocumentBuilderFactory { + val dbf = DocumentBuilderFactory.newInstance() + dbf.setFeature(FEATURE_SECURE_PROCESSING, false) + dbf.setAttribute(ACCESS_EXTERNAL_SCHEMA, "all") + disableProperty(dbf, ACCESS_EXTERNAL_DTD) + dbf.isExpandEntityReferences = true + dbf.isIgnoringComments = true + dbf.isNamespaceAware = true + dbf.isValidating = schemaResourceURL == null + dbf.setFeature("http://apache.org/xml/features/validation/schema", true) + schemaResourceURL?.let { + dbf.schema = getSchema(it) + } + return dbf + } + + fun newDocumentBuilder(resource: URL, schemaResourceURL: URL?): DocumentBuilder { + val db = newDocumentBuilderFactory(schemaResourceURL).newDocumentBuilder() + db.setErrorHandler(ErrorHandler(resource)) + return db + } + + fun parseXmlResource(resource: URL, schemaResourceURL: URL?): Document { + val db = newDocumentBuilder(resource, schemaResourceURL) + return resource.openStream().use(db::parse) + } + + fun parseXml(sourceURL: URL, sourceStream: InputStream? = null, schemaResourceURL: URL? = null): Document { + val db = newDocumentBuilder(sourceURL, schemaResourceURL) + return sourceStream?.let(db::parse) ?: sourceURL.openStream().use(db::parse) + } + + fun write(doc: Document, output: OutputStream) { + val transformerFactory = TransformerFactory.newInstance() + val transformer = transformerFactory.newTransformer() + transformer.setOutputProperty(OutputKeys.INDENT, "yes") + transformer.setOutputProperty(OutputKeys.INDENT, "yes") + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "4") + transformer.setOutputProperty(OutputKeys.STANDALONE, "yes") + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8") + val source = DOMSource(doc) + val result = StreamResult(output) + transformer.transform(source, result) + } + + fun of( + namespaceURI: String, + qualifiedName: String, + schemaResourceURL: URL? = null, + cb: Xml.(el: Element) -> Unit + ): Document { + val dbf = newDocumentBuilderFactory(schemaResourceURL) + val db = dbf.newDocumentBuilder() + val doc = db.newDocument() + val root = doc.createElementNS(namespaceURI, qualifiedName) + .also(doc::appendChild) + Xml(doc, root).cb(root) + return doc + } + + fun of(doc: Document, el: Element, cb: Xml.(el: Element) -> Unit): Element { + Xml(doc, el).cb(el) + return el + } + + fun Element.removeChildren() { + while (true) { + removeChild(firstChild ?: break) + } + } + } + + fun node( + name: String, + namespaceURI: String? = null, + attrs: Map = emptyMap(), + cb: Xml.(el: Element) -> Unit = {} + ): Element { + val child = doc.createElementNS(namespaceURI, name) + for ((key, value) in attrs) { + child.setAttribute(key, value) + } + return child + .also { + element.appendChild(it) + Xml(doc, it).cb(it) + } + } + + fun attr(key: String, value: String, namespaceURI: String? = null) { + element.setAttributeNS(namespaceURI, key, value) + } + + fun text(txt: String) { + element.appendChild(doc.createTextNode(txt)) + } +} diff --git a/rbcs-common/bin/test/net/woggioni/rbcs/common/CidrTest.kt b/rbcs-common/bin/test/net/woggioni/rbcs/common/CidrTest.kt new file mode 100644 index 0000000..5ab6d87 --- /dev/null +++ b/rbcs-common/bin/test/net/woggioni/rbcs/common/CidrTest.kt @@ -0,0 +1,17 @@ +package net.woggioni.rbcs.common + +import java.net.InetAddress +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class CidrTest { + class CidrTest { + @Test + fun test() { + val cidr = Cidr.from("2a02:4780:12:368b::1/128") + Assertions.assertTrue { + cidr.contains(InetAddress.ofLiteral("2a02:4780:12:368b::1")) + } + } + } +} \ No newline at end of file diff --git a/rbcs-common/bin/test/net/woggioni/rbcs/common/PasswordHashingTest.kt b/rbcs-common/bin/test/net/woggioni/rbcs/common/PasswordHashingTest.kt new file mode 100644 index 0000000..2b8f287 --- /dev/null +++ b/rbcs-common/bin/test/net/woggioni/rbcs/common/PasswordHashingTest.kt @@ -0,0 +1,38 @@ +package net.woggioni.rbcs.common + +import java.security.Provider +import java.security.Security +import java.util.Base64 +import net.woggioni.rbcs.common.PasswordSecurity.decodePasswordHash +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource + + +class PasswordHashingTest { + + @EnumSource(PasswordSecurity.Algorithm::class) + @ParameterizedTest + fun test(algo: PasswordSecurity.Algorithm) { + val password = "password" + val encoded = hashPassword(password, algorithm = algo) + val (_, salt) = decodePasswordHash(encoded, algo) + Assertions.assertEquals(encoded, + hashPassword(password, salt = salt.let(Base64.getEncoder()::encodeToString), algorithm = algo) + ) + } + + @Test + fun listAvailableAlgorithms() { + Security.getProviders().asSequence() + .flatMap { provider: Provider -> provider.services.asSequence() } + .filter { service: Provider.Service -> "SecretKeyFactory" == service.type } + .map(Provider.Service::getAlgorithm) + .forEach { + println(it) + } + + } +} \ No newline at end of file diff --git a/rbcs-server-memcache/.classpath b/rbcs-server-memcache/.classpath new file mode 100644 index 0000000..df4ae25 --- /dev/null +++ b/rbcs-server-memcache/.classpath @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-server-memcache/.project b/rbcs-server-memcache/.project new file mode 100644 index 0000000..19cdffb --- /dev/null +++ b/rbcs-server-memcache/.project @@ -0,0 +1,34 @@ + + + rbcs-server-memcache + Project rbcs-server-memcache created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776958 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-server-memcache/.settings/org.eclipse.buildship.core.prefs b/rbcs-server-memcache/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-server-memcache/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-server-memcache/.settings/org.eclipse.jdt.core.prefs b/rbcs-server-memcache/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-server-memcache/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-server-memcache/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider b/rbcs-server-memcache/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider new file mode 100644 index 0000000..5a04b38 --- /dev/null +++ b/rbcs-server-memcache/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider @@ -0,0 +1 @@ +net.woggioni.rbcs.server.memcache.MemcacheCacheProvider \ No newline at end of file diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/Exception.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/Exception.kt new file mode 100644 index 0000000..31ca929 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/Exception.kt @@ -0,0 +1,4 @@ +package net.woggioni.rbcs.server.memcache + +class MemcacheException(status : Short, msg : String? = null, cause : Throwable? = null) + : RuntimeException(msg ?: "Memcached status $status", cause) \ No newline at end of file diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheConfiguration.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheConfiguration.kt new file mode 100644 index 0000000..a789876 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheConfiguration.kt @@ -0,0 +1,105 @@ +package net.woggioni.rbcs.server.memcache + +import io.netty.channel.ChannelFactory +import io.netty.channel.EventLoopGroup +import io.netty.channel.pool.FixedChannelPool +import io.netty.channel.socket.DatagramChannel +import io.netty.channel.socket.SocketChannel +import java.time.Duration +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.CacheHandlerFactory +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.server.memcache.client.MemcacheClient + +data class MemcacheCacheConfiguration( + val servers: List, + val maxAge: Duration = Duration.ofDays(1), + val keyPrefix : String? = null, + val digestAlgorithm: String? = null, + val compressionMode: CompressionMode? = null, + val compressionLevel: Int, +) : Configuration.Cache { + + companion object { + private val log = createLogger() + } + + enum class CompressionMode { + /** + * Deflate mode + */ + DEFLATE + } + + data class Server( + val endpoint: HostAndPort, + val connectionTimeoutMillis: Int?, + val maxConnections: Int + ) + + override fun materialize() = object : CacheHandlerFactory { + + private val connectionPoolMap = ConcurrentHashMap() + + override fun newHandler( + cfg : Configuration, + eventLoop: EventLoopGroup, + socketChannelFactory: ChannelFactory, + datagramChannelFactory: ChannelFactory, + ): CacheHandler { + return MemcacheCacheHandler( + MemcacheClient( + this@MemcacheCacheConfiguration.servers, + cfg.connection.chunkSize, + eventLoop, + socketChannelFactory, + connectionPoolMap + ), + keyPrefix, + digestAlgorithm, + compressionMode != null, + compressionLevel, + cfg.connection.chunkSize, + maxAge + ) + } + + override fun asyncClose() = object : CompletableFuture() { + init { + val failure = AtomicReference(null) + val pools = connectionPoolMap.values.toList() + val npools = pools.size + val finished = AtomicInteger(0) + if (pools.isEmpty()) { + complete(null) + } else { + pools.forEach { pool -> + pool.closeAsync().addListener { + if (!it.isSuccess) { + failure.compareAndSet(null, it.cause()) + } + if (finished.incrementAndGet() == npools) { + when (val ex = failure.get()) { + null -> complete(null) + else -> completeExceptionally(ex) + } + } + } + } + } + } + } + + } + + override fun getNamespaceURI() = "urn:net.woggioni.rbcs.server.memcache" + + override fun getTypeName() = "memcacheCacheType" +} + diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheHandler.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheHandler.kt new file mode 100644 index 0000000..d488eb1 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheHandler.kt @@ -0,0 +1,442 @@ +package net.woggioni.rbcs.server.memcache + +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.CompositeByteBuf +import io.netty.channel.Channel as NettyChannel +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.memcache.DefaultLastMemcacheContent +import io.netty.handler.codec.memcache.DefaultMemcacheContent +import io.netty.handler.codec.memcache.LastMemcacheContent +import io.netty.handler.codec.memcache.MemcacheContent +import io.netty.handler.codec.memcache.binary.BinaryMemcacheOpcodes +import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponse +import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponseStatus +import io.netty.handler.codec.memcache.binary.DefaultBinaryMemcacheRequest +import java.io.ByteArrayOutputStream +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.nio.ByteBuffer +import java.nio.channels.Channels +import java.nio.channels.FileChannel +import java.nio.channels.ReadableByteChannel +import java.nio.file.Files +import java.nio.file.StandardOpenOption +import java.time.Duration +import java.time.Instant +import java.util.concurrent.CompletableFuture +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream +import java.util.zip.InflaterOutputStream +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.api.exception.ContentTooLargeException +import net.woggioni.rbcs.api.message.CacheMessage +import net.woggioni.rbcs.api.message.CacheMessage.CacheContent +import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent +import net.woggioni.rbcs.common.ByteBufInputStream +import net.woggioni.rbcs.common.ByteBufOutputStream +import net.woggioni.rbcs.common.RBCS.processCacheKey +import net.woggioni.rbcs.common.RBCS.toIntOrNull +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.extractChunk +import net.woggioni.rbcs.common.trace +import net.woggioni.rbcs.server.memcache.client.MemcacheClient +import net.woggioni.rbcs.server.memcache.client.MemcacheRequestController +import net.woggioni.rbcs.server.memcache.client.MemcacheResponseHandler + +class MemcacheCacheHandler( + private val client: MemcacheClient, + private val keyPrefix: String?, + private val digestAlgorithm: String?, + private val compressionEnabled: Boolean, + private val compressionLevel: Int, + private val chunkSize: Int, + private val maxAge: Duration +) : CacheHandler() { + companion object { + private val log = createLogger() + + private fun encodeExpiry(expiry: Duration): Int { + val expirySeconds = expiry.toSeconds() + return expirySeconds.toInt().takeIf { it.toLong() == expirySeconds } + ?: Instant.ofEpochSecond(expirySeconds).epochSecond.toInt() + } + } + + private interface InProgressRequest { + + } + + private inner class InProgressGetRequest( + val key: String, + private val ctx: ChannelHandlerContext + ) : InProgressRequest { + private val acc = ctx.alloc().compositeBuffer() + private val chunk = ctx.alloc().compositeBuffer() + private val outputStream = ByteBufOutputStream(chunk).let { + if (compressionEnabled) { + InflaterOutputStream(it) + } else { + it + } + } + private var responseSent = false + private var metadataSize: Int? = null + + fun write(buf: ByteBuf) { + acc.addComponent(true, buf.retain()) + if (metadataSize == null && acc.readableBytes() >= Int.SIZE_BYTES) { + metadataSize = acc.readInt() + } + metadataSize + ?.takeIf { !responseSent } + ?.takeIf { acc.readableBytes() >= it } + ?.let { mSize -> + val metadata = ObjectInputStream(ByteBufInputStream(acc)).use { + acc.retain() + it.readObject() as CacheValueMetadata + } + log.trace(ctx) { + "Sending response from cache" + } + sendMessageAndFlush(ctx, CacheValueFoundResponse(key, metadata)) + responseSent = true + acc.readerIndex(Int.SIZE_BYTES + mSize) + } + if (responseSent) { + acc.readBytes(outputStream, acc.readableBytes()) + if (acc.readableBytes() >= chunkSize) { + flush(false) + } + } + } + + private fun flush(last: Boolean) { + val toSend = extractChunk(chunk, ctx.alloc()) + val msg = if (last) { + log.trace(ctx) { + "Sending last chunk to client" + } + LastCacheContent(toSend) + } else { + log.trace(ctx) { + "Sending chunk to client" + } + CacheContent(toSend) + } + sendMessageAndFlush(ctx, msg) + } + + fun commit() { + acc.release() + chunk.retain() + outputStream.close() + flush(true) + chunk.release() + } + + fun rollback() { + acc.release() + outputStream.close() + } + } + + private inner class InProgressPutRequest( + private val ch: NettyChannel, + metadata: CacheValueMetadata, + val digest: ByteBuf, + val requestController: CompletableFuture, + private val alloc: ByteBufAllocator + ) : InProgressRequest { + private var totalSize = 0 + private var tmpFile: FileChannel? = null + private val accumulator = alloc.compositeBuffer() + private val stream = ByteBufOutputStream(accumulator).let { + if (compressionEnabled) { + DeflaterOutputStream(it, Deflater(compressionLevel)) + } else { + it + } + } + + init { + ByteArrayOutputStream().let { baos -> + ObjectOutputStream(baos).use { + it.writeObject(metadata) + } + val serializedBytes = baos.toByteArray() + accumulator.writeInt(serializedBytes.size) + accumulator.writeBytes(serializedBytes) + } + } + + fun write(buf: ByteBuf) { + totalSize += buf.readableBytes() + buf.readBytes(stream, buf.readableBytes()) + tmpFile?.let { + flushToDisk(it, accumulator) + } + if (accumulator.readableBytes() > 0x100000) { + log.debug(ch) { + "Entry is too big, buffering it into a file" + } + val opts = arrayOf( + StandardOpenOption.DELETE_ON_CLOSE, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING + ) + FileChannel.open(Files.createTempFile("rbcs-memcache", ".tmp"), *opts).let { fc -> + tmpFile = fc + flushToDisk(fc, accumulator) + } + } + } + + private fun flushToDisk(fc: FileChannel, buf: CompositeByteBuf) { + val chunk = extractChunk(buf, alloc) + fc.write(chunk.nioBuffer()) + chunk.release() + } + + fun commit(): Pair { + digest.release() + accumulator.retain() + stream.close() + val fileChannel = tmpFile + return if (fileChannel != null) { + flushToDisk(fileChannel, accumulator) + accumulator.release() + fileChannel.position(0) + val fileSize = fileChannel.size().toIntOrNull() ?: let { + fileChannel.close() + throw ContentTooLargeException("Request body is too large", null) + } + fileSize to fileChannel + } else { + accumulator.readableBytes() to Channels.newChannel(ByteBufInputStream(accumulator)) + } + } + + fun rollback() { + stream.close() + digest.release() + tmpFile?.close() + } + } + + private var inProgressRequest: InProgressRequest? = null + + override fun channelRead0(ctx: ChannelHandlerContext, msg: CacheMessage) { + when (msg) { + is CacheGetRequest -> handleGetRequest(ctx, msg) + is CachePutRequest -> handlePutRequest(ctx, msg) + is LastCacheContent -> handleLastCacheContent(ctx, msg) + is CacheContent -> handleCacheContent(ctx, msg) + else -> ctx.fireChannelRead(msg) + } + } + + private fun handleGetRequest(ctx: ChannelHandlerContext, msg: CacheGetRequest) { + log.debug(ctx) { + "Fetching ${msg.key} from memcache" + } + val key = ctx.alloc().buffer().also { + it.writeBytes(processCacheKey(msg.key, keyPrefix, digestAlgorithm)) + } + val responseHandler = object : MemcacheResponseHandler { + override fun responseReceived(response: BinaryMemcacheResponse) { + val status = response.status() + when (status) { + BinaryMemcacheResponseStatus.SUCCESS -> { + log.debug(ctx) { + "Cache hit for key ${msg.key} on memcache" + } + inProgressRequest = InProgressGetRequest(msg.key, ctx) + } + + BinaryMemcacheResponseStatus.KEY_ENOENT -> { + log.debug(ctx) { + "Cache miss for key ${msg.key} on memcache" + } + sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key)) + } + } + } + + override fun contentReceived(content: MemcacheContent) { + log.trace(ctx) { + "${if (content is LastMemcacheContent) "Last chunk" else "Chunk"} of ${ + content.content().readableBytes() + } bytes received from memcache for key ${msg.key}" + } + (inProgressRequest as? InProgressGetRequest)?.let { inProgressGetRequest -> + inProgressGetRequest.write(content.content()) + if (content is LastMemcacheContent) { + inProgressRequest = null + inProgressGetRequest.commit() + } + } + } + + override fun exceptionCaught(ex: Throwable) { + (inProgressRequest as? InProgressGetRequest).let { inProgressGetRequest -> + inProgressGetRequest?.let { + inProgressRequest = null + it.rollback() + } + } + this@MemcacheCacheHandler.exceptionCaught(ctx, ex) + } + } + client.sendRequest(key.retainedDuplicate(), responseHandler).thenAccept { requestHandle -> + log.trace(ctx) { + "Sending GET request for key ${msg.key} to memcache" + } + val request = DefaultBinaryMemcacheRequest(key).apply { + setOpcode(BinaryMemcacheOpcodes.GET) + } + requestHandle.sendRequest(request) + requestHandle.sendContent(LastMemcacheContent.EMPTY_LAST_CONTENT) + } + } + + private fun handlePutRequest(ctx: ChannelHandlerContext, msg: CachePutRequest) { + val key = ctx.alloc().buffer().also { + it.writeBytes(processCacheKey(msg.key, keyPrefix, digestAlgorithm)) + } + val responseHandler = object : MemcacheResponseHandler { + override fun responseReceived(response: BinaryMemcacheResponse) { + val status = response.status() + when (status) { + BinaryMemcacheResponseStatus.SUCCESS -> { + log.debug(ctx) { + "Inserted key ${msg.key} into memcache" + } + sendMessageAndFlush(ctx, CachePutResponse(msg.key)) + } + + else -> this@MemcacheCacheHandler.exceptionCaught(ctx, MemcacheException(status)) + } + } + + override fun contentReceived(content: MemcacheContent) {} + + override fun exceptionCaught(ex: Throwable) { + this@MemcacheCacheHandler.exceptionCaught(ctx, ex) + } + } + + val requestController = client.sendRequest(key.retainedDuplicate(), responseHandler).whenComplete { _, ex -> + ex?.let { + this@MemcacheCacheHandler.exceptionCaught(ctx, ex) + } + } + inProgressRequest = InProgressPutRequest(ctx.channel(), msg.metadata, key, requestController, ctx.alloc()) + } + + private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + log.trace(ctx) { + "Received chunk of ${msg.content().readableBytes()} bytes for memcache" + } + request.write(msg.content()) + } + + is InProgressGetRequest -> { + msg.release() + } + } + } + + private fun handleLastCacheContent(ctx: ChannelHandlerContext, msg: LastCacheContent) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + inProgressRequest = null + log.trace(ctx) { + "Received last chunk of ${msg.content().readableBytes()} bytes for memcache" + } + request.write(msg.content()) + val key = request.digest.retainedDuplicate() + val (payloadSize, payloadSource) = request.commit() + val extras = ctx.alloc().buffer(8, 8) + extras.writeInt(0) + extras.writeInt(encodeExpiry(maxAge)) + val totalBodyLength = request.digest.readableBytes() + extras.readableBytes() + payloadSize + log.trace(ctx) { + "Trying to send SET request to memcache" + } + request.requestController.whenComplete { requestController, ex -> + if (ex == null) { + log.trace(ctx) { + "Sending SET request to memcache" + } + requestController.sendRequest(DefaultBinaryMemcacheRequest().apply { + setOpcode(BinaryMemcacheOpcodes.SET) + setKey(key) + setExtras(extras) + setTotalBodyLength(totalBodyLength) + }) + log.trace(ctx) { + "Sending request payload to memcache" + } + payloadSource.use { source -> + val bb = ByteBuffer.allocate(chunkSize) + while (true) { + val read = source.read(bb) + bb.limit() + if (read >= 0 && bb.position() < chunkSize && bb.hasRemaining()) { + continue + } + val chunk = ctx.alloc().buffer(chunkSize) + bb.flip() + chunk.writeBytes(bb) + bb.clear() + log.trace(ctx) { + "Sending ${chunk.readableBytes()} bytes chunk to memcache" + } + if (read < 0) { + requestController.sendContent(DefaultLastMemcacheContent(chunk)) + break + } else { + requestController.sendContent(DefaultMemcacheContent(chunk)) + } + } + } + } else { + payloadSource.close() + } + } + } + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + inProgressRequest = null + request.requestController.thenAccept { controller -> + controller.exceptionCaught(cause) + } + request.rollback() + } + + is InProgressGetRequest -> { + inProgressRequest = null + request.rollback() + } + } + super.exceptionCaught(ctx, cause) + } +} \ No newline at end of file diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheProvider.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheProvider.kt new file mode 100644 index 0000000..0ff29a1 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/MemcacheCacheProvider.kt @@ -0,0 +1,102 @@ +package net.woggioni.rbcs.server.memcache + +import java.time.Duration +import java.time.temporal.ChronoUnit +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.api.exception.ConfigurationException +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.common.Xml.Companion.asIterable +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document +import org.w3c.dom.Element + + +class MemcacheCacheProvider : CacheProvider { + override fun getXmlSchemaLocation() = "jpms://net.woggioni.rbcs.server.memcache/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd" + + override fun getXmlType() = "memcacheCacheType" + + override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server.memcache" + + val xmlNamespacePrefix : String + get() = "rbcs-memcache" + + override fun deserialize(el: Element): MemcacheCacheConfiguration { + val servers = mutableListOf() + val maxAge = el.renderAttribute("max-age") + ?.let(Duration::parse) + ?: Duration.ofDays(1) + val compressionLevel = el.renderAttribute("compression-level") + ?.let(Integer::decode) + ?: -1 + val compressionMode = el.renderAttribute("compression-mode") + ?.let { + when (it) { + "deflate" -> MemcacheCacheConfiguration.CompressionMode.DEFLATE + else -> MemcacheCacheConfiguration.CompressionMode.DEFLATE + } + } + val keyPrefix = el.renderAttribute("key-prefix") + val digestAlgorithm = el.renderAttribute("digest") + for (child in el.asIterable()) { + when (child.nodeName) { + "server" -> { + val host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required") + val port = child.renderAttribute("port")?.toInt() ?: throw ConfigurationException("port attribute is required") + val maxConnections = child.renderAttribute("max-connections")?.toInt() ?: 1 + val connectionTimeout = child.renderAttribute("connection-timeout") + ?.let(Duration::parse) + ?.let(Duration::toMillis) + ?.let(Long::toInt) + ?: 10000 + servers.add(MemcacheCacheConfiguration.Server(HostAndPort(host, port), connectionTimeout, maxConnections)) + } + } + } + return MemcacheCacheConfiguration( + servers, + maxAge, + keyPrefix, + digestAlgorithm, + compressionMode, + compressionLevel + ) + } + + override fun serialize(doc: Document, cache: MemcacheCacheConfiguration) = cache.run { + val result = doc.createElement("cache") + Xml.of(doc, result) { + attr("xmlns:${xmlNamespacePrefix}", xmlNamespace, namespaceURI = "http://www.w3.org/2000/xmlns/") + attr("xs:type", "${xmlNamespacePrefix}:$xmlType", RBCS.XML_SCHEMA_NAMESPACE_URI) + for (server in servers) { + node("server") { + attr("host", server.endpoint.host) + attr("port", server.endpoint.port.toString()) + server.connectionTimeoutMillis?.let { connectionTimeoutMillis -> + attr("connection-timeout", Duration.of(connectionTimeoutMillis.toLong(), ChronoUnit.MILLIS).toString()) + } + attr("max-connections", server.maxConnections.toString()) + } + + } + attr("max-age", maxAge.toString()) + keyPrefix?.let { + attr("key-prefix", it) + } + digestAlgorithm?.let { digestAlgorithm -> + attr("digest", digestAlgorithm) + } + compressionMode?.let { compressionMode -> + attr( + "compression-mode", when (compressionMode) { + MemcacheCacheConfiguration.CompressionMode.DEFLATE -> "deflate" + } + ) + } + attr("compression-level", compressionLevel.toString()) + } + result + } +} diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheClient.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheClient.kt new file mode 100644 index 0000000..07e37ab --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheClient.kt @@ -0,0 +1,198 @@ +package net.woggioni.rbcs.server.memcache.client + + +import io.netty.bootstrap.Bootstrap +import io.netty.buffer.ByteBuf +import io.netty.channel.Channel +import io.netty.channel.ChannelFactory +import io.netty.channel.ChannelFutureListener +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelOption +import io.netty.channel.ChannelPipeline +import io.netty.channel.EventLoopGroup +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.channel.pool.AbstractChannelPoolHandler +import io.netty.channel.pool.FixedChannelPool +import io.netty.channel.socket.SocketChannel +import io.netty.handler.codec.memcache.LastMemcacheContent +import io.netty.handler.codec.memcache.MemcacheContent +import io.netty.handler.codec.memcache.MemcacheObject +import io.netty.handler.codec.memcache.binary.BinaryMemcacheClientCodec +import io.netty.handler.codec.memcache.binary.BinaryMemcacheRequest +import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponse +import io.netty.util.concurrent.Future as NettyFuture +import io.netty.util.concurrent.GenericFutureListener +import java.io.IOException +import java.net.InetSocketAddress +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.trace +import net.woggioni.rbcs.server.memcache.MemcacheCacheConfiguration +import net.woggioni.rbcs.server.memcache.MemcacheCacheHandler + + +class MemcacheClient( + private val servers: List, + private val chunkSize : Int, + private val group: EventLoopGroup, + private val channelFactory: ChannelFactory, + private val connectionPool: ConcurrentHashMap +) : AutoCloseable { + + private companion object { + private val log = createLogger() + } + + private fun newConnectionPool(server: MemcacheCacheConfiguration.Server): FixedChannelPool { + val bootstrap = Bootstrap().apply { + group(group) + channelFactory(channelFactory) + option(ChannelOption.SO_KEEPALIVE, true) + remoteAddress(InetSocketAddress(server.endpoint.host, server.endpoint.port)) + server.connectionTimeoutMillis?.let { + option(ChannelOption.CONNECT_TIMEOUT_MILLIS, it) + } + } + val channelPoolHandler = object : AbstractChannelPoolHandler() { + + override fun channelCreated(ch: Channel) { + val pipeline: ChannelPipeline = ch.pipeline() + pipeline.addLast(BinaryMemcacheClientCodec(chunkSize, true)) + } + } + return FixedChannelPool(bootstrap, channelPoolHandler, server.maxConnections) + } + + fun sendRequest( + key: ByteBuf, + responseHandler: MemcacheResponseHandler + ): CompletableFuture { + val server = if (servers.size > 1) { + var checksum = 0 + while (key.readableBytes() > 4) { + val byte = key.readInt() + checksum = checksum xor byte + } + while (key.readableBytes() > 0) { + val byte = key.readByte() + checksum = checksum xor byte.toInt() + } + servers[checksum % servers.size] + } else { + servers.first() + } + key.release() + + val response = CompletableFuture() + // Custom handler for processing responses + val pool = connectionPool.computeIfAbsent(server.endpoint) { + newConnectionPool(server) + } + pool.acquire().addListener(object : GenericFutureListener> { + override fun operationComplete(channelFuture: NettyFuture) { + if (channelFuture.isSuccess) { + val channel = channelFuture.now + var connectionClosedByTheRemoteServer = true + val closeCallback = { + if (connectionClosedByTheRemoteServer) { + val ex = IOException("The memcache server closed the connection") + val completed = response.completeExceptionally(ex) + if(!completed) responseHandler.exceptionCaught(ex) + } + } + val closeListener = ChannelFutureListener { + closeCallback() + } + channel.closeFuture().addListener(closeListener) + val pipeline = channel.pipeline() + val handler = object : SimpleChannelInboundHandler() { + + override fun handlerAdded(ctx: ChannelHandlerContext) { + channel.closeFuture().removeListener(closeListener) + } + + override fun channelRead0( + ctx: ChannelHandlerContext, + msg: MemcacheObject + ) { + when (msg) { + is BinaryMemcacheResponse -> { + responseHandler.responseReceived(msg) + } + + is LastMemcacheContent -> { + responseHandler.contentReceived(msg) + pipeline.remove(this) + } + + is MemcacheContent -> { + responseHandler.contentReceived(msg) + } + } + } + + override fun channelInactive(ctx: ChannelHandlerContext) { + closeCallback() + ctx.fireChannelInactive() + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + connectionClosedByTheRemoteServer = false + ctx.close() + responseHandler.exceptionCaught(cause) + } + } + + channel.pipeline().addLast(handler) + response.complete(object : MemcacheRequestController { + private var channelReleased = false + + override fun sendRequest(request: BinaryMemcacheRequest) { + channel.writeAndFlush(request) + } + + override fun sendContent(content: MemcacheContent) { + channel.writeAndFlush(content).addListener { + if(content is LastMemcacheContent) { + if(!channelReleased) { + pool.release(channel) + channelReleased = true + log.trace(channel) { + "Channel released" + } + } + } + } + } + + override fun exceptionCaught(ex: Throwable) { + log.warn(ex.message, ex) + connectionClosedByTheRemoteServer = false + channel.close() + if(!channelReleased) { + pool.release(channel) + channelReleased = true + log.trace(channel) { + "Channel released" + } + } + } + }) + } else { + response.completeExceptionally(channelFuture.cause()) + } + } + }) + return response + } + + fun shutDown(): NettyFuture<*> { + return group.shutdownGracefully() + } + + override fun close() { + shutDown().sync() + } +} \ No newline at end of file diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheRequestController.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheRequestController.kt new file mode 100644 index 0000000..06cc772 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheRequestController.kt @@ -0,0 +1,13 @@ +package net.woggioni.rbcs.server.memcache.client + +import io.netty.handler.codec.memcache.MemcacheContent +import io.netty.handler.codec.memcache.binary.BinaryMemcacheRequest + +interface MemcacheRequestController { + + fun sendRequest(request : BinaryMemcacheRequest) + + fun sendContent(content : MemcacheContent) + + fun exceptionCaught(ex : Throwable) +} diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheResponseHandler.kt b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheResponseHandler.kt new file mode 100644 index 0000000..19bb930 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/client/MemcacheResponseHandler.kt @@ -0,0 +1,14 @@ +package net.woggioni.rbcs.server.memcache.client + +import io.netty.handler.codec.memcache.MemcacheContent +import io.netty.handler.codec.memcache.binary.BinaryMemcacheResponse + +interface MemcacheResponseHandler { + + + fun responseReceived(response : BinaryMemcacheResponse) + + fun contentReceived(content : MemcacheContent) + + fun exceptionCaught(ex : Throwable) +} diff --git a/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd new file mode 100644 index 0000000..1083f44 --- /dev/null +++ b/rbcs-server-memcache/bin/main/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + Prepend this string to all the keys inserted in memcache, + useful in case the caching backend is shared with other applications + + + + + + + + + + + + + + + + + diff --git a/rbcs-server-memcache/bin/test/net/woggioni/rbcs/server/memcache/client/ByteBufferTest.kt b/rbcs-server-memcache/bin/test/net/woggioni/rbcs/server/memcache/client/ByteBufferTest.kt new file mode 100644 index 0000000..21df339 --- /dev/null +++ b/rbcs-server-memcache/bin/test/net/woggioni/rbcs/server/memcache/client/ByteBufferTest.kt @@ -0,0 +1,27 @@ +package net.woggioni.rbcs.server.memcache.client + +import io.netty.buffer.ByteBufUtil +import io.netty.buffer.Unpooled +import java.io.ByteArrayInputStream +import java.nio.ByteBuffer +import java.nio.channels.Channels +import kotlin.random.Random +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class ByteBufferTest { + + @Test + fun test() { + val byteBuffer = ByteBuffer.allocate(0x100) + val originalBytes = Random(101325).nextBytes(0x100) + Channels.newChannel(ByteArrayInputStream(originalBytes)).use { source -> + source.read(byteBuffer) + } + byteBuffer.flip() + val buf = Unpooled.buffer() + buf.writeBytes(byteBuffer) + val finalBytes = ByteBufUtil.getBytes(buf) + Assertions.assertArrayEquals(originalBytes, finalBytes) + } +} \ No newline at end of file diff --git a/rbcs-server-redis/.classpath b/rbcs-server-redis/.classpath new file mode 100644 index 0000000..f330fac --- /dev/null +++ b/rbcs-server-redis/.classpath @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-server-redis/.project b/rbcs-server-redis/.project new file mode 100644 index 0000000..7db4573 --- /dev/null +++ b/rbcs-server-redis/.project @@ -0,0 +1,34 @@ + + + rbcs-server-redis + Project rbcs-server-redis created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776964 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-server-redis/.settings/org.eclipse.buildship.core.prefs b/rbcs-server-redis/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-server-redis/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-server-redis/.settings/org.eclipse.jdt.core.prefs b/rbcs-server-redis/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-server-redis/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-server-redis/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider b/rbcs-server-redis/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider new file mode 100644 index 0000000..20dc9e6 --- /dev/null +++ b/rbcs-server-redis/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider @@ -0,0 +1 @@ +net.woggioni.rbcs.server.redis.RedisCacheProvider diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/Exception.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/Exception.kt new file mode 100644 index 0000000..9e55570 --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/Exception.kt @@ -0,0 +1,4 @@ +package net.woggioni.rbcs.server.redis + +class RedisException(msg: String, cause: Throwable? = null) + : RuntimeException(msg, cause) diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheConfiguration.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheConfiguration.kt new file mode 100644 index 0000000..3b25bde --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheConfiguration.kt @@ -0,0 +1,107 @@ +package net.woggioni.rbcs.server.redis + +import io.netty.channel.ChannelFactory +import io.netty.channel.EventLoopGroup +import io.netty.channel.pool.FixedChannelPool +import io.netty.channel.socket.DatagramChannel +import io.netty.channel.socket.SocketChannel + +import java.time.Duration +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference + +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.CacheHandlerFactory +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.server.redis.client.RedisClient + +data class RedisCacheConfiguration( + val servers: List, + val maxAge: Duration = Duration.ofDays(1), + val keyPrefix: String? = null, + val digestAlgorithm: String? = null, + val compressionMode: CompressionMode? = null, + val compressionLevel: Int, +) : Configuration.Cache { + + companion object { + private val log = createLogger() + } + + enum class CompressionMode { + /** + * Deflate mode + */ + DEFLATE + } + + data class Server( + val endpoint: HostAndPort, + val connectionTimeoutMillis: Int?, + val maxConnections: Int, + val password: String? = null, + ) + + override fun materialize() = object : CacheHandlerFactory { + + private val connectionPoolMap = ConcurrentHashMap() + + override fun newHandler( + cfg: Configuration, + eventLoop: EventLoopGroup, + socketChannelFactory: ChannelFactory, + datagramChannelFactory: ChannelFactory, + ): CacheHandler { + return RedisCacheHandler( + RedisClient( + this@RedisCacheConfiguration.servers, + cfg.connection.chunkSize, + eventLoop, + socketChannelFactory, + connectionPoolMap + ), + keyPrefix, + digestAlgorithm, + compressionMode != null, + compressionLevel, + cfg.connection.chunkSize, + maxAge + ) + } + + override fun asyncClose() = object : CompletableFuture() { + init { + val failure = AtomicReference(null) + val pools = connectionPoolMap.values.toList() + val npools = pools.size + val finished = AtomicInteger(0) + if (pools.isEmpty()) { + complete(null) + } else { + pools.forEach { pool -> + pool.closeAsync().addListener { + if (!it.isSuccess) { + failure.compareAndSet(null, it.cause()) + } + if (finished.incrementAndGet() == npools) { + when (val ex = failure.get()) { + null -> complete(null) + else -> completeExceptionally(ex) + } + } + } + } + } + } + } + + } + + override fun getNamespaceURI() = "urn:net.woggioni.rbcs.server.redis" + + override fun getTypeName() = "redisCacheType" +} diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheHandler.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheHandler.kt new file mode 100644 index 0000000..584dd7f --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheHandler.kt @@ -0,0 +1,438 @@ +package net.woggioni.rbcs.server.redis + +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.CompositeByteBuf +import io.netty.channel.Channel as NettyChannel +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.redis.ArrayRedisMessage +import io.netty.handler.codec.redis.ErrorRedisMessage +import io.netty.handler.codec.redis.FullBulkStringRedisMessage +import io.netty.handler.codec.redis.RedisMessage +import io.netty.handler.codec.redis.SimpleStringRedisMessage + +import java.io.ByteArrayOutputStream +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.nio.ByteBuffer +import java.nio.channels.Channels +import java.nio.channels.FileChannel +import java.nio.channels.ReadableByteChannel +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.StandardOpenOption +import java.time.Duration +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream +import java.util.zip.InflaterOutputStream + +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.api.exception.ContentTooLargeException +import net.woggioni.rbcs.api.message.CacheMessage +import net.woggioni.rbcs.api.message.CacheMessage.CacheContent +import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent +import net.woggioni.rbcs.common.ByteBufInputStream +import net.woggioni.rbcs.common.ByteBufOutputStream +import net.woggioni.rbcs.common.RBCS.processCacheKey +import net.woggioni.rbcs.common.RBCS.toIntOrNull +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.extractChunk +import net.woggioni.rbcs.common.trace +import net.woggioni.rbcs.common.warn +import net.woggioni.rbcs.server.redis.client.RedisClient +import net.woggioni.rbcs.server.redis.client.RedisResponseHandler + +class RedisCacheHandler( + private val client: RedisClient, + private val keyPrefix: String?, + private val digestAlgorithm: String?, + private val compressionEnabled: Boolean, + private val compressionLevel: Int, + private val chunkSize: Int, + private val maxAge: Duration, +) : CacheHandler() { + companion object { + private val log = createLogger() + } + + private interface InProgressRequest + + private inner class InProgressGetRequest( + val key: String, + private val ctx: ChannelHandlerContext, + ) : InProgressRequest { + private val chunk = ctx.alloc().compositeBuffer() + private val outputStream = ByteBufOutputStream(chunk).let { + if (compressionEnabled) { + InflaterOutputStream(it) + } else { + it + } + } + + fun processResponse(data: ByteBuf) { + if (data.readableBytes() < Int.SIZE_BYTES) { + log.debug(ctx) { + "Received empty or corrupt data from Redis for key $key" + } + sendMessageAndFlush(ctx, CacheValueNotFoundResponse(key)) + data.release() + return + } + + val metadataSize = data.readInt() + if (data.readableBytes() < metadataSize) { + log.debug(ctx) { + "Received incomplete metadata from Redis for key $key" + } + sendMessageAndFlush(ctx, CacheValueNotFoundResponse(key)) + data.release() + return + } + + val metadata = ObjectInputStream(ByteBufInputStream(data)).use { + data.retain() + it.readObject() as CacheValueMetadata + } + data.readerIndex(Int.SIZE_BYTES + metadataSize) + + log.trace(ctx) { + "Sending response from cache" + } + sendMessageAndFlush(ctx, CacheValueFoundResponse(key, metadata)) + + // Decompress and stream the remaining payload + data.readBytes(outputStream, data.readableBytes()) + data.release() + commit() + } + + private fun flush(last: Boolean) { + val toSend = extractChunk(chunk, ctx.alloc()) + val msg = if (last) { + log.trace(ctx) { + "Sending last chunk to client" + } + LastCacheContent(toSend) + } else { + log.trace(ctx) { + "Sending chunk to client" + } + CacheContent(toSend) + } + sendMessageAndFlush(ctx, msg) + } + + fun commit() { + chunk.retain() + outputStream.close() + flush(true) + chunk.release() + } + + fun rollback() { + outputStream.close() + } + } + + private inner class InProgressPutRequest( + private val ch: NettyChannel, + metadata: CacheValueMetadata, + val keyString: String, + val keyBytes: ByteBuf, + private val alloc: ByteBufAllocator, + ) : InProgressRequest { + private var totalSize = 0 + private var tmpFile: FileChannel? = null + private val accumulator = alloc.compositeBuffer() + private val stream = ByteBufOutputStream(accumulator).let { + if (compressionEnabled) { + DeflaterOutputStream(it, Deflater(compressionLevel)) + } else { + it + } + } + + init { + ByteArrayOutputStream().let { baos -> + ObjectOutputStream(baos).use { + it.writeObject(metadata) + } + val serializedBytes = baos.toByteArray() + accumulator.writeInt(serializedBytes.size) + accumulator.writeBytes(serializedBytes) + } + } + + fun write(buf: ByteBuf) { + totalSize += buf.readableBytes() + buf.readBytes(stream, buf.readableBytes()) + tmpFile?.let { + flushToDisk(it, accumulator) + } + if (accumulator.readableBytes() > 0x100000) { + log.debug(ch) { + "Entry is too big, buffering it into a file" + } + val opts = arrayOf( + StandardOpenOption.DELETE_ON_CLOSE, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING + ) + FileChannel.open(Files.createTempFile("rbcs-server-redis", ".tmp"), *opts).let { fc -> + tmpFile = fc + flushToDisk(fc, accumulator) + } + } + } + + private fun flushToDisk(fc: FileChannel, buf: CompositeByteBuf) { + val chunk = extractChunk(buf, alloc) + fc.write(chunk.nioBuffer()) + chunk.release() + } + + fun commit(): Pair { + keyBytes.release() + accumulator.retain() + stream.close() + val fileChannel = tmpFile + return if (fileChannel != null) { + flushToDisk(fileChannel, accumulator) + accumulator.release() + fileChannel.position(0) + val fileSize = fileChannel.size().toIntOrNull() ?: let { + fileChannel.close() + throw ContentTooLargeException("Request body is too large", null) + } + fileSize to fileChannel + } else { + accumulator.readableBytes() to Channels.newChannel(ByteBufInputStream(accumulator)) + } + } + + fun rollback() { + stream.close() + keyBytes.release() + tmpFile?.close() + } + } + + private var inProgressRequest: InProgressRequest? = null + + override fun channelRead0(ctx: ChannelHandlerContext, msg: CacheMessage) { + when (msg) { + is CacheGetRequest -> handleGetRequest(ctx, msg) + is CachePutRequest -> handlePutRequest(ctx, msg) + is LastCacheContent -> handleLastCacheContent(ctx, msg) + is CacheContent -> handleCacheContent(ctx, msg) + else -> ctx.fireChannelRead(msg) + } + } + + private fun handleGetRequest(ctx: ChannelHandlerContext, msg: CacheGetRequest) { + log.debug(ctx) { + "Fetching ${msg.key} from Redis" + } + val keyBytes = processCacheKey(msg.key, keyPrefix, digestAlgorithm) + val keyString = String(keyBytes, StandardCharsets.UTF_8) + val responseHandler = object : RedisResponseHandler { + override fun responseReceived(response: RedisMessage) { + when (response) { + is FullBulkStringRedisMessage -> { + if (response === FullBulkStringRedisMessage.NULL_INSTANCE || response.content().readableBytes() == 0) { + log.debug(ctx) { + "Cache miss for key ${msg.key} on Redis" + } + sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key)) + } else { + log.debug(ctx) { + "Cache hit for key ${msg.key} on Redis" + } + val getRequest = InProgressGetRequest(msg.key, ctx) + inProgressRequest = getRequest + getRequest.processResponse(response.content()) + inProgressRequest = null + } + } + + is ErrorRedisMessage -> { + this@RedisCacheHandler.exceptionCaught( + ctx, RedisException("Redis error for GET ${msg.key}: ${response.content()}") + ) + } + + else -> { + log.warn(ctx) { + "Unexpected response type from Redis for key ${msg.key}: ${response.javaClass.name}" + } + sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key)) + } + } + } + + override fun exceptionCaught(ex: Throwable) { + this@RedisCacheHandler.exceptionCaught(ctx, ex) + } + } + client.sendCommand(keyBytes, ctx.alloc(), responseHandler).thenAccept { channel -> + log.trace(ctx) { + "Sending GET request for key ${msg.key} to Redis" + } + val cmd = buildRedisCommand(ctx.alloc(), "GET", keyString) + channel.writeAndFlush(cmd) + } + } + + private fun handlePutRequest(ctx: ChannelHandlerContext, msg: CachePutRequest) { + val keyBytes = processCacheKey(msg.key, keyPrefix, digestAlgorithm) + val keyBuf = ctx.alloc().buffer().also { + it.writeBytes(keyBytes) + } + inProgressRequest = InProgressPutRequest(ctx.channel(), msg.metadata, msg.key, keyBuf, ctx.alloc()) + } + + private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + log.trace(ctx) { + "Received chunk of ${msg.content().readableBytes()} bytes for Redis" + } + request.write(msg.content()) + } + + is InProgressGetRequest -> { + msg.release() + } + } + } + + private fun handleLastCacheContent(ctx: ChannelHandlerContext, msg: LastCacheContent) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + inProgressRequest = null + log.trace(ctx) { + "Received last chunk of ${msg.content().readableBytes()} bytes for Redis" + } + request.write(msg.content()) + val keyBytes = processCacheKey(request.keyString, keyPrefix, digestAlgorithm) + val keyString = String(keyBytes, StandardCharsets.UTF_8) + val (payloadSize, payloadSource) = request.commit() + + // Read the entire payload into a single ByteBuf for the SET command + val valueBuf = ctx.alloc().buffer(payloadSize) + payloadSource.use { source -> + val bb = ByteBuffer.allocate(chunkSize) + while (true) { + val read = source.read(bb) + if (read < 0) break + bb.flip() + valueBuf.writeBytes(bb) + bb.clear() + } + } + + val expirySeconds = maxAge.toSeconds().toString() + + val responseHandler = object : RedisResponseHandler { + override fun responseReceived(response: RedisMessage) { + when (response) { + is SimpleStringRedisMessage -> { + log.debug(ctx) { + "Inserted key ${request.keyString} into Redis" + } + sendMessageAndFlush(ctx, CachePutResponse(request.keyString)) + } + + is ErrorRedisMessage -> { + this@RedisCacheHandler.exceptionCaught( + ctx, RedisException("Redis error for SET ${request.keyString}: ${response.content()}") + ) + } + + else -> { + this@RedisCacheHandler.exceptionCaught( + ctx, RedisException("Unexpected response for SET ${request.keyString}: ${response.javaClass.name}") + ) + } + } + } + + override fun exceptionCaught(ex: Throwable) { + this@RedisCacheHandler.exceptionCaught(ctx, ex) + } + } + + // Use a ByteBuf key for server selection + client.sendCommand(keyBytes, ctx.alloc(), responseHandler).thenAccept { channel -> + log.trace(ctx) { + "Sending SET request to Redis" + } + // Build SET key value EX seconds + val cmd = buildRedisSetCommand(ctx.alloc(), keyString, valueBuf, expirySeconds) + channel.writeAndFlush(cmd) + }.whenComplete { _, ex -> + if (ex != null) { + valueBuf.release() + this@RedisCacheHandler.exceptionCaught(ctx, ex) + } + } + } + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + val request = inProgressRequest + when (request) { + is InProgressPutRequest -> { + inProgressRequest = null + request.rollback() + } + + is InProgressGetRequest -> { + inProgressRequest = null + request.rollback() + } + } + super.exceptionCaught(ctx, cause) + } + + private fun buildRedisCommand(alloc: ByteBufAllocator, vararg args: String): ArrayRedisMessage { + val children = args.map { arg -> + FullBulkStringRedisMessage( + alloc.buffer(arg.toByteArray(StandardCharsets.UTF_8)) + ) + } + return ArrayRedisMessage(children) + } + + private fun ByteBufAllocator.buffer(bytes : ByteArray) = buffer().apply { + writeBytes(bytes) + } + + private fun buildRedisSetCommand( + alloc: ByteBufAllocator, + key: String, + value: ByteBuf, + expirySeconds: String, + ): ArrayRedisMessage { + val children = listOf( + FullBulkStringRedisMessage(alloc.buffer("SET".toByteArray(StandardCharsets.UTF_8))), + FullBulkStringRedisMessage(alloc.buffer(key.toByteArray(StandardCharsets.UTF_8))), + FullBulkStringRedisMessage(value), + FullBulkStringRedisMessage(alloc.buffer("EX".toByteArray(StandardCharsets.UTF_8))), + FullBulkStringRedisMessage(alloc.buffer(expirySeconds.toByteArray(StandardCharsets.UTF_8))), + ) + return ArrayRedisMessage(children) + } +} diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheProvider.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheProvider.kt new file mode 100644 index 0000000..bc4a5b8 --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/RedisCacheProvider.kt @@ -0,0 +1,108 @@ +package net.woggioni.rbcs.server.redis + +import java.time.Duration +import java.time.temporal.ChronoUnit + +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.api.exception.ConfigurationException +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.common.Xml.Companion.asIterable +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute + +import org.w3c.dom.Document +import org.w3c.dom.Element + + +class RedisCacheProvider : CacheProvider { + override fun getXmlSchemaLocation() = "jpms://net.woggioni.rbcs.server.redis/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd" + + override fun getXmlType() = "redisCacheType" + + override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server.redis" + + val xmlNamespacePrefix: String + get() = "rbcs-redis" + + override fun deserialize(el: Element): RedisCacheConfiguration { + val servers = mutableListOf() + val maxAge = el.renderAttribute("max-age") + ?.let(Duration::parse) + ?: Duration.ofDays(1) + val compressionLevel = el.renderAttribute("compression-level") + ?.let(Integer::decode) + ?: -1 + val compressionMode = el.renderAttribute("compression-mode") + ?.let { + when (it) { + "deflate" -> RedisCacheConfiguration.CompressionMode.DEFLATE + else -> RedisCacheConfiguration.CompressionMode.DEFLATE + } + } + val keyPrefix = el.renderAttribute("key-prefix") + val digestAlgorithm = el.renderAttribute("digest") + for (child in el.asIterable()) { + when (child.nodeName) { + "server" -> { + val host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required") + val port = child.renderAttribute("port")?.toInt() ?: throw ConfigurationException("port attribute is required") + val maxConnections = child.renderAttribute("max-connections")?.toInt() ?: 1 + val connectionTimeout = child.renderAttribute("connection-timeout") + ?.let(Duration::parse) + ?.let(Duration::toMillis) + ?.let(Long::toInt) + ?: 10000 + val password = child.renderAttribute("password") + servers.add(RedisCacheConfiguration.Server(HostAndPort(host, port), connectionTimeout, maxConnections, password)) + } + } + } + return RedisCacheConfiguration( + servers, + maxAge, + keyPrefix, + digestAlgorithm, + compressionMode, + compressionLevel + ) + } + + override fun serialize(doc: Document, cache: RedisCacheConfiguration) = cache.run { + val result = doc.createElement("cache") + Xml.of(doc, result) { + attr("xmlns:${xmlNamespacePrefix}", xmlNamespace, namespaceURI = "http://www.w3.org/2000/xmlns/") + attr("xs:type", "${xmlNamespacePrefix}:$xmlType", RBCS.XML_SCHEMA_NAMESPACE_URI) + for (server in servers) { + node("server") { + attr("host", server.endpoint.host) + attr("port", server.endpoint.port.toString()) + server.connectionTimeoutMillis?.let { connectionTimeoutMillis -> + attr("connection-timeout", Duration.of(connectionTimeoutMillis.toLong(), ChronoUnit.MILLIS).toString()) + } + attr("max-connections", server.maxConnections.toString()) + server.password?.let { password -> + attr("password", password) + } + } + + } + attr("max-age", maxAge.toString()) + keyPrefix?.let { + attr("key-prefix", it) + } + digestAlgorithm?.let { digestAlgorithm -> + attr("digest", digestAlgorithm) + } + compressionMode?.let { compressionMode -> + attr( + "compression-mode", when (compressionMode) { + RedisCacheConfiguration.CompressionMode.DEFLATE -> "deflate" + } + ) + } + attr("compression-level", compressionLevel.toString()) + } + result + } +} diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisClient.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisClient.kt new file mode 100644 index 0000000..d683377 --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisClient.kt @@ -0,0 +1,204 @@ +package net.woggioni.rbcs.server.redis.client + +import io.netty.bootstrap.Bootstrap +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.Unpooled +import io.netty.channel.Channel +import io.netty.channel.ChannelFactory +import io.netty.channel.ChannelFutureListener +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelOption +import io.netty.channel.ChannelPipeline +import io.netty.channel.EventLoopGroup +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.channel.pool.AbstractChannelPoolHandler +import io.netty.channel.pool.FixedChannelPool +import io.netty.channel.socket.SocketChannel +import io.netty.handler.codec.redis.ArrayRedisMessage +import io.netty.handler.codec.redis.ErrorRedisMessage +import io.netty.handler.codec.redis.FullBulkStringRedisMessage +import io.netty.handler.codec.redis.RedisArrayAggregator +import io.netty.handler.codec.redis.RedisBulkStringAggregator +import io.netty.handler.codec.redis.RedisDecoder +import io.netty.handler.codec.redis.RedisEncoder +import io.netty.handler.codec.redis.RedisMessage +import io.netty.util.concurrent.Future as NettyFuture +import io.netty.util.concurrent.GenericFutureListener + +import java.io.IOException +import java.net.InetSocketAddress +import java.nio.charset.StandardCharsets +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap + +import net.woggioni.rbcs.common.HostAndPort +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.trace +import net.woggioni.rbcs.server.redis.RedisCacheConfiguration +import net.woggioni.rbcs.server.redis.RedisCacheHandler + + +class RedisClient( + private val servers: List, + private val chunkSize: Int, + private val group: EventLoopGroup, + private val channelFactory: ChannelFactory, + private val connectionPool: ConcurrentHashMap, +) : AutoCloseable { + + private companion object { + private val log = createLogger() + } + + private fun newConnectionPool(server: RedisCacheConfiguration.Server): FixedChannelPool { + val bootstrap = Bootstrap().apply { + group(group) + channelFactory(channelFactory) + option(ChannelOption.SO_KEEPALIVE, true) + remoteAddress(InetSocketAddress(server.endpoint.host, server.endpoint.port)) + server.connectionTimeoutMillis?.let { + option(ChannelOption.CONNECT_TIMEOUT_MILLIS, it) + } + } + val channelPoolHandler = object : AbstractChannelPoolHandler() { + + override fun channelCreated(ch: Channel) { + val pipeline: ChannelPipeline = ch.pipeline() + pipeline.addLast(RedisEncoder()) + pipeline.addLast(RedisDecoder()) + pipeline.addLast(RedisBulkStringAggregator()) + pipeline.addLast(RedisArrayAggregator()) + server.password?.let { password -> + // Send AUTH command synchronously on new connections + val authCmd = buildCommand("AUTH", password) + ch.writeAndFlush(authCmd).addListener(ChannelFutureListener { future -> + if (!future.isSuccess) { + ch.close() + } + }) + // Install a one-shot handler to consume the AUTH response + pipeline.addLast(object : SimpleChannelInboundHandler() { + override fun channelRead0(ctx: ChannelHandlerContext, msg: RedisMessage) { + when (msg) { + is ErrorRedisMessage -> { + ctx.close() + } + else -> { + // AUTH succeeded, remove this one-shot handler + ctx.pipeline().remove(this) + } + } + } + }) + } + } + } + return FixedChannelPool(bootstrap, channelPoolHandler, server.maxConnections) + } + + private fun buildCommand(vararg args: String): ArrayRedisMessage { + val children = args.map { arg -> + FullBulkStringRedisMessage( + Unpooled.wrappedBuffer(arg.toByteArray(StandardCharsets.UTF_8)) + ) + } + return ArrayRedisMessage(children) + } + + fun sendCommand( + key: ByteArray, + alloc: ByteBufAllocator, + responseHandler: RedisResponseHandler, + ): CompletableFuture { + val server = if (servers.size > 1) { + val keyBuffer = alloc.buffer(key.size) + keyBuffer.writeBytes(key) + var checksum = 0 + while (keyBuffer.readableBytes() > 4) { + val byte = keyBuffer.readInt() + checksum = checksum xor byte + } + while (keyBuffer.readableBytes() > 0) { + val byte = keyBuffer.readByte() + checksum = checksum xor byte.toInt() + } + keyBuffer.release() + servers[Math.floorMod(checksum, servers.size)] + } else { + servers.first() + } + + val response = CompletableFuture() + val pool = connectionPool.computeIfAbsent(server.endpoint) { + newConnectionPool(server) + } + pool.acquire().addListener(object : GenericFutureListener> { + override fun operationComplete(channelFuture: NettyFuture) { + if (channelFuture.isSuccess) { + val channel = channelFuture.now + var connectionClosedByTheRemoteServer = true + val closeCallback = { + if (connectionClosedByTheRemoteServer) { + val ex = IOException("The Redis server closed the connection") + val completed = response.completeExceptionally(ex) + if (!completed) responseHandler.exceptionCaught(ex) + } + } + val closeListener = ChannelFutureListener { + closeCallback() + } + channel.closeFuture().addListener(closeListener) + val pipeline = channel.pipeline() + val handler = object : SimpleChannelInboundHandler(false) { + + override fun handlerAdded(ctx: ChannelHandlerContext) { + channel.closeFuture().removeListener(closeListener) + } + + override fun channelRead0( + ctx: ChannelHandlerContext, + msg: RedisMessage, + ) { + pipeline.remove(this) + pool.release(channel) + log.trace(channel) { + "Channel released" + } + responseHandler.responseReceived(msg) + } + + override fun channelInactive(ctx: ChannelHandlerContext) { + closeCallback() + ctx.fireChannelInactive() + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + connectionClosedByTheRemoteServer = false + pipeline.remove(this) + ctx.close() + pool.release(channel) + log.trace(channel) { + "Channel released after exception" + } + responseHandler.exceptionCaught(cause) + } + } + + channel.pipeline().addLast(handler) + response.complete(channel) + } else { + response.completeExceptionally(channelFuture.cause()) + } + } + }) + return response + } + + fun shutDown(): NettyFuture<*> { + return group.shutdownGracefully() + } + + override fun close() { + shutDown().sync() + } +} diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisResponseHandler.kt b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisResponseHandler.kt new file mode 100644 index 0000000..0aa6fb0 --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/client/RedisResponseHandler.kt @@ -0,0 +1,10 @@ +package net.woggioni.rbcs.server.redis.client + +import io.netty.handler.codec.redis.RedisMessage + +interface RedisResponseHandler { + + fun responseReceived(response: RedisMessage) + + fun exceptionCaught(ex: Throwable) +} diff --git a/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd new file mode 100644 index 0000000..3ee4dc8 --- /dev/null +++ b/rbcs-server-redis/bin/main/net/woggioni/rbcs/server/redis/schema/rbcs-redis.xsd @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + Password for Redis AUTH command, used when the Redis server requires authentication + + + + + + + + + + + + + + + + + Prepend this string to all the keys inserted in Redis, + useful in case the caching backend is shared with other applications + + + + + + + + + + + + + + + + + diff --git a/rbcs-server/.classpath b/rbcs-server/.classpath new file mode 100644 index 0000000..da0cd61 --- /dev/null +++ b/rbcs-server/.classpath @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-server/.project b/rbcs-server/.project new file mode 100644 index 0000000..d562929 --- /dev/null +++ b/rbcs-server/.project @@ -0,0 +1,34 @@ + + + rbcs-server + Project rbcs-server created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370776957 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-server/.settings/org.eclipse.buildship.core.prefs b/rbcs-server/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-server/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-server/.settings/org.eclipse.jdt.core.prefs b/rbcs-server/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-server/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-server/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider b/rbcs-server/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider new file mode 100644 index 0000000..46a7419 --- /dev/null +++ b/rbcs-server/bin/main/META-INF/services/net.woggioni.rbcs.api.CacheProvider @@ -0,0 +1,2 @@ +net.woggioni.rbcs.server.cache.FileSystemCacheProvider +net.woggioni.rbcs.server.cache.InMemoryCacheProvider \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt new file mode 100644 index 0000000..708d2ba --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt @@ -0,0 +1,562 @@ +package net.woggioni.rbcs.server + +import io.netty.bootstrap.ServerBootstrap +import io.netty.buffer.ByteBuf +import io.netty.channel.Channel +import io.netty.channel.ChannelFactory +import io.netty.channel.ChannelFuture +import io.netty.channel.ChannelHandler.Sharable +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.channel.ChannelInitializer +import io.netty.channel.ChannelOption +import io.netty.channel.ChannelPromise +import io.netty.channel.MultiThreadIoEventLoopGroup +import io.netty.channel.nio.NioIoHandler +import io.netty.channel.socket.DatagramChannel +import io.netty.channel.socket.ServerSocketChannel +import io.netty.channel.socket.SocketChannel +import io.netty.channel.socket.nio.NioDatagramChannel +import io.netty.channel.socket.nio.NioServerSocketChannel +import io.netty.channel.socket.nio.NioSocketChannel +import io.netty.handler.codec.compression.CompressionOptions +import io.netty.handler.codec.haproxy.HAProxyMessageDecoder +import io.netty.handler.codec.http.DefaultHttpContent +import io.netty.handler.codec.http.HttpContentCompressor +import io.netty.handler.codec.http.HttpDecoderConfig +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpRequest +import io.netty.handler.codec.http.HttpServerCodec +import io.netty.handler.ssl.ClientAuth +import io.netty.handler.ssl.SslContext +import io.netty.handler.ssl.SslContextBuilder +import io.netty.handler.ssl.SslHandler +import io.netty.handler.stream.ChunkedWriteHandler +import io.netty.handler.timeout.IdleState +import io.netty.handler.timeout.IdleStateEvent +import io.netty.handler.timeout.IdleStateHandler +import io.netty.util.AttributeKey +import io.netty.util.concurrent.EventExecutorGroup +import java.io.OutputStream +import java.net.InetSocketAddress +import java.nio.file.Files +import java.nio.file.Path +import java.security.PrivateKey +import java.security.cert.X509Certificate +import java.time.Duration +import java.time.Instant +import java.util.Arrays +import java.util.Base64 +import java.util.concurrent.CompletableFuture +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException +import java.util.regex.Matcher +import java.util.regex.Pattern +import javax.naming.ldap.LdapName +import javax.net.ssl.SSLPeerUnverifiedException +import net.woggioni.rbcs.api.AsyncCloseable +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.exception.ConfigurationException +import net.woggioni.rbcs.common.Cidr +import net.woggioni.rbcs.common.PasswordSecurity.decodePasswordHash +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import net.woggioni.rbcs.common.RBCS.getTrustManager +import net.woggioni.rbcs.common.RBCS.loadKeystore +import net.woggioni.rbcs.common.RBCS.toUrl +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.info +import net.woggioni.rbcs.server.otel.OtelIntegration +import net.woggioni.rbcs.server.auth.AbstractNettyHttpAuthenticator +import net.woggioni.rbcs.server.auth.Authorizer +import net.woggioni.rbcs.server.auth.RoleAuthorizer +import net.woggioni.rbcs.server.configuration.Parser +import net.woggioni.rbcs.server.configuration.Serializer +import net.woggioni.rbcs.server.exception.ExceptionHandler +import net.woggioni.rbcs.server.handler.MaxRequestSizeHandler +import net.woggioni.rbcs.server.handler.ProxyProtocolHandler +import net.woggioni.rbcs.server.handler.ReadTriggerDuplexHandler +import net.woggioni.rbcs.server.handler.ServerHandler +import net.woggioni.rbcs.server.throttling.BucketManager +import net.woggioni.rbcs.server.throttling.ThrottlingHandler + +class RemoteBuildCacheServer(private val cfg: Configuration) { + + companion object { + private val log = createLogger() + + val userAttribute: AttributeKey = AttributeKey.valueOf("user") + val groupAttribute: AttributeKey> = AttributeKey.valueOf("group") + val clientIp: AttributeKey = AttributeKey.valueOf("client-ip") + + val DEFAULT_CONFIGURATION_URL by lazy { "jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/rbcs-default.xml".toUrl() } + private const val SSL_HANDLER_NAME = "sslHandler" + + fun loadConfiguration(configurationFile: Path): Configuration { + val doc = Files.newInputStream(configurationFile).use { + Xml.parseXml(configurationFile.toUri().toURL(), it) + } + return Parser.parse(doc) + } + + fun dumpConfiguration(conf: Configuration, outputStream: OutputStream) { + Xml.write(Serializer.serialize(conf), outputStream) + } + } + + private class HttpChunkContentCompressor( + threshold: Int, + vararg compressionOptions: CompressionOptions = emptyArray() + ) : HttpContentCompressor(threshold, *compressionOptions) { + override fun write(ctx: ChannelHandlerContext, msg: Any, promise: ChannelPromise) { + var message: Any? = msg + if (message is ByteBuf) { + // convert ByteBuf to HttpContent to make it work with compression. This is needed as we use the + // ChunkedWriteHandler to send files when compression is enabled. + val buff = message + if (buff.isReadable) { + // We only encode non empty buffers, as empty buffers can be used for determining when + // the content has been flushed and it confuses the HttpContentCompressor + // if we let it go + message = DefaultHttpContent(buff) + } + } + super.write(ctx, message, promise) + } + } + + @Sharable + private class ClientCertificateAuthenticator( + authorizer: Authorizer, + private val anonymousUserGroups: Set?, + private val userExtractor: Configuration.UserExtractor?, + private val groupExtractor: Configuration.GroupExtractor?, + ) : AbstractNettyHttpAuthenticator(authorizer) { + + override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? { + return try { + val sslHandler = (ctx.pipeline().get(SSL_HANDLER_NAME) as? SslHandler) + ?: throw ConfigurationException("Client certificate authentication cannot be used when TLS is disabled") + val sslEngine = sslHandler.engine() + sslEngine.session.peerCertificates.takeIf { + it.isNotEmpty() + }?.let { peerCertificates -> + val clientCertificate = peerCertificates.first() as X509Certificate + val user = userExtractor?.extract(clientCertificate) + val group = groupExtractor?.extract(clientCertificate) + val allGroups = + ((user?.groups ?: emptySet()).asSequence() + sequenceOf(group).filterNotNull()).toSet() + AuthenticationResult(user, allGroups) + } ?: anonymousUserGroups?.let { AuthenticationResult(null, it) } + } catch (ex: SSLPeerUnverifiedException) { + log.debug(ctx) { + ex.message ?: "Error witch client certificate authentication" + } + anonymousUserGroups?.let { AuthenticationResult(null, it) } + } + } + } + + @Sharable + private class ForwardedClientCertificateAuthenticator( + authorizer: Authorizer, + private val anonymousUserGroups: Set?, + private val subjectDnUserExtractor: SubjectDnExtractor?, + private val subjectDnGroupExtractor: SubjectDnExtractor?, + private val headerName: String, + private val trustedProxyIPs: List, + private val users: Map, + private val groups: Map, + ) : AbstractNettyHttpAuthenticator(authorizer) { + + companion object { + private val log = createLogger() + } + + override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? { + val clientIp = ctx.channel().attr(clientIp).get() + if (clientIp == null || trustedProxyIPs.none { it.contains(clientIp.address) }) { + log.debug(ctx) { + "Rejecting forwarded client certificate authentication from untrusted address: $clientIp" + } + return null + } + val subjectDn = req.headers()[headerName] + ?: return anonymousUserGroups?.let { AuthenticationResult(null, it) } + val ldapName = try { + LdapName(subjectDn) + } catch (e: Exception) { + log.debug(ctx) { + "Invalid subject DN in header $headerName: $subjectDn" + } + return anonymousUserGroups?.let { AuthenticationResult(null, it) } + } + val user = subjectDnUserExtractor?.extract(ldapName)?.let { userName -> + users[userName] ?: throw RuntimeException("Failed to extract user '$userName'") + } + val group = subjectDnGroupExtractor?.extract(ldapName)?.let { groupName -> + groups[groupName] ?: throw RuntimeException("Failed to extract group '$groupName'") + } + val allGroups = ((user?.groups ?: emptySet()).asSequence() + sequenceOf(group).filterNotNull()).toSet() + return AuthenticationResult(user, allGroups) + } + } + + private data class SubjectDnExtractor(val rdnType: String, val pattern: Pattern) { + fun extract(ldapName: LdapName): String? { + return ldapName.rdns.find { it.type == rdnType } + ?.let { pattern.matcher(it.value.toString()) } + ?.takeIf(Matcher::matches)?.group(1) + } + } + + @Sharable + private class NettyHttpBasicAuthenticator( + private val users: Map, authorizer: Authorizer + ) : AbstractNettyHttpAuthenticator(authorizer) { + companion object { + private val log = createLogger() + } + + override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? { + val authorizationHeader = req.headers()[HttpHeaderNames.AUTHORIZATION] ?: let { + log.debug(ctx) { + "Missing Authorization header" + } + return users[""]?.let { AuthenticationResult(it, it.groups) } + } + val cursor = authorizationHeader.indexOf(' ') + if (cursor < 0) { + log.debug(ctx) { + "Invalid Authorization header: '$authorizationHeader'" + } + return users[""]?.let { AuthenticationResult(it, it.groups) } + } + val authenticationType = authorizationHeader.substring(0, cursor) + if ("Basic" != authenticationType) { + log.debug(ctx) { + "Invalid authentication type header: '$authenticationType'" + } + return users[""]?.let { AuthenticationResult(it, it.groups) } + } + val (username, password) = Base64.getDecoder().decode(authorizationHeader.substring(cursor + 1)) + .let(::String) + .let { + val colon = it.indexOf(':') + if (colon < 0) { + log.debug(ctx) { + "Missing colon from authentication" + } + return null + } + it.substring(0, colon) to it.substring(colon + 1) + } + + return username.let(users::get)?.takeIf { user -> + user.password?.let { passwordAndSalt -> + val (_, salt) = decodePasswordHash(passwordAndSalt) + hashPassword(password, Base64.getEncoder().encodeToString(salt)) == passwordAndSalt + } ?: false + }?.let { user -> + AuthenticationResult(user, user.groups) + } + } + } + + private class ServerInitializer( + private val cfg: Configuration, + private val channelFactory : ChannelFactory, + private val datagramChannelFactory : ChannelFactory, + ) : ChannelInitializer(), AsyncCloseable { + + companion object { + private fun createSslCtx(tls: Configuration.Tls): SslContext { + val keyStore = tls.keyStore + return if (keyStore == null) { + throw IllegalArgumentException("No keystore configured") + } else { + val javaKeyStore = loadKeystore(keyStore.file, keyStore.password) + val serverKey = javaKeyStore.getKey( + keyStore.keyAlias, (keyStore.keyPassword ?: "").let(String::toCharArray) + ) as PrivateKey + val serverCert: Array = + Arrays.stream(javaKeyStore.getCertificateChain(keyStore.keyAlias)) + .map { it as X509Certificate } + .toArray { size -> Array(size) { null } } + SslContextBuilder.forServer(serverKey, *serverCert).apply { + val clientAuth = tls.trustStore?.let { trustStore -> + val ts = loadKeystore(trustStore.file, trustStore.password) + trustManager( + getTrustManager(ts, trustStore.isCheckCertificateStatus) + ) + if (trustStore.isRequireClientCertificate) ClientAuth.REQUIRE + else ClientAuth.OPTIONAL + } ?: ClientAuth.NONE + clientAuth(clientAuth) + + }.build() + } + } + + private val log = createLogger() + } + + private val cacheHandlerFactory = cfg.cache.materialize() + + private val bucketManager = BucketManager.from(cfg) + + private val authenticator = when (val auth = cfg.authentication) { + is Configuration.BasicAuthentication -> NettyHttpBasicAuthenticator(cfg.users, RoleAuthorizer()) + is Configuration.ClientCertificateAuthentication -> { + ClientCertificateAuthenticator( + RoleAuthorizer(), + cfg.users[""]?.groups, + userExtractor(auth), + groupExtractor(auth) + ) + } + + is Configuration.ForwardedClientCertificateAuthentication -> { + ForwardedClientCertificateAuthenticator( + RoleAuthorizer(), + cfg.users[""]?.groups, + auth.userExtractor?.let { extractor -> + SubjectDnExtractor(extractor.rdnType, Pattern.compile(extractor.pattern)) + }, + auth.groupExtractor?.let { extractor -> + SubjectDnExtractor(extractor.rdnType, Pattern.compile(extractor.pattern)) + }, + auth.headerName, + cfg.trustedProxyIPs, + cfg.users, + cfg.groups, + ) + } + + else -> null + } + + private val proxyProtocolEnabled: Boolean = cfg.isProxyProtocolEnabled + private val trustedProxyIPs: List = cfg.trustedProxyIPs + + private val sslContext: SslContext? = cfg.tls?.let(Companion::createSslCtx) + + private fun userExtractor(authentication: Configuration.ClientCertificateAuthentication) = + authentication.userExtractor?.let { extractor -> + val pattern = Pattern.compile(extractor.pattern) + val rdnType = extractor.rdnType + Configuration.UserExtractor { cert: X509Certificate -> + val userName = LdapName(cert.subjectX500Principal.name).rdns.find { + it.type == rdnType + }?.let { + pattern.matcher(it.value.toString()) + }?.takeIf(Matcher::matches)?.group(1) + cfg.users[userName] ?: throw java.lang.RuntimeException("Failed to extract user") + } + } + + private fun groupExtractor(authentication: Configuration.ClientCertificateAuthentication) = + authentication.groupExtractor?.let { extractor -> + val pattern = Pattern.compile(extractor.pattern) + val rdnType = extractor.rdnType + Configuration.GroupExtractor { cert: X509Certificate -> + val groupName = LdapName(cert.subjectX500Principal.name).rdns.find { + it.type == rdnType + }?.let { + pattern.matcher(it.value.toString()) + }?.takeIf(Matcher::matches)?.group(1) + cfg.groups[groupName] ?: throw java.lang.RuntimeException("Failed to extract group") + } + } + + override fun initChannel(ch: Channel) { + ch.attr(clientIp).set(ch.remoteAddress() as InetSocketAddress) + log.debug { + "Created connection ${ch.id().asShortText()} with ${ch.remoteAddress()}" + } + ch.closeFuture().addListener { + log.debug { + "Closed connection ${ch.id().asShortText()} with ${ch.remoteAddress()}" + } + } + ch.config().isAutoRead = false + val pipeline = ch.pipeline() + cfg.connection.also { conn -> + val readIdleTimeout = conn.readIdleTimeout.toMillis() + val writeIdleTimeout = conn.writeIdleTimeout.toMillis() + val idleTimeout = conn.idleTimeout.toMillis() + if (readIdleTimeout > 0 || writeIdleTimeout > 0 || idleTimeout > 0) { + pipeline.addLast( + IdleStateHandler( + true, + readIdleTimeout, + writeIdleTimeout, + idleTimeout, + TimeUnit.MILLISECONDS + ) + ) + } + } + pipeline.addLast(object : ChannelInboundHandlerAdapter() { + override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) { + if (evt is IdleStateEvent) { + when (evt.state()) { + IdleState.READER_IDLE -> log.debug { + "Read timeout reached on channel ${ch.id().asShortText()}, closing the connection" + } + + IdleState.WRITER_IDLE -> log.debug { + "Write timeout reached on channel ${ch.id().asShortText()}, closing the connection" + } + + IdleState.ALL_IDLE -> log.debug { + "Idle timeout reached on channel ${ch.id().asShortText()}, closing the connection" + } + + null -> throw IllegalStateException("This should never happen") + } + ctx.close() + } + } + }) + if(proxyProtocolEnabled) { + pipeline.addLast(HAProxyMessageDecoder()) + pipeline.addLast(ProxyProtocolHandler(trustedProxyIPs)) + } + sslContext?.newHandler(ch.alloc())?.also { + pipeline.addLast(SSL_HANDLER_NAME, it) + } + val httpDecoderConfig = HttpDecoderConfig().apply { + maxChunkSize = cfg.connection.chunkSize + } + pipeline.addLast(HttpServerCodec(httpDecoderConfig)) + OtelIntegration.createHandler()?.let { pipeline.addLast(it) } + pipeline.addLast(ReadTriggerDuplexHandler.NAME, ReadTriggerDuplexHandler()) + pipeline.addLast(MaxRequestSizeHandler.NAME, MaxRequestSizeHandler(cfg.connection.maxRequestSize)) + pipeline.addLast(HttpChunkContentCompressor(1024)) + pipeline.addLast(ChunkedWriteHandler()) + authenticator?.let { + pipeline.addLast(it) + } + pipeline.addLast(ThrottlingHandler(bucketManager,cfg.rateLimiter, cfg.connection)) + + val serverHandler = let { + val prefix = Path.of("/").resolve(Path.of(cfg.serverPath ?: "/")) + ServerHandler(prefix) { + cacheHandlerFactory.newHandler(cfg, ch.eventLoop(), channelFactory, datagramChannelFactory) + } + } + pipeline.addLast(ServerHandler.NAME, serverHandler) + pipeline.addLast(ExceptionHandler.NAME, ExceptionHandler) + } + + override fun asyncClose() = cacheHandlerFactory.asyncClose() + } + + class ServerHandle( + closeFuture: ChannelFuture, + private val bossGroup: EventExecutorGroup, + private val executorGroups: Iterable, + private val serverInitializer: AsyncCloseable, + ) : Future by from(closeFuture, bossGroup, executorGroups, serverInitializer) { + + companion object { + private val log = createLogger() + + private fun from( + closeFuture: ChannelFuture, + bossGroup: EventExecutorGroup, + executorGroups: Iterable, + serverInitializer: AsyncCloseable + ): CompletableFuture { + val result = CompletableFuture() + closeFuture.addListener { + val errors = mutableListOf() + val deadline = Instant.now().plusSeconds(20) + + serverInitializer.asyncClose().whenCompleteAsync { _, ex -> + if(ex != null) { + log.error(ex.message, ex) + errors.addLast(ex) + } + + executorGroups.forEach(EventExecutorGroup::shutdownGracefully) + bossGroup.terminationFuture().sync() + + for (executorGroup in executorGroups) { + val future = executorGroup.terminationFuture() + try { + val now = Instant.now() + if (now > deadline) { + future.get(0, TimeUnit.SECONDS) + } else { + future.get(Duration.between(now, deadline).toMillis(), TimeUnit.MILLISECONDS) + } + } + catch (te: TimeoutException) { + errors.addLast(te) + log.warn("Timeout while waiting for shutdown of $executorGroup", te) + } catch (ex: Throwable) { + log.warn(ex.message, ex) + errors.addLast(ex) + } + } + + if(errors.isEmpty()) { + result.complete(null) + } else { + result.completeExceptionally(errors.first()) + } + } + } + + return result.thenAccept { + log.info { + "RemoteBuildCacheServer has been gracefully shut down" + } + } + } + } + + + fun sendShutdownSignal() { + bossGroup.shutdownGracefully() + } + } + + fun run(): ServerHandle { + // Create the multithreaded event loops for the server + val bossGroup = MultiThreadIoEventLoopGroup(1, NioIoHandler.newFactory()) + val channelFactory = ChannelFactory { NioSocketChannel() } + val datagramChannelFactory = ChannelFactory { NioDatagramChannel() } + val serverChannelFactory = ChannelFactory { NioServerSocketChannel() } + val workerGroup = MultiThreadIoEventLoopGroup(0, NioIoHandler.newFactory()) + + val serverInitializer = ServerInitializer(cfg, channelFactory, datagramChannelFactory) + val bootstrap = ServerBootstrap().apply { + // Configure the server + group(bossGroup, workerGroup) + channelFactory(serverChannelFactory) + childHandler(serverInitializer) + option(ChannelOption.SO_BACKLOG, cfg.incomingConnectionsBacklogSize) + childOption(ChannelOption.SO_KEEPALIVE, true) + } + + + // Bind and start to accept incoming connections. + val bindAddress = InetSocketAddress(cfg.host, cfg.port) + val httpChannel = bootstrap.bind(bindAddress).sync().channel() + log.info { + "RemoteBuildCacheServer is listening on ${cfg.host}:${cfg.port}" + } + + return ServerHandle( + httpChannel.closeFuture(), + bossGroup, + setOf(workerGroup), + serverInitializer + ) + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authenticator.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authenticator.kt new file mode 100644 index 0000000..202551e --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authenticator.kt @@ -0,0 +1,88 @@ +package net.woggioni.rbcs.server.auth + +import io.netty.buffer.Unpooled +import io.netty.channel.ChannelFutureListener +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.handler.codec.http.DefaultFullHttpResponse +import io.netty.handler.codec.http.FullHttpResponse +import io.netty.handler.codec.http.HttpContent +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpRequest +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import io.netty.util.ReferenceCountUtil +import java.net.InetSocketAddress +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Configuration.Group +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.server.RemoteBuildCacheServer + +abstract class AbstractNettyHttpAuthenticator(private val authorizer: Authorizer) : ChannelInboundHandlerAdapter() { + + companion object { + private val log = createLogger() + + private val AUTHENTICATION_FAILED: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.UNAUTHORIZED, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val NOT_AUTHORIZED: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + } + + class AuthenticationResult(val user: Configuration.User?, val groups: Set) + + abstract fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + if (msg is HttpRequest) { + val result = authenticate(ctx, msg) ?: return authenticationFailure(ctx, msg) + ctx.channel().attr(RemoteBuildCacheServer.userAttribute).set(result.user) + ctx.channel().attr(RemoteBuildCacheServer.groupAttribute).set(result.groups) + + val roles = ( + (result.user?.let { user -> + user.groups.asSequence().flatMap { group -> + group.roles.asSequence() + } + } ?: emptySequence()) + + result.groups.asSequence().flatMap { it.roles.asSequence() } + ).toSet() + val authorized = authorizer.authorize(roles, msg) + if(log.isDebugEnabled) { + val authorizedMessage = if(authorized) { "Authorized" } else { "Forbidden" } + val clientAddress = ctx.channel().attr(RemoteBuildCacheServer.clientIp).get() + val roleString = "[" + roles.asSequence().map { "\"" + it + "\""}.joinToString(", ") + "]" + result.user?.name?.takeUnless(String::isEmpty)?.let { username -> + log.debug("$authorizedMessage ${msg.method()} request from user $username with address $clientAddress, granted roles $roleString") + } ?: { + log.debug("$authorizedMessage anonymous ${msg.method()} request with address $clientAddress, granted roles $roleString") + } + } + if (authorized) { + super.channelRead(ctx, msg) + } else { + authorizationFailure(ctx, msg) + } + } else if(msg is HttpContent) { + ctx.fireChannelRead(msg) + } + } + + private fun authenticationFailure(ctx: ChannelHandlerContext, msg: Any) { + ReferenceCountUtil.release(msg) + ctx.writeAndFlush(AUTHENTICATION_FAILED.retainedDuplicate()).addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + + private fun authorizationFailure(ctx: ChannelHandlerContext, msg: Any) { + ReferenceCountUtil.release(msg) + ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate()).addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authorizer.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authorizer.kt new file mode 100644 index 0000000..1e777db --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/Authorizer.kt @@ -0,0 +1,8 @@ +package net.woggioni.rbcs.server.auth + +import io.netty.handler.codec.http.HttpRequest +import net.woggioni.rbcs.api.Role + +fun interface Authorizer { + fun authorize(roles : Set, request: HttpRequest) : Boolean +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/UserAuthorizer.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/UserAuthorizer.kt new file mode 100644 index 0000000..541e528 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/auth/UserAuthorizer.kt @@ -0,0 +1,24 @@ +package net.woggioni.rbcs.server.auth + +import io.netty.handler.codec.http.HttpMethod +import io.netty.handler.codec.http.HttpRequest +import net.woggioni.rbcs.api.Role + +class RoleAuthorizer : Authorizer { + + companion object { + private val METHOD_MAP = mapOf( + Role.Reader to setOf(HttpMethod.GET, HttpMethod.HEAD), + Role.Writer to setOf(HttpMethod.PUT, HttpMethod.POST), + Role.Healthcheck to setOf(HttpMethod.TRACE) + ) + } + + override fun authorize(roles: Set, request: HttpRequest) : Boolean { + val allowedMethods = roles.asSequence() + .mapNotNull(METHOD_MAP::get) + .flatten() + .toSet() + return request.method() in allowedMethods + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCache.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCache.kt new file mode 100644 index 0000000..cefab14 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCache.kt @@ -0,0 +1,169 @@ +package net.woggioni.rbcs.server.cache + +import java.io.ByteArrayOutputStream +import java.io.InputStream +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.io.Serializable +import java.nio.ByteBuffer +import java.nio.channels.Channels +import java.nio.channels.FileChannel +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.nio.file.StandardOpenOption +import java.nio.file.attribute.BasicFileAttributes +import java.time.Duration +import java.time.Instant +import java.util.concurrent.CompletableFuture +import net.woggioni.jwo.JWO +import net.woggioni.rbcs.api.AsyncCloseable +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.common.createLogger + +class FileSystemCache( + val root: Path, + val maxAge: Duration +) : AsyncCloseable { + + class EntryValue(val metadata: CacheValueMetadata, val channel : FileChannel, val offset : Long, val size : Long) : Serializable + + private companion object { + private val log = createLogger() + } + + init { + Files.createDirectories(root) + } + + @Volatile + private var running = true + + private var nextGc = Instant.now() + + fun get(key: String): EntryValue? = + root.resolve(key).takeIf(Files::exists) + ?.let { file -> + val size = Files.size(file) + val channel = FileChannel.open(file, StandardOpenOption.READ) + val source = Channels.newInputStream(channel) + val tmp = ByteArray(Integer.BYTES) + val buffer = ByteBuffer.wrap(tmp) + source.read(tmp) + buffer.rewind() + val offset = (Integer.BYTES + buffer.getInt()).toLong() + var count = 0 + val wrapper = object : InputStream() { + override fun read(): Int { + return source.read().also { + if (it > 0) count += it + } + } + + override fun read(b: ByteArray, off: Int, len: Int): Int { + return source.read(b, off, len).also { + if (it > 0) count += it + } + } + + override fun close() { + } + } + val metadata = ObjectInputStream(wrapper).use { ois -> + ois.readObject() as CacheValueMetadata + } + EntryValue(metadata, channel, offset, size) + } + + class FileSink(metadata: CacheValueMetadata, private val path: Path, private val tmpFile: Path) { + val channel: FileChannel + + init { + val baos = ByteArrayOutputStream() + ObjectOutputStream(baos).use { + it.writeObject(metadata) + } + Files.newOutputStream(tmpFile).use { + val bytes = baos.toByteArray() + val buffer = ByteBuffer.allocate(Integer.BYTES) + buffer.putInt(bytes.size) + buffer.rewind() + it.write(buffer.array()) + it.write(bytes) + } + channel = FileChannel.open(tmpFile, StandardOpenOption.APPEND) + } + + fun commit() { + channel.close() + Files.move(tmpFile, path, StandardCopyOption.ATOMIC_MOVE) + } + + fun rollback() { + channel.close() + Files.delete(path) + } + } + + fun put( + key: String, + metadata: CacheValueMetadata, + ): FileSink { + val file = root.resolve(key) + val tmpFile = Files.createTempFile(root, null, ".tmp") + return FileSink(metadata, file, tmpFile) + } + + private val closeFuture = object : CompletableFuture() { + init { + Thread.ofVirtual().name("file-system-cache-gc").start { + try { + while (running) { + gc() + } + complete(null) + } catch (ex : Throwable) { + completeExceptionally(ex) + } + } + } + } + + private fun gc() { + val now = Instant.now() + if (nextGc < now) { + val oldestEntry = actualGc(now) + nextGc = (oldestEntry ?: now).plus(maxAge) + } + Thread.sleep(minOf(Duration.between(now, nextGc), Duration.ofSeconds(1))) + } + + /** + * Returns the creation timestamp of the oldest cache entry (if any) + */ + private fun actualGc(now: Instant): Instant? { + var result: Instant? = null + Files.list(root) + .filter { path -> + JWO.splitExtension(path) + .map { it._2 } + .map { it != ".tmp" } + .orElse(true) + } + .filter { + val creationTimeStamp = Files.readAttributes(it, BasicFileAttributes::class.java) + .creationTime() + .toInstant() + if (result == null || creationTimeStamp < result) { + result = creationTimeStamp + } + now > creationTimeStamp.plus(maxAge) + }.forEach(Files::delete) + return result + } + + override fun asyncClose() : CompletableFuture { + running = false + return closeFuture + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheConfiguration.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheConfiguration.kt new file mode 100644 index 0000000..9a867c8 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheConfiguration.kt @@ -0,0 +1,38 @@ +package net.woggioni.rbcs.server.cache + +import io.netty.channel.ChannelFactory +import io.netty.channel.EventLoopGroup +import io.netty.channel.socket.DatagramChannel +import io.netty.channel.socket.SocketChannel +import java.nio.file.Path +import java.time.Duration +import net.woggioni.jwo.Application +import net.woggioni.rbcs.api.CacheHandlerFactory +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.RBCS + +data class FileSystemCacheConfiguration( + val root: Path?, + val maxAge: Duration, + val digestAlgorithm : String?, + val compressionEnabled: Boolean, + val compressionLevel: Int, +) : Configuration.Cache { + + override fun materialize() = object : CacheHandlerFactory { + private val cache = FileSystemCache(root ?: Application.builder("rbcs").build().computeCacheDirectory(), maxAge) + + override fun asyncClose() = cache.asyncClose() + + override fun newHandler( + cfg : Configuration, + eventLoop: EventLoopGroup, + socketChannelFactory: ChannelFactory, + datagramChannelFactory: ChannelFactory + ) = FileSystemCacheHandler(cache, digestAlgorithm, compressionEnabled, compressionLevel, cfg.connection.chunkSize) + } + + override fun getNamespaceURI() = RBCS.RBCS_NAMESPACE_URI + + override fun getTypeName() = "fileSystemCacheType" +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheHandler.kt new file mode 100644 index 0000000..ea6ee09 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheHandler.kt @@ -0,0 +1,137 @@ +package net.woggioni.rbcs.server.cache + +import io.netty.buffer.ByteBuf +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.http.LastHttpContent +import io.netty.handler.stream.ChunkedNioFile +import java.nio.channels.Channels +import java.util.Base64 +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream +import java.util.zip.InflaterInputStream +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.message.CacheMessage +import net.woggioni.rbcs.api.message.CacheMessage.CacheContent +import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent +import net.woggioni.rbcs.common.RBCS.processCacheKey + +class FileSystemCacheHandler( + private val cache: FileSystemCache, + private val digestAlgorithm: String?, + private val compressionEnabled: Boolean, + private val compressionLevel: Int, + private val chunkSize: Int +) : CacheHandler() { + + private interface InProgressRequest{ + + } + + private class InProgressGetRequest(val request : CacheGetRequest) : InProgressRequest + + private inner class InProgressPutRequest( + val key : String, + private val fileSink : FileSystemCache.FileSink + ) : InProgressRequest { + + private val stream = Channels.newOutputStream(fileSink.channel).let { + if (compressionEnabled) { + DeflaterOutputStream(it, Deflater(compressionLevel)) + } else { + it + } + } + + fun write(buf: ByteBuf) { + buf.readBytes(stream, buf.readableBytes()) + } + + fun commit() { + stream.close() + fileSink.commit() + } + + fun rollback() { + fileSink.rollback() + } + } + + private var inProgressRequest: InProgressRequest? = null + + override fun channelRead0(ctx: ChannelHandlerContext, msg: CacheMessage) { + when (msg) { + is CacheGetRequest -> handleGetRequest(ctx, msg) + is CachePutRequest -> handlePutRequest(ctx, msg) + is LastCacheContent -> handleLastCacheContent(ctx, msg) + is CacheContent -> handleCacheContent(ctx, msg) + else -> ctx.fireChannelRead(msg) + } + } + + private fun handleGetRequest(ctx: ChannelHandlerContext, msg: CacheGetRequest) { + inProgressRequest = InProgressGetRequest(msg) + + } + + private fun handlePutRequest(ctx: ChannelHandlerContext, msg: CachePutRequest) { + val key = String(Base64.getUrlEncoder().encode(processCacheKey(msg.key, null, digestAlgorithm))) + val sink = cache.put(key, msg.metadata) + inProgressRequest = InProgressPutRequest(msg.key, sink) + } + + private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) { + val request = inProgressRequest + if(request is InProgressPutRequest) { + request.write(msg.content()) + } + } + + private fun handleLastCacheContent(ctx: ChannelHandlerContext, msg: LastCacheContent) { + when(val request = inProgressRequest) { + is InProgressPutRequest -> { + inProgressRequest = null + request.write(msg.content()) + request.commit() + sendMessageAndFlush(ctx, CachePutResponse(request.key)) + } + is InProgressGetRequest -> { + val key = String(Base64.getUrlEncoder().encode(processCacheKey(request.request.key, null, digestAlgorithm))) + cache.get(key)?.also { entryValue -> + sendMessageAndFlush(ctx, CacheValueFoundResponse(request.request.key, entryValue.metadata)) + entryValue.channel.let { channel -> + if(compressionEnabled) { + InflaterInputStream(Channels.newInputStream(channel)).use { stream -> + + outerLoop@ + while (true) { + val buf = ctx.alloc().heapBuffer(chunkSize) + while(buf.readableBytes() < chunkSize) { + val read = buf.writeBytes(stream, chunkSize) + if(read < 0) { + sendMessageAndFlush(ctx, LastCacheContent(buf)) + break@outerLoop + } + } + sendMessageAndFlush(ctx, CacheContent(buf)) + } + } + } else { + sendMessage(ctx, ChunkedNioFile(channel, entryValue.offset, entryValue.size - entryValue.offset, chunkSize)) + sendMessageAndFlush(ctx, LastHttpContent.EMPTY_LAST_CONTENT) + } + } + } ?: sendMessageAndFlush(ctx, CacheValueNotFoundResponse(key)) + } + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + (inProgressRequest as? InProgressPutRequest)?.rollback() + super.exceptionCaught(ctx, cause) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheProvider.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheProvider.kt new file mode 100644 index 0000000..d5112ce --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/FileSystemCacheProvider.kt @@ -0,0 +1,65 @@ +package net.woggioni.rbcs.server.cache + +import java.nio.file.Path +import java.time.Duration +import java.util.zip.Deflater +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document +import org.w3c.dom.Element + +class FileSystemCacheProvider : CacheProvider { + + override fun getXmlSchemaLocation() = "classpath:net/woggioni/rbcs/server/schema/rbcs-server.xsd" + + override fun getXmlType() = "fileSystemCacheType" + + override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server" + + override fun deserialize(el: Element): FileSystemCacheConfiguration { + val path = el.renderAttribute("path") + ?.let(Path::of) + val maxAge = el.renderAttribute("max-age") + ?.let(Duration::parse) + ?: Duration.ofDays(1) + val enableCompression = el.renderAttribute("enable-compression") + ?.let(String::toBoolean) + ?: true + val compressionLevel = el.renderAttribute("compression-level") + ?.let(String::toInt) + ?: Deflater.DEFAULT_COMPRESSION + val digestAlgorithm = el.renderAttribute("digest") + + return FileSystemCacheConfiguration( + path, + maxAge, + digestAlgorithm, + enableCompression, + compressionLevel, + ) + } + + override fun serialize(doc: Document, cache : FileSystemCacheConfiguration) = cache.run { + val result = doc.createElement("cache") + Xml.of(doc, result) { + val prefix = doc.lookupPrefix(RBCS.RBCS_NAMESPACE_URI) + attr("xs:type", "${prefix}:fileSystemCacheType", RBCS.XML_SCHEMA_NAMESPACE_URI) + root?.let { + attr("path", it.toString()) + } + attr("max-age", maxAge.toString()) + digestAlgorithm?.let { digestAlgorithm -> + attr("digest", digestAlgorithm) + } + attr("enable-compression", compressionEnabled.toString()) + compressionLevel.takeIf { + it != Deflater.DEFAULT_COMPRESSION + }?.let { + attr("compression-level", it.toString()) + } + } + result + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCache.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCache.kt new file mode 100644 index 0000000..4f3acbc --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCache.kt @@ -0,0 +1,131 @@ +package net.woggioni.rbcs.server.cache + +import java.time.Duration +import java.time.Instant +import java.util.PriorityQueue +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantReadWriteLock +import kotlin.concurrent.withLock +import net.woggioni.rbcs.api.AsyncCloseable +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.common.createLogger + +private class CacheKey(private val value: ByteArray) { + override fun equals(other: Any?) = if (other is CacheKey) { + value.contentEquals(other.value) + } else false + + override fun hashCode() = value.contentHashCode() +} + +class CacheEntry( + val metadata: CacheValueMetadata, + val content: ByteArray +) + +class InMemoryCache( + private val maxAge: Duration, + private val maxSize: Long +) : AsyncCloseable { + + companion object { + private val log = createLogger() + } + + private var mapSize : Long = 0 + private val map = HashMap() + private val lock = ReentrantReadWriteLock() + private val cond = lock.writeLock().newCondition() + + private class RemovalQueueElement(val key: CacheKey, val value: CacheEntry, val expiry: Instant) : + Comparable { + override fun compareTo(other: RemovalQueueElement) = expiry.compareTo(other.expiry) + } + + private val removalQueue = PriorityQueue() + + @Volatile + private var running = true + + private val closeFuture = object : CompletableFuture() { + init { + Thread.ofVirtual().name("in-memory-cache-gc").start { + try { + lock.writeLock().withLock { + while (running) { + val el = removalQueue.poll() + if(el == null) { + cond.await(1000, TimeUnit.MILLISECONDS) + continue + } + val value = el.value + val now = Instant.now() + if (now > el.expiry) { + val removed = map.remove(el.key, value) + if (removed) { + updateSizeAfterRemoval(value.content) + } + } else { + removalQueue.offer(el) + val interval = minOf(Duration.between(now, el.expiry), Duration.ofSeconds(1)) + cond.await(interval.toMillis(), TimeUnit.MILLISECONDS) + } + } + map.clear() + } + complete(null) + } catch (ex: Throwable) { + completeExceptionally(ex) + } + } + } + } + + fun removeEldest(): Long { + while (true) { + val el = removalQueue.poll() ?: return mapSize + val value = el.value + val removed = map.remove(el.key, value) + if (removed) { + val newSize = updateSizeAfterRemoval(value.content) + return newSize + } + } + } + + private fun updateSizeAfterRemoval(removed: ByteArray): Long { + mapSize -= removed.size + return mapSize + } + + override fun asyncClose() : CompletableFuture { + running = false + lock.writeLock().withLock { + cond.signal() + } + return closeFuture + } + + fun get(key: ByteArray) = lock.readLock().withLock { + map[CacheKey(key)]?.run { + CacheEntry(metadata, content) + } + } + + fun put( + key: ByteArray, + value: CacheEntry, + ) { + val cacheKey = CacheKey(key) + lock.writeLock().withLock { + val oldSize = map.put(cacheKey, value)?.content?.size ?: 0 + val delta = value.content.size - oldSize + mapSize += delta + removalQueue.offer(RemovalQueueElement(cacheKey, value, Instant.now().plus(maxAge))) + while (mapSize > maxSize) { + removeEldest() + } + } + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheConfiguration.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheConfiguration.kt new file mode 100644 index 0000000..99491f7 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheConfiguration.kt @@ -0,0 +1,35 @@ +package net.woggioni.rbcs.server.cache + +import io.netty.channel.ChannelFactory +import io.netty.channel.EventLoopGroup +import io.netty.channel.socket.DatagramChannel +import io.netty.channel.socket.SocketChannel +import java.time.Duration +import net.woggioni.rbcs.api.CacheHandlerFactory +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.RBCS + +data class InMemoryCacheConfiguration( + val maxAge: Duration, + val maxSize: Long, + val digestAlgorithm : String?, + val compressionEnabled: Boolean, + val compressionLevel: Int, +) : Configuration.Cache { + override fun materialize() = object : CacheHandlerFactory { + private val cache = InMemoryCache(maxAge, maxSize) + + override fun asyncClose() = cache.asyncClose() + + override fun newHandler( + cfg : Configuration, + eventLoop: EventLoopGroup, + socketChannelFactory: ChannelFactory, + datagramChannelFactory: ChannelFactory + ) = InMemoryCacheHandler(cache, digestAlgorithm, compressionEnabled, compressionLevel) + } + + override fun getNamespaceURI() = RBCS.RBCS_NAMESPACE_URI + + override fun getTypeName() = "inMemoryCacheType" +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheHandler.kt new file mode 100644 index 0000000..3a4d37d --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheHandler.kt @@ -0,0 +1,155 @@ +package net.woggioni.rbcs.server.cache + +import io.netty.buffer.ByteBuf +import io.netty.channel.ChannelHandlerContext +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream +import java.util.zip.InflaterOutputStream +import net.woggioni.rbcs.api.CacheHandler +import net.woggioni.rbcs.api.message.CacheMessage +import net.woggioni.rbcs.api.message.CacheMessage.CacheContent +import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent +import net.woggioni.rbcs.common.ByteBufOutputStream +import net.woggioni.rbcs.common.RBCS.processCacheKey + +class InMemoryCacheHandler( + private val cache: InMemoryCache, + private val digestAlgorithm: String?, + private val compressionEnabled: Boolean, + private val compressionLevel: Int +) : CacheHandler() { + + private interface InProgressRequest : AutoCloseable { + } + + private class InProgressGetRequest(val request: CacheGetRequest) : InProgressRequest { + override fun close() { + } + } + + private interface InProgressPutRequest : InProgressRequest { + val request: CachePutRequest + val buf: ByteBuf + + fun append(buf: ByteBuf) + } + + private inner class InProgressPlainPutRequest(ctx: ChannelHandlerContext, override val request: CachePutRequest) : + InProgressPutRequest { + override val buf = ctx.alloc().compositeHeapBuffer() + + override fun append(buf: ByteBuf) { + if (buf.isDirect) { + this.buf.writeBytes(buf) + } else { + this.buf.addComponent(true, buf.retain()) + } + } + + override fun close() { + buf.release() + } + } + + private inner class InProgressCompressedPutRequest( + ctx: ChannelHandlerContext, + override val request: CachePutRequest + ) : InProgressPutRequest { + + override val buf = ctx.alloc().heapBuffer() + + private val stream = ByteBufOutputStream(buf).let { + DeflaterOutputStream(it, Deflater(compressionLevel)) + } + + override fun append(buf: ByteBuf) { + buf.readBytes(stream, buf.readableBytes()) + } + + override fun close() { + stream.close() + } + } + + private var inProgressRequest: InProgressRequest? = null + + override fun channelRead0(ctx: ChannelHandlerContext, msg: CacheMessage) { + when (msg) { + is CacheGetRequest -> handleGetRequest(ctx, msg) + is CachePutRequest -> handlePutRequest(ctx, msg) + is LastCacheContent -> handleLastCacheContent(ctx, msg) + is CacheContent -> handleCacheContent(ctx, msg) + else -> ctx.fireChannelRead(msg) + } + } + + private fun handleGetRequest(ctx: ChannelHandlerContext, msg: CacheGetRequest) { + inProgressRequest = InProgressGetRequest(msg) + } + + private fun handlePutRequest(ctx: ChannelHandlerContext, msg: CachePutRequest) { + inProgressRequest = if (compressionEnabled) { + InProgressCompressedPutRequest(ctx, msg) + } else { + InProgressPlainPutRequest(ctx, msg) + } + } + + private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) { + val req = inProgressRequest + if (req is InProgressPutRequest) { + req.append(msg.content()) + } + } + + private fun handleLastCacheContent(ctx: ChannelHandlerContext, msg: LastCacheContent) { + handleCacheContent(ctx, msg) + when (val req = inProgressRequest) { + is InProgressGetRequest -> { +// this.inProgressRequest = null + cache.get(processCacheKey(req.request.key, null, digestAlgorithm))?.let { value -> + sendMessageAndFlush(ctx, CacheValueFoundResponse(req.request.key, value.metadata)) + if (compressionEnabled) { + val buf = ctx.alloc().heapBuffer() + InflaterOutputStream(ByteBufOutputStream(buf)).use { + it.write(value.content) + buf.retain() + } + sendMessage(ctx, LastCacheContent(buf)) + } else { + val buf = ctx.alloc().heapBuffer() + ByteBufOutputStream(buf).use { + it.write(value.content) + buf.retain() + } + sendMessage(ctx, LastCacheContent(buf)) + } + } ?: sendMessage(ctx, CacheValueNotFoundResponse(req.request.key)) + } + + is InProgressPutRequest -> { + this.inProgressRequest = null + val buf = req.buf + buf.retain() + req.close() + + val bytes = ByteArray(buf.readableBytes()).also(buf::readBytes) + buf.release() + val cacheKey = processCacheKey(req.request.key, null, digestAlgorithm) + cache.put(cacheKey, CacheEntry(req.request.metadata, bytes)) + sendMessageAndFlush(ctx, CachePutResponse(req.request.key)) + } + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + inProgressRequest?.close() + inProgressRequest = null + super.exceptionCaught(ctx, cause) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheProvider.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheProvider.kt new file mode 100644 index 0000000..c231bc0 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/cache/InMemoryCacheProvider.kt @@ -0,0 +1,62 @@ +package net.woggioni.rbcs.server.cache + +import java.time.Duration +import java.util.zip.Deflater +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document +import org.w3c.dom.Element + +class InMemoryCacheProvider : CacheProvider { + + override fun getXmlSchemaLocation() = "classpath:net/woggioni/rbcs/server/schema/rbcs-server.xsd" + + override fun getXmlType() = "inMemoryCacheType" + + override fun getXmlNamespace() = "urn:net.woggioni.rbcs.server" + + override fun deserialize(el: Element): InMemoryCacheConfiguration { + val maxAge = el.renderAttribute("max-age") + ?.let(Duration::parse) + ?: Duration.ofDays(1) + val maxSize = el.renderAttribute("max-size") + ?.let(java.lang.Long::decode) + ?: 0x1000000 + val enableCompression = el.renderAttribute("enable-compression") + ?.let(String::toBoolean) + ?: true + val compressionLevel = el.renderAttribute("compression-level") + ?.let(String::toInt) + ?: Deflater.DEFAULT_COMPRESSION + val digestAlgorithm = el.renderAttribute("digest") + return InMemoryCacheConfiguration( + maxAge, + maxSize, + digestAlgorithm, + enableCompression, + compressionLevel, + ) + } + + override fun serialize(doc: Document, cache : InMemoryCacheConfiguration) = cache.run { + val result = doc.createElement("cache") + Xml.of(doc, result) { + val prefix = doc.lookupPrefix(RBCS.RBCS_NAMESPACE_URI) + attr("xs:type", "${prefix}:inMemoryCacheType", RBCS.XML_SCHEMA_NAMESPACE_URI) + attr("max-age", maxAge.toString()) + attr("max-size", maxSize.toString()) + digestAlgorithm?.let { digestAlgorithm -> + attr("digest", digestAlgorithm) + } + attr("enable-compression", compressionEnabled.toString()) + compressionLevel.takeIf { + it != Deflater.DEFAULT_COMPRESSION + }?.let { + attr("compression-level", it.toString()) + } + } + result + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/CacheSerializers.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/CacheSerializers.kt new file mode 100644 index 0000000..2cc708c --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/CacheSerializers.kt @@ -0,0 +1,15 @@ +package net.woggioni.rbcs.server.configuration + +import java.util.ServiceLoader +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.api.Configuration + +object CacheSerializers { + val index = (Configuration::class.java.module.layer?.let { layer -> + ServiceLoader.load(layer, CacheProvider::class.java) + } ?: ServiceLoader.load(CacheProvider::class.java)) + .asSequence() + .map { + (it.xmlNamespace to it.xmlType) to it + }.toMap() +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Parser.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Parser.kt new file mode 100644 index 0000000..f1ac5c6 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Parser.kt @@ -0,0 +1,361 @@ +package net.woggioni.rbcs.server.configuration + +import java.nio.file.Paths +import java.time.Duration +import java.time.temporal.ChronoUnit +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Configuration.Authentication +import net.woggioni.rbcs.api.Configuration.BasicAuthentication +import net.woggioni.rbcs.api.Configuration.Cache +import net.woggioni.rbcs.api.Configuration.ClientCertificateAuthentication +import net.woggioni.rbcs.api.Configuration.ForwardedClientCertificateAuthentication +import net.woggioni.rbcs.api.Configuration.Group +import net.woggioni.rbcs.api.Configuration.KeyStore +import net.woggioni.rbcs.api.Configuration.Tls +import net.woggioni.rbcs.api.Configuration.TlsCertificateExtractor +import net.woggioni.rbcs.api.Configuration.TrustStore +import net.woggioni.rbcs.api.Configuration.User +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.api.exception.ConfigurationException +import net.woggioni.rbcs.common.Cidr +import net.woggioni.rbcs.common.Xml.Companion.asIterable +import net.woggioni.rbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document +import org.w3c.dom.Element +import org.w3c.dom.TypeInfo + +object Parser { + fun parse(document: Document): Configuration { + val root = document.documentElement + val anonymousUser = User("", null, emptySet(), null) + var connection: Configuration.Connection = Configuration.Connection( + Duration.of(30, ChronoUnit.SECONDS), + Duration.of(60, ChronoUnit.SECONDS), + Duration.of(60, ChronoUnit.SECONDS), + 0x4000000, + 0x10000 + ) + var rateLimiter = Configuration.RateLimiter(false, 0x100000, 100) + var eventExecutor: Configuration.EventExecutor = Configuration.EventExecutor(true) + var cache: Cache? = null + var host = "127.0.0.1" + var port = 11080 + var proxyProtocolEnabled = false + var trustedProxies = emptyList() + var users: Map = mapOf(anonymousUser.name to anonymousUser) + var groups = emptyMap() + var tls: Tls? = null + val serverPath = root.renderAttribute("path") + var incomingConnectionsBacklogSize = 1024 + var authentication: Authentication? = null + for (child in root.asIterable()) { + val tagName = child.localName + when (tagName) { + "authentication" -> { + for (gchild in child.asIterable()) { + when (gchild.localName) { + "basic" -> { + authentication = BasicAuthentication() + } + + "client-certificate" -> { + var tlsExtractorUser: TlsCertificateExtractor? = null + var tlsExtractorGroup: TlsCertificateExtractor? = null + for (ggchild in gchild.asIterable()) { + when (ggchild.localName) { + "group-extractor" -> { + val attrName = ggchild.renderAttribute("attribute-name") + val pattern = ggchild.renderAttribute("pattern") + tlsExtractorGroup = TlsCertificateExtractor(attrName, pattern) + } + + "user-extractor" -> { + val attrName = ggchild.renderAttribute("attribute-name") + val pattern = ggchild.renderAttribute("pattern") + tlsExtractorUser = TlsCertificateExtractor(attrName, pattern) + } + } + } + authentication = ClientCertificateAuthentication(tlsExtractorUser, tlsExtractorGroup) + } + + "forwarded-client-certificate" -> { + val headerName = gchild.renderAttribute("header-name") ?: "X-Client-Cert-Subject-DN" + var tlsExtractorUser: TlsCertificateExtractor? = null + var tlsExtractorGroup: TlsCertificateExtractor? = null + for (ggchild in gchild.asIterable()) { + when (ggchild.localName) { + "group-extractor" -> { + val attrName = ggchild.renderAttribute("attribute-name") + val pattern = ggchild.renderAttribute("pattern") + tlsExtractorGroup = TlsCertificateExtractor(attrName, pattern) + } + + "user-extractor" -> { + val attrName = ggchild.renderAttribute("attribute-name") + val pattern = ggchild.renderAttribute("pattern") + tlsExtractorUser = TlsCertificateExtractor(attrName, pattern) + } + } + } + authentication = ForwardedClientCertificateAuthentication(headerName, tlsExtractorUser, tlsExtractorGroup) + } + } + } + } + + "authorization" -> { + var knownUsers = sequenceOf(anonymousUser) + for (gchild in child.asIterable()) { + when (gchild.localName) { + "users" -> { + knownUsers += parseUsers(gchild) + } + + "groups" -> { + val pair = parseGroups(gchild, knownUsers) + users = pair.first + groups = pair.second + } + } + } + } + + "bind" -> { + host = child.renderAttribute("host") ?: throw ConfigurationException("host attribute is required") + port = Integer.parseInt(child.renderAttribute("port")) + proxyProtocolEnabled = child.renderAttribute("proxy-protocol") + ?.let(String::toBoolean) ?: false + incomingConnectionsBacklogSize = child.renderAttribute("incoming-connections-backlog-size") + ?.let(Integer::parseInt) + ?: 1024 + + for(grandChild in child.asIterable()) { + when(grandChild.localName) { + "trusted-proxies" -> { + trustedProxies = parseTrustedProxies(grandChild) + } + } + } + child.asIterable().filter { + it.localName == "trusted-proxies" + }.firstOrNull()?.let(::parseTrustedProxies) + + } + + "cache" -> { + cache = (child as TypeInfo).let { tf -> + val typeNamespace = tf.typeNamespace + val typeName = tf.typeName + CacheSerializers.index[typeNamespace to typeName] + ?: throw IllegalArgumentException("Cache provider for namespace '$typeNamespace' with name '$typeName' not found") + }.deserialize(child) + } + + "connection" -> { + val idleTimeout = child.renderAttribute("idle-timeout") + ?.let(Duration::parse) ?: Duration.of(30, ChronoUnit.SECONDS) + val readIdleTimeout = child.renderAttribute("read-idle-timeout") + ?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS) + val writeIdleTimeout = child.renderAttribute("write-idle-timeout") + ?.let(Duration::parse) ?: Duration.of(60, ChronoUnit.SECONDS) + val maxRequestSize = child.renderAttribute("max-request-size") + ?.let(Integer::decode) ?: 0x4000000 + val chunkSize = child.renderAttribute("chunk-size") + ?.let(Integer::decode) ?: 0x10000 + connection = Configuration.Connection( + idleTimeout, + readIdleTimeout, + writeIdleTimeout, + maxRequestSize, + chunkSize + ) + } + + "event-executor" -> { + val useVirtualThread = child.renderAttribute("use-virtual-threads") + ?.let(String::toBoolean) ?: true + eventExecutor = Configuration.EventExecutor(useVirtualThread) + } + + "rate-limiter" -> { + val delayResponse = child.renderAttribute("delay-response") + ?.let(String::toBoolean) + ?: false + val messageBufferSize = child.renderAttribute("message-buffer-size") + ?.let(Integer::decode) + ?: 0x100000 + val maxQueuedMessages = child.renderAttribute("max-queued-messages") + ?.let(Integer::decode) + ?: 100 + rateLimiter = Configuration.RateLimiter(delayResponse, messageBufferSize, maxQueuedMessages) + } + + "tls" -> { + var keyStore: KeyStore? = null + var trustStore: TrustStore? = null + + for (granChild in child.asIterable()) { + when (granChild.localName) { + "keystore" -> { + val keyStoreFile = Paths.get(granChild.renderAttribute("file")) + val keyStorePassword = granChild.renderAttribute("password") + val keyAlias = granChild.renderAttribute("key-alias") + val keyPassword = granChild.renderAttribute("key-password") + keyStore = KeyStore( + keyStoreFile, + keyStorePassword, + keyAlias, + keyPassword + ) + } + + "truststore" -> { + val trustStoreFile = Paths.get(granChild.renderAttribute("file")) + val trustStorePassword = granChild.renderAttribute("password") + val checkCertificateStatus = granChild.renderAttribute("check-certificate-status") + ?.let(String::toBoolean) + ?: false + val requireClientCertificate = child.renderAttribute("require-client-certificate") + ?.let(String::toBoolean) ?: false + + trustStore = TrustStore( + trustStoreFile, + trustStorePassword, + checkCertificateStatus, + requireClientCertificate + ) + } + } + } + tls = Tls(keyStore, trustStore) + } + } + } + return Configuration.of( + host, + port, + proxyProtocolEnabled, + trustedProxies, + incomingConnectionsBacklogSize, + serverPath, + eventExecutor, + rateLimiter, + connection, + users, + groups, + cache!!, + authentication, + tls, + ) + } + + private fun parseRoles(root: Element) = root.asIterable().asSequence().map { + when (it.localName) { + "reader" -> Role.Reader + "writer" -> Role.Writer + "healthcheck" -> Role.Healthcheck + else -> throw UnsupportedOperationException("Illegal node '${it.localName}'") + } + }.toSet() + + private fun parseTrustedProxies(root: Element) = root.asIterable().asSequence().map { + when (it.localName) { + "allow" -> it.renderAttribute("cidr") + ?.let(Cidr::from) + ?: throw ConfigurationException("Missing 'cidr' attribute") + else -> throw ConfigurationException("Unrecognized tag '${it.localName}'") + } + }.toList() + + private fun parseUserRefs(root: Element) = root.asIterable().asSequence().map { + when (it.localName) { + "user" -> it.renderAttribute("ref") + "anonymous" -> "" + else -> ConfigurationException("Unrecognized tag '${it.localName}'") + } + } + + private fun parseQuota(el: Element): Configuration.Quota { + val calls = el.renderAttribute("calls") + ?.let(String::toLong) + ?: throw ConfigurationException("Missing attribute 'calls'") + val maxAvailableCalls = el.renderAttribute("max-available-calls") + ?.let(String::toLong) + ?: calls + val initialAvailableCalls = el.renderAttribute("initial-available-calls") + ?.let(String::toLong) + ?: maxAvailableCalls + val period = el.renderAttribute("period") + ?.let(Duration::parse) + ?: throw ConfigurationException("Missing attribute 'period'") + return Configuration.Quota(calls, period, initialAvailableCalls, maxAvailableCalls) + } + + private fun parseUsers(root: Element): Sequence { + return root.asIterable().asSequence().mapNotNull { child -> + when (child.localName) { + "user" -> { + val username = child.renderAttribute("name") + val password = child.renderAttribute("password") + var quota: Configuration.Quota? = null + for (gchild in child.asIterable()) { + if (gchild.localName == "quota") { + quota = parseQuota(gchild) + } + } + User(username, password, emptySet(), quota) + } + "anonymous" -> { + var quota: Configuration.Quota? = null + for (gchild in child.asIterable()) { + if (gchild.localName == "quota") { + quota= parseQuota(gchild) + } + } + User("", null, emptySet(), quota) + } + else -> null + } + } + } + + private fun parseGroups(root: Element, knownUsers: Sequence): Pair, Map> { + val knownUsersMap = knownUsers.associateBy(User::getName) + val userGroups = mutableMapOf>() + val groups = root.asIterable().asSequence().filter { + it.localName == "group" + }.map { el -> + val groupName = el.renderAttribute("name") ?: throw ConfigurationException("Group name is required") + var roles = emptySet() + var userQuota: Configuration.Quota? = null + var groupQuota: Configuration.Quota? = null + for (child in el.asIterable()) { + when (child.localName) { + "users" -> { + parseUserRefs(child).mapNotNull(knownUsersMap::get).forEach { user -> + userGroups.computeIfAbsent(user.name) { + mutableSetOf() + }.add(groupName) + } + } + + "roles" -> { + roles = parseRoles(child) + } + "group-quota" -> { + userQuota = parseQuota(child) + } + "user-quota" -> { + groupQuota = parseQuota(child) + } + } + } + groupName to Group(groupName, roles, userQuota, groupQuota) + }.toMap() + val users = knownUsersMap.map { (name, user) -> + name to User(name, user.password, userGroups[name]?.mapNotNull { groups[it] }?.toSet() ?: emptySet(), user.quota) + }.toMap() + return users to groups + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Serializer.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Serializer.kt new file mode 100644 index 0000000..67e78d3 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/configuration/Serializer.kt @@ -0,0 +1,218 @@ +package net.woggioni.rbcs.server.configuration + +import net.woggioni.rbcs.api.CacheProvider +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.RBCS +import net.woggioni.rbcs.common.Xml +import org.w3c.dom.Document + +object Serializer { + + private fun Xml.serializeQuota(quota : Configuration.Quota) { + attr("calls", quota.calls.toString()) + attr("period", quota.period.toString()) + attr("max-available-calls", quota.maxAvailableCalls.toString()) + attr("initial-available-calls", quota.initialAvailableCalls.toString()) + } + + fun serialize(conf : Configuration) : Document { + val schemaLocations = CacheSerializers.index.values.asSequence().map { + it.xmlNamespace to it.xmlSchemaLocation + }.toMap() + return Xml.of(RBCS.RBCS_NAMESPACE_URI, RBCS.RBCS_PREFIX + ":server") { +// attr("xmlns:xs", GradleBuildCacheServer.XML_SCHEMA_NAMESPACE_URI) + val value = schemaLocations.asSequence().map { (k, v) -> "$k $v" }.joinToString(" ") + attr("xs:schemaLocation", value , namespaceURI = RBCS.XML_SCHEMA_NAMESPACE_URI) + + conf.serverPath + ?.takeIf(String::isNotEmpty) + ?.let { serverPath -> + attr("path", serverPath) + } + node("bind") { + attr("host", conf.host) + attr("port", conf.port.toString()) + attr("incoming-connections-backlog-size", conf.incomingConnectionsBacklogSize.toString()) + attr("proxy-protocol", conf.isProxyProtocolEnabled.toString()) + + if (conf.trustedProxyIPs.isNotEmpty()) { + node("trusted-proxies") { + for(trustedProxy in conf.trustedProxyIPs) { + node("allow") { + attr("cidr", trustedProxy.toString()) + } + } + } + } + } + node("connection") { + conf.connection.let { connection -> + attr("idle-timeout", connection.idleTimeout.toString()) + attr("read-idle-timeout", connection.readIdleTimeout.toString()) + attr("write-idle-timeout", connection.writeIdleTimeout.toString()) + attr("max-request-size", connection.maxRequestSize.toString()) + attr("chunk-size", connection.chunkSize.toString()) + } + } + node("event-executor") { + attr("use-virtual-threads", conf.eventExecutor.isUseVirtualThreads.toString()) + } + node("rate-limiter") { + attr("delay-response", conf.rateLimiter.isDelayRequest.toString()) + attr("max-queued-messages", conf.rateLimiter.maxQueuedMessages.toString()) + attr("message-buffer-size", conf.rateLimiter.messageBufferSize.toString()) + } + val cache = conf.cache + val serializer : CacheProvider = + (CacheSerializers.index[cache.namespaceURI to cache.typeName] as? CacheProvider) ?: throw NotImplementedError() + element.appendChild(serializer.serialize(doc, cache)) + node("authorization") { + node("users") { + for(user in conf.users.values) { + if(user.name.isNotEmpty()) { + node("user") { + attr("name", user.name) + user.password?.let { password -> + attr("password", password) + } + user.quota?.let { quota -> + node("quota") { + serializeQuota(quota) + } + } + } + } + } + conf.users[""] + ?.let { anonymousUser -> + anonymousUser.quota?.let { quota -> + node("anonymous") { + node("quota") { + serializeQuota(quota) + } + } + } + } + } + node("groups") { + val groups = conf.users.values.asSequence() + .flatMap { + user -> user.groups.map { it to user } + }.groupBy(Pair::first, Pair::second) + for(pair in groups) { + val group = pair.key + val users = pair.value + node("group") { + attr("name", group.name) + if(users.isNotEmpty()) { + node("users") { + var anonymousUser : Configuration.User? = null + for(user in users) { + if(user.name.isNotEmpty()) { + node("user") { + attr("ref", user.name) + } + } else { + anonymousUser = user + } + } + if(anonymousUser != null) { + node("anonymous") + } + } + } + if(group.roles.isNotEmpty()) { + node("roles") { + for(role in group.roles) { + node(role.toString().lowercase()) + } + } + } + group.userQuota?.let { quota -> + node("user-quota") { + serializeQuota(quota) + } + } + group.groupQuota?.let { quota -> + node("group-quota") { + serializeQuota(quota) + } + } + } + } + } + } + + conf.authentication?.let { authentication -> + node("authentication") { + when(authentication) { + is Configuration.BasicAuthentication -> { + node("basic") + } + is Configuration.ClientCertificateAuthentication -> { + node("client-certificate") { + authentication.groupExtractor?.let { extractor -> + node("group-extractor") { + attr("attribute-name", extractor.rdnType) + attr("pattern", extractor.pattern) + } + } + authentication.userExtractor?.let { extractor -> + node("user-extractor") { + attr("attribute-name", extractor.rdnType) + attr("pattern", extractor.pattern) + } + } + } + } + is Configuration.ForwardedClientCertificateAuthentication -> { + node("forwarded-client-certificate") { + attr("header-name", authentication.headerName) + authentication.groupExtractor?.let { extractor -> + node("group-extractor") { + attr("attribute-name", extractor.rdnType) + attr("pattern", extractor.pattern) + } + } + authentication.userExtractor?.let { extractor -> + node("user-extractor") { + attr("attribute-name", extractor.rdnType) + attr("pattern", extractor.pattern) + } + } + } + } + } + } + } + + conf.tls?.let { tlsConfiguration -> + node("tls") { + tlsConfiguration.keyStore?.let { keyStore -> + node("keystore") { + attr("file", keyStore.file.toString()) + keyStore.password?.let { keyStorePassword -> + attr("password", keyStorePassword) + } + attr("key-alias", keyStore.keyAlias) + keyStore.keyPassword?.let { keyPassword -> + attr("key-password", keyPassword) + } + } + } + + tlsConfiguration.trustStore?.let { trustStore -> + node("truststore") { + attr("file", trustStore.file.toString()) + trustStore.password?.let { password -> + attr("password", password) + } + attr("check-certificate-status", trustStore.isCheckCertificateStatus.toString()) + attr("require-client-certificate", trustStore.isRequireClientCertificate.toString()) + } + } + } + } + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/exception/ExceptionHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/exception/ExceptionHandler.kt new file mode 100644 index 0000000..0553952 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/exception/ExceptionHandler.kt @@ -0,0 +1,139 @@ +package net.woggioni.rbcs.server.exception + +import io.netty.buffer.Unpooled +import io.netty.channel.ChannelDuplexHandler +import io.netty.channel.ChannelFutureListener +import io.netty.channel.ChannelHandler.Sharable +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.DecoderException +import io.netty.handler.codec.http.DefaultFullHttpResponse +import io.netty.handler.codec.http.FullHttpResponse +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import io.netty.handler.timeout.ReadTimeoutException +import io.netty.handler.timeout.WriteTimeoutException +import java.net.ConnectException +import java.net.SocketException +import javax.net.ssl.SSLException +import javax.net.ssl.SSLPeerUnverifiedException +import net.woggioni.rbcs.api.exception.CacheException +import net.woggioni.rbcs.api.exception.ContentTooLargeException +import net.woggioni.rbcs.common.contextLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.log +import net.woggioni.rbcs.server.RemoteBuildCacheServer +import org.slf4j.event.Level +import org.slf4j.spi.LoggingEventBuilder + +@Sharable +object ExceptionHandler : ChannelDuplexHandler() { + + val NAME : String = this::class.java.name + + private val log = contextLogger() + + private val NOT_AUTHORIZED: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val NOT_AVAILABLE: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.SERVICE_UNAVAILABLE, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val SERVER_ERROR: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val TOO_BIG: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + when (cause) { + is DecoderException -> { + if(log.isDebugEnabled) { + log.debug(cause.message, cause) + } + ctx.close() + } + + is ConnectException -> { + if(log.isErrorEnabled) { + log.error(cause.message, cause) + } + ctx.writeAndFlush(SERVER_ERROR.retainedDuplicate()) + } + + is SocketException -> { + if(log.isDebugEnabled) { + log.debug(cause.message, cause) + } + ctx.close() + } + + is SSLPeerUnverifiedException -> { + if(log.isDebugEnabled) { + log.debug(cause.message, cause) + } + ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + + is SSLException -> { + if(log.isDebugEnabled) { + log.debug(cause.message, cause) + } + ctx.close() + } + + is ContentTooLargeException -> { + log.log(Level.DEBUG, ctx.channel()) { builder : LoggingEventBuilder -> + builder.setMessage("Request body is too large") + } + ctx.writeAndFlush(TOO_BIG.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + + is ReadTimeoutException -> { + log.debug { + val channelId = ctx.channel().id().asShortText() + "Read timeout on channel $channelId, closing the connection" + } + ctx.close() + } + + is WriteTimeoutException -> { + log.debug { + val channelId = ctx.channel().id().asShortText() + "Write timeout on channel $channelId, closing the connection" + } + ctx.close() + } + + is CacheException -> { + if(log.isErrorEnabled) { + log.error(cause.message, cause) + } + ctx.writeAndFlush(NOT_AVAILABLE.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + + else -> { + if(log.isErrorEnabled) { + log.error(cause.message, cause) + } + ctx.writeAndFlush(SERVER_ERROR.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/BlackHoleRequestHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/BlackHoleRequestHandler.kt new file mode 100644 index 0000000..d912c40 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/BlackHoleRequestHandler.kt @@ -0,0 +1,13 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.handler.codec.http.HttpContent + +class BlackHoleRequestHandler : SimpleChannelInboundHandler() { + companion object { + val NAME = BlackHoleRequestHandler::class.java.name + } + override fun channelRead0(ctx: ChannelHandlerContext, msg: HttpContent) { + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/MaxRequestSizeHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/MaxRequestSizeHandler.kt new file mode 100644 index 0000000..e5babeb --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/MaxRequestSizeHandler.kt @@ -0,0 +1,40 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.handler.codec.http.HttpContent +import io.netty.handler.codec.http.HttpRequest +import net.woggioni.rbcs.api.exception.ContentTooLargeException + + +class MaxRequestSizeHandler(private val maxRequestSize : Int) : ChannelInboundHandlerAdapter() { + companion object { + val NAME = MaxRequestSizeHandler::class.java.name + } + + private var cumulativeSize = 0 + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + when(msg) { + is HttpRequest -> { + cumulativeSize = 0 + ctx.fireChannelRead(msg) + } + is HttpContent -> { + val exceeded = cumulativeSize > maxRequestSize + if(!exceeded) { + cumulativeSize += msg.content().readableBytes() + } + if(cumulativeSize > maxRequestSize) { + msg.release() + if(!exceeded) { + ctx.fireExceptionCaught(ContentTooLargeException("Request body is too large", null)) + } + } else { + ctx.fireChannelRead(msg) + } + } + else -> ctx.fireChannelRead(msg) + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ProxyProtocolHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ProxyProtocolHandler.kt new file mode 100644 index 0000000..0a9860f --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ProxyProtocolHandler.kt @@ -0,0 +1,47 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.SimpleChannelInboundHandler +import io.netty.handler.codec.haproxy.HAProxyMessage +import java.net.InetAddress +import java.net.InetSocketAddress +import net.woggioni.rbcs.common.Cidr +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.trace +import net.woggioni.rbcs.server.RemoteBuildCacheServer + + +class ProxyProtocolHandler(private val trustedProxyIPs : List) : SimpleChannelInboundHandler() { + + companion object { + private val log = createLogger() + } + + override fun channelRead0( + ctx: ChannelHandlerContext, + msg: HAProxyMessage + ) { + val sourceAddress = ctx.channel().remoteAddress() + if (sourceAddress is InetSocketAddress && + trustedProxyIPs.isEmpty() || + trustedProxyIPs.any { it.contains((sourceAddress as InetSocketAddress).address) }.also { + if(!it && log.isTraceEnabled) { + log.trace { + "Received a proxied connection request from $sourceAddress which is not a trusted proxy address, " + + "the proxy server address will be used instead" + } + } + }) { + val proxiedClientAddress = InetSocketAddress( + InetAddress.ofLiteral(msg.sourceAddress()), + msg.sourcePort() + ) + if(log.isTraceEnabled) { + log.trace { + "Received proxied connection request from $sourceAddress forwarded for $proxiedClientAddress" + } + } + ctx.channel().attr(RemoteBuildCacheServer.clientIp).set(proxiedClientAddress) + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ReadTriggerDuplexHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ReadTriggerDuplexHandler.kt new file mode 100644 index 0000000..dc53ae2 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ReadTriggerDuplexHandler.kt @@ -0,0 +1,65 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.buffer.ByteBufHolder +import io.netty.channel.ChannelDuplexHandler +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelPromise +import io.netty.handler.codec.http.LastHttpContent +import net.woggioni.rbcs.common.createLogger + +class ReadTriggerDuplexHandler : ChannelDuplexHandler() { + companion object { + val NAME = ReadTriggerDuplexHandler::class.java.name + private val log = createLogger() + } + + private var inFlight = 0 + private val messageBuffer = ArrayDeque() + + override fun handlerAdded(ctx: ChannelHandlerContext) { + ctx.read() + } + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + if(inFlight > 0) { + messageBuffer.addLast(msg) + } else { + super.channelRead(ctx, msg) + if(msg !is LastHttpContent) { + invokeRead(ctx) + } else { + inFlight += 1 + } + } + } + + private fun invokeRead(ctx : ChannelHandlerContext) { + if(messageBuffer.isEmpty()) { + ctx.read() + } else { + this.channelRead(ctx, messageBuffer.removeFirst()) + } + } + + override fun write( + ctx: ChannelHandlerContext, + msg: Any, + promise: ChannelPromise + ) { + super.write(ctx, msg, promise) + if(msg is LastHttpContent) { + inFlight -= 1 + invokeRead(ctx) + } + } + + override fun channelInactive(ctx: ChannelHandlerContext) { + while(messageBuffer.isNotEmpty()) { + val msg = messageBuffer.removeFirst() + if(msg is ByteBufHolder) { + msg.release() + } + } + super.channelInactive(ctx) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ServerHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ServerHandler.kt new file mode 100644 index 0000000..48ca18c --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/ServerHandler.kt @@ -0,0 +1,227 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.channel.ChannelDuplexHandler +import io.netty.channel.ChannelHandler +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelPromise +import io.netty.handler.codec.http.DefaultFullHttpResponse +import io.netty.handler.codec.http.DefaultHttpContent +import io.netty.handler.codec.http.DefaultHttpResponse +import io.netty.handler.codec.http.DefaultLastHttpContent +import io.netty.handler.codec.http.HttpContent +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpHeaderValues +import io.netty.handler.codec.http.HttpHeaders +import io.netty.handler.codec.http.HttpMethod +import io.netty.handler.codec.http.HttpRequest +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpUtil +import io.netty.handler.codec.http.HttpVersion +import io.netty.handler.codec.http.LastHttpContent +import java.nio.file.Path +import net.woggioni.rbcs.api.CacheValueMetadata +import net.woggioni.rbcs.api.message.CacheMessage +import net.woggioni.rbcs.api.message.CacheMessage.CacheContent +import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest +import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse +import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.common.warn +import net.woggioni.rbcs.server.exception.ExceptionHandler + +class ServerHandler(private val serverPrefix: Path, private val cacheHandlerSupplier : () -> ChannelHandler) : + ChannelDuplexHandler() { + + companion object { + private val log = createLogger() + val NAME = ServerHandler::class.java.name + } + + private var httpVersion = HttpVersion.HTTP_1_1 + private var keepAlive = true + + private fun resetRequestMetadata() { + httpVersion = HttpVersion.HTTP_1_1 + keepAlive = true + } + + private fun setRequestMetadata(req: HttpRequest) { + httpVersion = req.protocolVersion() + keepAlive = HttpUtil.isKeepAlive(req) + } + + private fun setKeepAliveHeader(headers: HttpHeaders) { + if (!keepAlive) { + headers.set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE) + } else { + headers.set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE) + } + } + + private var cacheRequestInProgress : Boolean = false + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + when (msg) { + is HttpRequest -> handleRequest(ctx, msg) + is HttpContent -> { + if(cacheRequestInProgress) { + if(msg is LastHttpContent) { + super.channelRead(ctx, LastCacheContent(msg.content().retain())) + cacheRequestInProgress = false + } else { + super.channelRead(ctx, CacheContent(msg.content().retain())) + } + msg.release() + } else { + super.channelRead(ctx, msg) + } + } + else -> super.channelRead(ctx, msg) + } + } + + override fun write(ctx: ChannelHandlerContext, msg: Any, promise: ChannelPromise?) { + if (msg is CacheMessage) { + try { + when (msg) { + is CachePutResponse -> { + log.debug(ctx) { + "Added value for key '${msg.key}' to build cache" + } + val response = DefaultFullHttpResponse(httpVersion, HttpResponseStatus.CREATED) + val keyBytes = msg.key.toByteArray(Charsets.UTF_8) + response.headers().apply { + set(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.TEXT_PLAIN) + set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED) + } + setKeepAliveHeader(response.headers()) + ctx.write(response) + val buf = ctx.alloc().buffer(keyBytes.size).apply { + writeBytes(keyBytes) + } + ctx.writeAndFlush(DefaultLastHttpContent(buf)) + } + + is CacheValueNotFoundResponse -> { + log.debug(ctx) { + "Value not found for key '${msg.key}'" + } + val response = DefaultFullHttpResponse(httpVersion, HttpResponseStatus.NOT_FOUND) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0 + setKeepAliveHeader(response.headers()) + ctx.writeAndFlush(response) + } + + is CacheValueFoundResponse -> { + log.debug(ctx) { + "Retrieved value for key '${msg.key}'" + } + val response = DefaultHttpResponse(httpVersion, HttpResponseStatus.OK) + response.headers().apply { + set(HttpHeaderNames.CONTENT_TYPE, msg.metadata.mimeType ?: HttpHeaderValues.APPLICATION_OCTET_STREAM) + msg.metadata.contentDisposition?.let { contentDisposition -> + set(HttpHeaderNames.CONTENT_DISPOSITION, contentDisposition) + } + } + setKeepAliveHeader(response.headers()) + response.headers().set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED) + ctx.writeAndFlush(response) + } + + is LastCacheContent -> { + ctx.writeAndFlush(DefaultLastHttpContent(msg.content())) + } + + is CacheContent -> { + ctx.writeAndFlush(DefaultHttpContent(msg.content())) + } + + else -> throw UnsupportedOperationException("This should never happen") + }.let { channelFuture -> + if (promise != null) { + channelFuture.addListener { + if (it.isSuccess) promise.setSuccess() + else promise.setFailure(it.cause()) + } + } + } + } finally { + resetRequestMetadata() + } + } else if(msg is LastHttpContent) { + ctx.write(msg, promise) + } else super.write(ctx, msg, promise) + } + + + private fun handleRequest(ctx: ChannelHandlerContext, msg: HttpRequest) { + setRequestMetadata(msg) + val method = msg.method() + if (method === HttpMethod.GET) { + val path = Path.of(msg.uri()).normalize() + if (path.startsWith(serverPrefix)) { + cacheRequestInProgress = true + val relativePath = serverPrefix.relativize(path) + val key : String = relativePath.toString() + val cacheHandler = cacheHandlerSupplier() + ctx.pipeline().addBefore(ExceptionHandler.NAME, null, cacheHandler) + key.let(::CacheGetRequest) + .let(ctx::fireChannelRead) + ?: ctx.channel().write(CacheValueNotFoundResponse(key)) + } else { + cacheRequestInProgress = false + log.warn(ctx) { + "Got request for unhandled path '${msg.uri()}'" + } + val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.BAD_REQUEST) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0 + ctx.writeAndFlush(response) + } + } else if (method === HttpMethod.PUT) { + val path = Path.of(msg.uri()).normalize() + if (path.startsWith(serverPrefix)) { + cacheRequestInProgress = true + val relativePath = serverPrefix.relativize(path) + val key = relativePath.toString() + val cacheHandler = cacheHandlerSupplier() + ctx.pipeline().addAfter(NAME, null, cacheHandler) + + path.fileName?.toString() + ?.let { + val mimeType = HttpUtil.getMimeType(msg)?.toString() + CachePutRequest(key, CacheValueMetadata(msg.headers().get(HttpHeaderNames.CONTENT_DISPOSITION), mimeType)) + } + ?.let(ctx::fireChannelRead) + ?: ctx.channel().write(CacheValueNotFoundResponse(key)) + } else { + cacheRequestInProgress = false + log.warn(ctx) { + "Got request for unhandled path '${msg.uri()}'" + } + val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.BAD_REQUEST) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + ctx.writeAndFlush(response) + } + } else if (method == HttpMethod.TRACE) { + cacheRequestInProgress = false + ctx.pipeline().addAfter(NAME, null, TraceHandler) + super.channelRead(ctx, msg) + } else { + cacheRequestInProgress = false + log.warn(ctx) { + "Got request with unhandled method '${msg.method().name()}'" + } + val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.METHOD_NOT_ALLOWED) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + ctx.writeAndFlush(response) + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + super.exceptionCaught(ctx, cause) + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/TraceHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/TraceHandler.kt new file mode 100644 index 0000000..35f197f --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/handler/TraceHandler.kt @@ -0,0 +1,55 @@ +package net.woggioni.rbcs.server.handler + +import io.netty.channel.ChannelHandler.Sharable +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.handler.codec.http.DefaultHttpResponse +import io.netty.handler.codec.http.HttpContent +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpHeaderValues +import io.netty.handler.codec.http.HttpRequest +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.LastHttpContent +import java.nio.file.Path + +@Sharable +object TraceHandler : ChannelInboundHandlerAdapter() { + val NAME = this::class.java.name + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + when(msg) { + is HttpRequest -> { + val response = DefaultHttpResponse(msg.protocolVersion(), HttpResponseStatus.OK) + response.headers().apply { + set(HttpHeaderNames.CONTENT_TYPE, "message/http") + set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED) + } + ctx.write(response) + val replayedRequestHead = ctx.alloc().buffer() + replayedRequestHead.writeCharSequence( + "TRACE ${Path.of(msg.uri())} ${msg.protocolVersion().text()}\r\n", + Charsets.US_ASCII + ) + msg.headers().forEach { (key, value) -> + replayedRequestHead.apply { + writeCharSequence(key, Charsets.US_ASCII) + writeCharSequence(": ", Charsets.US_ASCII) + writeCharSequence(value, Charsets.UTF_8) + writeCharSequence("\r\n", Charsets.US_ASCII) + } + } + replayedRequestHead.writeCharSequence("\r\n", Charsets.US_ASCII) + ctx.writeAndFlush(replayedRequestHead) + } + is LastHttpContent -> { + ctx.writeAndFlush(msg) + ctx.pipeline().remove(this) + } + is HttpContent -> ctx.writeAndFlush(msg) + else -> super.channelRead(ctx, msg) + } + } + + override fun exceptionCaught(ctx: ChannelHandlerContext?, cause: Throwable?) { + super.exceptionCaught(ctx, cause) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/otel/OtelIntegration.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/otel/OtelIntegration.kt new file mode 100644 index 0000000..fcf9144 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/otel/OtelIntegration.kt @@ -0,0 +1,32 @@ +package net.woggioni.rbcs.server.otel + +import io.netty.channel.ChannelHandler +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.instrumentation.netty.v4_1.NettyServerTelemetry +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.warn + +object OtelIntegration { + + private val log = createLogger() + + val isAvailable: Boolean by lazy { + runCatching { + Class.forName("io.opentelemetry.api.OpenTelemetry") + }.fold( + onSuccess = { true }, + onFailure = { + log.warn { "OpenTelemetry classes not on classpath, instrumentation disabled" } + false + }, + ) + } + + fun createHandler(): ChannelHandler? { + return if (isAvailable) createHandlerInternal() else null + } + + private fun createHandlerInternal(): ChannelHandler { + return NettyServerTelemetry.create(GlobalOpenTelemetry.get()).createCombinedHandler() + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/rbcs-default.xml b/rbcs-server/bin/main/net/woggioni/rbcs/server/rbcs-default.xml new file mode 100644 index 0000000..6d45ea1 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/rbcs-default.xml @@ -0,0 +1,8 @@ + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/schema/rbcs-server.xsd b/rbcs-server/bin/main/net/woggioni/rbcs/server/schema/rbcs-server.xsd new file mode 100644 index 0000000..65aa616 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/schema/rbcs-server.xsd @@ -0,0 +1,767 @@ + + + + + + Root element containing the server configuration + + + + + + + + + + + + + + Cache storage backend implementation to use, more implementations can be added through + the use of plugins + + + + + + + + + + + + + + + + + Mechanism to use to assign a username to a specific client + + + + + + + Use TLS to encrypt all the communications + + + + + + + + URI path prefix, if your rbcs is hosted at "http://www.example.com" + and this parameter is set to "cache", then all the requests will need to be sent at + "http://www.example.com/cache/KEY", where "KEY" is the cache entry KEY + + + + + + + + + + + + Server bind address + + + + + Server port number + + + + + Enable proxy protocol + + + + + + + The maximum queue length for incoming connection indications (a request to connect) is set to + the backlog parameter. If a connection indication arrives when the queue is full, + the connection is refused. + + + + + + + + + + + + + + + + + + + + The server will close the connection with the client + when neither a read nor a write was performed for the specified period of time. + + + + + + + The server will close the connection with the client + when no read was performed for the specified period of time. + + + + + + + The server will close the connection with the client + when no write was performed for the specified period of time. + + + + + + + The maximum request body size the server will accept from a client + (if exceeded the server returns 413 HTTP status code) + + + + + + + Maximum byte size of socket write calls + (reduce it to reduce memory consumption, increase it for increased throughput) + + + + + + + + + + Whether or not to use virtual threads for the execution of the core server handler + (not for the I/O operations) + + + + + + + + + + If set to true, the server will delay responses to meet user quotas, otherwise it will simply + return an immediate 429 status code to all requests that exceed the configured quota + + + + + + + Only meaningful when "delay-response" is set to "true", + when a request is delayed, it and all the following messages are queued + as long as "max-queued-messages" is not crossed, all requests that would exceed the + max-queued-message limit are instead discarded and responded with a 429 status code + + + + + + + Only meaningful when "delay-response" is set to "true", + when a request is delayed, it and all the following requests are buffered + as long as "message-buffer-size" is not crossed, all requests that would exceed the buffer + size are instead discarded and responded with a 429 status code + + + + + + + + + + + A simple cache implementation that uses a java.util.ConcurrentHashMap as a storage backend + + + + + + + + Values will be removed from the cache after this amount of time + + + + + + + The maximum allowed total size of the cache in bytes, old values will be purged from the cache + when the insertion of a new value causes this limit to be exceeded + + + + + + + Hashing algorithm to apply to the key. If omitted, no hashing is performed. + + + + + + + Enable deflate compression for stored cache elements + + + + + + + Deflate compression level to use for cache compression, + use -1 to use the default compression level of java.util.zip.Deflater + + + + + + + + + + + A simple cache implementation that stores data in a folder on the filesystem + + + + + + + + File system path that will be used to store the cache data files + (it will be created if it doesn't already exist) + + + + + + + Values will be removed from the cache after this amount of time + + + + + + + Hashing algorithm to apply to the key. If omitted, no hashing is performed. + + + + + + + Enable deflate compression for stored cache elements + + + + + + + Deflate compression level to use for cache compression, + use -1 to use the default compression level of java.util.zip.Deflater + + + + + + + + + + + + + A regex based extractor that will be used to determine which group the client belongs to, + based on the X.500 name of the subject field in the client's TLS certificate. + When this is set RBAC works even if the user isn't listed in the <users/> section as + the client will be assigned role solely based on the group he is found to belong to. + Note that this does not allow for a client to be part of multiple groups. + + + + + + + A regex based extractor that will be used to assign a user to a connected client, + based on the X.500 name of the subject field in the client's TLS certificate. + + + + + + + + + + Authenticate clients based on a custom HTTP header containing the client TLS certificate + subject DN, forwarded by a reverse proxy that performs TLS termination. The proxy must be + listed in the trusted-proxies configuration for the header to be accepted. + + + + + + + A regex based extractor that will be used to determine which group the client belongs to, + based on the X.500 name of the subject DN forwarded by the reverse proxy. + When this is set RBAC works even if the user isn't listed in the <users/> section as + the client will be assigned role solely based on the group he is found to belong to. + Note that this does not allow for a client to be part of multiple groups. + + + + + + + A regex based extractor that will be used to assign a user to a connected client, + based on the X.500 name of the subject DN forwarded by the reverse proxy. + + + + + + + + Name of the HTTP header containing the client certificate subject DN + forwarded by the reverse proxy. Defaults to "X-Client-Cert-Subject-DN". + + + + + + + + + Extract informations from a client TLS certificates using + regular expressions applied to the X.500 name "Subject" field + + + + + + X.500 name attribute to apply the regex + + + + + + + Regex that wil be applied to the attribute value, + use regex groups to extract relevant data + (note that only the first group that appears in the regex is used) + + + + + + + + + + + List of users registered in the application + + + + + + + List of user groups registered in the application + + + + + + + + + + + + + + Authentication mechanism to assign usernames and groups to clients + + + + + + + Enable HTTP basic authentication + + + + + + + Enable TLS certificate authentication + + + + + + + Enable forwarded client certificate authentication. Authenticates clients based on + a custom HTTP header containing the client certificate subject DN, forwarded by a + reverse proxy that performs TLS termination. Requires trusted-proxies to be configured. + + + + + + + Disable authentication altogether + + + + + + + + + + Defines a quota for a user or a group + + + + + + Maximum number of allowed calls in a given period + + + + + + + The period length + + + + + + + Maximum number of available calls that can be accumulated + + + + + + + Number of available calls for users at their first call + + + + + + + + + Placeholder for a client that is not authenticated + + + + + + + Calls quota for the user + + + + + + + + + + An authenticated user + + + + + + + Calls quota for the user + + + + + + + + User's name + + + + + + + User's password hash used for HTTP basic authentication, this has to be generated with + the `password` subcommand of `rbcs-cli` + + + + + + + + + List of registered users, add an <anonymous> tag to enable authenticated user access + when authentication is enabled + + + + + + + + + + + + List of registered user groups + + + + + + + + + + + The definition of a user group, with the list of its member users + + + + + + + + + + + + + The list of application roles awarded to all the members of this group + + + + + + + The call quota for each user in this group + + + + + + + The cumulative call quota for all users in this group + + + + + + + + The group's name + + + + + + + + + + + + + + + + + + + A list of references to users in the <users> section + + + + + + + + + + + + A reference to a user in the <users> section + + + + + + Name of the referenced user + + + + + + + + + Enable TLS protocol + + + + + + + Path to the keystore file that contains the server's key and certificate + + + + + + + Path to the truststore file that contains the trusted CAs + for TLS client certificate verification + + + + + + + + + + + System path to the keystore file + + + + + + + Password to open they keystore file + + + + + + + Alias of the keystore entry containing the private key + + + + + + + Private key entry's encryption password + + + + + + + + + + Path to the trustore file + + + + + + + Trustore file password + + + + + + + Whether or not check the certificate validity using CRL/OCSP + + + + + + + If true, the server requires a TLS client certificate from the client and simply refuses to connect + when a client certificate isn't provided + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/BucketManager.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/BucketManager.kt new file mode 100644 index 0000000..24b0d60 --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/BucketManager.kt @@ -0,0 +1,91 @@ +package net.woggioni.rbcs.server.throttling + +import java.net.InetSocketAddress +import java.util.Arrays +import java.util.concurrent.ConcurrentHashMap +import java.util.function.Function +import net.woggioni.jwo.Bucket +import net.woggioni.rbcs.api.Configuration + +class BucketManager private constructor( + private val bucketsByUser: Map> = HashMap(), + private val bucketsByGroup: Map = HashMap(), + loader: Function? +) { + + private class BucketsByAddress( + private val map: MutableMap, + private val loader: Function + ) { + fun getBucket(socketAddress : InetSocketAddress) = map.computeIfAbsent(ByteArrayKey(socketAddress.address.address)) { + loader.apply(socketAddress) + } + } + + private val bucketsByAddress: BucketsByAddress? = loader?.let { + BucketsByAddress(ConcurrentHashMap(), it) + } + + private class ByteArrayKey(val array: ByteArray) { + override fun equals(other: Any?) = (other as? ByteArrayKey)?.let { bak -> + array contentEquals bak.array + } ?: false + + override fun hashCode() = Arrays.hashCode(array) + } + + fun getBucketByAddress(address : InetSocketAddress) : Bucket? { + return bucketsByAddress?.getBucket(address) + } + + fun getBucketByUser(user : Configuration.User) = bucketsByUser[user] + fun getBucketByGroup(group : Configuration.Group) = bucketsByGroup[group] + + companion object { + fun from(cfg : Configuration) : BucketManager { + val bucketsByUser = cfg.users.values.asSequence().map { user -> + val buckets = ( + user.quota + ?.let { quota -> + sequenceOf(quota) + } ?: user.groups.asSequence() + .mapNotNull(Configuration.Group::getUserQuota) + ).map { quota -> + Bucket.local( + quota.maxAvailableCalls, + quota.calls, + quota.period, + quota.initialAvailableCalls + ) + }.toList() + user to buckets + }.toMap() + val bucketsByGroup = cfg.groups.values.asSequence().filter { + it.groupQuota != null + }.map { group -> + val quota = group.groupQuota + val bucket = Bucket.local( + quota.maxAvailableCalls, + quota.calls, + quota.period, + quota.initialAvailableCalls + ) + group to bucket + }.toMap() + return BucketManager( + bucketsByUser, + bucketsByGroup, + cfg.users[""]?.quota?.let { anonymousUserQuota -> + Function { + Bucket.local( + anonymousUserQuota.maxAvailableCalls, + anonymousUserQuota.calls, + anonymousUserQuota.period, + anonymousUserQuota.initialAvailableCalls + ) + } + } + ) + } + } +} diff --git a/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/ThrottlingHandler.kt b/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/ThrottlingHandler.kt new file mode 100644 index 0000000..1a695de --- /dev/null +++ b/rbcs-server/bin/main/net/woggioni/rbcs/server/throttling/ThrottlingHandler.kt @@ -0,0 +1,228 @@ +package net.woggioni.rbcs.server.throttling + +import io.netty.buffer.ByteBufHolder +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.handler.codec.http.DefaultFullHttpResponse +import io.netty.handler.codec.http.FullHttpMessage +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpRequest +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import io.netty.handler.codec.http.LastHttpContent +import java.net.InetSocketAddress +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.ArrayDeque +import java.util.concurrent.TimeUnit +import net.woggioni.jwo.Bucket +import net.woggioni.jwo.LongMath +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.createLogger +import net.woggioni.rbcs.common.debug +import net.woggioni.rbcs.server.RemoteBuildCacheServer + +class ThrottlingHandler( + private val bucketManager: BucketManager, + rateLimiterConfiguration: Configuration.RateLimiter, + connectionConfiguration: Configuration.Connection +) : ChannelInboundHandlerAdapter() { + + private companion object { + private val log = createLogger() + + fun nextAttemptIsWithinThreshold(nextAttemptNanos : Long, waitThreshold : Duration) : Boolean { + val waitDuration = Duration.of(LongMath.ceilDiv(nextAttemptNanos, 100_000_000L) * 100L, ChronoUnit.MILLIS) + return waitDuration < waitThreshold + } + } + + private class RefusedRequest + + private val maxMessageBufferSize = rateLimiterConfiguration.messageBufferSize + private val maxQueuedMessages = rateLimiterConfiguration.maxQueuedMessages + private val delayRequests = rateLimiterConfiguration.isDelayRequest + private var requestBufferSize : Int = 0 + private var valveClosed = false + private var queuedContent = ArrayDeque() + + /** + * If the suggested waiting time from the bucket is lower than this + * amount, then the server will simply wait by itself before sending a response + * instead of replying with 429 + */ + private val waitThreshold = minOf( + connectionConfiguration.idleTimeout, + connectionConfiguration.readIdleTimeout, + connectionConfiguration.writeIdleTimeout + ).dividedBy(2) + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + if(valveClosed) { + if(msg !is HttpRequest && msg is ByteBufHolder) { + val newBufferSize = requestBufferSize + msg.content().readableBytes() + if(newBufferSize > maxMessageBufferSize || queuedContent.size + 1 > maxQueuedMessages) { + log.debug { + if (newBufferSize > maxMessageBufferSize) { + "New message part exceeds maxMessageBufferSize, removing previous chunks" + } else { + "New message part exceeds maxQueuedMessages, removing previous chunks" + } + } + // If this message overflows the maxMessageBufferSize, + // then remove the previously enqueued chunks of the request from the deque, + // then discard the message + while(true) { + val tail = queuedContent.last() + if(tail is ByteBufHolder) { + requestBufferSize -= tail.content().readableBytes() + tail.release() + } + queuedContent.removeLast() + if(tail is HttpRequest) { + break + } + } + msg.release() + //Add a placeholder to remember to return a 429 response corresponding to this request + queuedContent.addLast(RefusedRequest()) + } else { + //If the message does not overflow maxMessageBufferSize, just add it to the deque + queuedContent.addLast(msg) + requestBufferSize = newBufferSize + } + } else if(msg is HttpRequest && msg is FullHttpMessage){ + val newBufferSize = requestBufferSize + msg.content().readableBytes() + + // If this message overflows the maxMessageBufferSize, + // discard the message + if(newBufferSize > maxMessageBufferSize || queuedContent.size + 1 > maxQueuedMessages) { + log.debug { + if (newBufferSize > maxMessageBufferSize) { + "New message exceeds maxMessageBufferSize, discarding it" + } else { + "New message exceeds maxQueuedMessages, discarding it" + } + } + msg.release() + //Add a placeholder to remember to return a 429 response corresponding to this request + queuedContent.addLast(RefusedRequest()) + } else { + //If the message does not exceed maxMessageBufferSize or maxQueuedMessages, just add it to the deque + queuedContent.addLast(msg) + requestBufferSize = newBufferSize + } + } else { + queuedContent.addLast(msg) + } + } else { + entryPoint(ctx, msg) + } + } + + private fun entryPoint(ctx : ChannelHandlerContext, msg : Any) { + if(msg is RefusedRequest) { + sendThrottledResponse(ctx, null) + if(queuedContent.isEmpty()) { + valveClosed = false + } else { + val head = queuedContent.poll() + if(head is ByteBufHolder) { + requestBufferSize -= head.content().readableBytes() + } + entryPoint(ctx, head) + } + } else if(msg is HttpRequest) { + val nextAttempt = getNextAttempt(ctx) + if (nextAttempt < 0) { + super.channelRead(ctx, msg) + if(msg !is LastHttpContent) { + while (true) { + val head = queuedContent.poll() ?: break + if(head is ByteBufHolder) { + requestBufferSize -= head.content().readableBytes() + } + super.channelRead(ctx, head) + if (head is LastHttpContent) break + } + } + if(queuedContent.isEmpty()) { + valveClosed = false + } else { + val head = queuedContent.poll() + if(head is ByteBufHolder) { + requestBufferSize -= head.content().readableBytes() + } + entryPoint(ctx, head) + } + } else { + val waitDuration = Duration.of(LongMath.ceilDiv(nextAttempt, 100_000_000L) * 100L, ChronoUnit.MILLIS) + if (delayRequests && nextAttemptIsWithinThreshold(nextAttempt, waitThreshold)) { + valveClosed = true + ctx.executor().schedule({ + entryPoint(ctx, msg) + }, waitDuration.toMillis(), TimeUnit.MILLISECONDS) + } else { + sendThrottledResponse(ctx, waitDuration) + if(queuedContent.isEmpty()) { + valveClosed = false + } else { + val head = queuedContent.poll() + if(head is ByteBufHolder) { + requestBufferSize -= head.content().readableBytes() + } + entryPoint(ctx, head) + } + } + } + } else { + super.channelRead(ctx, msg) + } + } + + /** + * Returns the number amount of milliseconds to wait before the requests can be processed + * or -1 if the request can be performed immediately + */ + private fun getNextAttempt(ctx : ChannelHandlerContext) : Long { + val buckets = mutableListOf() + val user = ctx.channel().attr(RemoteBuildCacheServer.userAttribute).get() + if (user != null) { + bucketManager.getBucketByUser(user)?.let(buckets::addAll) + } + val groups = ctx.channel().attr(RemoteBuildCacheServer.groupAttribute).get() ?: emptySet() + if (groups.isNotEmpty()) { + groups.forEach { group -> + bucketManager.getBucketByGroup(group)?.let(buckets::add) + } + } + if (user == null && groups.isEmpty()) { + val clientAddress = ctx.channel().attr(RemoteBuildCacheServer.clientIp).get() + bucketManager.getBucketByAddress(clientAddress)?.let(buckets::add) + } + + var nextAttempt = -1L + for (bucket in buckets) { + val bucketNextAttempt = bucket.removeTokensWithEstimate(1) + if (bucketNextAttempt > nextAttempt) { + nextAttempt = bucketNextAttempt + } + } + return nextAttempt + } + + private fun sendThrottledResponse(ctx: ChannelHandlerContext, retryAfter: Duration?) { + val response = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, + HttpResponseStatus.TOO_MANY_REQUESTS + ) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0 + retryAfter?.seconds?.takeIf { + it > 0 + }?.let { + response.headers()[HttpHeaderNames.RETRY_AFTER] = it + } + + ctx.writeAndFlush(response) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/logback.xml b/rbcs-server/bin/test/logback.xml new file mode 100644 index 0000000..c6f9111 --- /dev/null +++ b/rbcs-server/bin/test/logback.xml @@ -0,0 +1,21 @@ + + + + + + + + + System.err + + %d [%highlight(%-5level)] \(%thread\) %logger{36} -%kvp- %msg %n + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt new file mode 100644 index 0000000..31d024c --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt @@ -0,0 +1,89 @@ +package net.woggioni.rbcs.server.test + +import java.net.URI +import java.net.http.HttpRequest +import java.nio.charset.StandardCharsets +import java.nio.file.Path +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.Base64 +import java.util.zip.Deflater +import kotlin.random.Random +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.common.RBCS.getFreePort +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration +import net.woggioni.rbcs.server.configuration.Serializer + + +abstract class AbstractBasicAuthServerTest : AbstractServerTest() { + + private lateinit var cacheDir : Path + + protected val random = Random(101325) + protected val keyValuePair = newEntry(random) + protected val serverPath = "rbcs" + protected val readersGroup = Configuration.Group("readers", setOf(Role.Reader), null, null) + protected val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null, null) + + abstract protected val users : List + + override fun setUp() { + this.cacheDir = testDir.resolve("cache") + cfg = Configuration.of( + "127.0.0.1", + getFreePort(), + false, + emptyList(), + 50, + serverPath, + Configuration.EventExecutor(false), + Configuration.RateLimiter(true, 0x100000, 50), + Configuration.Connection( + Duration.of(60, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + 0x1000, + 0x10000 + ), + users.asSequence().map { it.name to it}.toMap(), + sequenceOf(writersGroup, readersGroup).map { it.name to it}.toMap(), + FileSystemCacheConfiguration( + this.cacheDir, + maxAge = Duration.ofSeconds(3600 * 24), + digestAlgorithm = "MD5", + compressionLevel = Deflater.DEFAULT_COMPRESSION, + compressionEnabled = false + ), + Configuration.BasicAuthentication(), + null, + ) + Xml.write(Serializer.serialize(cfg), System.out) + } + + override fun tearDown() { + } + + protected fun buildAuthorizationHeader(user : Configuration.User, password : String) : String { + val b64 = Base64.getEncoder().encode("${user.name}:${password}".toByteArray(Charsets.UTF_8)).let{ + String(it, StandardCharsets.UTF_8) + } + return "Basic $b64" + } + + protected fun newRequestBuilder(key : String) = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/$serverPath/$key")) + + + protected fun newEntry(random : Random) : Pair { + val key = ByteArray(0x10).let { + random.nextBytes(it) + Base64.getUrlEncoder().encodeToString(it) + } + val value = ByteArray(0x1000).also { + random.nextBytes(it) + } + return key to value + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractServerTest.kt new file mode 100644 index 0000000..960fb4f --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractServerTest.kt @@ -0,0 +1,51 @@ +package net.woggioni.rbcs.server.test + +import java.nio.file.Path +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.server.RemoteBuildCacheServer +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.MethodOrderer +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.TestMethodOrder +import org.junit.jupiter.api.io.TempDir + + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation::class) +abstract class AbstractServerTest { + + protected lateinit var cfg : Configuration + + protected lateinit var testDir : Path + + private var serverHandle : RemoteBuildCacheServer.ServerHandle? = null + + @BeforeAll + fun setUp0(@TempDir tmpDir : Path) { + this.testDir = tmpDir + setUp() + startServer(cfg) + } + + @AfterAll + fun tearDown0() { + tearDown() + stopServer() + } + + abstract fun setUp() + + abstract fun tearDown() + + private fun startServer(cfg : Configuration) { + this.serverHandle = RemoteBuildCacheServer(cfg).run() + } + + private fun stopServer() { + this.serverHandle?.let { + it.sendShutdownSignal() + it.get() + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt new file mode 100644 index 0000000..ad8d660 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt @@ -0,0 +1,206 @@ +package net.woggioni.rbcs.server.test + +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.security.KeyStore +import java.security.KeyStore.PasswordProtection +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.Base64 +import java.util.zip.Deflater +import javax.net.ssl.KeyManagerFactory +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManagerFactory +import kotlin.random.Random +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.common.RBCS.getFreePort +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration +import net.woggioni.rbcs.server.configuration.Serializer +import net.woggioni.rbcs.server.test.utils.CertificateUtils +import net.woggioni.rbcs.server.test.utils.CertificateUtils.X509Credentials +import org.bouncycastle.asn1.x500.X500Name + + +abstract class AbstractTlsServerTest : AbstractServerTest() { + + companion object { + private const val CA_CERTIFICATE_ENTRY = "rbcs-ca" + private const val CLIENT_CERTIFICATE_ENTRY = "rbcs-client" + private const val SERVER_CERTIFICATE_ENTRY = "rbcs-server" + private const val PASSWORD = "password" + } + + private lateinit var cacheDir: Path + private lateinit var serverKeyStoreFile: Path + private lateinit var clientKeyStoreFile: Path + private lateinit var trustStoreFile: Path + private lateinit var serverKeyStore: KeyStore + private lateinit var clientKeyStore: KeyStore + private lateinit var trustStore: KeyStore + protected lateinit var ca: X509Credentials + + protected val readersGroup = Configuration.Group("readers", setOf(Role.Reader), null, null) + protected val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null, null) + protected val healthCheckGroup = Configuration.Group("healthcheckers", setOf(Role.Healthcheck), null, null) + protected val random = Random(101325) + protected val keyValuePair = newEntry(random) + private val serverPath : String? = null + + protected abstract val users : List + + protected fun createKeyStoreAndTrustStore() { + ca = CertificateUtils.createCertificateAuthority(CA_CERTIFICATE_ENTRY, 30) + val serverCert = CertificateUtils.createServerCertificate(ca, X500Name("CN=$SERVER_CERTIFICATE_ENTRY"), 30) + val clientCert = CertificateUtils.createClientCertificate(ca, X500Name("CN=$CLIENT_CERTIFICATE_ENTRY"), 30) + + serverKeyStore = KeyStore.getInstance("PKCS12").apply { + load(null, null) + setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null)) + setEntry( + SERVER_CERTIFICATE_ENTRY, + KeyStore.PrivateKeyEntry( + serverCert.keyPair().private, + arrayOf(serverCert.certificate(), ca.certificate) + ), + PasswordProtection(PASSWORD.toCharArray()) + ) + } + Files.newOutputStream(this.serverKeyStoreFile).use { + serverKeyStore.store(it, null) + } + + clientKeyStore = KeyStore.getInstance("PKCS12").apply { + load(null, null) + setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null)) + setEntry( + CLIENT_CERTIFICATE_ENTRY, + KeyStore.PrivateKeyEntry( + clientCert.keyPair().private, + arrayOf(clientCert.certificate(), ca.certificate) + ), + PasswordProtection(PASSWORD.toCharArray()) + ) + } + Files.newOutputStream(this.clientKeyStoreFile).use { + clientKeyStore.store(it, null) + } + + trustStore = KeyStore.getInstance("PKCS12").apply { + load(null, null) + setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null)) + } + Files.newOutputStream(this.trustStoreFile).use { + trustStore.store(it, null) + } + } + + protected fun getClientKeyStore(ca: X509Credentials, subject: X500Name) = KeyStore.getInstance("PKCS12").apply { + val clientCert = CertificateUtils.createClientCertificate(ca, subject, 30) + + load(null, null) + setEntry(CA_CERTIFICATE_ENTRY, KeyStore.TrustedCertificateEntry(ca.certificate), PasswordProtection(null)) + setEntry( + CLIENT_CERTIFICATE_ENTRY, + KeyStore.PrivateKeyEntry(clientCert.keyPair().private, arrayOf(clientCert.certificate(), ca.certificate)), + PasswordProtection(PASSWORD.toCharArray()) + ) + } + + protected fun getHttpClient(clientKeyStore: KeyStore?): HttpClient { + val kmf = clientKeyStore?.let { + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()).apply { + init(it, PASSWORD.toCharArray()) + } + } + + + // Set up trust manager factory with the truststore + val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) + tmf.init(trustStore) + + // Create SSL context with the key and trust managers + val sslContext = SSLContext.getInstance("TLS").apply { + init(kmf?.keyManagers ?: emptyArray(), tmf.trustManagers, null) + } + return HttpClient.newBuilder().sslContext(sslContext).build() + } + + override fun setUp() { + this.clientKeyStoreFile = testDir.resolve("client-keystore.p12") + this.serverKeyStoreFile = testDir.resolve("server-keystore.p12") + this.trustStoreFile = testDir.resolve("truststore.p12") + this.cacheDir = testDir.resolve("cache") + createKeyStoreAndTrustStore() + cfg = Configuration( + "127.0.0.1", + getFreePort(), + false, + emptyList(), + + 100, + serverPath, + Configuration.EventExecutor(false), + Configuration.RateLimiter(true, 0x100000, 50), + Configuration.Connection( + Duration.of(60, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + 0x1000, + 0x10000 + ), + users.asSequence().map { it.name to it }.toMap(), + sequenceOf(writersGroup, readersGroup).map { it.name to it }.toMap(), + FileSystemCacheConfiguration(this.cacheDir, + maxAge = Duration.ofSeconds(3600 * 24), + compressionEnabled = false, + compressionLevel = Deflater.DEFAULT_COMPRESSION, + digestAlgorithm = "MD5", + ), +// InMemoryCacheConfiguration( +// maxAge = Duration.ofSeconds(3600 * 24), +// compressionEnabled = true, +// compressionLevel = Deflater.DEFAULT_COMPRESSION, +// digestAlgorithm = "MD5" +// ), + Configuration.ClientCertificateAuthentication( + Configuration.TlsCertificateExtractor("CN", "(.*)"), + null + ), + Configuration.Tls( + Configuration.KeyStore(this.serverKeyStoreFile, null, SERVER_CERTIFICATE_ENTRY, PASSWORD), + Configuration.TrustStore(this.trustStoreFile, null, false, false), + ) + ) + Xml.write(Serializer.serialize(cfg), System.out) + } + + override fun tearDown() { + } + + protected fun newRequestBuilder(key: String) = HttpRequest.newBuilder() + .uri(URI.create("https://${cfg.host}:${cfg.port}/${serverPath ?: ""}/$key")) + + private fun buildAuthorizationHeader(user: Configuration.User, password: String): String { + val b64 = Base64.getEncoder().encode("${user.name}:${password}".toByteArray(Charsets.UTF_8)).let { + String(it, StandardCharsets.UTF_8) + } + return "Basic $b64" + } + + protected fun newEntry(random: Random): Pair { + val key = ByteArray(0x10).let { + random.nextBytes(it) + Base64.getUrlEncoder().encodeToString(it) + } + val value = ByteArray(0x1000).also { + random.nextBytes(it) + } + return key to value + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/BasicAuthServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/BasicAuthServerTest.kt new file mode 100644 index 0000000..316455c --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/BasicAuthServerTest.kt @@ -0,0 +1,192 @@ +package net.woggioni.rbcs.server.test + +import io.netty.handler.codec.http.HttpResponseStatus +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.time.Duration +import java.time.temporal.ChronoUnit +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Role +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test + + +class BasicAuthServerTest : AbstractBasicAuthServerTest() { + + companion object { + private const val PASSWORD = "password" + } + + override val users = listOf( + Configuration.User("user1", hashPassword(PASSWORD), setOf(readersGroup), null), + Configuration.User("user2", hashPassword(PASSWORD), setOf(writersGroup), null), + Configuration.User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup), null), + Configuration.User("", null, setOf(readersGroup), null), + Configuration.User("user4", hashPassword(PASSWORD), setOf(readersGroup), + Configuration.Quota(1, Duration.of(1, ChronoUnit.DAYS), 0, 1) + ), + Configuration.User("user5", hashPassword(PASSWORD), setOf(readersGroup), + Configuration.Quota(1, Duration.of(5, ChronoUnit.SECONDS), 0, 1) + ) + ) + + @Test + @Order(1) + fun putWithNoAuthorizationHeader() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(2) + fun putAsAReaderUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Reader in it.roles && Role.Writer !in it.roles + } ?: throw RuntimeException("Reader user not found") + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(3) + fun getAsAWriterUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, _) = keyValuePair + val user = cfg.users.values.find { + Role.Writer in it.roles + } ?: throw RuntimeException("Reader user not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(4) + fun putAsAWriterUser() { + val client: HttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Writer in it.roles + } ?: throw RuntimeException("Reader user not found") + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.CREATED.code(), response.statusCode()) + } + + @Test + @Order(5) + fun getAsAReaderUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Reader in it.roles + } ?: throw RuntimeException("Reader user not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(6) + fun getAsAnonymousUser() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + + val requestBuilder = newRequestBuilder(key) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(7) + fun getMissingKeyAsAReaderUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, _) = newEntry(random) + val user = cfg.users.values.find { + Role.Reader in it.roles + } ?: throw RuntimeException("Reader user not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.statusCode()) + } + + @Test + @Order(8) + fun getAsAThrottledUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + it.name == "user4" + } ?: throw RuntimeException("user4 not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.TOO_MANY_REQUESTS.code(), response.statusCode()) + } + + @Test + @Order(9) + fun getAsAThrottledUser2() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + it.name == "user5" + } ?: throw RuntimeException("user5 not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/ConfigurationTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/ConfigurationTest.kt new file mode 100644 index 0000000..17ab632 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/ConfigurationTest.kt @@ -0,0 +1,58 @@ +package net.woggioni.rbcs.server.test + +import java.nio.file.Files +import java.nio.file.Path +import net.woggioni.rbcs.common.RBCS.toUrl +import net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.server.configuration.Parser +import net.woggioni.rbcs.server.configuration.Serializer +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import org.xml.sax.SAXParseException + +class ConfigurationTest { + + @ValueSource( + strings = [ + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-default.xml", + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml", + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-tls.xml", + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml", + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-redis.xml", + "classpath:net/woggioni/rbcs/server/test/valid/rbcs-redis-tls.xml", + ] + ) + @ParameterizedTest + fun test(configurationUrl: String, @TempDir testDir: Path) { + RbcsUrlStreamHandlerFactory.install() + val doc = Xml.parseXml(configurationUrl.toUrl()) + val cfg = Parser.parse(doc) + val configFile = testDir.resolve("rbcs.xml") + Files.newOutputStream(configFile).use { + Xml.write(Serializer.serialize(cfg), it) + } + Xml.write(Serializer.serialize(cfg), System.out) + + val parsed = Parser.parse(Xml.parseXml(configFile.toUri().toURL())) + Assertions.assertEquals(cfg, parsed) + } + + @ValueSource( + strings = [ + "classpath:net/woggioni/rbcs/server/test/invalid/invalid-user-ref.xml", + "classpath:net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user.xml", + "classpath:net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user2.xml", + "classpath:net/woggioni/rbcs/server/test/invalid/multiple-user-quota.xml", + ] + ) + @ParameterizedTest + fun invalidConfigurationTest(configurationUrl: String) { + RbcsUrlStreamHandlerFactory.install() + Assertions.assertThrows(SAXParseException::class.java) { + Xml.parseXml(configurationUrl.toUrl()) + } + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt new file mode 100644 index 0000000..f4e59f6 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt @@ -0,0 +1,52 @@ +package net.woggioni.rbcs.server.test + +import io.netty.handler.codec.http.HttpResponseStatus +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.PasswordSecurity.hashPassword +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test + + +class NoAnonymousUserBasicAuthServerTest : AbstractBasicAuthServerTest() { + + companion object { + private const val PASSWORD = "anotherPassword" + } + + override val users = listOf( + Configuration.User("user1", hashPassword(PASSWORD), setOf(readersGroup), null), + Configuration.User("user2", hashPassword(PASSWORD), setOf(writersGroup), null), + Configuration.User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup), null), + ) + + @Test + @Order(1) + fun putWithNoAuthorizationHeader() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.UNAUTHORIZED.code(), response.statusCode()) + } + + @Test + @Order(2) + fun getWithNoAuthorizationHeader() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + + val requestBuilder = newRequestBuilder(key) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.UNAUTHORIZED.code(), response.statusCode()) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserTlsServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserTlsServerTest.kt new file mode 100644 index 0000000..a0fbd2f --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAnonymousUserTlsServerTest.kt @@ -0,0 +1,47 @@ +package net.woggioni.rbcs.server.test + +import io.netty.handler.codec.http.HttpResponseStatus +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import net.woggioni.rbcs.api.Configuration +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test + +class NoAnonymousUserTlsServerTest : AbstractTlsServerTest() { + + override val users = listOf( + Configuration.User("user1", null, setOf(readersGroup), null), + Configuration.User("user2", null, setOf(writersGroup), null), + Configuration.User("user3", null, setOf(readersGroup, writersGroup), null), + ) + + @Test + @Order(1) + fun getAsAnonymousUser() { + val (key, _) = keyValuePair + val client: HttpClient = getHttpClient(null) + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.UNAUTHORIZED.code(), response.statusCode()) + } + + @Test + @Order(2) + fun putAsAnonymousUser() { + val (key, value) = keyValuePair + val client: HttpClient = getHttpClient(null) + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.UNAUTHORIZED.code(), response.statusCode()) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAuthServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAuthServerTest.kt new file mode 100644 index 0000000..d81f4d6 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/NoAuthServerTest.kt @@ -0,0 +1,186 @@ +package net.woggioni.rbcs.server.test + +import io.netty.handler.codec.http.HttpResponseStatus +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.nio.file.Path +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.Base64 +import java.util.zip.Deflater +import kotlin.random.Random +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.common.RBCS.getFreePort +import net.woggioni.rbcs.common.Xml +import net.woggioni.rbcs.server.cache.InMemoryCacheConfiguration +import net.woggioni.rbcs.server.configuration.Serializer +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test + + +class NoAuthServerTest : AbstractServerTest() { + + private lateinit var cacheDir: Path + + private val random = Random(101325) + private val keyValuePair = newEntry(random) + private val serverPath = "/some/nested/path" + + override fun setUp() { + this.cacheDir = testDir.resolve("cache") + cfg = Configuration( + "127.0.0.1", + getFreePort(), + false, + emptyList(), + 100, + serverPath, + Configuration.EventExecutor(false), + Configuration.RateLimiter(true, 0x100000, 50), + Configuration.Connection( + Duration.of(60, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + Duration.of(30, ChronoUnit.SECONDS), + 0x1000, + 0x10000 + ), + emptyMap(), + emptyMap(), + InMemoryCacheConfiguration( + maxAge = Duration.ofSeconds(3600 * 24), + compressionEnabled = true, + digestAlgorithm = "MD5", + compressionLevel = Deflater.DEFAULT_COMPRESSION, + maxSize = 0x1000000, + ), + null, + null, + ) + Xml.write(Serializer.serialize(cfg), System.out) + } + + override fun tearDown() { + } + + fun newRequestBuilder(key: String) = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/$serverPath/$key")) + + fun newEntry(random: Random): Pair { + val key = ByteArray(0x10).let { + random.nextBytes(it) + Base64.getUrlEncoder().encodeToString(it) + } + val value = ByteArray(0x1000).also { + random.nextBytes(it) + } + return key to value + } + + @Test + @Order(1) + fun putWithNoAuthorizationHeader() { + val client: HttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build() + val (key, value) = keyValuePair + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.CREATED.code(), response.statusCode()) + } + + @Test + @Order(2) + fun getWithNoAuthorizationHeader() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + val requestBuilder = newRequestBuilder(key) + .GET() + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(3) + fun getMissingKey() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, _) = newEntry(random) + val requestBuilder = newRequestBuilder(key).GET() + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.statusCode()) + } + + @Test + @Order(4) + fun getUnhandledPath() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, _) = newEntry(random) + val requestBuilder = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/some/other/path/$key")) + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.statusCode()) + } + + @Test + @Order(5) + fun putUnhandledPath() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = newEntry(random) + val requestBuilder = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/some/other/path/$key")) + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.statusCode()) + } + + @Test + @Order(6) + fun getRelativeUnhandledPath() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, _) = newEntry(random) + val requestBuilder = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/some/nested/path/../../../some/other/path/$key")) + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.statusCode()) + } + + @Test + @Order(7) + fun getRelativePath() { + val client: HttpClient = HttpClient.newHttpClient() + val (key, value) = keyValuePair + val requestBuilder = HttpRequest.newBuilder() + .uri(URI.create("http://${cfg.host}:${cfg.port}/some/other/path/../../nested/path/$key")) + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(10) + fun traceTest() { + val client: HttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build() + val requestBuilder = newRequestBuilder("").method( + "TRACE", + HttpRequest.BodyPublishers.ofByteArray("sfgsdgfaiousfiuhsd".toByteArray()) + ) + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + println(String(response.body())) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/TlsServerTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/TlsServerTest.kt new file mode 100644 index 0000000..2ac8315 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/TlsServerTest.kt @@ -0,0 +1,182 @@ +package net.woggioni.rbcs.server.test + +import io.netty.handler.codec.http.HttpResponseStatus +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import net.woggioni.rbcs.api.Configuration +import net.woggioni.rbcs.api.Role +import org.bouncycastle.asn1.x500.X500Name +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Order +import org.junit.jupiter.api.Test + + +class TlsServerTest : AbstractTlsServerTest() { + + override val users = listOf( + Configuration.User("user1", null, setOf(readersGroup), null), + Configuration.User("user2", null, setOf(writersGroup), null), + Configuration.User("user3", null, setOf(readersGroup, writersGroup), null), + Configuration.User("user4", null, setOf(healthCheckGroup), null), + Configuration.User("", null, setOf(readersGroup), null) + ) + + @Test + @Order(1) + fun putAsAReaderUser() { + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Reader in it.roles && Role.Writer !in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(2) + fun getAsAWriterUser() { + val (key, _) = keyValuePair + val user = cfg.users.values.find { + Role.Writer in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + + val requestBuilder = newRequestBuilder(key) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(3) + fun putAsAWriterUser() { + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Writer in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.CREATED.code(), response.statusCode()) + } + + @Test + @Order(4) + fun getAsAReaderUser() { + val (key, value) = keyValuePair + val user = cfg.users.values.find { + Role.Reader in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + + val requestBuilder = newRequestBuilder(key) + .GET() + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(5) + fun getMissingKeyAsAReaderUser() { + val (key, _) = newEntry(random) + val user = cfg.users.values.find { + Role.Reader in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + + val requestBuilder = newRequestBuilder(key) + .GET() + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.statusCode()) + } + + @Test + @Order(6) + fun getAsAnonymousUser() { + val (key, value) = keyValuePair + val client: HttpClient = getHttpClient(null) + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } + + @Test + @Order(7) + fun putAsAnonymousUser() { + val (key, value) = keyValuePair + val client: HttpClient = getHttpClient(null) + + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(8) + fun traceAsAnonymousUser() { + val client: HttpClient = getHttpClient(null) + val requestBuilder = newRequestBuilder("").method( + "TRACE", + HttpRequest.BodyPublishers.ofByteArray("this is an healthcheck".toByteArray()) + ) + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.FORBIDDEN.code(), response.statusCode()) + } + + @Test + @Order(9) + fun traceAsHealthcheckUser() { + val user = cfg.users.values.find { + Role.Healthcheck in it.roles + } ?: throw RuntimeException("Reader user not found") + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=${user.name}"))) + val requestBuilder = newRequestBuilder("").method( + "TRACE", + HttpRequest.BodyPublishers.ofByteArray("this is an healthcheck".toByteArray()) + ) + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + println(String(response.body())) + } + + @Test + @Order(10) + fun putAsUnknownUserUser() { + val (key, value) = keyValuePair + val client: HttpClient = getHttpClient(getClientKeyStore(ca, X500Name("CN=Unknown user"))) + val requestBuilder = newRequestBuilder(key) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofByteArray(value)) + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()) + Assertions.assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), response.statusCode()) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/X500NameTest.kt b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/X500NameTest.kt new file mode 100644 index 0000000..ca2a1fe --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/X500NameTest.kt @@ -0,0 +1,19 @@ +package net.woggioni.rbcs.server.test + +import javax.naming.ldap.LdapName +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class X500NameTest { + + @Test + fun test() { + val name = + "C=SG, L=Bugis, CN=woggioni@f6aa5663ef26, emailAddress=oggioni.walter@gmail.com, street=1 Fraser Street\\, Duo Residences #23-05, postalCode=189350, GN=Walter, SN=Oggioni, pseudonym=woggioni" + val ldapName = LdapName(name) + val value = ldapName.rdns.asSequence().find { + it.type == "CN" + }!!.value + Assertions.assertEquals("woggioni@f6aa5663ef26", value) + } +} \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user.xml new file mode 100644 index 0000000..d4f7f89 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user2.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user2.xml new file mode 100644 index 0000000..0c28def --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/duplicate-anonymous-user2.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/invalid-user-ref.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/invalid-user-ref.xml new file mode 100644 index 0000000..3d0971e --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/invalid-user-ref.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/multiple-user-quota.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/multiple-user-quota.xml new file mode 100644 index 0000000..5384928 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/invalid/multiple-user-quota.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$CertificateAuthority.class b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$CertificateAuthority.class new file mode 100644 index 0000000000000000000000000000000000000000..b2e9bd79744080b2e55fdfa437cb0b41febdd2b7 GIT binary patch literal 986 zcmb_a-)j>=5dJ2YCQW)xY;65eYqe5pk`~toeGo+`L9sMcK@Ilta<1u?^G?JhEnJ4jOc6R3b=G&RsU%!9+1n>-Zswfc-Vrj+aWHeGqtb`thMi{Bb zQVT1M6{%H`5k0A``k=zV%C|1CoeS)z_9W5Do>fsMTsjTLfrx^5Bwh|rWoQZYPg;$c zc__~a>-`L1WSBaRcy*C`OjrtMI)v>4)Fs8chxZ@l#FCl zJ9=V@5b!v-=%>clVr~y>qO5h($ literal 0 HcmV?d00001 diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$X509Credentials.class b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/utils/CertificateUtils$X509Credentials.class new file mode 100644 index 0000000000000000000000000000000000000000..843a12609b2e93d9af0e530be0d57432cc003599 GIT binary patch literal 1795 zcmb_cQB%`E5dN-EQcAU@fQTZX7HA6+P*Aamjzx!#6&XRsfu}S*OQ0n6a>e04`5-eg z&hX$5@JBiBrNPEhd2o10F4@~}zrF8v_vf$E9{^tBxrPwKip{0|*{M`4$F_7)GF{!} z;*bkna#!kgX;oc4&xN!;TBafSdu2#{SeRYR3tr~7w2Z2&p_3u8XB--O)vzo2J8n9n z%%JV_qjkd)41+~)*X3qiSn^1J-Pp)6^q9@y45=1BwW)h-O)mlsr86y#&48VGr_8&- z5Q`yXsoa_3s-e&c$p?t|IrwpdrE#kI_DoeNJEM28t$$-H$z7{Q&7A&Vi?3thQej>78ttt zr#f9X!$7(rzn8_^(jGTuF0-v@M`IYq7{ksb(YE8EM3yYulFJOsej?`ztED{JUS(!0 ziVmb=xQ%g7xGk#UH(c&I<#j_C2V8PNhmCGHbz$;XmMWcre6tX}UYJpZO}H11BV8$s zS|flP#T4#om}VGlHioEE!yItEA(`(P!whtW?$TKVhU1{VmVM~#bG_lcYS`r}cT+{j z*{|1f0eSqI3s`*M=24BWaMu)8O*$e7$yW{64I2CyJLiK0+5f6R|Mufu$VPCVq0jI6 zoRZuh8gXiKdJ@!=1>5GrbR2PHwGK#Xa;mpv4^O&X~fW1QVj%JyJ#m2H6x<*H%=>~HJts1p0D%~!WG|% zIt5of3&=%7ybUDL3K{zY=_STKA2LZ{6^oWVML*CAVgb+~uD1aV0bg2@3xbAm%Lk>2 zDMX%;?9?enEG0ut7m35hR2l!4Qxofq;sW-60Iw-3ha^A;B9_ z)Oz%;^{7X++SXb`+#snasI8)Sptkj>wXHp@y|mXq{rT(j%}h3vWRg~X{!#K{XXZQK z_kEuC_`U<5?AZ&T7XMT*LSR-}x1*cQ)>gwz8&Rtzo{463t3$VYbk>yjG1PW@*gdWC7gj5uw zNMKaGk=7fssTSQ@skJ0^fwFott|gl^%aGq*V}ZTF$OxQKe>k#L2^htDZET3~`REThM>Qpo6ede?Hzu+G)fx~18sr6Pi31WHJF-s7MqPT6`Ub3##ziZ zQhLcEWxZvl1Wxj_>o?b1 zFF&O!glg0%SRim*pDebXYBwz{8Lio1nQ60;kgHgTTCTmYx;Z8=(N)V?I$$tGoQ-o; zEJhutv?sH~=gjgYL6cT`Tgg4Dg0xj|p1@gyXcA|1iEfN*8{>L(NnDdo)p5Gdc^71j zWJ0&XSc+u|8feS@xOoIsu^i_Mgy=^~@Xl`DM@2K%Fgmwr z86)mEm8BVuTTL56lImE)OgDnpIgxC=p<8t63j|IbX2S&o)(>MXE|T_KCxgiv=^7g1 z3R(na3|hv4d?=t*B*2KIYKcT%D}il8VH7C-nqhRUO~=iIOgALyr<05>)`V6S8>EGc z+w`u+jMl0v*f?DDnS%1Q%Pj;|eNU%g%sC6+B2}ISDcduyJ|p$ri83sS<|>hCeyaN`oNgY!p!aH zmrUT2Ia$EK%FeMJZrU^8NDtMLJp6s*n=MWiIG^>*1pa?HOhllAg6=Z zp=oyE2^qik3QR9wJ2avC@6XCx!BYa0g5_OUyCSA<%IZ8PDcDc52M_CHr5SajQc~Kv z7|$>S%slPP85R7S8OA>RNXqtOU-|C`K-M@<;(0mb0F@q6d|9FRiVn_t5icn?IJ~`m zf-$K_@Up;M|0%~U*@9}AK3n_OaEg~#RUDFh8JBTMU7cj6O>4H%p-T%$AA4QJE%E?3 zBCZK6`fLIPJ=WXM<#Vaa7B4aIu-PY$9Kh5P5Eh3}D<9};k_Ia!U!apL{ke*FeJqVT zV_cS&x*vHV!Z9;z#q~voyqGx3w@#zH^OzxBmuz2Pns&yvwDzSs?-UZ5Fy6;+75s)_ z*@-F+L5y^V*``O`gDNW;o^cmxah~M6gmWV=xV9pV}ZKEs)u|q`7U|J1~-qAM-JbG z4QbiHTdt+LyphUa5_nqY4FyoZH{MwBSK(KbPr+y5)NT|%!j};U^H+8Q<59w2)g8cR zDZfHE0;4!;G~eY?W8^a8H59WS=qWpLH;!65ZOmx$^Vn%QOl;`EiOo~@;3S|MQz{F3 zFs*qE?(V^i=5ADUV`gRHZp^AIiWKceWu+1+I`*2Kn4iNU`B~h9CC!n7NMS^Y6h{iC z=TP5`6_ue#XeS!E)Y|5PZd{nd#j9k?dbSwNql>!HCYySYYA!Rov5C{H96Bn)J=nrc zFO7sE;T$f@;mXRA9&Br#9x3U@wL2)jtNgXJ7I?u?ggI2X5@ndnmU)x9{1$wH z=uX0I_#(bUSY~nUFLT{O%Da`Ce3j$p;}U!gUngA4DABEaE2fl<_y%W%xc3Cga0kKP z!F}(W1lKiY3&v@k9wOvCiS19PZEIdlK)$!FUtsSp-^f z0FEf*eRzi)ybB{mL_2h9q`<>F90~X0J+c>XwzzncOdjBJ)FYH(S98G=$Tg4LiQTbo z{LtmXld^GN^T;0lj8@8??#8q1d~P>hs4S5aUnz)`^kWV6c+De@+Dn{pMD>VMDv3j8 zmiA-LepC>nB(0qybJ92xn{YBLqL3kfZ1N>b{5!E4UBr6}alQl>;Zj^oPVg{DJUYpk zt(<)|+L^C5;Tn$I%HHj`0@v~FdiLBv-s0{bVxq%DmC~pV6HCdsw*#0sj_ooez4w2^#4Cqn;yZmz^rDiFUyMR- zUTdX$>)qV2ocvYYmPoR;208~4z + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml new file mode 100644 index 0000000..ef308b5 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached-tls.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml new file mode 100644 index 0000000..0f654b6 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-memcached.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis-tls.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis-tls.xml new file mode 100644 index 0000000..7cd5c41 --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis-tls.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis.xml new file mode 100644 index 0000000..eb7995d --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-redis.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + diff --git a/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-tls.xml b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-tls.xml new file mode 100644 index 0000000..5001fdf --- /dev/null +++ b/rbcs-server/bin/test/net/woggioni/rbcs/server/test/valid/rbcs-tls.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt index 708d2ba..6118875 100644 --- a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt +++ b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/RemoteBuildCacheServer.kt @@ -432,7 +432,7 @@ class RemoteBuildCacheServer(private val cfg: Configuration) { maxChunkSize = cfg.connection.chunkSize } pipeline.addLast(HttpServerCodec(httpDecoderConfig)) - OtelIntegration.createHandler()?.let { pipeline.addLast(it) } + OtelIntegration.createHandler(cfg.isEnableTelemetry)?.let { pipeline.addLast(it) } pipeline.addLast(ReadTriggerDuplexHandler.NAME, ReadTriggerDuplexHandler()) pipeline.addLast(MaxRequestSizeHandler.NAME, MaxRequestSizeHandler(cfg.connection.maxRequestSize)) pipeline.addLast(HttpChunkContentCompressor(1024)) diff --git a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Parser.kt b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Parser.kt index f1ac5c6..355fc6e 100644 --- a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Parser.kt +++ b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Parser.kt @@ -46,6 +46,8 @@ object Parser { var groups = emptyMap() var tls: Tls? = null val serverPath = root.renderAttribute("path") + var enableTelemetry = root.renderAttribute("enable-telemetry") + ?.let(String::toBoolean) ?: false var incomingConnectionsBacklogSize = 1024 var authentication: Authentication? = null for (child in root.asIterable()) { @@ -233,6 +235,7 @@ object Parser { } } return Configuration.of( + enableTelemetry, host, port, proxyProtocolEnabled, diff --git a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Serializer.kt b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Serializer.kt index 67e78d3..f7506fb 100644 --- a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Serializer.kt +++ b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/configuration/Serializer.kt @@ -29,6 +29,7 @@ object Serializer { ?.let { serverPath -> attr("path", serverPath) } + attr("enable-telemetry", conf.isEnableTelemetry.toString()) node("bind") { attr("host", conf.host) attr("port", conf.port.toString()) diff --git a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/otel/OtelIntegration.kt b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/otel/OtelIntegration.kt index fcf9144..fff39de 100644 --- a/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/otel/OtelIntegration.kt +++ b/rbcs-server/src/main/kotlin/net/woggioni/rbcs/server/otel/OtelIntegration.kt @@ -22,8 +22,8 @@ object OtelIntegration { ) } - fun createHandler(): ChannelHandler? { - return if (isAvailable) createHandlerInternal() else null + fun createHandler(enabled: Boolean): ChannelHandler? { + return if (enabled && isAvailable) createHandlerInternal() else null } private fun createHandlerInternal(): ChannelHandler { diff --git a/rbcs-server/src/main/resources/net/woggioni/rbcs/server/rbcs-default.xml b/rbcs-server/src/main/resources/net/woggioni/rbcs/server/rbcs-default.xml index 6d45ea1..55f3710 100644 --- a/rbcs-server/src/main/resources/net/woggioni/rbcs/server/rbcs-default.xml +++ b/rbcs-server/src/main/resources/net/woggioni/rbcs/server/rbcs-default.xml @@ -2,7 +2,8 @@ + xs:schemaLocation="urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd" + enable-telemetry="false"> \ No newline at end of file diff --git a/rbcs-server/src/main/resources/net/woggioni/rbcs/server/schema/rbcs-server.xsd b/rbcs-server/src/main/resources/net/woggioni/rbcs/server/schema/rbcs-server.xsd index 65aa616..b4c5399 100644 --- a/rbcs-server/src/main/resources/net/woggioni/rbcs/server/schema/rbcs-server.xsd +++ b/rbcs-server/src/main/resources/net/woggioni/rbcs/server/schema/rbcs-server.xsd @@ -59,6 +59,14 @@ + + + + Enable OpenTelemetry distributed tracing for the server. + Even when enabled, telemetry only activates if OpenTelemetry classes are present on the classpath. + + + diff --git a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt index 31d024c..acb5474 100644 --- a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt +++ b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractBasicAuthServerTest.kt @@ -32,6 +32,7 @@ abstract class AbstractBasicAuthServerTest : AbstractServerTest() { override fun setUp() { this.cacheDir = testDir.resolve("cache") cfg = Configuration.of( + false, "127.0.0.1", getFreePort(), false, diff --git a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt index ad8d660..f160c50 100644 --- a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt +++ b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/AbstractTlsServerTest.kt @@ -138,6 +138,7 @@ abstract class AbstractTlsServerTest : AbstractServerTest() { this.cacheDir = testDir.resolve("cache") createKeyStoreAndTrustStore() cfg = Configuration( + false, "127.0.0.1", getFreePort(), false, diff --git a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/NoAuthServerTest.kt b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/NoAuthServerTest.kt index d81f4d6..63b2a61 100644 --- a/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/NoAuthServerTest.kt +++ b/rbcs-server/src/test/kotlin/net/woggioni/rbcs/server/test/NoAuthServerTest.kt @@ -32,6 +32,7 @@ class NoAuthServerTest : AbstractServerTest() { override fun setUp() { this.cacheDir = testDir.resolve("cache") cfg = Configuration( + false, "127.0.0.1", getFreePort(), false, diff --git a/rbcs-servlet/.classpath b/rbcs-servlet/.classpath new file mode 100644 index 0000000..86ae890 --- /dev/null +++ b/rbcs-servlet/.classpath @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/rbcs-servlet/.project b/rbcs-servlet/.project new file mode 100644 index 0000000..b60a501 --- /dev/null +++ b/rbcs-servlet/.project @@ -0,0 +1,44 @@ + + + rbcs-servlet + Project rbcs-servlet created by Buildship. + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.wst.common.project.facet.core.builder + + + + + org.eclipse.wst.validation.validationbuilder + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature + + + + 1777370777023 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/rbcs-servlet/.settings/org.eclipse.buildship.core.prefs b/rbcs-servlet/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 0000000..b1886ad --- /dev/null +++ b/rbcs-servlet/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,2 @@ +connection.project.dir=.. +eclipse.preferences.version=1 diff --git a/rbcs-servlet/.settings/org.eclipse.jdt.core.prefs b/rbcs-servlet/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..96e40c8 --- /dev/null +++ b/rbcs-servlet/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=25 +org.eclipse.jdt.core.compiler.compliance=25 +org.eclipse.jdt.core.compiler.source=25 diff --git a/rbcs-servlet/bin/main/META-INF/beans.xml b/rbcs-servlet/bin/main/META-INF/beans.xml new file mode 100644 index 0000000..253b7a3 --- /dev/null +++ b/rbcs-servlet/bin/main/META-INF/beans.xml @@ -0,0 +1,5 @@ + + \ No newline at end of file diff --git a/rbcs-servlet/bin/main/META-INF/context.xml b/rbcs-servlet/bin/main/META-INF/context.xml new file mode 100644 index 0000000..85bd8de --- /dev/null +++ b/rbcs-servlet/bin/main/META-INF/context.xml @@ -0,0 +1,7 @@ + + + + \ No newline at end of file diff --git a/rbcs-servlet/bin/main/logging.properties b/rbcs-servlet/bin/main/logging.properties new file mode 100644 index 0000000..436e24a --- /dev/null +++ b/rbcs-servlet/bin/main/logging.properties @@ -0,0 +1,8 @@ +handlers = java.util.logging.ConsoleHandler +.level=INFO +net.woggioni.rbcs.servlet.level=FINEST +java.util.logging.ConsoleHandler.level=INFO +java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format = %1$tF %1$tT [%4$s] %2$s %5$s %6$s%n +org.apache.catalina.core.ContainerBase.[Catalina].level=ALL +org.apache.catalina.core.ContainerBase.[Catalina].handlers=java.util.logging.ConsoleHandler diff --git a/rbcs-servlet/bin/main/net/woggioni/rbcs/servlet/CacheServlet.kt b/rbcs-servlet/bin/main/net/woggioni/rbcs/servlet/CacheServlet.kt new file mode 100644 index 0000000..bd5713a --- /dev/null +++ b/rbcs-servlet/bin/main/net/woggioni/rbcs/servlet/CacheServlet.kt @@ -0,0 +1,169 @@ +package net.woggioni.rbcs.servlet + +import jakarta.annotation.PreDestroy +import jakarta.enterprise.context.ApplicationScoped +import jakarta.inject.Inject +import jakarta.servlet.annotation.WebServlet +import jakarta.servlet.http.HttpServlet +import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.nio.file.Path +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.PriorityBlockingQueue +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong +import java.util.logging.Logger +import net.woggioni.jwo.HttpClient.HttpStatus +import net.woggioni.jwo.JWO + + +private class CacheKey(private val value: ByteArray) { + override fun equals(other: Any?) = if (other is CacheKey) { + value.contentEquals(other.value) + } else false + + override fun hashCode() = value.contentHashCode() +} + + +@ApplicationScoped +open class InMemoryServletCache : AutoCloseable { + + private val maxAge= Duration.ofDays(7) + private val maxSize = 0x8000000 + + companion object { + @JvmStatic + private val log = Logger.getLogger(this::class.java.name) + } + + private val size = AtomicLong() + private val map = ConcurrentHashMap() + + private class RemovalQueueElement(val key: CacheKey, val value: ByteArray, val expiry: Instant) : + Comparable { + override fun compareTo(other: RemovalQueueElement) = expiry.compareTo(other.expiry) + } + + private val removalQueue = PriorityBlockingQueue() + + @Volatile + private var running = false + + private val garbageCollector = Thread.ofVirtual().name("in-memory-cache-gc").start { + while (running) { + val el = removalQueue.poll(1, TimeUnit.SECONDS) ?: continue + val value = el.value + val now = Instant.now() + if (now > el.expiry) { + val removed = map.remove(el.key, value) + if (removed) { + updateSizeAfterRemoval(value) + } + } else { + removalQueue.put(el) + Thread.sleep(minOf(Duration.between(now, el.expiry), Duration.ofSeconds(1))) + } + } + } + + private fun removeEldest(): Long { + while (true) { + val el = removalQueue.take() + val value = el.value + val removed = map.remove(el.key, value) + if (removed) { + val newSize = updateSizeAfterRemoval(value) + return newSize + } + } + } + + private fun updateSizeAfterRemoval(removed: ByteArray): Long { + return size.updateAndGet { currentSize: Long -> + currentSize - removed.size + } + } + + @PreDestroy + override fun close() { + running = false + garbageCollector.join() + } + + open fun get(key: ByteArray) = map[CacheKey(key)] + + open fun put( + key: ByteArray, + value: ByteArray, + ) { + val cacheKey = CacheKey(key) + val oldSize = map.put(cacheKey, value)?.let { old -> + val result = old.size + result + } ?: 0 + val delta = value.size - oldSize + var newSize = size.updateAndGet { currentSize: Long -> + currentSize + delta + } + removalQueue.put(RemovalQueueElement(cacheKey, value, Instant.now().plus(maxAge))) + while (newSize > maxSize) { + newSize = removeEldest() + } + } +} + + +@WebServlet(urlPatterns = ["/cache/*"]) +class CacheServlet : HttpServlet() { + companion object { + @JvmStatic + private val log = Logger.getLogger(this::class.java.name) + } + + @Inject + private lateinit var cache : InMemoryServletCache + + private fun getKey(req : HttpServletRequest) : String { + return Path.of(req.pathInfo).fileName.toString() + } + + override fun doPut(req: HttpServletRequest, resp: HttpServletResponse) { + val baos = ByteArrayOutputStream() + baos.use { + JWO.copy(req.inputStream, baos) + } + val key = getKey(req) + cache.put(key.toByteArray(Charsets.UTF_8), baos.toByteArray()) + resp.status = 201 + resp.setContentLength(0) + log.fine { + "[${Thread.currentThread().name}] Added value for key $key" + } + } + + override fun doGet(req: HttpServletRequest, resp: HttpServletResponse) { + val key = getKey(req) + val value = cache.get(key.toByteArray(Charsets.UTF_8)) + if (value == null) { + log.fine { + "[${Thread.currentThread().name}] Cache miss for key $key" + } + resp.status = HttpStatus.NOT_FOUND.code + resp.setContentLength(0) + } else { + log.fine { + "[${Thread.currentThread().name}] Cache hit for key $key" + } + resp.status = HttpStatus.OK.code + resp.setContentLength(value.size) + ByteArrayInputStream(value).use { + JWO.copy(it, resp.outputStream) + } + } + } +} \ No newline at end of file