From 423b749db96403e05303f2381f7ed3948b677f8d Mon Sep 17 00:00:00 2001 From: Walter Oggioni Date: Fri, 24 Jan 2025 16:42:48 +0800 Subject: [PATCH] added throttling --- README.md | 0 .../java/net/woggioni/gbcs/api/Cache.java | 1 + .../net/woggioni/gbcs/api/Configuration.java | 18 ++- .../gbcs/api/exception/CacheException.java | 11 ++ gbcs-cli/build.gradle | 17 ++- gbcs-cli/native-image/native-image.properties | 2 +- .../gbcs/cli/GradleBuildCacheServerCli.kt | 1 + .../cli/impl/commands/BenchmarkCommand.kt | 3 +- .../net/woggioni/gbcs/cli}/logging.properties | 0 .../kotlin/net/woggioni/gbcs/client/Client.kt | 2 +- gbcs-common/src/main/java/module-info.java | 1 + .../common/GbcsUrlStreamHandlerFactory.kt | 3 +- .../java.net.spi.URLStreamHandlerProvider | 1 + gbcs-server/build.gradle | 5 + gbcs-server/src/main/java/module-info.java | 3 +- .../gbcs/server/GradleBuildCacheServer.kt | 108 +++++------------- .../gbcs/server/auth/Authenticator.kt | 32 ++++-- .../gbcs/server/auth/UserAuthorizer.kt | 2 +- .../woggioni/gbcs/server/cache/CacheUtils.kt | 21 ++++ .../gbcs/server/cache/FileSystemCache.kt | 91 +++++++-------- .../gbcs/server/cache/InMemoryCache.kt | 106 +++++++++++++++++ .../cache/InMemoryCacheConfiguration.kt | 23 ++++ .../server/cache/InMemoryCacheProvider.kt | 59 ++++++++++ .../gbcs/server/configuration/Parser.kt | 65 +++++++++-- .../gbcs/server/configuration/Serializer.kt | 29 +++++ .../gbcs/server/exception/ExceptionHandler.kt | 92 +++++++++++++++ .../gbcs/server/handler/ServerHandler.kt | 51 +++++++-- .../gbcs/server/throttling/BucketManager.kt | 86 ++++++++++++++ .../server/throttling/ThrottlingHandler.kt | 86 ++++++++++++++ .../net.woggioni.gbcs.api.CacheProvider | 3 +- .../net/woggioni/gbcs/server/schema/gbcs.xsd | 33 +++++- .../test/AbstractBasicAuthServerTest.kt | 4 +- .../gbcs/server/test/AbstractTlsServerTest.kt | 11 +- .../gbcs/server/test/BasicAuthServerTest.kt | 53 ++++++++- .../gbcs/server/test/ConfigurationTest.kt | 23 +++- .../NoAnonymousUserBasicAuthServerTest.kt | 6 +- .../test/NoAnonymousUserTlsServerTest.kt | 6 +- .../gbcs/server/test/NoAuthServerTest.kt | 55 ++++----- .../gbcs/server/test/TlsServerTest.kt | 8 +- .../test/invalid/duplicate-anonymous-user.xml | 19 +++ .../invalid/duplicate-anonymous-user2.xml | 25 ++++ .../server/test/invalid/invalid-user-ref.xml | 24 ++++ .../test/invalid/multiple-user-quota.xml | 15 +++ .../server/test/{ => valid}/gbcs-default.xml | 0 .../test/{ => valid}/gbcs-memcached.xml | 0 .../gbcs/server/test/{ => valid}/gbcs-tls.xml | 12 +- gradle.properties | 4 +- 47 files changed, 988 insertions(+), 232 deletions(-) create mode 100644 README.md create mode 100644 gbcs-api/src/main/java/net/woggioni/gbcs/api/exception/CacheException.java rename {gbcs-server/src/main/resources => gbcs-cli/src/main/resources/net/woggioni/gbcs/cli}/logging.properties (100%) create mode 100644 gbcs-common/src/main/resources/META-INF/services/java.net.spi.URLStreamHandlerProvider create mode 100644 gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/CacheUtils.kt create mode 100644 gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCache.kt create mode 100644 gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCacheConfiguration.kt create mode 100644 gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCacheProvider.kt create mode 100644 gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/exception/ExceptionHandler.kt create mode 100644 gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/throttling/BucketManager.kt create mode 100644 gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/throttling/ThrottlingHandler.kt create mode 100644 gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/duplicate-anonymous-user.xml create mode 100644 gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/duplicate-anonymous-user2.xml create mode 100644 gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/invalid-user-ref.xml create mode 100644 gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/multiple-user-quota.xml rename gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/{ => valid}/gbcs-default.xml (100%) rename gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/{ => valid}/gbcs-memcached.xml (100%) rename gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/{ => valid}/gbcs-tls.xml (84%) diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/gbcs-api/src/main/java/net/woggioni/gbcs/api/Cache.java b/gbcs-api/src/main/java/net/woggioni/gbcs/api/Cache.java index c655bfa..c6018b0 100644 --- a/gbcs-api/src/main/java/net/woggioni/gbcs/api/Cache.java +++ b/gbcs-api/src/main/java/net/woggioni/gbcs/api/Cache.java @@ -4,6 +4,7 @@ import net.woggioni.gbcs.api.exception.ContentTooLargeException; import java.nio.channels.ReadableByteChannel; + public interface Cache extends AutoCloseable { ReadableByteChannel get(String key); diff --git a/gbcs-api/src/main/java/net/woggioni/gbcs/api/Configuration.java b/gbcs-api/src/main/java/net/woggioni/gbcs/api/Configuration.java index 1251376..58a317f 100644 --- a/gbcs-api/src/main/java/net/woggioni/gbcs/api/Configuration.java +++ b/gbcs-api/src/main/java/net/woggioni/gbcs/api/Configuration.java @@ -43,11 +43,20 @@ public class Configuration { int maxRequestSize; } + @Value + public static class Quota { + long calls; + Duration period; + long initialAvailableCalls; + long maxAvailableCalls; + } + @Value public static class Group { @EqualsAndHashCode.Include String name; Set roles; + Quota quota; } @Value @@ -56,7 +65,7 @@ public class Configuration { String name; String password; Set groups; - + Quota quota; public Set getRoles() { return groups.stream() @@ -75,6 +84,13 @@ public class Configuration { Group extract(X509Certificate cert); } + @Value + public static class Throttling { + KeyStore keyStore; + TrustStore trustStore; + boolean verifyClients; + } + @Value public static class Tls { KeyStore keyStore; diff --git a/gbcs-api/src/main/java/net/woggioni/gbcs/api/exception/CacheException.java b/gbcs-api/src/main/java/net/woggioni/gbcs/api/exception/CacheException.java new file mode 100644 index 0000000..3d9f96c --- /dev/null +++ b/gbcs-api/src/main/java/net/woggioni/gbcs/api/exception/CacheException.java @@ -0,0 +1,11 @@ +package net.woggioni.gbcs.api.exception; + +public class CacheException extends GbcsException { + public CacheException(String message, Throwable cause) { + super(message, cause); + } + + public CacheException(String message) { + this(message, null); + } +} diff --git a/gbcs-cli/build.gradle b/gbcs-cli/build.gradle index a90bfb6..6c8f61f 100644 --- a/gbcs-cli/build.gradle +++ b/gbcs-cli/build.gradle @@ -16,6 +16,8 @@ import net.woggioni.gradle.graalvm.NativeImageTask import net.woggioni.gradle.graalvm.JlinkPlugin import net.woggioni.gradle.graalvm.JlinkTask +Property mainModuleName = objects.property(String.class) +mainModuleName.set('net.woggioni.gbcs.cli') Property mainClassName = objects.property(String.class) mainClassName.set('net.woggioni.gbcs.cli.GradleBuildCacheServerCli') @@ -33,7 +35,7 @@ configurations { } envelopeJar { - mainModule = 'net.woggioni.gbcs.cli' + mainModule = mainModuleName mainClass = mainClassName extraClasspath = ["plugins"] @@ -50,20 +52,31 @@ dependencies { // runtimeOnly catalog.slf4j.jdk14 runtimeOnly catalog.logback.classic +// runtimeOnly catalog.slf4j.simple } Provider envelopeJarTaskProvider = tasks.named('envelopeJar', EnvelopeJarTask.class) { // systemProperties['java.util.logging.config.class'] = 'net.woggioni.gbcs.LoggingConfig' -// systemProperties['log.config.source'] = 'logging.properties' +// systemProperties['log.config.source'] = 'net/woggioni/gbcs/cli/logging.properties' +// systemProperties['java.util.logging.config.file'] = 'classpath:net/woggioni/gbcs/cli/logging.properties' systemProperties['logback.configurationFile'] = 'classpath:net/woggioni/gbcs/cli/logback.xml' + systemProperties['io.netty.leakDetectionLevel'] = 'DISABLED' + +// systemProperties['org.slf4j.simpleLogger.showDateTime'] = 'true' +// systemProperties['org.slf4j.simpleLogger.defaultLogLevel'] = 'debug' +// systemProperties['org.slf4j.simpleLogger.log.com.google.code.yanf4j'] = 'warn' +// systemProperties['org.slf4j.simpleLogger.log.net.rubyeye.xmemcached'] = 'warn' +// systemProperties['org.slf4j.simpleLogger.dateTimeFormat'] = 'yyyy-MM-dd\'T\'HH:mm:ss.SSSZ' } tasks.named(NativeImagePlugin.CONFIGURE_NATIVE_IMAGE_TASK_NAME, NativeImageConfigurationTask) { mainClass = mainClassName + mainModule = mainModuleName } tasks.named(NativeImagePlugin.NATIVE_IMAGE_TASK_NAME, NativeImageTask) { mainClass = mainClassName + mainModule = mainModuleName useMusl = true buildStaticImage = true } diff --git a/gbcs-cli/native-image/native-image.properties b/gbcs-cli/native-image/native-image.properties index 150581f..52d1cbf 100644 --- a/gbcs-cli/native-image/native-image.properties +++ b/gbcs-cli/native-image/native-image.properties @@ -1,2 +1,2 @@ -Args=-H:Optimize=3 --gc=serial +Args=-H:Optimize=3 --gc=serial --initialize-at-run-time=io.netty #-H:TraceClassInitialization=io.netty.handler.ssl.BouncyCastleAlpnSslUtils \ No newline at end of file diff --git a/gbcs-cli/src/main/kotlin/net/woggioni/gbcs/cli/GradleBuildCacheServerCli.kt b/gbcs-cli/src/main/kotlin/net/woggioni/gbcs/cli/GradleBuildCacheServerCli.kt index fa492e0..7f099cd 100644 --- a/gbcs-cli/src/main/kotlin/net/woggioni/gbcs/cli/GradleBuildCacheServerCli.kt +++ b/gbcs-cli/src/main/kotlin/net/woggioni/gbcs/cli/GradleBuildCacheServerCli.kt @@ -13,6 +13,7 @@ import net.woggioni.gbcs.cli.impl.commands.ServerCommand import net.woggioni.jwo.Application import picocli.CommandLine import picocli.CommandLine.Model.CommandSpec +import java.net.URI @CommandLine.Command( diff --git a/gbcs-cli/src/main/kotlin/net/woggioni/gbcs/cli/impl/commands/BenchmarkCommand.kt b/gbcs-cli/src/main/kotlin/net/woggioni/gbcs/cli/impl/commands/BenchmarkCommand.kt index bd25e08..6ad3425 100644 --- a/gbcs-cli/src/main/kotlin/net/woggioni/gbcs/cli/impl/commands/BenchmarkCommand.kt +++ b/gbcs-cli/src/main/kotlin/net/woggioni/gbcs/cli/impl/commands/BenchmarkCommand.kt @@ -5,6 +5,7 @@ import net.woggioni.gbcs.common.error import net.woggioni.gbcs.common.info import net.woggioni.gbcs.cli.impl.GbcsCommand import net.woggioni.gbcs.client.GradleBuildCacheClient +import net.woggioni.jwo.JWO import picocli.CommandLine import java.security.SecureRandom import java.time.Duration @@ -45,7 +46,7 @@ class BenchmarkCommand : GbcsCommand() { val entryGenerator = sequence { val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong()) while (true) { - val key = Base64.getUrlEncoder().encode(random.nextBytes(16)).toString(Charsets.UTF_8) + val key = JWO.bytesToHex(random.nextBytes(16)) val content = random.nextInt().toByte() val value = ByteArray(0x1000, { _ -> content }) yield(key to value) diff --git a/gbcs-server/src/main/resources/logging.properties b/gbcs-cli/src/main/resources/net/woggioni/gbcs/cli/logging.properties similarity index 100% rename from gbcs-server/src/main/resources/logging.properties rename to gbcs-cli/src/main/resources/net/woggioni/gbcs/cli/logging.properties diff --git a/gbcs-client/src/main/kotlin/net/woggioni/gbcs/client/Client.kt b/gbcs-client/src/main/kotlin/net/woggioni/gbcs/client/Client.kt index 4213f41..864d434 100644 --- a/gbcs-client/src/main/kotlin/net/woggioni/gbcs/client/Client.kt +++ b/gbcs-client/src/main/kotlin/net/woggioni/gbcs/client/Client.kt @@ -184,7 +184,7 @@ class GradleBuildCacheClient(private val profile: Configuration.Profile) : AutoC fun put(key: String, content: ByteArray): CompletableFuture { return sendRequest(profile.serverURI.resolve(key), HttpMethod.PUT, content).thenApply { val status = it.status() - if (it.status() != HttpResponseStatus.CREATED) { + if (it.status() != HttpResponseStatus.CREATED && it.status() != HttpResponseStatus.OK) { throw HttpException(status) } } diff --git a/gbcs-common/src/main/java/module-info.java b/gbcs-common/src/main/java/module-info.java index 201ea3a..492b855 100644 --- a/gbcs-common/src/main/java/module-info.java +++ b/gbcs-common/src/main/java/module-info.java @@ -5,5 +5,6 @@ module net.woggioni.gbcs.common { requires kotlin.stdlib; requires net.woggioni.jwo; + provides java.net.spi.URLStreamHandlerProvider with net.woggioni.gbcs.common.GbcsUrlStreamHandlerFactory; exports net.woggioni.gbcs.common; } \ No newline at end of file diff --git a/gbcs-common/src/main/kotlin/net/woggioni/gbcs/common/GbcsUrlStreamHandlerFactory.kt b/gbcs-common/src/main/kotlin/net/woggioni/gbcs/common/GbcsUrlStreamHandlerFactory.kt index e74d169..47a5e0a 100644 --- a/gbcs-common/src/main/kotlin/net/woggioni/gbcs/common/GbcsUrlStreamHandlerFactory.kt +++ b/gbcs-common/src/main/kotlin/net/woggioni/gbcs/common/GbcsUrlStreamHandlerFactory.kt @@ -6,12 +6,13 @@ import java.net.URL import java.net.URLConnection import java.net.URLStreamHandler import java.net.URLStreamHandlerFactory +import java.net.spi.URLStreamHandlerProvider import java.util.Optional import java.util.concurrent.atomic.AtomicBoolean import java.util.stream.Collectors -class GbcsUrlStreamHandlerFactory : URLStreamHandlerFactory { +class GbcsUrlStreamHandlerFactory : URLStreamHandlerProvider() { private class ClasspathHandler(private val classLoader: ClassLoader = GbcsUrlStreamHandlerFactory::class.java.classLoader) : URLStreamHandler() { diff --git a/gbcs-common/src/main/resources/META-INF/services/java.net.spi.URLStreamHandlerProvider b/gbcs-common/src/main/resources/META-INF/services/java.net.spi.URLStreamHandlerProvider new file mode 100644 index 0000000..c5aac89 --- /dev/null +++ b/gbcs-common/src/main/resources/META-INF/services/java.net.spi.URLStreamHandlerProvider @@ -0,0 +1 @@ +net.woggioni.gbcs.common.GbcsUrlStreamHandlerFactory \ No newline at end of file diff --git a/gbcs-server/build.gradle b/gbcs-server/build.gradle index 3254f1a..80c84a9 100644 --- a/gbcs-server/build.gradle +++ b/gbcs-server/build.gradle @@ -25,6 +25,11 @@ dependencies { testRuntimeOnly project(":gbcs-server-memcached") } +test { + systemProperty("io.netty.leakDetectionLevel", "PARANOID") + systemProperty("jdk.httpclient.redirects.retrylimit", "1") +} + publishing { publications { maven(MavenPublication) { diff --git a/gbcs-server/src/main/java/module-info.java b/gbcs-server/src/main/java/module-info.java index 1044452..4e00a8c 100644 --- a/gbcs-server/src/main/java/module-info.java +++ b/gbcs-server/src/main/java/module-info.java @@ -1,5 +1,6 @@ import net.woggioni.gbcs.api.CacheProvider; import net.woggioni.gbcs.server.cache.FileSystemCacheProvider; +import net.woggioni.gbcs.server.cache.InMemoryCacheProvider; module net.woggioni.gbcs.server { requires java.sql; @@ -24,5 +25,5 @@ module net.woggioni.gbcs.server { opens net.woggioni.gbcs.server.schema; uses CacheProvider; - provides CacheProvider with FileSystemCacheProvider; + provides CacheProvider with FileSystemCacheProvider, InMemoryCacheProvider; } \ No newline at end of file diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/GradleBuildCacheServer.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/GradleBuildCacheServer.kt index 2784357..4ad79eb 100644 --- a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/GradleBuildCacheServer.kt +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/GradleBuildCacheServer.kt @@ -2,11 +2,8 @@ package net.woggioni.gbcs.server import io.netty.bootstrap.ServerBootstrap import io.netty.buffer.ByteBuf -import io.netty.buffer.Unpooled import io.netty.channel.Channel -import io.netty.channel.ChannelDuplexHandler import io.netty.channel.ChannelFuture -import io.netty.channel.ChannelFutureListener import io.netty.channel.ChannelHandler.Sharable import io.netty.channel.ChannelHandlerContext import io.netty.channel.ChannelInboundHandlerAdapter @@ -15,18 +12,13 @@ import io.netty.channel.ChannelOption import io.netty.channel.ChannelPromise import io.netty.channel.nio.NioEventLoopGroup import io.netty.channel.socket.nio.NioServerSocketChannel -import io.netty.handler.codec.DecoderException import io.netty.handler.codec.compression.CompressionOptions -import io.netty.handler.codec.http.DefaultFullHttpResponse import io.netty.handler.codec.http.DefaultHttpContent -import io.netty.handler.codec.http.FullHttpResponse import io.netty.handler.codec.http.HttpContentCompressor import io.netty.handler.codec.http.HttpHeaderNames import io.netty.handler.codec.http.HttpObjectAggregator import io.netty.handler.codec.http.HttpRequest -import io.netty.handler.codec.http.HttpResponseStatus import io.netty.handler.codec.http.HttpServerCodec -import io.netty.handler.codec.http.HttpVersion import io.netty.handler.ssl.ClientAuth import io.netty.handler.ssl.SslContext import io.netty.handler.ssl.SslContextBuilder @@ -34,16 +26,13 @@ import io.netty.handler.ssl.SslHandler import io.netty.handler.stream.ChunkedWriteHandler import io.netty.handler.timeout.IdleStateEvent import io.netty.handler.timeout.IdleStateHandler -import io.netty.handler.timeout.ReadTimeoutException import io.netty.handler.timeout.ReadTimeoutHandler -import io.netty.handler.timeout.WriteTimeoutException import io.netty.handler.timeout.WriteTimeoutHandler +import io.netty.util.AttributeKey import io.netty.util.concurrent.DefaultEventExecutorGroup import io.netty.util.concurrent.EventExecutorGroup import net.woggioni.gbcs.api.Configuration -import net.woggioni.gbcs.api.Role import net.woggioni.gbcs.api.exception.ConfigurationException -import net.woggioni.gbcs.api.exception.ContentTooLargeException import net.woggioni.gbcs.common.GBCS.toUrl import net.woggioni.gbcs.common.PasswordSecurity.decodePasswordHash import net.woggioni.gbcs.common.PasswordSecurity.hashPassword @@ -57,7 +46,9 @@ import net.woggioni.gbcs.server.auth.ClientCertificateValidator import net.woggioni.gbcs.server.auth.RoleAuthorizer import net.woggioni.gbcs.server.configuration.Parser import net.woggioni.gbcs.server.configuration.Serializer +import net.woggioni.gbcs.server.exception.ExceptionHandler import net.woggioni.gbcs.server.handler.ServerHandler +import net.woggioni.gbcs.server.throttling.ThrottlingHandler import net.woggioni.jwo.JWO import net.woggioni.jwo.Tuple2 import java.io.OutputStream @@ -75,12 +66,14 @@ import java.util.regex.Pattern import javax.naming.ldap.LdapName import javax.net.ssl.SSLPeerUnverifiedException - class GradleBuildCacheServer(private val cfg: Configuration) { private val log = contextLogger() companion object { + val userAttribute: AttributeKey = AttributeKey.valueOf("user") + val groupAttribute: AttributeKey> = AttributeKey.valueOf("group") + val DEFAULT_CONFIGURATION_URL by lazy { "classpath:net/woggioni/gbcs/gbcs-default.xml".toUrl() } private const val SSL_HANDLER_NAME = "sslHandler" @@ -120,12 +113,12 @@ class GradleBuildCacheServer(private val cfg: Configuration) { @Sharable private class ClientCertificateAuthenticator( authorizer: Authorizer, - private val anonymousUserRoles: Set?, + private val anonymousUserGroups: Set?, private val userExtractor: Configuration.UserExtractor?, private val groupExtractor: Configuration.GroupExtractor?, ) : AbstractNettyHttpAuthenticator(authorizer) { - override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): Set? { + override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? { return try { val sslHandler = (ctx.pipeline().get(SSL_HANDLER_NAME) as? SslHandler) ?: throw ConfigurationException("Client certificate authentication cannot be used when TLS is disabled") @@ -136,10 +129,11 @@ class GradleBuildCacheServer(private val cfg: Configuration) { val clientCertificate = peerCertificates.first() as X509Certificate val user = userExtractor?.extract(clientCertificate) val group = groupExtractor?.extract(clientCertificate) - (group?.roles ?: emptySet()) + (user?.roles ?: emptySet()) - } ?: anonymousUserRoles + val allGroups = ((user?.groups ?: emptySet()).asSequence() + sequenceOf(group).filterNotNull()).toSet() + AuthenticationResult(user, allGroups) + } ?: anonymousUserGroups?.let{ AuthenticationResult(null, it) } } catch (es: SSLPeerUnverifiedException) { - anonymousUserRoles + anonymousUserGroups?.let{ AuthenticationResult(null, it) } } } } @@ -150,26 +144,26 @@ class GradleBuildCacheServer(private val cfg: Configuration) { ) : AbstractNettyHttpAuthenticator(authorizer) { private val log = contextLogger() - override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): Set? { + override fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? { val authorizationHeader = req.headers()[HttpHeaderNames.AUTHORIZATION] ?: let { log.debug(ctx) { "Missing Authorization header" } - return users[""]?.roles + return users[""]?.let { AuthenticationResult(it, it.groups) } } val cursor = authorizationHeader.indexOf(' ') if (cursor < 0) { log.debug(ctx) { "Invalid Authorization header: '$authorizationHeader'" } - return users[""]?.roles + return users[""]?.let { AuthenticationResult(it, it.groups) } } val authenticationType = authorizationHeader.substring(0, cursor) if ("Basic" != authenticationType) { log.debug(ctx) { "Invalid authentication type header: '$authenticationType'" } - return users[""]?.roles + return users[""]?.let { AuthenticationResult(it, it.groups) } } val (username, password) = Base64.getDecoder().decode(authorizationHeader.substring(cursor + 1)) .let(::String) @@ -189,7 +183,9 @@ class GradleBuildCacheServer(private val cfg: Configuration) { val (_, salt) = decodePasswordHash(passwordAndSalt) hashPassword(password, Base64.getEncoder().encodeToString(salt)) == passwordAndSalt } ?: false - }?.roles + }?.let { user -> + AuthenticationResult(user, user.groups) + } } } @@ -257,13 +253,14 @@ class GradleBuildCacheServer(private val cfg: Configuration) { } private val exceptionHandler = ExceptionHandler() + private val throttlingHandler = ThrottlingHandler(cfg) private val authenticator = when (val auth = cfg.authentication) { is Configuration.BasicAuthentication -> NettyHttpBasicAuthenticator(cfg.users, RoleAuthorizer()) is Configuration.ClientCertificateAuthentication -> { ClientCertificateAuthenticator( RoleAuthorizer(), - cfg.users[""]?.roles, + cfg.users[""]?.groups, userExtractor(auth), groupExtractor(auth) ) @@ -312,10 +309,10 @@ class GradleBuildCacheServer(private val cfg: Configuration) { } } val pipeline = ch.pipeline() - cfg.connection.apply { - pipeline.addLast(ReadTimeoutHandler(readTimeout.toMillis(), TimeUnit.MILLISECONDS)) - pipeline.addLast(WriteTimeoutHandler(writeTimeout.toMillis(), TimeUnit.MILLISECONDS)) - pipeline.addLast(IdleStateHandler(false, 0, 0, idleTimeout.toMillis(), TimeUnit.MILLISECONDS)) + cfg.connection.also { conn -> + pipeline.addLast(ReadTimeoutHandler(conn.readTimeout.toMillis(), TimeUnit.MILLISECONDS)) + pipeline.addLast(WriteTimeoutHandler(conn.writeTimeout.toMillis(), TimeUnit.MILLISECONDS)) + pipeline.addLast(IdleStateHandler(false, 0, 0, conn.idleTimeout.toMillis(), TimeUnit.MILLISECONDS)) } pipeline.addLast(object : ChannelInboundHandlerAdapter() { override fun userEventTriggered(ctx: ChannelHandlerContext, evt: Any) { @@ -337,65 +334,12 @@ class GradleBuildCacheServer(private val cfg: Configuration) { authenticator?.let { pipeline.addLast(it) } + pipeline.addLast(throttlingHandler) pipeline.addLast(eventExecutorGroup, serverHandler) pipeline.addLast(exceptionHandler) } } - @Sharable - private class ExceptionHandler : ChannelDuplexHandler() { - private val log = contextLogger() - - private val NOT_AUTHORIZED: FullHttpResponse = DefaultFullHttpResponse( - HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER - ).apply { - headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" - } - - private val TOO_BIG: FullHttpResponse = DefaultFullHttpResponse( - HttpVersion.HTTP_1_1, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, Unpooled.EMPTY_BUFFER - ).apply { - headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" - } - - override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { - when (cause) { - is DecoderException -> { - log.error(cause.message, cause) - ctx.close() - } - - is SSLPeerUnverifiedException -> { - ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate()) - .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) - } - - is ContentTooLargeException -> { - ctx.writeAndFlush(TOO_BIG.retainedDuplicate()) - .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) - } - is ReadTimeoutException -> { - log.debug { - val channelId = ctx.channel().id().asShortText() - "Read timeout on channel $channelId, closing the connection" - } - ctx.close() - } - is WriteTimeoutException -> { - log.debug { - val channelId = ctx.channel().id().asShortText() - "Write timeout on channel $channelId, closing the connection" - } - ctx.close() - } - else -> { - log.error(cause.message, cause) - ctx.close() - } - } - } - } - class ServerHandle( httpChannelFuture: ChannelFuture, private val executorGroups: Iterable diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/auth/Authenticator.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/auth/Authenticator.kt index 5536df4..bb9c9a3 100644 --- a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/auth/Authenticator.kt +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/auth/Authenticator.kt @@ -11,32 +11,48 @@ import io.netty.handler.codec.http.HttpRequest import io.netty.handler.codec.http.HttpResponseStatus import io.netty.handler.codec.http.HttpVersion import io.netty.util.ReferenceCountUtil +import net.woggioni.gbcs.api.Configuration +import net.woggioni.gbcs.api.Configuration.Group import net.woggioni.gbcs.api.Role +import net.woggioni.gbcs.server.GradleBuildCacheServer -abstract class AbstractNettyHttpAuthenticator(private val authorizer : Authorizer) - : ChannelInboundHandlerAdapter() { +abstract class AbstractNettyHttpAuthenticator(private val authorizer: Authorizer) : ChannelInboundHandlerAdapter() { companion object { private val AUTHENTICATION_FAILED: FullHttpResponse = DefaultFullHttpResponse( - HttpVersion.HTTP_1_1, HttpResponseStatus.UNAUTHORIZED, Unpooled.EMPTY_BUFFER).apply { + HttpVersion.HTTP_1_1, HttpResponseStatus.UNAUTHORIZED, Unpooled.EMPTY_BUFFER + ).apply { headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" } private val NOT_AUTHORIZED: FullHttpResponse = DefaultFullHttpResponse( - HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER).apply { + HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER + ).apply { headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" } } + class AuthenticationResult(val user: Configuration.User?, val groups: Set) - abstract fun authenticate(ctx : ChannelHandlerContext, req : HttpRequest) : Set? + abstract fun authenticate(ctx: ChannelHandlerContext, req: HttpRequest): AuthenticationResult? override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { - if(msg is HttpRequest) { - val roles = authenticate(ctx, msg) ?: return authenticationFailure(ctx, msg) + if (msg is HttpRequest) { + val result = authenticate(ctx, msg) ?: return authenticationFailure(ctx, msg) + ctx.channel().attr(GradleBuildCacheServer.userAttribute).set(result.user) + ctx.channel().attr(GradleBuildCacheServer.groupAttribute).set(result.groups) + + val roles = ( + (result.user?.let { user -> + user.groups.asSequence().flatMap { group -> + group.roles.asSequence() + } + } ?: emptySequence()) + + result.groups.asSequence().flatMap { it.roles.asSequence() } + ).toSet() val authorized = authorizer.authorize(roles, msg) - if(authorized) { + if (authorized) { super.channelRead(ctx, msg) } else { authorizationFailure(ctx, msg) diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/auth/UserAuthorizer.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/auth/UserAuthorizer.kt index c7c20f9..aedf65f 100644 --- a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/auth/UserAuthorizer.kt +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/auth/UserAuthorizer.kt @@ -8,7 +8,7 @@ class RoleAuthorizer : Authorizer { companion object { private val METHOD_MAP = mapOf( - Role.Reader to setOf(HttpMethod.GET, HttpMethod.HEAD), + Role.Reader to setOf(HttpMethod.GET, HttpMethod.HEAD, HttpMethod.TRACE), Role.Writer to setOf(HttpMethod.PUT, HttpMethod.POST) ) } diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/CacheUtils.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/CacheUtils.kt new file mode 100644 index 0000000..97f02c2 --- /dev/null +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/CacheUtils.kt @@ -0,0 +1,21 @@ +package net.woggioni.gbcs.server.cache + +import net.woggioni.jwo.JWO +import java.security.MessageDigest + +object CacheUtils { + fun digest( + data: ByteArray, + md: MessageDigest = MessageDigest.getInstance("MD5") + ): ByteArray { + md.update(data) + return md.digest() + } + + fun digestString( + data: ByteArray, + md: MessageDigest = MessageDigest.getInstance("MD5") + ): String { + return JWO.bytesToHex(digest(data, md)) + } +} \ No newline at end of file diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/FileSystemCache.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/FileSystemCache.kt index a61e1ae..4deed97 100644 --- a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/FileSystemCache.kt +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/FileSystemCache.kt @@ -1,7 +1,8 @@ package net.woggioni.gbcs.server.cache import net.woggioni.gbcs.api.Cache -import net.woggioni.jwo.JWO +import net.woggioni.gbcs.common.contextLogger +import net.woggioni.gbcs.server.cache.CacheUtils.digestString import net.woggioni.jwo.LockFile import java.nio.channels.Channels import java.nio.channels.FileChannel @@ -19,7 +20,6 @@ import java.util.zip.DeflaterOutputStream import java.util.zip.Inflater import java.util.zip.InflaterInputStream - class FileSystemCache( val root: Path, val maxAge: Duration, @@ -28,7 +28,7 @@ class FileSystemCache( val compressionLevel: Int ) : Cache { - private fun lockFilePath(key: String): Path = root.resolve("$key.lock") + private val log = contextLogger() init { Files.createDirectories(root) @@ -41,18 +41,28 @@ class FileSystemCache( ?.let { md -> digestString(key.toByteArray(), md) } ?: key).let { digest -> - LockFile.acquire(lockFilePath(digest), true).use { - root.resolve(digest).takeIf(Files::exists)?.let { file -> - if (compressionEnabled) { - val inflater = Inflater() - Channels.newChannel(InflaterInputStream(Files.newInputStream(file), inflater)) - } else { - FileChannel.open(file, StandardOpenOption.READ) + root.resolve(digest).takeIf(Files::exists) + ?.let { file -> + file.takeIf(Files::exists)?.let { file -> + if (compressionEnabled) { + val inflater = Inflater() + Channels.newChannel( + InflaterInputStream( + Channels.newInputStream( + FileChannel.open( + file, + StandardOpenOption.READ + ) + ), inflater + ) + ) + } else { + FileChannel.open(file, StandardOpenOption.READ) + } } + }.also { + gc() } - }.also { - gc() - } } override fun put(key: String, content: ByteArray) { @@ -61,25 +71,23 @@ class FileSystemCache( ?.let { md -> digestString(key.toByteArray(), md) } ?: key).let { digest -> - LockFile.acquire(lockFilePath(digest), false).use { - val file = root.resolve(digest) - val tmpFile = Files.createTempFile(root, null, ".tmp") - try { - Files.newOutputStream(tmpFile).let { - if (compressionEnabled) { - val deflater = Deflater(compressionLevel) - DeflaterOutputStream(it, deflater) - } else { - it - } - }.use { - it.write(content) + val file = root.resolve(digest) + val tmpFile = Files.createTempFile(root, null, ".tmp") + try { + Files.newOutputStream(tmpFile).let { + if (compressionEnabled) { + val deflater = Deflater(compressionLevel) + DeflaterOutputStream(it, deflater) + } else { + it } - Files.move(tmpFile, file, StandardCopyOption.ATOMIC_MOVE) - } catch (t: Throwable) { - Files.delete(tmpFile) - throw t + }.use { + it.write(content) } + Files.move(tmpFile, file, StandardCopyOption.ATOMIC_MOVE) + } catch (t: Throwable) { + Files.delete(tmpFile) + throw t } }.also { gc() @@ -97,37 +105,16 @@ class FileSystemCache( @Synchronized private fun actualGc(now: Instant) { Files.list(root).filter { - !it.fileName.toString().endsWith(".lock") - }.filter { val creationTimeStamp = Files.readAttributes(it, BasicFileAttributes::class.java) .creationTime() .toInstant() now > creationTimeStamp.plus(maxAge) }.forEach { file -> - val lockFile = lockFilePath(file.fileName.toString()) - LockFile.acquire(lockFile, false).use { + LockFile.acquire(file, false).use { Files.delete(file) } - Files.delete(lockFile) } } override fun close() {} - - companion object { - fun digest( - data: ByteArray, - md: MessageDigest = MessageDigest.getInstance("MD5") - ): ByteArray { - md.update(data) - return md.digest() - } - - fun digestString( - data: ByteArray, - md: MessageDigest = MessageDigest.getInstance("MD5") - ): String { - return JWO.bytesToHex(digest(data, md)) - } - } } \ No newline at end of file diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCache.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCache.kt new file mode 100644 index 0000000..9e1e669 --- /dev/null +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCache.kt @@ -0,0 +1,106 @@ +package net.woggioni.gbcs.server.cache + +import net.woggioni.gbcs.api.Cache +import net.woggioni.gbcs.server.cache.CacheUtils.digestString +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.channels.Channels +import java.security.MessageDigest +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.PriorityBlockingQueue +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream +import java.util.zip.Inflater +import java.util.zip.InflaterInputStream + +class InMemoryCache( + val maxAge: Duration, + val digestAlgorithm: String?, + val compressionEnabled: Boolean, + val compressionLevel: Int +) : Cache { + + private val map = ConcurrentHashMap() + + private class MapValue(val rc: AtomicInteger, val payload : AtomicReference) + + private class RemovalQueueElement(val key: String, val expiry : Instant) : Comparable { + override fun compareTo(other: RemovalQueueElement)= expiry.compareTo(other.expiry) + } + + private val removalQueue = PriorityBlockingQueue() + + private var running = true + private val garbageCollector = Thread({ + while(true) { + val el = removalQueue.take() + val now = Instant.now() + if(now > el.expiry) { + val value = map[el.key] ?: continue + val rc = value.rc.decrementAndGet() + if(rc == 0) { + map.remove(el.key) + } + } else { + removalQueue.put(el) + Thread.sleep(minOf(Duration.between(now, el.expiry), Duration.ofSeconds(1))) + } + } + }).apply { + start() + } + + override fun close() { + running = false + garbageCollector.join() + } + + override fun get(key: String) = + (digestAlgorithm + ?.let(MessageDigest::getInstance) + ?.let { md -> + digestString(key.toByteArray(), md) + } ?: key + ).let { digest -> + map[digest] + ?.let(MapValue::payload) + ?.let(AtomicReference::get) + ?.let { value -> + if (compressionEnabled) { + val inflater = Inflater() + Channels.newChannel(InflaterInputStream(ByteArrayInputStream(value), inflater)) + } else { + Channels.newChannel(ByteArrayInputStream(value)) + } + } + } + + override fun put(key: String, content: ByteArray) { + (digestAlgorithm + ?.let(MessageDigest::getInstance) + ?.let { md -> + digestString(key.toByteArray(), md) + } ?: key).let { digest -> + val value = if (compressionEnabled) { + val deflater = Deflater(compressionLevel) + val baos = ByteArrayOutputStream() + DeflaterOutputStream(baos, deflater).use { stream -> + stream.write(content) + } + baos.toByteArray() + } else { + content + } + val mapValue = map.computeIfAbsent(digest) { + MapValue(AtomicInteger(0), AtomicReference()) + } + mapValue.payload.set(value) + removalQueue.put(RemovalQueueElement(digest, Instant.now().plus(maxAge))) + } + } +} \ No newline at end of file diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCacheConfiguration.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCacheConfiguration.kt new file mode 100644 index 0000000..92a31be --- /dev/null +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCacheConfiguration.kt @@ -0,0 +1,23 @@ +package net.woggioni.gbcs.server.cache + +import net.woggioni.gbcs.api.Configuration +import net.woggioni.gbcs.common.GBCS +import java.time.Duration + +data class InMemoryCacheConfiguration( + val maxAge: Duration, + val digestAlgorithm : String?, + val compressionEnabled: Boolean, + val compressionLevel: Int, +) : Configuration.Cache { + override fun materialize() = InMemoryCache( + maxAge, + digestAlgorithm, + compressionEnabled, + compressionLevel + ) + + override fun getNamespaceURI() = GBCS.GBCS_NAMESPACE_URI + + override fun getTypeName() = "inMemoryCacheType" +} \ No newline at end of file diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCacheProvider.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCacheProvider.kt new file mode 100644 index 0000000..7d27528 --- /dev/null +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/cache/InMemoryCacheProvider.kt @@ -0,0 +1,59 @@ +package net.woggioni.gbcs.server.cache + +import net.woggioni.gbcs.api.CacheProvider +import net.woggioni.gbcs.common.GBCS +import net.woggioni.gbcs.common.Xml +import net.woggioni.gbcs.common.Xml.Companion.renderAttribute +import org.w3c.dom.Document +import org.w3c.dom.Element +import java.nio.file.Path +import java.time.Duration +import java.util.zip.Deflater + +class InMemoryCacheProvider : CacheProvider { + + override fun getXmlSchemaLocation() = "classpath:net/woggioni/gbcs/server/schema/gbcs.xsd" + + override fun getXmlType() = "inMemoryCacheType" + + override fun getXmlNamespace() = "urn:net.woggioni.gbcs.server" + + override fun deserialize(el: Element): InMemoryCacheConfiguration { + val maxAge = el.renderAttribute("max-age") + ?.let(Duration::parse) + ?: Duration.ofDays(1) + val enableCompression = el.renderAttribute("enable-compression") + ?.let(String::toBoolean) + ?: true + val compressionLevel = el.renderAttribute("compression-level") + ?.let(String::toInt) + ?: Deflater.DEFAULT_COMPRESSION + val digestAlgorithm = el.renderAttribute("digest") ?: "MD5" + + return InMemoryCacheConfiguration( + maxAge, + digestAlgorithm, + enableCompression, + compressionLevel + ) + } + + override fun serialize(doc: Document, cache : InMemoryCacheConfiguration) = cache.run { + val result = doc.createElement("cache") + Xml.of(doc, result) { + val prefix = doc.lookupPrefix(GBCS.GBCS_NAMESPACE_URI) + attr("xs:type", "${prefix}:inMemoryCacheType", GBCS.XML_SCHEMA_NAMESPACE_URI) + attr("max-age", maxAge.toString()) + digestAlgorithm?.let { digestAlgorithm -> + attr("digest", digestAlgorithm) + } + attr("enable-compression", compressionEnabled.toString()) + compressionLevel.takeIf { + it != Deflater.DEFAULT_COMPRESSION + }?.let { + attr("compression-level", it.toString()) + } + } + result + } +} diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/configuration/Parser.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/configuration/Parser.kt index e64ee92..f38467b 100644 --- a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/configuration/Parser.kt +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/configuration/Parser.kt @@ -25,7 +25,7 @@ import java.time.temporal.ChronoUnit object Parser { fun parse(document: Document): Configuration { val root = document.documentElement - val anonymousUser = User("", null, emptySet()) + val anonymousUser = User("", null, emptySet(), null) var connection: Configuration.Connection = Configuration.Connection( Duration.of(10, ChronoUnit.SECONDS), Duration.of(10, ChronoUnit.SECONDS), @@ -38,7 +38,7 @@ object Parser { var cache: Cache? = null var host = "127.0.0.1" var port = 11080 - var users : Map = mapOf(anonymousUser.name to anonymousUser) + var users: Map = mapOf(anonymousUser.name to anonymousUser) var groups = emptyMap() var tls: Tls? = null val serverPath = root.renderAttribute("path") @@ -85,6 +85,7 @@ object Parser { "users" -> { knownUsers += parseUsers(gchild) } + "groups" -> { val pair = parseGroups(gchild, knownUsers) users = pair.first @@ -107,7 +108,7 @@ object Parser { val typeNamespace = tf.typeNamespace val typeName = tf.typeName CacheSerializers.index[typeNamespace to typeName] - ?: throw IllegalArgumentException("Cache provider for namespace '$typeNamespace' not found") + ?: throw IllegalArgumentException("Cache provider for namespace '$typeNamespace' with name '$typeName' not found") }.deserialize(child) } @@ -133,11 +134,13 @@ object Parser { maxRequestSize ) } + "event-executor" -> { val useVirtualThread = root.renderAttribute("use-virtual-threads") ?.let(String::toBoolean) ?: true eventExecutor = Configuration.EventExecutor(useVirtualThread) } + "tls" -> { val verifyClients = child.renderAttribute("verify-clients") ?.let(String::toBoolean) ?: false @@ -200,20 +203,54 @@ object Parser { }.toSet() private fun parseUserRefs(root: Element) = root.asIterable().asSequence().map { - when(it.localName) { + when (it.localName) { "user" -> it.renderAttribute("ref") "anonymous" -> "" else -> ConfigurationException("Unrecognized tag '${it.localName}'") } } + private fun parseQuota(el: Element): Configuration.Quota { + val calls = el.renderAttribute("calls") + ?.let(String::toLong) + ?: throw ConfigurationException("Missing attribute 'calls'") + val maxAvailableCalls = el.renderAttribute("max-available-calls") + ?.let(String::toLong) + ?: calls + val initialAvailableCalls = el.renderAttribute("initial-available-calls") + ?.let(String::toLong) + ?: maxAvailableCalls + val period = el.renderAttribute("period") + ?.let(Duration::parse) + ?: throw ConfigurationException("Missing attribute 'period'") + return Configuration.Quota(calls, period, initialAvailableCalls, maxAvailableCalls) + } + private fun parseUsers(root: Element): Sequence { - return root.asIterable().asSequence().filter { - it.localName == "user" - }.map { el -> - val username = el.renderAttribute("name") - val password = el.renderAttribute("password") - User(username, password, emptySet()) + return root.asIterable().asSequence().mapNotNull { child -> + when (child.localName) { + "user" -> { + val username = child.renderAttribute("name") + val password = child.renderAttribute("password") + var quota: Configuration.Quota? = null + for (gchild in child.asIterable()) { + if (gchild.localName == "quota") { + quota = parseQuota(gchild) + } + } + User(username, password, emptySet(), quota) + } + "anonymous" -> { + var quota: Configuration.Quota? = null + for (gchild in child.asIterable()) { + if (gchild.localName == "quota") { + quota= parseQuota(gchild) + } + } + User("", null, emptySet(), quota) + } + else -> null + } } } @@ -225,6 +262,7 @@ object Parser { }.map { el -> val groupName = el.renderAttribute("name") ?: throw ConfigurationException("Group name is required") var roles = emptySet() + var quota: Configuration.Quota? = null for (child in el.asIterable()) { when (child.localName) { "users" -> { @@ -238,12 +276,15 @@ object Parser { "roles" -> { roles = parseRoles(child) } + "quota" -> { + quota = parseQuota(child) + } } } - groupName to Group(groupName, roles) + groupName to Group(groupName, roles, quota) }.toMap() val users = knownUsersMap.map { (name, user) -> - name to User(name, user.password, userGroups[name]?.mapNotNull { groups[it] }?.toSet() ?: emptySet()) + name to User(name, user.password, userGroups[name]?.mapNotNull { groups[it] }?.toSet() ?: emptySet(), user.quota) }.toMap() return users to groups } diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/configuration/Serializer.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/configuration/Serializer.kt index e1f1b46..de4a9c6 100644 --- a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/configuration/Serializer.kt +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/configuration/Serializer.kt @@ -54,9 +54,30 @@ object Serializer { user.password?.let { password -> attr("password", password) } + user.quota?.let { quota -> + node("quota") { + attr("calls", quota.calls.toString()) + attr("period", quota.period.toString()) + attr("max-available-calls", quota.maxAvailableCalls.toString()) + attr("initial-available-calls", quota.initialAvailableCalls.toString()) + } + } } } } + conf.users[""] + ?.let { anonymousUser -> + anonymousUser.quota?.let { quota -> + node("anonymous") { + node("quota") { + attr("calls", quota.calls.toString()) + attr("period", quota.period.toString()) + attr("max-available-calls", quota.maxAvailableCalls.toString()) + attr("initial-available-calls", quota.initialAvailableCalls.toString()) + } + } + } + } } node("groups") { val groups = conf.users.values.asSequence() @@ -92,6 +113,14 @@ object Serializer { } } } + group.quota?.let { quota -> + node("quota") { + attr("calls", quota.calls.toString()) + attr("period", quota.period.toString()) + attr("max-available-calls", quota.maxAvailableCalls.toString()) + attr("initial-available-calls", quota.initialAvailableCalls.toString()) + } + } } } } diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/exception/ExceptionHandler.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/exception/ExceptionHandler.kt new file mode 100644 index 0000000..bf8e4d6 --- /dev/null +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/exception/ExceptionHandler.kt @@ -0,0 +1,92 @@ +package net.woggioni.gbcs.server.exception + +import io.netty.buffer.Unpooled +import io.netty.channel.ChannelDuplexHandler +import io.netty.channel.ChannelFutureListener +import io.netty.channel.ChannelHandler +import io.netty.channel.ChannelHandlerContext +import io.netty.handler.codec.DecoderException +import io.netty.handler.codec.http.DefaultFullHttpResponse +import io.netty.handler.codec.http.FullHttpResponse +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import io.netty.handler.timeout.ReadTimeoutException +import io.netty.handler.timeout.WriteTimeoutException +import net.woggioni.gbcs.api.exception.CacheException +import net.woggioni.gbcs.api.exception.ContentTooLargeException +import net.woggioni.gbcs.common.contextLogger +import net.woggioni.gbcs.common.debug +import javax.net.ssl.SSLPeerUnverifiedException + +@ChannelHandler.Sharable +class ExceptionHandler : ChannelDuplexHandler() { + private val log = contextLogger() + + private val NOT_AUTHORIZED: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val TOO_BIG: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val NOT_AVAILABLE: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.SERVICE_UNAVAILABLE, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + private val SERVER_ERROR: FullHttpResponse = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR, Unpooled.EMPTY_BUFFER + ).apply { + headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" + } + + override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + when (cause) { + is DecoderException -> { + log.error(cause.message, cause) + ctx.close() + } + + is SSLPeerUnverifiedException -> { + ctx.writeAndFlush(NOT_AUTHORIZED.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + + is ContentTooLargeException -> { + ctx.writeAndFlush(TOO_BIG.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + is ReadTimeoutException -> { + log.debug { + val channelId = ctx.channel().id().asShortText() + "Read timeout on channel $channelId, closing the connection" + } + ctx.close() + } + is WriteTimeoutException -> { + log.debug { + val channelId = ctx.channel().id().asShortText() + "Write timeout on channel $channelId, closing the connection" + } + ctx.close() + } + is CacheException -> { + log.error(cause.message, cause) + ctx.writeAndFlush(NOT_AVAILABLE.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + else -> { + log.error(cause.message, cause) + ctx.writeAndFlush(SERVER_ERROR.retainedDuplicate()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE) + } + } + } +} \ No newline at end of file diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/handler/ServerHandler.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/handler/ServerHandler.kt index cdc39d8..2cb2d23 100644 --- a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/handler/ServerHandler.kt +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/handler/ServerHandler.kt @@ -15,9 +15,9 @@ import io.netty.handler.codec.http.HttpMethod import io.netty.handler.codec.http.HttpResponseStatus import io.netty.handler.codec.http.HttpUtil import io.netty.handler.codec.http.LastHttpContent -import io.netty.handler.stream.ChunkedNioFile import io.netty.handler.stream.ChunkedNioStream import net.woggioni.gbcs.api.Cache +import net.woggioni.gbcs.api.exception.CacheException import net.woggioni.gbcs.common.contextLogger import net.woggioni.gbcs.server.debug import net.woggioni.gbcs.server.warn @@ -38,7 +38,11 @@ class ServerHandler(private val cache: Cache, private val serverPrefix: Path) : val prefix = path.parent val key = path.fileName.toString() if (serverPrefix == prefix) { - cache.get(key)?.let { channel -> + try { + cache.get(key) + } catch(ex : Throwable) { + throw CacheException("Error accessing the cache backend", ex) + }?.let { channel -> log.debug(ctx) { "Cache hit for key '$key'" } @@ -55,17 +59,18 @@ class ServerHandler(private val cache: Cache, private val serverPrefix: Path) : when (channel) { is FileChannel -> { if (keepAlive) { - ctx.write(ChunkedNioFile(channel)) - ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT) + ctx.write(DefaultFileRegion(channel, 0, channel.size())) + ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT.retainedDuplicate()) } else { ctx.writeAndFlush(DefaultFileRegion(channel, 0, channel.size())) .addListener(ChannelFutureListener.CLOSE) } } - else -> { - ctx.write(ChunkedNioStream(channel)) - ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT) + ctx.write(ChunkedNioStream(channel)).addListener { evt -> + channel.close() + } + ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT.retainedDuplicate()) } } } ?: let { @@ -102,7 +107,11 @@ class ServerHandler(private val cache: Cache, private val serverPrefix: Path) : array() } } - cache.put(key, bodyBytes) + try { + cache.put(key, bodyBytes) + } catch(ex : Throwable) { + throw CacheException("Error accessing the cache backend", ex) + } val response = DefaultFullHttpResponse( msg.protocolVersion(), HttpResponseStatus.CREATED, Unpooled.copiedBuffer(key.toByteArray()) @@ -117,11 +126,35 @@ class ServerHandler(private val cache: Cache, private val serverPrefix: Path) : response.headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" ctx.writeAndFlush(response) } + } else if(method == HttpMethod.TRACE) { + val replayedRequestHead = ctx.alloc().buffer() + replayedRequestHead.writeCharSequence("TRACE ${Path.of(msg.uri())} ${msg.protocolVersion().text()}\r\n", Charsets.US_ASCII) + msg.headers().forEach { (key, value) -> + replayedRequestHead.apply { + writeCharSequence(key, Charsets.US_ASCII) + writeCharSequence(": ", Charsets.US_ASCII) + writeCharSequence(value, Charsets.UTF_8) + writeCharSequence("\r\n", Charsets.US_ASCII) + } + } + replayedRequestHead.writeCharSequence("\r\n", Charsets.US_ASCII) + val requestBody = msg.content() + requestBody.retain() + val responseBody = ctx.alloc().compositeBuffer(2).apply { + addComponents(true, replayedRequestHead) + addComponents(true, requestBody) + } + val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.OK, responseBody) + response.headers().apply { + set(HttpHeaderNames.CONTENT_TYPE, "message/http") + set(HttpHeaderNames.CONTENT_LENGTH, responseBody.readableBytes()) + } + ctx.writeAndFlush(response) } else { log.warn(ctx) { "Got request with unhandled method '${msg.method().name()}'" } - val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.BAD_REQUEST) + val response = DefaultFullHttpResponse(msg.protocolVersion(), HttpResponseStatus.METHOD_NOT_ALLOWED) response.headers()[HttpHeaderNames.CONTENT_LENGTH] = "0" ctx.writeAndFlush(response) } diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/throttling/BucketManager.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/throttling/BucketManager.kt new file mode 100644 index 0000000..1196bd6 --- /dev/null +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/throttling/BucketManager.kt @@ -0,0 +1,86 @@ +package net.woggioni.gbcs.server.throttling + +import net.woggioni.gbcs.api.Configuration +import net.woggioni.jwo.Bucket +import java.net.InetSocketAddress +import java.util.Arrays +import java.util.concurrent.ConcurrentHashMap +import java.util.function.Function + +class BucketManager private constructor( + private val bucketsByUser: Map = HashMap(), + private val bucketsByGroup: Map = HashMap(), + loader: Function? +) { + + private class BucketsByAddress( + private val map: MutableMap, + private val loader: Function + ) { + fun getBucket(socketAddress : InetSocketAddress) = map.computeIfAbsent(ByteArrayKey(socketAddress.address.address)) { + loader.apply(socketAddress) + } + } + + private val bucketsByAddress: BucketsByAddress? = loader?.let { + BucketsByAddress(ConcurrentHashMap(), it) + } + + private class ByteArrayKey(val array: ByteArray) { + override fun equals(other: Any?) = (other as? ByteArrayKey)?.let { bak -> + array contentEquals bak.array + } ?: false + + override fun hashCode() = Arrays.hashCode(array) + } + + fun getBucketByAddress(address : InetSocketAddress) : Bucket? { + return bucketsByAddress?.getBucket(address) + } + + fun getBucketByUser(user : Configuration.User) = bucketsByUser[user] + fun getBucketByGroup(group : Configuration.Group) = bucketsByGroup[group] + + companion object { + fun from(cfg : Configuration) : BucketManager { + val bucketsByUser = cfg.users.values.asSequence().filter { + it.quota != null + }.map { user -> + val quota = user.quota + val bucket = Bucket.local( + quota.maxAvailableCalls, + quota.calls, + quota.period, + quota.initialAvailableCalls + ) + user to bucket + }.toMap() + val bucketsByGroup = cfg.groups.values.asSequence().filter { + it.quota != null + }.map { group -> + val quota = group.quota + val bucket = Bucket.local( + quota.maxAvailableCalls, + quota.calls, + quota.period, + quota.initialAvailableCalls + ) + group to bucket + }.toMap() + return BucketManager( + bucketsByUser, + bucketsByGroup, + cfg.users[""]?.quota?.let { anonymousUserQuota -> + Function { + Bucket.local( + anonymousUserQuota.maxAvailableCalls, + anonymousUserQuota.calls, + anonymousUserQuota.period, + anonymousUserQuota.initialAvailableCalls + ) + } + } + ) + } + } +} diff --git a/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/throttling/ThrottlingHandler.kt b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/throttling/ThrottlingHandler.kt new file mode 100644 index 0000000..ea70a20 --- /dev/null +++ b/gbcs-server/src/main/kotlin/net/woggioni/gbcs/server/throttling/ThrottlingHandler.kt @@ -0,0 +1,86 @@ +package net.woggioni.gbcs.server.throttling + +import io.netty.channel.ChannelHandler.Sharable +import io.netty.channel.ChannelHandlerContext +import io.netty.channel.ChannelInboundHandlerAdapter +import io.netty.handler.codec.http.DefaultFullHttpResponse +import io.netty.handler.codec.http.HttpHeaderNames +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import net.woggioni.gbcs.api.Configuration +import net.woggioni.gbcs.common.contextLogger +import net.woggioni.gbcs.server.GradleBuildCacheServer +import net.woggioni.jwo.Bucket +import java.net.InetSocketAddress +import java.time.Duration +import java.util.concurrent.TimeUnit + + +@Sharable +class ThrottlingHandler(cfg: Configuration) : + ChannelInboundHandlerAdapter() { + + private val log = contextLogger() + private val bucketManager = BucketManager.from(cfg) + + private val connectionConfiguration = cfg.connection + + /** + * If the suggested waiting time from the bucket is lower than this + * amount, then the server will simply wait by itself before sending a response + * instead of replying with 429 + */ + private val waitThreshold = minOf( + connectionConfiguration.idleTimeout, + connectionConfiguration.readIdleTimeout, + connectionConfiguration.writeIdleTimeout + ).dividedBy(2) + + override fun channelRead(ctx: ChannelHandlerContext, msg: Any) { + val buckets = mutableListOf() + val user = ctx.channel().attr(GradleBuildCacheServer.userAttribute).get() + if (user != null) { + bucketManager.getBucketByUser(user)?.let(buckets::add) + } + val groups = ctx.channel().attr(GradleBuildCacheServer.groupAttribute).get() ?: emptySet() + if (groups.isNotEmpty()) { + groups.forEach { group -> + bucketManager.getBucketByGroup(group)?.let(buckets::add) + } + } + if (user == null && groups.isEmpty()) { + bucketManager.getBucketByAddress(ctx.channel().remoteAddress() as InetSocketAddress)?.let(buckets::add) + } + if (buckets.isEmpty()) { + return super.channelRead(ctx, msg) + } else { + var nextAttempt = Long.MAX_VALUE + for (bucket in buckets) { + val bucketNextAttempt = bucket.removeTokensWithEstimate(1) + if (bucketNextAttempt < 0) { + return super.channelRead(ctx, msg) + } else if (bucketNextAttempt < nextAttempt) { + nextAttempt = bucketNextAttempt + } + } + val waitDuration = Duration.ofNanos(nextAttempt) + if (waitDuration < waitThreshold) { + ctx.executor().schedule({ + ctx.fireChannelRead(msg) + }, waitDuration.toNanos(), TimeUnit.NANOSECONDS) + } else { + sendThrottledResponse(ctx, waitDuration) + } + } + } + + private fun sendThrottledResponse(ctx: ChannelHandlerContext, retryAfter: Duration) { + val response = DefaultFullHttpResponse( + HttpVersion.HTTP_1_1, + HttpResponseStatus.TOO_MANY_REQUESTS + ) + response.headers()[HttpHeaderNames.CONTENT_LENGTH] = 0 + response.headers()[HttpHeaderNames.RETRY_AFTER] = retryAfter.seconds + ctx.writeAndFlush(response) + } +} \ No newline at end of file diff --git a/gbcs-server/src/main/resources/META-INF/services/net.woggioni.gbcs.api.CacheProvider b/gbcs-server/src/main/resources/META-INF/services/net.woggioni.gbcs.api.CacheProvider index e249e1e..a681f5b 100644 --- a/gbcs-server/src/main/resources/META-INF/services/net.woggioni.gbcs.api.CacheProvider +++ b/gbcs-server/src/main/resources/META-INF/services/net.woggioni.gbcs.api.CacheProvider @@ -1 +1,2 @@ -net.woggioni.gbcs.server.cache.FileSystemCacheProvider \ No newline at end of file +net.woggioni.gbcs.server.cache.FileSystemCacheProvider +net.woggioni.gbcs.server.cache.InMemoryCacheProvider \ No newline at end of file diff --git a/gbcs-server/src/main/resources/net/woggioni/gbcs/server/schema/gbcs.xsd b/gbcs-server/src/main/resources/net/woggioni/gbcs/server/schema/gbcs.xsd index ee680b6..835147d 100644 --- a/gbcs-server/src/main/resources/net/woggioni/gbcs/server/schema/gbcs.xsd +++ b/gbcs-server/src/main/resources/net/woggioni/gbcs/server/schema/gbcs.xsd @@ -48,6 +48,17 @@ + + + + + + + + + + + @@ -92,17 +103,34 @@ - + + + + + + + + - + + + + + + + + + + + @@ -118,6 +146,7 @@ + diff --git a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/AbstractBasicAuthServerTest.kt b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/AbstractBasicAuthServerTest.kt index ee772f1..1b17a9e 100644 --- a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/AbstractBasicAuthServerTest.kt +++ b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/AbstractBasicAuthServerTest.kt @@ -24,8 +24,8 @@ abstract class AbstractBasicAuthServerTest : AbstractServerTest() { protected val random = Random(101325) protected val keyValuePair = newEntry(random) protected val serverPath = "gbcs" - protected val readersGroup = Configuration.Group("readers", setOf(Role.Reader)) - protected val writersGroup = Configuration.Group("writers", setOf(Role.Writer)) + protected val readersGroup = Configuration.Group("readers", setOf(Role.Reader), null) + protected val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null) abstract protected val users : List diff --git a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/AbstractTlsServerTest.kt b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/AbstractTlsServerTest.kt index 9611973..5c90385 100644 --- a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/AbstractTlsServerTest.kt +++ b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/AbstractTlsServerTest.kt @@ -4,6 +4,7 @@ import net.woggioni.gbcs.api.Configuration import net.woggioni.gbcs.api.Role import net.woggioni.gbcs.common.Xml import net.woggioni.gbcs.server.cache.FileSystemCacheConfiguration +import net.woggioni.gbcs.server.cache.InMemoryCacheConfiguration import net.woggioni.gbcs.server.configuration.Serializer import net.woggioni.gbcs.server.test.utils.CertificateUtils import net.woggioni.gbcs.server.test.utils.CertificateUtils.X509Credentials @@ -45,8 +46,8 @@ abstract class AbstractTlsServerTest : AbstractServerTest() { private lateinit var trustStore: KeyStore protected lateinit var ca: X509Credentials - protected val readersGroup = Configuration.Group("readers", setOf(Role.Reader)) - protected val writersGroup = Configuration.Group("writers", setOf(Role.Writer)) + protected val readersGroup = Configuration.Group("readers", setOf(Role.Reader), null) + protected val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null) protected val random = Random(101325) protected val keyValuePair = newEntry(random) private val serverPath : String? = null @@ -158,6 +159,12 @@ abstract class AbstractTlsServerTest : AbstractServerTest() { compressionLevel = Deflater.DEFAULT_COMPRESSION, digestAlgorithm = "MD5" ), +// InMemoryCacheConfiguration( +// maxAge = Duration.ofSeconds(3600 * 24), +// compressionEnabled = true, +// compressionLevel = Deflater.DEFAULT_COMPRESSION, +// digestAlgorithm = "MD5" +// ), Configuration.ClientCertificateAuthentication( Configuration.TlsCertificateExtractor("CN", "(.*)"), null diff --git a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/BasicAuthServerTest.kt b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/BasicAuthServerTest.kt index cb18894..91ef79b 100644 --- a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/BasicAuthServerTest.kt +++ b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/BasicAuthServerTest.kt @@ -10,6 +10,8 @@ import org.junit.jupiter.api.Test import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.time.Duration +import java.time.temporal.ChronoUnit class BasicAuthServerTest : AbstractBasicAuthServerTest() { @@ -19,10 +21,16 @@ class BasicAuthServerTest : AbstractBasicAuthServerTest() { } override val users = listOf( - Configuration.User("user1", hashPassword(PASSWORD), setOf(readersGroup)), - Configuration.User("user2", hashPassword(PASSWORD), setOf(writersGroup)), - Configuration.User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup)), - Configuration.User("", null, setOf(readersGroup)) + Configuration.User("user1", hashPassword(PASSWORD), setOf(readersGroup), null), + Configuration.User("user2", hashPassword(PASSWORD), setOf(writersGroup), null), + Configuration.User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup), null), + Configuration.User("", null, setOf(readersGroup), null), + Configuration.User("user4", hashPassword(PASSWORD), setOf(readersGroup), + Configuration.Quota(1, Duration.of(1, ChronoUnit.DAYS), 0, 1) + ), + Configuration.User("user5", hashPassword(PASSWORD), setOf(readersGroup), + Configuration.Quota(1, Duration.of(5, ChronoUnit.SECONDS), 0, 1) + ) ) @Test @@ -144,4 +152,41 @@ class BasicAuthServerTest : AbstractBasicAuthServerTest() { val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.statusCode()) } + + @Test + @Order(6) + fun getAsAThrottledUser() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + it.name == "user4" + } ?: throw RuntimeException("user4 not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.TOO_MANY_REQUESTS.code(), response.statusCode()) + } + + @Test + @Order(7) + fun getAsAThrottledUser2() { + val client: HttpClient = HttpClient.newHttpClient() + + val (key, value) = keyValuePair + val user = cfg.users.values.find { + it.name == "user5" + } ?: throw RuntimeException("user5 not found") + + val requestBuilder = newRequestBuilder(key) + .header("Authorization", buildAuthorizationHeader(user, PASSWORD)) + .GET() + + val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + Assertions.assertArrayEquals(value, response.body()) + } } \ No newline at end of file diff --git a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/ConfigurationTest.kt b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/ConfigurationTest.kt index 1675a05..5206987 100644 --- a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/ConfigurationTest.kt +++ b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/ConfigurationTest.kt @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.io.TempDir import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource +import org.xml.sax.SAXParseException import java.nio.file.Files import java.nio.file.Path @@ -16,9 +17,9 @@ class ConfigurationTest { @ValueSource( strings = [ - "classpath:net/woggioni/gbcs/server/test/gbcs-default.xml", - "classpath:net/woggioni/gbcs/server/test/gbcs-memcached.xml", - "classpath:net/woggioni/gbcs/server/test/gbcs-tls.xml", + "classpath:net/woggioni/gbcs/server/test/valid/gbcs-default.xml", + "classpath:net/woggioni/gbcs/server/test/valid/gbcs-memcached.xml", + "classpath:net/woggioni/gbcs/server/test/valid/gbcs-tls.xml", ] ) @ParameterizedTest @@ -35,4 +36,20 @@ class ConfigurationTest { val parsed = Parser.parse(Xml.parseXml(configFile.toUri().toURL())) Assertions.assertEquals(cfg, parsed) } + + @ValueSource( + strings = [ + "classpath:net/woggioni/gbcs/server/test/invalid/invalid-user-ref.xml", + "classpath:net/woggioni/gbcs/server/test/invalid/duplicate-anonymous-user.xml", + "classpath:net/woggioni/gbcs/server/test/invalid/duplicate-anonymous-user2.xml", + "classpath:net/woggioni/gbcs/server/test/invalid/multiple-user-quota.xml", + ] + ) + @ParameterizedTest + fun invalidConfigurationTest(configurationUrl: String) { + GbcsUrlStreamHandlerFactory.install() + Assertions.assertThrows(SAXParseException::class.java) { + Xml.parseXml(configurationUrl.toUrl()) + } + } } \ No newline at end of file diff --git a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt index f9d2fc6..1672b7f 100644 --- a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt +++ b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAnonymousUserBasicAuthServerTest.kt @@ -18,9 +18,9 @@ class NoAnonymousUserBasicAuthServerTest : AbstractBasicAuthServerTest() { } override val users = listOf( - Configuration.User("user1", hashPassword(PASSWORD), setOf(readersGroup)), - Configuration.User("user2", hashPassword(PASSWORD), setOf(writersGroup)), - Configuration.User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup)), + Configuration.User("user1", hashPassword(PASSWORD), setOf(readersGroup), null), + Configuration.User("user2", hashPassword(PASSWORD), setOf(writersGroup), null), + Configuration.User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup), null), ) @Test diff --git a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAnonymousUserTlsServerTest.kt b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAnonymousUserTlsServerTest.kt index d62fe46..5ebab76 100644 --- a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAnonymousUserTlsServerTest.kt +++ b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAnonymousUserTlsServerTest.kt @@ -12,9 +12,9 @@ import java.net.http.HttpResponse class NoAnonymousUserTlsServerTest : AbstractTlsServerTest() { override val users = listOf( - Configuration.User("user1", null, setOf(readersGroup)), - Configuration.User("user2", null, setOf(writersGroup)), - Configuration.User("user3", null, setOf(readersGroup, writersGroup)), + Configuration.User("user1", null, setOf(readersGroup), null), + Configuration.User("user2", null, setOf(writersGroup), null), + Configuration.User("user3", null, setOf(readersGroup, writersGroup), null), ) @Test diff --git a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAuthServerTest.kt b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAuthServerTest.kt index 3c71c68..f2759ef 100644 --- a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAuthServerTest.kt +++ b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/NoAuthServerTest.kt @@ -4,6 +4,7 @@ import io.netty.handler.codec.http.HttpResponseStatus import net.woggioni.gbcs.api.Configuration import net.woggioni.gbcs.common.Xml import net.woggioni.gbcs.server.cache.FileSystemCacheConfiguration +import net.woggioni.gbcs.server.cache.InMemoryCacheConfiguration import net.woggioni.gbcs.server.configuration.Serializer import net.woggioni.gbcs.server.test.utils.NetworkUtils import org.junit.jupiter.api.Assertions @@ -23,7 +24,7 @@ import kotlin.random.Random class NoAuthServerTest : AbstractServerTest() { - private lateinit var cacheDir : Path + private lateinit var cacheDir: Path private val random = Random(101325) private val keyValuePair = newEntry(random) @@ -47,8 +48,7 @@ class NoAuthServerTest : AbstractServerTest() { ), emptyMap(), emptyMap(), - FileSystemCacheConfiguration( - this.cacheDir, + InMemoryCacheConfiguration( maxAge = Duration.ofSeconds(3600 * 24), compressionEnabled = true, digestAlgorithm = "MD5", @@ -63,10 +63,10 @@ class NoAuthServerTest : AbstractServerTest() { override fun tearDown() { } - fun newRequestBuilder(key : String) = HttpRequest.newBuilder() + fun newRequestBuilder(key: String) = HttpRequest.newBuilder() .uri(URI.create("http://${cfg.host}:${cfg.port}/$serverPath/$key")) - fun newEntry(random : Random) : Pair { + fun newEntry(random: Random): Pair { val key = ByteArray(0x10).let { random.nextBytes(it) Base64.getUrlEncoder().encodeToString(it) @@ -95,10 +95,11 @@ class NoAuthServerTest : AbstractServerTest() { @Order(2) fun getWithNoAuthorizationHeader() { val client: HttpClient = HttpClient.newHttpClient() - val (key, value ) = keyValuePair + val (key, value) = keyValuePair val requestBuilder = newRequestBuilder(key) .GET() - val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) Assertions.assertArrayEquals(value, response.body()) } @@ -111,31 +112,23 @@ class NoAuthServerTest : AbstractServerTest() { val (key, _) = newEntry(random) val requestBuilder = newRequestBuilder(key).GET() - val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.statusCode()) } -// @Test -// @Order(4) -// fun manyRequestsTest() { -// val client: HttpClient = HttpClient.newHttpClient() -// -// for(i in 0 until 100000) { -// -// val newEntry = random.nextBoolean() -// val (key, _) = if(newEntry) { -// newEntry(random) -// } else { -// keyValuePair -// } -// val requestBuilder = newRequestBuilder(key).GET() -// -// val response: HttpResponse = client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) -// if(newEntry) { -// Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.statusCode()) -// } else { -// Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) -// } -// } -// } + @Test + @Order(4) + fun traceTest() { + val client: HttpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build() + val requestBuilder = newRequestBuilder("").method( + "TRACE", + HttpRequest.BodyPublishers.ofByteArray("sfgsdgfaiousfiuhsd".toByteArray()) + ) + + val response: HttpResponse = + client.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofByteArray()) + Assertions.assertEquals(HttpResponseStatus.OK.code(), response.statusCode()) + println(String(response.body())) + } } \ No newline at end of file diff --git a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/TlsServerTest.kt b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/TlsServerTest.kt index ecef412..b6b93c3 100644 --- a/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/TlsServerTest.kt +++ b/gbcs-server/src/test/kotlin/net/woggioni/gbcs/server/test/TlsServerTest.kt @@ -15,10 +15,10 @@ import java.net.http.HttpResponse class TlsServerTest : AbstractTlsServerTest() { override val users = listOf( - Configuration.User("user1", null, setOf(readersGroup)), - Configuration.User("user2", null, setOf(writersGroup)), - Configuration.User("user3", null, setOf(readersGroup, writersGroup)), - Configuration.User("", null, setOf(readersGroup)) + Configuration.User("user1", null, setOf(readersGroup), null), + Configuration.User("user2", null, setOf(writersGroup), null), + Configuration.User("user3", null, setOf(readersGroup, writersGroup), null), + Configuration.User("", null, setOf(readersGroup), null) ) @Test diff --git a/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/duplicate-anonymous-user.xml b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/duplicate-anonymous-user.xml new file mode 100644 index 0000000..01bebab --- /dev/null +++ b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/duplicate-anonymous-user.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/duplicate-anonymous-user2.xml b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/duplicate-anonymous-user2.xml new file mode 100644 index 0000000..0a2f507 --- /dev/null +++ b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/duplicate-anonymous-user2.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/invalid-user-ref.xml b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/invalid-user-ref.xml new file mode 100644 index 0000000..0669ea4 --- /dev/null +++ b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/invalid-user-ref.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/multiple-user-quota.xml b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/multiple-user-quota.xml new file mode 100644 index 0000000..75da0ac --- /dev/null +++ b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/invalid/multiple-user-quota.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/gbcs-default.xml b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/valid/gbcs-default.xml similarity index 100% rename from gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/gbcs-default.xml rename to gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/valid/gbcs-default.xml diff --git a/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/gbcs-memcached.xml b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/valid/gbcs-memcached.xml similarity index 100% rename from gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/gbcs-memcached.xml rename to gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/valid/gbcs-memcached.xml diff --git a/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/gbcs-tls.xml b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/valid/gbcs-tls.xml similarity index 84% rename from gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/gbcs-tls.xml rename to gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/valid/gbcs-tls.xml index e72736b..6f52c1c 100644 --- a/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/gbcs-tls.xml +++ b/gbcs-server/src/test/resources/net/woggioni/gbcs/server/test/valid/gbcs-tls.xml @@ -11,23 +11,28 @@ idle-timeout="PT30M" max-request-size="4096"/> - + - + + + + + + - + @@ -45,6 +50,7 @@ + diff --git a/gradle.properties b/gradle.properties index cf9f63b..cf33dcb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,9 +2,9 @@ org.gradle.configuration-cache=false org.gradle.parallel=true org.gradle.caching=true -gbcs.version = 0.0.8 +gbcs.version = 0.0.11 -lys.version = 2025.01.17 +lys.version = 2025.01.24 gitea.maven.url = https://gitea.woggioni.net/api/packages/woggioni/maven docker.registry.url=gitea.woggioni.net