implemented streaming request/response streaming

added metadata to cache values added cache servlet for comparison
2025-02-19 22:36:31 +08:00
parent 0463038aaa
commit f048a60540
66 changed files with 2474 additions and 1078 deletions
--- a/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/BenchmarkCommand.kt
+++ b/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/BenchmarkCommand.kt
@@ -1,17 +1,20 @@
 package net.woggioni.rbcs.cli.impl.commands

-import net.woggioni.rbcs.cli.impl.RbcsCommand
-import net.woggioni.rbcs.client.RemoteBuildCacheClient
-import net.woggioni.rbcs.common.contextLogger
-import net.woggioni.rbcs.common.error
-import net.woggioni.rbcs.common.info
 import net.woggioni.jwo.JWO
 import net.woggioni.jwo.LongMath
+import net.woggioni.rbcs.api.CacheValueMetadata
+import net.woggioni.rbcs.cli.impl.RbcsCommand
+import net.woggioni.rbcs.cli.impl.converters.ByteSizeConverter
+import net.woggioni.rbcs.client.RemoteBuildCacheClient
+import net.woggioni.rbcs.common.createLogger
 import net.woggioni.rbcs.common.debug
+import net.woggioni.rbcs.common.error
+import net.woggioni.rbcs.common.info
 import picocli.CommandLine
 import java.security.SecureRandom
 import java.time.Duration
 import java.time.Instant
+import java.time.temporal.ChronoUnit
 import java.util.concurrent.LinkedBlockingQueue
 import java.util.concurrent.Semaphore
 import java.util.concurrent.atomic.AtomicLong
@@ -23,7 +26,9 @@ import kotlin.random.Random
    showDefaultValues = true
 )
 class BenchmarkCommand : RbcsCommand() {
-    private val log = contextLogger()
+    companion object{
+        private val log = createLogger<BenchmarkCommand>()
+    }

    @CommandLine.Spec
    private lateinit var spec: CommandLine.Model.CommandSpec
@@ -38,10 +43,17 @@ class BenchmarkCommand : RbcsCommand() {
    @CommandLine.Option(
        names = ["-s", "--size"],
        description = ["Size of a cache value in bytes"],
-        paramLabel = "SIZE"
+        paramLabel = "SIZE",
+        converter = [ByteSizeConverter::class]
    )
    private var size = 0x1000

+    @CommandLine.Option(
+        names = ["-r", "--random"],
+        description = ["Insert completely random byte values"]
+    )
+    private var randomValues = false
+
    override fun run() {
        val clientCommand = spec.parent().userObject() as ClientCommand
        val profile = clientCommand.profileName.let { profileName ->
@@ -55,8 +67,12 @@ class BenchmarkCommand : RbcsCommand() {
                val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong())
                while (true) {
                    val key = JWO.bytesToHex(random.nextBytes(16))
-                    val content = random.nextInt().toByte()
-                    val value = ByteArray(size, { _ -> content })
+                    val value = if(randomValues) {
+                        random.nextBytes(size)
+                    } else {
+                        val byteValue = random.nextInt().toByte()
+                        ByteArray(size) {_ -> byteValue}
+                    }
                    yield(key to value)
                }
            }
@@ -68,13 +84,13 @@ class BenchmarkCommand : RbcsCommand() {
                val completionCounter = AtomicLong(0)
                val completionQueue = LinkedBlockingQueue<Pair<String, ByteArray>>(numberOfEntries)
                val start = Instant.now()
-                val semaphore = Semaphore(profile.maxConnections * 3)
+                val semaphore = Semaphore(profile.maxConnections * 5)
                val iterator = entryGenerator.take(numberOfEntries).iterator()
                while (completionCounter.get() < numberOfEntries) {
                    if (iterator.hasNext()) {
                        val entry = iterator.next()
                        semaphore.acquire()
-                        val future = client.put(entry.first, entry.second).thenApply { entry }
+                        val future = client.put(entry.first, entry.second, CacheValueMetadata(null, null)).thenApply { entry }
                        future.whenComplete { result, ex ->
                            if (ex != null) {
                                log.error(ex.message, ex)
@@ -90,7 +106,7 @@ class BenchmarkCommand : RbcsCommand() {
                            }
                        }
                    } else {
-                        Thread.sleep(0)
+                        Thread.sleep(Duration.of(500, ChronoUnit.MILLIS))
                    }
                }

@@ -111,12 +127,13 @@ class BenchmarkCommand : RbcsCommand() {
            }
            if (entries.isNotEmpty()) {
                val completionCounter = AtomicLong(0)
-                val semaphore = Semaphore(profile.maxConnections * 3)
+                val semaphore = Semaphore(profile.maxConnections * 5)
                val start = Instant.now()
                val it = entries.iterator()
                while (completionCounter.get() < entries.size) {
                    if (it.hasNext()) {
                        val entry = it.next()
+                        semaphore.acquire()
                        val future = client.get(entry.first).thenApply {
                            if (it == null) {
                                log.error {
@@ -138,7 +155,7 @@ class BenchmarkCommand : RbcsCommand() {
                            semaphore.release()
                        }
                    } else {
-                        Thread.sleep(0)
+                        Thread.sleep(Duration.of(500, ChronoUnit.MILLIS))
                    }
                }
                val end = Instant.now()
--- a/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/GetCommand.kt
+++ b/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/GetCommand.kt
@@ -2,7 +2,7 @@ package net.woggioni.rbcs.cli.impl.commands

 import net.woggioni.rbcs.cli.impl.RbcsCommand
 import net.woggioni.rbcs.client.RemoteBuildCacheClient
-import net.woggioni.rbcs.common.contextLogger
+import net.woggioni.rbcs.common.createLogger
 import picocli.CommandLine
 import java.nio.file.Files
 import java.nio.file.Path
@@ -13,7 +13,9 @@ import java.nio.file.Path
    showDefaultValues = true
 )
 class GetCommand : RbcsCommand() {
-    private val log = contextLogger()
+    companion object{
+        private val log = createLogger<GetCommand>()
+    }

    @CommandLine.Spec
    private lateinit var spec: CommandLine.Model.CommandSpec
--- a/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/HealthCheckCommand.kt
+++ b/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/HealthCheckCommand.kt
@@ -3,6 +3,7 @@ package net.woggioni.rbcs.cli.impl.commands
 import net.woggioni.rbcs.cli.impl.RbcsCommand
 import net.woggioni.rbcs.client.RemoteBuildCacheClient
 import net.woggioni.rbcs.common.contextLogger
+import net.woggioni.rbcs.common.createLogger
 import picocli.CommandLine
 import java.security.SecureRandom
 import kotlin.random.Random
@@ -13,7 +14,9 @@ import kotlin.random.Random
    showDefaultValues = true
 )
 class HealthCheckCommand : RbcsCommand() {
-    private val log = contextLogger()
+    companion object{
+        private val log = createLogger<HealthCheckCommand>()
+    }

    @CommandLine.Spec
    private lateinit var spec: CommandLine.Model.CommandSpec
@@ -32,11 +35,12 @@ class HealthCheckCommand : RbcsCommand() {
                if(value == null) {
                    throw IllegalStateException("Empty response from server")
                }
+                val offset = value.size - nonce.size
                for(i in 0 until nonce.size) {
-                    for(j in value.size - nonce.size until nonce.size) {
-                        if(nonce[i] != value[j]) {
-                            throw IllegalStateException("Server nonce does not match")
-                        }
+                    val a = nonce[i]
+                    val b = value[offset + i]
+                    if(a != b) {
+                        throw IllegalStateException("Server nonce does not match")
                    }
                }
            }.get()
--- a/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/PutCommand.kt
+++ b/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/PutCommand.kt
@@ -1,11 +1,21 @@
 package net.woggioni.rbcs.cli.impl.commands

+import net.woggioni.jwo.Hash
+import net.woggioni.jwo.JWO
+import net.woggioni.jwo.NullOutputStream
+import net.woggioni.rbcs.api.CacheValueMetadata
 import net.woggioni.rbcs.cli.impl.RbcsCommand
-import net.woggioni.rbcs.cli.impl.converters.InputStreamConverter
 import net.woggioni.rbcs.client.RemoteBuildCacheClient
 import net.woggioni.rbcs.common.contextLogger
+import net.woggioni.rbcs.common.createLogger
 import picocli.CommandLine
 import java.io.InputStream
+import java.nio.file.Files
+import java.nio.file.Path
+import java.security.DigestInputStream
+import java.security.MessageDigest
+import java.util.UUID
+import kotlin.io.encoding.decodingWith

@CommandLine.Command(
    name = "put",
@@ -13,25 +23,41 @@ import java.io.InputStream
    showDefaultValues = true
 )
 class PutCommand : RbcsCommand() {
-    private val log = contextLogger()
+    companion object{
+        private val log = createLogger<PutCommand>()
+    }
+

    @CommandLine.Spec
    private lateinit var spec: CommandLine.Model.CommandSpec

    @CommandLine.Option(
        names = ["-k", "--key"],
-        description = ["The key for the new value"],
+        description = ["The key for the new value, randomly generated if omitted"],
        paramLabel = "KEY"
    )
-    private var key : String = ""
+    private var key : String? = null
+
+    @CommandLine.Option(
+        names = ["-i", "--inline"],
+        description = ["File is to be displayed in the browser"],
+        paramLabel = "INLINE",
+    )
+    private var inline : Boolean = false
+
+    @CommandLine.Option(
+        names = ["-t", "--type"],
+        description = ["File mime type"],
+        paramLabel = "MIME_TYPE",
+    )
+    private var mimeType : String? = null

    @CommandLine.Option(
        names = ["-v", "--value"],
        description = ["Path to a file containing the value to be added (defaults to stdin)"],
        paramLabel = "VALUE_FILE",
-        converter = [InputStreamConverter::class]
    )
-    private var value : InputStream = System.`in`
+    private var value : Path? = null

    override fun run() {
        val clientCommand = spec.parent().userObject() as ClientCommand
@@ -40,9 +66,40 @@ class PutCommand : RbcsCommand() {
                ?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
        }
        RemoteBuildCacheClient(profile).use { client ->
-            value.use {
-                client.put(key, it.readAllBytes())
+            val inputStream : InputStream
+            val mimeType : String?
+            val contentDisposition : String?
+            val valuePath = value
+            val actualKey : String?
+            if(valuePath != null) {
+                inputStream = Files.newInputStream(valuePath)
+                mimeType = this.mimeType ?: Files.probeContentType(valuePath)
+                contentDisposition = if(inline) {
+                    "inline"
+                } else {
+                    "attachment; filename=\"${valuePath.fileName}\""
+                }
+                actualKey = key ?: let {
+                    val md = Hash.Algorithm.SHA512.newInputStream(Files.newInputStream(valuePath)).use {
+                        JWO.copy(it, NullOutputStream())
+                        it.messageDigest
+                    }
+                    UUID.nameUUIDFromBytes(md.digest()).toString()
+                }
+            } else {
+                inputStream = System.`in`
+                mimeType = this.mimeType
+                contentDisposition = if(inline) {
+                    "inline"
+                } else {
+                    null
+                }
+                actualKey = key ?: UUID.randomUUID().toString()
+            }
+            inputStream.use {
+                client.put(actualKey, it.readAllBytes(), CacheValueMetadata(contentDisposition, mimeType))
            }.get()
+            println(profile.serverURI.resolve(actualKey))
        }
    }
 }
--- a/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/ServerCommand.kt
+++ b/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/ServerCommand.kt
@@ -1,19 +1,20 @@
 package net.woggioni.rbcs.cli.impl.commands

+import net.woggioni.jwo.Application
+import net.woggioni.jwo.JWO
 import net.woggioni.rbcs.cli.impl.RbcsCommand
 import net.woggioni.rbcs.cli.impl.converters.DurationConverter
-import net.woggioni.rbcs.common.contextLogger
+import net.woggioni.rbcs.common.createLogger
 import net.woggioni.rbcs.common.debug
 import net.woggioni.rbcs.common.info
 import net.woggioni.rbcs.server.RemoteBuildCacheServer
 import net.woggioni.rbcs.server.RemoteBuildCacheServer.Companion.DEFAULT_CONFIGURATION_URL
-import net.woggioni.jwo.Application
-import net.woggioni.jwo.JWO
 import picocli.CommandLine
 import java.io.ByteArrayOutputStream
 import java.nio.file.Files
 import java.nio.file.Path
 import java.time.Duration
+import java.util.concurrent.TimeUnit

@CommandLine.Command(
    name = "server",
@@ -21,8 +22,9 @@ import java.time.Duration
    showDefaultValues = true
 )
 class ServerCommand(app : Application) : RbcsCommand() {
-
-    private val log = contextLogger()
+    companion object {
+        private val log = createLogger<ServerCommand>()
+    }

    private fun createDefaultConfigurationFile(configurationFile: Path) {
        log.info {
@@ -66,11 +68,20 @@ class ServerCommand(app : Application) : RbcsCommand() {
            }
        }
        val server = RemoteBuildCacheServer(configuration)
-        server.run().use { server ->
-            timeout?.let {
-                Thread.sleep(it)
-                server.shutdown()
+        val handle = server.run()
+        val shutdownHook = Thread.ofPlatform().unstarted {
+            handle.sendShutdownSignal()
+            try {
+                handle.get(60, TimeUnit.SECONDS)
+            } catch (ex : Throwable) {
+                log.warn(ex.message, ex)
            }
        }
+        Runtime.getRuntime().addShutdownHook(shutdownHook)
+        if(timeout != null) {
+            Thread.sleep(timeout)
+            handle.sendShutdownSignal()
+        }
+        handle.get()
    }
 }
--- a/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/converters/ByteSizeConverter.kt
+++ b/rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/converters/ByteSizeConverter.kt
@@ -0,0 +1,10 @@
+package net.woggioni.rbcs.cli.impl.converters
+
+import picocli.CommandLine
+
+
+class ByteSizeConverter : CommandLine.ITypeConverter<Int> {
+    override fun convert(value: String): Int {
+        return Integer.decode(value)
+    }
+}