woggioni/rbcs
1
0
Fork 0
Code Actions Packages Releases 4 Wiki Activity

added GraalVM native image executable build

Browse Source
This commit is contained in:
Walter Oggioni
2025-02-25 15:30:58 +08:00
parent 23f2a351a6
commit ca18b63f27
25 changed files with 1420 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
rbcs-cli/src/graal/kotlin/net/woggioni/rbcs/cli/graal/GraalNativeImageConfiguration.kt Normal file
View File

@@ -0,0 +1,161 @@
package net.woggioni.rbcs.cli.graal
import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.api.Configuration.User
import net.woggioni.rbcs.api.Role
import net.woggioni.rbcs.cli.RemoteBuildCacheServerCli
import net.woggioni.rbcs.cli.impl.commands.BenchmarkCommand
import net.woggioni.rbcs.cli.impl.commands.HealthCheckCommand
import net.woggioni.rbcs.client.RemoteBuildCacheClient
import net.woggioni.rbcs.common.HostAndPort
import net.woggioni.rbcs.common.PasswordSecurity.hashPassword
import net.woggioni.rbcs.common.RBCS
import net.woggioni.rbcs.common.Xml
import net.woggioni.rbcs.server.RemoteBuildCacheServer
import net.woggioni.rbcs.server.cache.FileSystemCacheConfiguration
import net.woggioni.rbcs.server.cache.InMemoryCacheConfiguration
import net.woggioni.rbcs.server.configuration.Parser
import net.woggioni.rbcs.server.memcache.MemcacheCacheConfiguration
import java.net.URI
import java.nio.file.Path
import java.time.Duration
import java.time.temporal.ChronoUnit
import java.util.concurrent.ExecutionException
import java.util.zip.Deflater
object GraalNativeImageConfiguration {
@JvmStatic
fun main(vararg args : String) {
val serverDoc = RemoteBuildCacheServer.DEFAULT_CONFIGURATION_URL.openStream().use {
Xml.parseXml(RemoteBuildCacheServer.DEFAULT_CONFIGURATION_URL, it)
}
Parser.parse(doc)
val clientDoc = RemoteBuildCacheClient.Configuration.openStream().use {
Xml.parseXml(RemoteBuildCacheServer.DEFAULT_CONFIGURATION_URL, it)
}
Parser.parse(doc)
val PASSWORD = "password"
val readersGroup = Configuration.Group("readers", setOf(Role.Reader), null, null)
val writersGroup = Configuration.Group("writers", setOf(Role.Writer), null, null)
val users = listOf(
User("user1", hashPassword(PASSWORD), setOf(readersGroup), null),
User("user2", hashPassword(PASSWORD), setOf(writersGroup), null),
User("user3", hashPassword(PASSWORD), setOf(readersGroup, writersGroup), null),
User("", null, setOf(readersGroup), null),
User("user4", hashPassword(PASSWORD), setOf(readersGroup),
Configuration.Quota(1, Duration.of(1, ChronoUnit.DAYS), 0, 1)
),
User("user5", hashPassword(PASSWORD), setOf(readersGroup),
Configuration.Quota(1, Duration.of(5, ChronoUnit.SECONDS), 0, 1)
)
)
val serverPort = RBCS.getFreePort()
val caches = listOf<Configuration.Cache>(
InMemoryCacheConfiguration(
maxAge = Duration.ofSeconds(3600),
digestAlgorithm = "MD5",
compressionLevel = Deflater.DEFAULT_COMPRESSION,
compressionEnabled = false,
maxSize = 0x1000000,
chunkSize = 0x1000
),
FileSystemCacheConfiguration(
Path.of(System.getProperty("java.io.tmpdir")).resolve("rbcs"),
maxAge = Duration.ofSeconds(3600),
digestAlgorithm = "MD5",
compressionLevel = Deflater.DEFAULT_COMPRESSION,
compressionEnabled = false,
chunkSize = 0x1000
),
MemcacheCacheConfiguration(
listOf(MemcacheCacheConfiguration.Server(
HostAndPort("127.0.0.1", 11211),
1000,
4)
),
Duration.ofSeconds(60),
"MD5",
null,
1,
0x1000
)
)
for (cache in caches) {
val serverConfiguration = Configuration(
"127.0.0.1",
serverPort,
100,
null,
Configuration.EventExecutor(true),
Configuration.Connection(
Duration.ofSeconds(10),
Duration.ofSeconds(15),
Duration.ofSeconds(15),
0x10000,
),
users.asSequence().map { it.name to it }.toMap(),
sequenceOf(writersGroup, readersGroup).map { it.name to it }.toMap(),
cache,
Configuration.BasicAuthentication(),
null,
)
MemcacheCacheConfiguration(
listOf(
MemcacheCacheConfiguration.Server(
HostAndPort("127.0.0.1", 11211),
1000,
4
)
),
Duration.ofSeconds(60),
"MD5",
null,
1,
0x1000
)
val serverHandle = RemoteBuildCacheServer(serverConfiguration).run()
val clientProfile = RemoteBuildCacheClient.Configuration.Profile(
URI.create("http://127.0.0.1:$serverPort/"),
null,
RemoteBuildCacheClient.Configuration.Authentication.BasicAuthenticationCredentials("user3", PASSWORD),
Duration.ofSeconds(3),
10,
true,
RemoteBuildCacheClient.Configuration.RetryPolicy(
3,
1000,
1.2
),
RemoteBuildCacheClient.Configuration.TrustStore(null, null, false, false)
)
HealthCheckCommand.run(clientProfile)
BenchmarkCommand.run(
clientProfile,
1000,
0x100,
true
)
serverHandle.sendShutdownSignal()
try {
serverHandle.get()
} catch (ee : ExecutionException) {
}
}
RemoteBuildCacheServerCli.main("--help")
}
}

5
rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/RemoteBuildCacheServerCli.kt
View File

@@ -23,8 +23,13 @@ class RemoteBuildCacheServerCli : RbcsCommand() {
class VersionProvider : AbstractVersionProvider()
companion object {
private fun setPropertyIfNotPresent(key: String, value: String) {
System.getProperty(key) ?: System.setProperty(key, value)
}
@JvmStatic
fun main(vararg args: String) {
setPropertyIfNotPresent("logback.configurationFile", "net/woggioni/rbcs/cli/logback.xml")
setPropertyIfNotPresent("io.netty.leakDetectionLevel", "DISABLED")
val currentClassLoader = RemoteBuildCacheServerCli::class.java.classLoader
Thread.currentThread().contextClassLoader = currentClassLoader
if(currentClassLoader.javaClass.name == "net.woggioni.envelope.loader.ModuleClassLoader") {

231
rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/BenchmarkCommand.kt
View File

@@ -26,8 +26,123 @@ import kotlin.random.Random
showDefaultValues = true
)
class BenchmarkCommand : RbcsCommand() {
companion object{
companion object {
private val log = createLogger<BenchmarkCommand>()
fun run(profile : RemoteBuildCacheClient.Configuration.Profile,
numberOfEntries : Int,
entrySize : Int,
useRandomValue : Boolean,
) {
val progressThreshold = LongMath.ceilDiv(numberOfEntries.toLong(), 20)
RemoteBuildCacheClient(profile).use { client ->
val entryGenerator = sequence {
val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong())
while (true) {
val key = JWO.bytesToHex(random.nextBytes(16))
val value = if (useRandomValue) {
random.nextBytes(entrySize)
} else {
val byteValue = random.nextInt().toByte()
ByteArray(entrySize) { _ -> byteValue }
}
yield(key to value)
}
}
log.info {
"Starting insertion"
}
val entries = let {
val completionCounter = AtomicLong(0)
val completionQueue = LinkedBlockingQueue<Pair<String, ByteArray>>(numberOfEntries)
val start = Instant.now()
val semaphore = Semaphore(profile.maxConnections * 5)
val iterator = entryGenerator.take(numberOfEntries).iterator()
while (completionCounter.get() < numberOfEntries) {
if (iterator.hasNext()) {
val entry = iterator.next()
semaphore.acquire()
val future =
client.put(entry.first, entry.second, CacheValueMetadata(null, null)).thenApply { entry }
future.whenComplete { result, ex ->
if (ex != null) {
log.error(ex.message, ex)
} else {
completionQueue.put(result)
}
semaphore.release()
val completed = completionCounter.incrementAndGet()
if (completed.mod(progressThreshold) == 0L) {
log.debug {
"Inserted $completed / $numberOfEntries"
}
}
}
} else {
Thread.sleep(Duration.of(500, ChronoUnit.MILLIS))
}
}
val inserted = completionQueue.toList()
val end = Instant.now()
log.info {
val elapsed = Duration.between(start, end).toMillis()
val opsPerSecond = String.format("%.2f", numberOfEntries.toDouble() / elapsed * 1000)
"Insertion rate: $opsPerSecond ops/s"
}
inserted
}
log.info {
"Inserted ${entries.size} entries"
}
log.info {
"Starting retrieval"
}
if (entries.isNotEmpty()) {
val completionCounter = AtomicLong(0)
val semaphore = Semaphore(profile.maxConnections * 5)
val start = Instant.now()
val it = entries.iterator()
while (completionCounter.get() < entries.size) {
if (it.hasNext()) {
val entry = it.next()
semaphore.acquire()
val future = client.get(entry.first).thenApply {
if (it == null) {
log.error {
"Missing entry for key '${entry.first}'"
}
} else if (!entry.second.contentEquals(it)) {
log.error {
"Retrieved a value different from what was inserted for key '${entry.first}'"
}
}
}
future.whenComplete { _, _ ->
val completed = completionCounter.incrementAndGet()
if (completed.mod(progressThreshold) == 0L) {
log.debug {
"Retrieved $completed / ${entries.size}"
}
}
semaphore.release()
}
} else {
Thread.sleep(Duration.of(500, ChronoUnit.MILLIS))
}
}
val end = Instant.now()
log.info {
val elapsed = Duration.between(start, end).toMillis()
val opsPerSecond = String.format("%.2f", entries.size.toDouble() / elapsed * 1000)
"Retrieval rate: $opsPerSecond ops/s"
}
} else {
log.error("Skipping retrieval benchmark as it was not possible to insert any entry in the cache")
}
}
}
}
@CommandLine.Spec
@@ -60,113 +175,11 @@ class BenchmarkCommand : RbcsCommand() {
clientCommand.configuration.profiles[profileName]
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
}
val progressThreshold = LongMath.ceilDiv(numberOfEntries.toLong(), 20)
RemoteBuildCacheClient(profile).use { client ->
val entryGenerator = sequence {
val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong())
while (true) {
val key = JWO.bytesToHex(random.nextBytes(16))
val value = if(randomValues) {
random.nextBytes(size)
} else {
val byteValue = random.nextInt().toByte()
ByteArray(size) {_ -> byteValue}
}
yield(key to value)
}
}
log.info {
"Starting insertion"
}
val entries = let {
val completionCounter = AtomicLong(0)
val completionQueue = LinkedBlockingQueue<Pair<String, ByteArray>>(numberOfEntries)
val start = Instant.now()
val semaphore = Semaphore(profile.maxConnections * 5)
val iterator = entryGenerator.take(numberOfEntries).iterator()
while (completionCounter.get() < numberOfEntries) {
if (iterator.hasNext()) {
val entry = iterator.next()
semaphore.acquire()
val future = client.put(entry.first, entry.second, CacheValueMetadata(null, null)).thenApply { entry }
future.whenComplete { result, ex ->
if (ex != null) {
log.error(ex.message, ex)
} else {
completionQueue.put(result)
}
semaphore.release()
val completed = completionCounter.incrementAndGet()
if(completed.mod(progressThreshold) == 0L) {
log.debug {
"Inserted $completed / $numberOfEntries"
}
}
}
} else {
Thread.sleep(Duration.of(500, ChronoUnit.MILLIS))
}
}
val inserted = completionQueue.toList()
val end = Instant.now()
log.info {
val elapsed = Duration.between(start, end).toMillis()
val opsPerSecond = String.format("%.2f", numberOfEntries.toDouble() / elapsed * 1000)
"Insertion rate: $opsPerSecond ops/s"
}
inserted
}
log.info {
"Inserted ${entries.size} entries"
}
log.info {
"Starting retrieval"
}
if (entries.isNotEmpty()) {
val completionCounter = AtomicLong(0)
val semaphore = Semaphore(profile.maxConnections * 5)
val start = Instant.now()
val it = entries.iterator()
while (completionCounter.get() < entries.size) {
if (it.hasNext()) {
val entry = it.next()
semaphore.acquire()
val future = client.get(entry.first).thenApply {
if (it == null) {
log.error {
"Missing entry for key '${entry.first}'"
}
} else if (!entry.second.contentEquals(it)) {
log.error {
"Retrieved a value different from what was inserted for key '${entry.first}'"
}
}
}
future.whenComplete { _, _ ->
val completed = completionCounter.incrementAndGet()
if(completed.mod(progressThreshold) == 0L) {
log.debug {
"Retrieved $completed / ${entries.size}"
}
}
semaphore.release()
}
} else {
Thread.sleep(Duration.of(500, ChronoUnit.MILLIS))
}
}
val end = Instant.now()
log.info {
val elapsed = Duration.between(start, end).toMillis()
val opsPerSecond = String.format("%.2f", entries.size.toDouble() / elapsed * 1000)
"Retrieval rate: $opsPerSecond ops/s"
}
} else {
log.error("Skipping retrieval benchmark as it was not possible to insert any entry in the cache")
}
}
run(
profile,
numberOfEntries,
size,
randomValues
)
}
}

40
rbcs-cli/src/main/kotlin/net/woggioni/rbcs/cli/impl/commands/HealthCheckCommand.kt
View File

@@ -15,6 +15,27 @@ import kotlin.random.Random
class HealthCheckCommand : RbcsCommand() {
companion object{
private val log = createLogger<HealthCheckCommand>()
fun run(profile : RemoteBuildCacheClient.Configuration.Profile) {
RemoteBuildCacheClient(profile).use { client ->
val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong())
val nonce = ByteArray(0xa0)
random.nextBytes(nonce)
client.healthCheck(nonce).thenApply { value ->
if(value == null) {
throw IllegalStateException("Empty response from server")
}
val offset = value.size - nonce.size
for(i in 0 until nonce.size) {
val a = nonce[i]
val b = value[offset + i]
if(a != b) {
throw IllegalStateException("Server nonce does not match")
}
}
}.get()
}
}
}
@CommandLine.Spec
@@ -26,23 +47,6 @@ class HealthCheckCommand : RbcsCommand() {
clientCommand.configuration.profiles[profileName]
?: throw IllegalArgumentException("Profile $profileName does not exist in configuration")
}
RemoteBuildCacheClient(profile).use { client ->
val random = Random(SecureRandom.getInstance("NativePRNGNonBlocking").nextLong())
val nonce = ByteArray(0xa0)
random.nextBytes(nonce)
client.healthCheck(nonce).thenApply { value ->
if(value == null) {
throw IllegalStateException("Empty response from server")
}
val offset = value.size - nonce.size
for(i in 0 until nonce.size) {
val a = nonce[i]
val b = value[offset + i]
if(a != b) {
throw IllegalStateException("Server nonce does not match")
}
}
}.get()
}
run(profile)
}
}