Compare commits

...

9 Commits

Author SHA1 Message Date
303828392e updated Netty to 4.2.1
All checks were successful
CI / build (push) Successful in 26m57s
2025-05-07 14:46:02 +08:00
5d8cbe34ef updated tomcat configuration 2025-05-01 03:42:37 +08:00
85c0d4a384 update Netty to 4.2.0 2025-04-16 01:06:28 +08:00
ae8817ad2a updated benchmarks 2025-03-24 15:01:56 +08:00
69f215e68f tuned GC parameters in Docker images
All checks were successful
CI / build (push) Successful in 16m35s
2025-03-24 14:42:04 +08:00
222b475223 ensured in-memory-cache is allocated to heap memory
All checks were successful
CI / build (push) Successful in 12m29s
2025-03-11 12:29:43 +08:00
ede515e2ca rebuild native image with wider ISA compatibility
All checks were successful
CI / build (push) Successful in 14m13s
2025-03-10 22:28:55 +08:00
974fdb7a91 added k8s benchmark files
All checks were successful
CI / build (push) Successful in 15m39s
2025-03-10 10:09:30 +08:00
a294229ff0 fixed memory leak in MemcachedCacheHandler 2025-03-09 22:16:03 +08:00
27 changed files with 656 additions and 283 deletions

View File

@@ -0,0 +1,80 @@
name: CI
on:
push:
branches:
- 'dev'
jobs:
build:
runs-on: hostinger
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v3
- name: Execute Gradle build
run: ./gradlew build
- name: Prepare Docker image build
run: ./gradlew prepareDockerBuild
- name: Get project version
id: retrieve-version
run: ./gradlew -q version >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: Login to Gitea container registry
uses: docker/login-action@v3
with:
registry: gitea.woggioni.net
username: woggioni
password: ${{ secrets.PUBLISHER_TOKEN }}
-
name: Build rbcs Docker image
uses: docker/build-push-action@v5.3.0
with:
context: "docker/build/docker"
platforms: linux/amd64,linux/arm64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:vanilla-dev
target: release-vanilla
cache-from: type=registry,ref=gitea.woggioni.net/woggioni/rbcs:buildx
-
name: Build rbcs memcache Docker image
uses: docker/build-push-action@v5.3.0
with:
context: "docker/build/docker"
platforms: linux/amd64,linux/arm64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:memcache-dev
target: release-memcache
cache-from: type=registry,ref=gitea.woggioni.net/woggioni/rbcs:buildx
cache-to: type=registry,mode=max,compression=zstd,image-manifest=true,oci-mediatypes=true,ref=gitea.woggioni.net/woggioni/rbcs:buildx
-
name: Build rbcs native Docker image
uses: docker/build-push-action@v5.3.0
with:
context: "docker/build/docker"
platforms: linux/amd64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:native-dev
target: release-native
-
name: Build rbcs jlink Docker image
uses: docker/build-push-action@v5.3.0
with:
context: "docker/build/docker"
platforms: linux/amd64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:jlink-dev
target: release-jlink

View File

@@ -52,6 +52,8 @@ jobs:
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:latest
gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}
gitea.woggioni.net/woggioni/rbcs:memcache
gitea.woggioni.net/woggioni/rbcs:memcache-${{ steps.retrieve-version.outputs.VERSION }}
target: release-memcache
@@ -66,11 +68,21 @@ jobs:
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:latest
gitea.woggioni.net/woggioni/rbcs:${{ steps.retrieve-version.outputs.VERSION }}
gitea.woggioni.net/woggioni/rbcs:native
gitea.woggioni.net/woggioni/rbcs:native-${{ steps.retrieve-version.outputs.VERSION }}
target: release-native
-
name: Build rbcs jlink Docker image
uses: docker/build-push-action@v5.3.0
with:
context: "docker/build/docker"
platforms: linux/amd64
push: true
pull: true
tags: |
gitea.woggioni.net/woggioni/rbcs:jlink
gitea.woggioni.net/woggioni/rbcs:jlink-${{ steps.retrieve-version.outputs.VERSION }}-jlink
target: release-jlink
- name: Publish artifacts
env:
PUBLISHER_TOKEN: ${{ secrets.PUBLISHER_TOKEN }}

View File

@@ -0,0 +1,94 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: rbcs-server
data:
rbcs-server.xml: |
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server"
xmlns:rbcs-memcache="urn:net.woggioni.rbcs.server.memcache"
xs:schemaLocation="urn:net.woggioni.rbcs.server.memcache jpms://net.woggioni.rbcs.server.memcache/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"
>
<bind host="0.0.0.0" port="8080" incoming-connections-backlog-size="128"/>
<connection
max-request-size="0xd000000"
idle-timeout="PT15S"
read-idle-timeout="PT30S"
write-idle-timeout="PT30S"/>
<event-executor use-virtual-threads="true"/>
<cache xs:type="rbcs:fileSystemCacheType" max-age="P7D" enable-compression="false" path="/home/luser/cache" digest="SHA-224"/>
</rbcs:server>
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rbcs-pvc
namespace: default
spec:
accessModes:
- ReadWriteOnce
storageClassName: local-path
resources:
requests:
storage: 16Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rbcs-deployment
labels:
app: rbcs
spec:
replicas: 1
selector:
matchLabels:
app: rbcs
template:
metadata:
labels:
app: rbcs
spec:
containers:
- name: rbcs
image: gitea.woggioni.net/woggioni/rbcs:memcache
imagePullPolicy: Always
command: ["java", "-Dlogback.configurationFile=logback.xml", "-XX:MaxRAMPercentage=75","-jar", "/home/luser/rbcs.jar"]
args: ['server', '-c', 'rbcs-server.xml']
ports:
- containerPort: 8080
volumeMounts:
- name: config-volume
mountPath: /home/luser/rbcs-server.xml
subPath: rbcs-server.xml
- name: cache-volume
mountPath: /home/luser/cache
resources:
requests:
memory: "0.25Gi"
cpu: "1"
limits:
memory: "0.5Gi"
cpu: "1"
volumes:
- name: config-volume
configMap:
name: rbcs-server
- name: cache-volume
persistentVolumeClaim:
claimName: rbcs-pvc
---
apiVersion: v1
kind: Service
metadata:
name: rbcs-service
spec:
type: LoadBalancer
ports:
- port: 8080
targetPort: 8080
protocol: TCP
selector:
app: rbcs

View File

@@ -0,0 +1,77 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: rbcs-server
data:
rbcs-server.xml: |
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server"
xmlns:rbcs-memcache="urn:net.woggioni.rbcs.server.memcache"
xs:schemaLocation="urn:net.woggioni.rbcs.server.memcache jpms://net.woggioni.rbcs.server.memcache/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"
>
<bind host="0.0.0.0" port="8080" incoming-connections-backlog-size="128"/>
<connection
max-request-size="0xd000000"
idle-timeout="PT15S"
read-idle-timeout="PT30S"
write-idle-timeout="PT30S"/>
<event-executor use-virtual-threads="true"/>
<cache xs:type="rbcs:inMemoryCacheType" max-age="P7D" enable-compression="false" max-size="0x40000000" digest="SHA-224"/>
</rbcs:server>
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rbcs-deployment
labels:
app: rbcs
spec:
replicas: 1
selector:
matchLabels:
app: rbcs
template:
metadata:
labels:
app: rbcs
spec:
containers:
- name: rbcs
image: gitea.woggioni.net/woggioni/rbcs:memcache
imagePullPolicy: Always
command: ["java", "-Dlogback.configurationFile=logback.xml", "-XX:MaxRAMPercentage=75","-jar", "/home/luser/rbcs.jar"]
args: ['server', '-c', 'rbcs-server.xml']
ports:
- containerPort: 8080
volumeMounts:
- name: config-volume
mountPath: /home/luser/rbcs-server.xml
subPath: rbcs-server.xml
resources:
requests:
memory: "0.5Gi"
cpu: "1"
limits:
memory: "4Gi"
cpu: "1"
volumes:
- name: config-volume
configMap:
name: rbcs-server
---
apiVersion: v1
kind: Service
metadata:
name: rbcs-service
spec:
type: LoadBalancer
ports:
- port: 8080
targetPort: 8080
protocol: TCP
selector:
app: rbcs

118
benchmark/rbcs-memcache.yml Normal file
View File

@@ -0,0 +1,118 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: rbcs-server
data:
rbcs-server.xml: |
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<rbcs:server xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
xmlns:rbcs="urn:net.woggioni.rbcs.server"
xmlns:rbcs-memcache="urn:net.woggioni.rbcs.server.memcache"
xs:schemaLocation="urn:net.woggioni.rbcs.server.memcache jpms://net.woggioni.rbcs.server.memcache/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd urn:net.woggioni.rbcs.server jpms://net.woggioni.rbcs.server/net/woggioni/rbcs/server/schema/rbcs-server.xsd"
>
<bind host="0.0.0.0" port="8080" incoming-connections-backlog-size="128"/>
<connection
max-request-size="0xd000000"
idle-timeout="PT15S"
read-idle-timeout="PT30S"
write-idle-timeout="PT30S"/>
<event-executor use-virtual-threads="true"/>
<!--cache xs:type="rbcs:inMemoryCacheType" max-age="P7D" enable-compression="false" max-size="0x10000000" /-->
<cache xs:type="rbcs-memcache:memcacheCacheType" max-age="P7D" chunk-size="0x1000" digest="SHA-224">
<server host="memcached-service" port="11211" max-connections="256"/>
</cache>
</rbcs:server>
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rbcs-deployment
labels:
app: rbcs
spec:
replicas: 1
selector:
matchLabels:
app: rbcs
template:
metadata:
labels:
app: rbcs
spec:
containers:
- name: rbcs
image: gitea.woggioni.net/woggioni/rbcs:memcache
imagePullPolicy: Always
command: ["java", "-Dlogback.configurationFile=logback.xml", "-XX:MaxRAMPercentage=75","-jar", "/home/luser/rbcs.jar"]
args: ['server', '-c', 'rbcs-server.xml']
ports:
- containerPort: 8080
volumeMounts:
- name: config-volume
mountPath: /home/luser/rbcs-server.xml
subPath: rbcs-server.xml
resources:
requests:
memory: "0.5Gi"
cpu: "1"
limits:
memory: "0.5Gi"
cpu: "3.5"
volumes:
- name: config-volume
configMap:
name: rbcs-server
---
apiVersion: v1
kind: Service
metadata:
name: rbcs-service
spec:
type: LoadBalancer
ports:
- port: 8080
targetPort: 8080
protocol: TCP
selector:
app: rbcs
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: memcached-deployment
spec:
replicas: 1
selector:
matchLabels:
app: memcached
template:
metadata:
labels:
app: memcached
spec:
containers:
- name: memcached
image: memcached
args: ["-I", "128m", "-m", "4096", "-t", "1"]
resources:
requests:
memory: "1Gi"
cpu: "500m" # 0.5 CPU
limits:
memory: "5Gi"
cpu: "500m" # 0.5 CP
---
apiVersion: v1
kind: Service
metadata:
name: memcached-service
spec:
type: ClusterIP # ClusterIP makes it accessible only within the cluster
ports:
- port: 11211 # Default memcached port
targetPort: 11211
protocol: TCP
selector:
app: memcached

View File

@@ -16,22 +16,22 @@ All test were executed under the following conditions:
| Cache backend | CPU | CPU quota | Memory quota (GB) | Request size (b) | Client connections | PUT (req/s) | GET (req/s) |
|----------------|---------------------|-----------|-------------------|------------------|--------------------|-------------|-------------|
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 128 | 10 | 3691 | 4037 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 128 | 100 | 6881 | 7483 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 512 | 10 | 3790 | 4069 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 512 | 100 | 6716 | 7408 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 4096 | 10 | 3399 | 1974 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 4096 | 100 | 5341 | 6402 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 65536 | 10 | 1099 | 1116 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 65536 | 100 | 1379 | 1703 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 128 | 10 | 4443 | 5170 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 128 | 100 | 12813 | 13568 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 512 | 10 | 4450 | 4383 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 512 | 100 | 12212 | 13586 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 4096 | 10 | 3441 | 3012 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 4096 | 100 | 8982 | 10452 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 65536 | 10 | 1391 | 1167 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 65536 | 100 | 1303 | 1151 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 128 | 10 | 7867 | 13762 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 128 | 100 | 7728 | 14180 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 512 | 10 | 7964 | 10992 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 512 | 100 | 8415 | 12478 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 4096 | 10 | 4268 | 5395 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 4096 | 100 | 5585 | 8259 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 65536 | 10 | 1063 | 1185 |
| in-memory | Intel Celeron J3455 | 1.00 | 4 | 65536 | 100 | 1522 | 1366 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 128 | 10 | 11271 | 14092 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 128 | 100 | 16064 | 24201 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 512 | 10 | 11504 | 13077 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 512 | 100 | 17379 | 22094 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 4096 | 10 | 9151 | 9489 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 4096 | 100 | 13194 | 18268 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 65536 | 10 | 1590 | 1174 |
| in-memory | Intel Celeron J3455 | 3.50 | 4 | 65536 | 100 | 1539 | 1561 |
### Filesystem cache backend
@@ -42,23 +42,22 @@ TLS: disabled
| Cache backend | CPU | CPU quota | Memory quota (GB) | Request size (b) | Client connections | PUT (req/s) | GET (req/s) |
|---------------|---------------------|-----------|-------------------|------------------|--------------------|-------------|-------------|
| filesystem | Intel Celeron J3455 | 1.00 | 0.25 | 128 | 10 | 1208 | 2048 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.25 | 128 | 100 | 1304 | 2394 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.25 | 512 | 10 | 1408 | 2157 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.25 | 512 | 100 | 1282 | 1888 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.25 | 4096 | 10 | 1291 | 1256 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.25 | 4096 | 100 | 1170 | 1423 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.25 | 65536 | 10 | 313 | 606 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.25 | 65536 | 100 | 298 | 609 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.25 | 128 | 10 | 2195 | 3477 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.25 | 128 | 100 | 2480 | 6207 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.25 | 512 | 10 | 2164 | 3413 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.25 | 512 | 100 | 2842 | 6218 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.25 | 4096 | 10 | 1302 | 2591 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.25 | 4096 | 100 | 2270 | 3045 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.25 | 65536 | 10 | 375 | 394 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.25 | 65536 | 100 | 364 | 462 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.5 | 128 | 10 | 1478 | 5771 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.5 | 128 | 100 | 3166 | 8070 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.5 | 512 | 10 | 1717 | 5895 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.5 | 512 | 100 | 1125 | 6564 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.5 | 4096 | 10 | 819 | 2509 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.5 | 4096 | 100 | 1136 | 2365 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.5 | 65536 | 10 | 584 | 632 |
| filesystem | Intel Celeron J3455 | 1.00 | 0.5 | 65536 | 100 | 529 | 635 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.5 | 128 | 10 | 1227 | 3342 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.5 | 128 | 100 | 1156 | 4035 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.5 | 512 | 10 | 979 | 3294 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.5 | 512 | 100 | 1217 | 3888 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.5 | 4096 | 10 | 535 | 1805 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.5 | 4096 | 100 | 555 | 1910 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.5 | 65536 | 10 | 301 | 494 |
| filesystem | Intel Celeron J3455 | 3.50 | 0.5 | 65536 | 100 | 353 | 595 |
### Memcache cache backend
@@ -69,19 +68,19 @@ TLS: disabled
| Cache backend | CPU | CPU quota | Memory quota (GB) | Request size (b) | Client connections | PUT (req/s) | GET (req/s) |
|---------------|---------------------|-----------|-------------------|------------------|--------------------|-------------|-------------|
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 128 | 10 | 2505 | 2578 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 128 | 100 | 3582 | 3935 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 512 | 10 | 2495 | 2784 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 512 | 100 | 3565 | 3883 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 4096 | 10 | 2174 | 2505 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 4096 | 100 | 2937 | 3563 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 65536 | 10 | 648 | 1074 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 65536 | 100 | 724 | 1548 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 128 | 10 | 2362 | 2927 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 128 | 100 | 5491 | 6531 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 512 | 10 | 2125 | 2807 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 512 | 100 | 5173 | 6242 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 4096 | 10 | 1720 | 2397 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 4096 | 100 | 3871 | 5859 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 65536 | 10 | 616 | 1016 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 65536 | 100 | 820 | 1677 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 128 | 10 | 3380 | 6083 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 128 | 100 | 3323 | 4998 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 512 | 10 | 3924 | 6086 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 512 | 100 | 3440 | 5049 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 4096 | 10 | 3347 | 5255 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 4096 | 100 | 3685 | 4693 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 65536 | 10 | 1304 | 1343 |
| memcache | Intel Celeron J3455 | 1.00 | 0.25 | 65536 | 100 | 1481 | 1541 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 128 | 10 | 4667 | 7984 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 128 | 100 | 4044 | 8358 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 512 | 10 | 4177 | 7828 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 512 | 100 | 4079 | 8794 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 4096 | 10 | 4588 | 6869 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 4096 | 100 | 5343 | 7797 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 65536 | 10 | 1624 | 1317 |
| memcache | Intel Celeron J3455 | 3.50 | 0.25 | 65536 | 100 | 1633 | 1317 |

View File

@@ -5,7 +5,7 @@ WORKDIR /home/luser
FROM base-release AS release-vanilla
ADD rbcs-cli-envelope-*.jar rbcs.jar
ENTRYPOINT ["java", "-XX:+UseSerialGC", "-XX:GCTimeRatio=24", "-jar", "/home/luser/rbcs.jar", "server"]
ENTRYPOINT ["java", "-Dlogback.configurationFile=logback.xml", "-XX:MaxRAMPercentage=70", "-XX:GCTimeRatio=24", "-XX:+UseZGC", "-XX:+ZGenerational", "-jar", "/home/luser/rbcs.jar"]
FROM base-release AS release-memcache
ADD --chown=luser:luser rbcs-cli-envelope-*.jar rbcs.jar
@@ -14,10 +14,21 @@ WORKDIR /home/luser/plugins
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-server-memcache*.tar
WORKDIR /home/luser
ADD logback.xml .
ENTRYPOINT ["java", "-Dlogback.configurationFile=logback.xml", "-XX:+UseSerialGC", "-XX:GCTimeRatio=24", "-jar", "/home/luser/rbcs.jar", "server"]
ENTRYPOINT ["java", "-Dlogback.configurationFile=logback.xml", "-XX:MaxRAMPercentage=70", "-XX:GCTimeRatio=24", "-XX:+UseZGC", "-XX:+ZGenerational", "-jar", "/home/luser/rbcs.jar"]
FROM scratch AS release-native
ADD rbcs-cli.upx /rbcs/rbcs-cli
ENV RBCS_CONFIGURATION_DIR="/rbcs"
WORKDIR /rbcs
ENTRYPOINT ["/rbcs/rbcs-cli"]
ENTRYPOINT ["/rbcs/rbcs-cli", "-XX:MaximumHeapSizePercent=70"]
FROM debian:12-slim AS release-jlink
RUN mkdir -p /usr/share/java/rbcs
RUN --mount=type=bind,source=.,target=/build/distributions tar -xf /build/distributions/rbcs-cli*.tar -C /usr/share/java/rbcs
ADD --chmod=755 rbcs-cli.sh /usr/local/bin/rbcs-cli
RUN adduser -u 1000 luser
USER luser
WORKDIR /home/luser
ADD logback.xml .
ENV JAVA_OPTS=-XX:-UseJVMCICompiler\ -Dlogback.configurationFile=logback.xml\ -XX:MaxRAMPercentage=70\ -XX:GCTimeRatio=24\ -XX:+UseZGC\ -XX:+ZGenerational
ENTRYPOINT ["/usr/local/bin/rbcs-cli"]

View File

@@ -11,11 +11,15 @@ The `memcache` image is similar to the `vanilla` image, except that it also cont
the `rbcs-server-memcache` plugin in the `plugins` folder, use this image if you don't want to use the `native`
image and want to use memcache as the cache backend
The `native` image contains a native, statically-linked executable created with GraalVM
The `native` image contains a native, statically-linked executable created with GraalVM Native Image
that has no userspace dependencies. It also embeds the memcache plugin inside the executable.
Use this image for maximum efficiency and minimal memory footprint.
## Which image shoud I use?
The `jlink` image contains a custom Java runtime created with GraalVM's Jlink
that only depends on glibc. It also contains the memcache plugin in the module path.
Use this image for best performance.
## Which image should I use?
The `native` image uses Java's SerialGC, so it's ideal for constrained environment like containers or small servers,
if you have a lot of resources and want to squeeze out the maximum throughput you should consider the
`vanilla` or `memcache` image, then choose and fine tune the garbage collector.

View File

@@ -29,7 +29,7 @@ Provider<Copy> prepareDockerBuild = tasks.register('prepareDockerBuild', Copy) {
group = 'docker'
into project.layout.buildDirectory.file('docker')
from(configurations.docker)
from(file('Dockerfile'))
from(files('Dockerfile', 'rbcs-cli.sh'))
from(rootProject.file('conf')) {
include 'logback.xml'
}

3
docker/rbcs-cli.sh Normal file
View File

@@ -0,0 +1,3 @@
#!/bin/sh
DIR=/usr/share/java/rbcs
$DIR/bin/java $JAVA_OPTS -m net.woggioni.rbcs.cli "$@"

View File

@@ -2,9 +2,9 @@ org.gradle.configuration-cache=false
org.gradle.parallel=true
org.gradle.caching=true
rbcs.version = 0.2.0
rbcs.version = 0.3.0-SNAPSHOT
lys.version = 2025.03.08
lys.version = 2025.05.07
gitea.maven.url = https://gitea.woggioni.net/api/packages/woggioni/maven
docker.registry.url=gitea.woggioni.net

View File

@@ -1,10 +1,9 @@
package net.woggioni.rbcs.api;
import java.io.Serializable;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import java.io.Serializable;
@Getter
@RequiredArgsConstructor
public class CacheValueMetadata implements Serializable {

View File

@@ -1,16 +1,15 @@
package net.woggioni.rbcs.api;
import lombok.EqualsAndHashCode;
import lombok.NonNull;
import lombok.Value;
import java.nio.file.Path;
import java.security.cert.X509Certificate;
import java.time.Duration;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.EqualsAndHashCode;
import lombok.NonNull;
import lombok.Value;
@Value
public class Configuration {

View File

@@ -18,6 +18,7 @@ import net.woggioni.gradle.graalvm.UpxTask
import net.woggioni.gradle.graalvm.JlinkPlugin
import net.woggioni.gradle.graalvm.JlinkTask
sourceSets {
configureNativeImage {
java {
@@ -29,6 +30,7 @@ sourceSets {
}
configurations {
release {
transitive = false
canBeConsumed = true
@@ -120,16 +122,36 @@ nativeImage {
linkAtBuildTime = false
classpath = project.files(jarTaskProvider, configurations.nativeImage)
compressExecutable = true
compressionLevel = 10
compressionLevel = 6
useLZMA = false
}
Provider<UpxTask> upxTaskProvider = tasks.named(NativeImagePlugin.UPX_TASK_NAME, UpxTask) {
}
tasks.named(JlinkPlugin.JLINK_TASK_NAME, JlinkTask) {
Provider<JlinkTask> jlinkTaskProvider = tasks.named(JlinkPlugin.JLINK_TASK_NAME, JlinkTask) {
toolchain {
languageVersion = JavaLanguageVersion.of(21)
vendor = JvmVendorSpec.GRAAL_VM
}
mainClass = mainClassName
mainModule = 'net.woggioni.rbcs.cli'
classpath = project.files(
configurations.configureNativeImageRuntimeClasspath,
sourceSets.configureNativeImage.output
)
additionalModules = [
'net.woggioni.rbcs.server.memcache',
'ch.qos.logback.classic',
'jdk.crypto.ec'
]
compressionLevel = 2
stripDebug = false
}
Provider<Tar> jlinkDistTarTaskProvider = tasks.named(JlinkPlugin.JLINK_DIST_TAR_TASK_NAME, Tar) {
exclude 'lib/libjvmcicompiler.so'
}
tasks.named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, ProcessResources) {
@@ -143,6 +165,7 @@ tasks.named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, ProcessResources) {
artifacts {
release(envelopeJarTaskProvider)
release(upxTaskProvider)
release(jlinkDistTarTaskProvider)
}
publishing {

View File

@@ -1,2 +1,2 @@
Args=-O3 -march=skylake --gc=serial --install-exit-handlers --initialize-at-run-time=io.netty --enable-url-protocols=jpms --initialize-at-build-time=net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory,net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory$JpmsHandler
Args=-O3 -march=x86-64-v2 --gc=serial --install-exit-handlers --initialize-at-run-time=io.netty --enable-url-protocols=jpms --initialize-at-build-time=net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory,net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory$JpmsHandler
#-H:TraceClassInitialization=io.netty.handler.ssl.BouncyCastleAlpnSslUtils

View File

@@ -120,6 +120,14 @@
"name":"io.netty.buffer.AbstractReferenceCountedByteBuf",
"fields":[{"name":"refCnt"}]
},
{
"name":"io.netty.buffer.AdaptivePoolingAllocator$Chunk",
"fields":[{"name":"refCnt"}]
},
{
"name":"io.netty.buffer.AdaptivePoolingAllocator$Magazine",
"fields":[{"name":"nextInLine"}]
},
{
"name":"io.netty.channel.AbstractChannelHandlerContext",
"fields":[{"name":"handlerState"}]
@@ -284,20 +292,13 @@
"fields":[{"name":"producerIndex"}]
},
{
"name":"io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField",
"name":"io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField",
"fields":[{"name":"consumerIndex"}]
},
{
"name":"io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField",
"name":"io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField",
"fields":[{"name":"producerIndex"}]
},
{
"name":"io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField",
"fields":[{"name":"producerLimit"}]
},
{
"name":"java.io.FilePermission"
},
{
"name":"java.lang.Object",
"allDeclaredFields":true,
@@ -307,26 +308,14 @@
"name":"java.lang.ProcessHandle",
"methods":[{"name":"current","parameterTypes":[] }, {"name":"pid","parameterTypes":[] }]
},
{
"name":"java.lang.RuntimePermission"
},
{
"name":"java.lang.System",
"methods":[{"name":"console","parameterTypes":[] }]
},
{
"name":"java.lang.Thread",
"fields":[{"name":"threadLocalRandomProbe"}]
},
{
"name":"java.net.NetPermission"
},
{
"name":"java.net.SocketPermission"
},
{
"name":"java.net.URLPermission",
"methods":[{"name":"<init>","parameterTypes":["java.lang.String","java.lang.String"] }]
"fields":[{"name":"threadLocalRandomProbe"}],
"methods":[{"name":"isVirtual","parameterTypes":[] }]
},
{
"name":"java.nio.Bits",
@@ -358,18 +347,12 @@
{
"name":"java.security.AlgorithmParametersSpi"
},
{
"name":"java.security.AllPermission"
},
{
"name":"java.security.KeyStoreSpi"
},
{
"name":"java.security.SecureRandomParameters"
},
{
"name":"java.security.SecurityPermission"
},
{
"name":"java.sql.Connection"
},
@@ -444,9 +427,6 @@
"name":"java.time.ZonedDateTime",
"methods":[{"name":"parse","parameterTypes":["java.lang.CharSequence"] }]
},
{
"name":"java.util.PropertyPermission"
},
{
"name":"java.util.concurrent.ForkJoinTask",
"fields":[{"name":"aux"}, {"name":"status"}]
@@ -467,22 +447,11 @@
"name":"java.util.concurrent.atomic.Striped64$Cell",
"fields":[{"name":"value"}]
},
{
"name":"java.util.zip.Adler32",
"methods":[{"name":"update","parameterTypes":["java.nio.ByteBuffer"] }]
},
{
"name":"java.util.zip.CRC32",
"methods":[{"name":"update","parameterTypes":["java.nio.ByteBuffer"] }]
},
{
"name":"javax.security.auth.x500.X500Principal",
"fields":[{"name":"thisX500Name"}],
"methods":[{"name":"<init>","parameterTypes":["sun.security.x509.X500Name"] }]
},
{
"name":"javax.smartcardio.CardPermission"
},
{
"name":"jdk.internal.misc.Unsafe",
"methods":[{"name":"getUnsafe","parameterTypes":[] }]
@@ -588,6 +557,9 @@
"name":"net.woggioni.rbcs.server.exception.ExceptionHandler",
"methods":[{"name":"exceptionCaught","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Throwable"] }]
},
{
"name":"net.woggioni.rbcs.server.handler.BlackHoleRequestHandler"
},
{
"name":"net.woggioni.rbcs.server.handler.MaxRequestSizeHandler",
"methods":[{"name":"channelRead","parameterTypes":["io.netty.channel.ChannelHandlerContext","java.lang.Object"] }]
@@ -615,7 +587,7 @@
{
"name":"sun.misc.Unsafe",
"fields":[{"name":"theUnsafe"}],
"methods":[{"name":"copyMemory","parameterTypes":["java.lang.Object","long","java.lang.Object","long","long"] }, {"name":"getAndAddLong","parameterTypes":["java.lang.Object","long","long"] }, {"name":"getAndSetObject","parameterTypes":["java.lang.Object","long","java.lang.Object"] }, {"name":"invokeCleaner","parameterTypes":["java.nio.ByteBuffer"] }, {"name":"storeFence","parameterTypes":[] }]
"methods":[{"name":"copyMemory","parameterTypes":["java.lang.Object","long","java.lang.Object","long","long"] }, {"name":"getAndAddLong","parameterTypes":["java.lang.Object","long","long"] }, {"name":"getAndSetObject","parameterTypes":["java.lang.Object","long","java.lang.Object"] }, {"name":"invokeCleaner","parameterTypes":["java.nio.ByteBuffer"] }]
},
{
"name":"sun.nio.ch.SelectorImpl",
@@ -637,10 +609,6 @@
"name":"sun.security.provider.DSA$SHA256withDSA",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"name":"sun.security.provider.JavaKeyStore$JKS",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"name":"sun.security.provider.MD5",
"methods":[{"name":"<init>","parameterTypes":[] }]
@@ -729,6 +697,14 @@
"name":"sun.security.x509.CertificatePoliciesExtension",
"methods":[{"name":"<init>","parameterTypes":["java.lang.Boolean","java.lang.Object"] }]
},
{
"name":"sun.security.x509.ExtendedKeyUsageExtension",
"methods":[{"name":"<init>","parameterTypes":["java.lang.Boolean","java.lang.Object"] }]
},
{
"name":"sun.security.x509.IssuerAlternativeNameExtension",
"methods":[{"name":"<init>","parameterTypes":["java.lang.Boolean","java.lang.Object"] }]
},
{
"name":"sun.security.x509.KeyUsageExtension",
"methods":[{"name":"<init>","parameterTypes":["java.lang.Boolean","java.lang.Object"] }]

View File

@@ -32,14 +32,12 @@
"pattern":"\\Qnet/woggioni/rbcs/cli/logback.xml\\E"
}, {
"pattern":"\\Qnet/woggioni/rbcs/client/schema/rbcs-client.xsd\\E"
}, {
"pattern":"\\Qnet/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd\\E"
}, {
"pattern":"\\Qnet/woggioni/rbcs/server/rbcs-default.xml\\E"
}, {
"pattern":"\\Qnet/woggioni/rbcs/server/schema/rbcs-server.xsd\\E"
}, {
"pattern":"\\Q/net/woggioni/rbcs/server/memcache/schema/rbcs-memcache.xsd\\E"
}, {
"pattern":"java.base:\\Qsun/text/resources/LineBreakIteratorData\\E"
}]},
"bundles":[{
"name":"com.sun.org.apache.xerces.internal.impl.xpath.regex.message",

View File

@@ -6,7 +6,6 @@ import net.woggioni.rbcs.client.Configuration
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.debug
import picocli.CommandLine
import java.lang.IllegalArgumentException
import java.nio.file.Path
@CommandLine.Command(

View File

@@ -7,8 +7,10 @@ import io.netty.channel.Channel
import io.netty.channel.ChannelHandlerContext
import io.netty.channel.ChannelOption
import io.netty.channel.ChannelPipeline
import io.netty.channel.IoEventLoopGroup
import io.netty.channel.MultiThreadIoEventLoopGroup
import io.netty.channel.SimpleChannelInboundHandler
import io.netty.channel.nio.NioEventLoopGroup
import io.netty.channel.nio.NioIoHandler
import io.netty.channel.pool.AbstractChannelPoolHandler
import io.netty.channel.pool.ChannelPool
import io.netty.channel.pool.FixedChannelPool
@@ -32,12 +34,8 @@ import io.netty.handler.timeout.IdleState
import io.netty.handler.timeout.IdleStateEvent
import io.netty.handler.timeout.IdleStateHandler
import io.netty.util.concurrent.Future
import io.netty.util.concurrent.Future as NettyFuture
import io.netty.util.concurrent.GenericFutureListener
import net.woggioni.rbcs.api.CacheValueMetadata
import net.woggioni.rbcs.common.RBCS.loadKeystore
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.debug
import net.woggioni.rbcs.common.trace
import java.io.IOException
import java.net.InetSocketAddress
import java.net.URI
@@ -50,19 +48,23 @@ import java.util.concurrent.atomic.AtomicInteger
import javax.net.ssl.TrustManagerFactory
import javax.net.ssl.X509TrustManager
import kotlin.random.Random
import io.netty.util.concurrent.Future as NettyFuture
import net.woggioni.rbcs.api.CacheValueMetadata
import net.woggioni.rbcs.common.RBCS.loadKeystore
import net.woggioni.rbcs.common.createLogger
import net.woggioni.rbcs.common.debug
import net.woggioni.rbcs.common.trace
class RemoteBuildCacheClient(private val profile: Configuration.Profile) : AutoCloseable {
companion object {
private val log = createLogger<RemoteBuildCacheClient>()
}
private val group: NioEventLoopGroup
private val group: IoEventLoopGroup
private val sslContext: SslContext
private val pool: ChannelPool
init {
group = NioEventLoopGroup()
group = MultiThreadIoEventLoopGroup(NioIoHandler.newFactory())
sslContext = SslContextBuilder.forClient().also { builder ->
(profile.authentication as? Configuration.Authentication.TlsClientAuthenticationCredentials)?.let { tlsClientAuthenticationCredentials ->
builder.apply {

View File

@@ -1,7 +1,6 @@
package net.woggioni.rbcs.server.memcache
import io.netty.channel.ChannelFactory
import io.netty.channel.ChannelHandler
import io.netty.channel.EventLoopGroup
import io.netty.channel.pool.FixedChannelPool
import io.netty.channel.socket.DatagramChannel

View File

@@ -111,17 +111,17 @@ class MemcacheCacheHandler(
}
if (responseSent) {
acc.readBytes(outputStream, acc.readableBytes())
if(acc.readableBytes() >= chunkSize) {
if (acc.readableBytes() >= chunkSize) {
flush(false)
}
}
}
private fun flush(last : Boolean) {
private fun flush(last: Boolean) {
val toSend = extractChunk(chunk, ctx.alloc())
val msg = if(last) {
val msg = if (last) {
log.trace(ctx) {
"Sending last chunk to client on channel"
"Sending last chunk to client"
}
LastCacheContent(toSend)
} else {
@@ -148,14 +148,14 @@ class MemcacheCacheHandler(
}
private inner class InProgressPutRequest(
private val ch : NettyChannel,
metadata : CacheValueMetadata,
val digest : ByteBuf,
private val ch: NettyChannel,
metadata: CacheValueMetadata,
val digest: ByteBuf,
val requestController: CompletableFuture<MemcacheRequestController>,
private val alloc: ByteBufAllocator
) : InProgressRequest {
private var totalSize = 0
private var tmpFile : FileChannel? = null
private var tmpFile: FileChannel? = null
private val accumulator = alloc.compositeBuffer()
private val stream = ByteBufOutputStream(accumulator).let {
if (compressionEnabled) {
@@ -182,7 +182,7 @@ class MemcacheCacheHandler(
tmpFile?.let {
flushToDisk(it, accumulator)
}
if(accumulator.readableBytes() > 0x100000) {
if (accumulator.readableBytes() > 0x100000) {
log.debug(ch) {
"Entry is too big, buffering it into a file"
}
@@ -199,18 +199,18 @@ class MemcacheCacheHandler(
}
}
private fun flushToDisk(fc : FileChannel, buf : CompositeByteBuf) {
private fun flushToDisk(fc: FileChannel, buf: CompositeByteBuf) {
val chunk = extractChunk(buf, alloc)
fc.write(chunk.nioBuffer())
chunk.release()
}
fun commit() : Pair<Int, ReadableByteChannel> {
fun commit(): Pair<Int, ReadableByteChannel> {
digest.release()
accumulator.retain()
stream.close()
val fileChannel = tmpFile
return if(fileChannel != null) {
return if (fileChannel != null) {
flushToDisk(fileChannel, accumulator)
accumulator.release()
fileChannel.position(0)
@@ -244,7 +244,67 @@ class MemcacheCacheHandler(
}
private fun handleGetRequest(ctx: ChannelHandlerContext, msg: CacheGetRequest) {
inProgressRequest = InProgressGetRequest(msg.key, ctx)
log.debug(ctx) {
"Fetching ${msg.key} from memcache"
}
val key = ctx.alloc().buffer().also {
it.writeBytes(processCacheKey(msg.key, digestAlgorithm))
}
val responseHandler = object : MemcacheResponseHandler {
override fun responseReceived(response: BinaryMemcacheResponse) {
val status = response.status()
when (status) {
BinaryMemcacheResponseStatus.SUCCESS -> {
log.debug(ctx) {
"Cache hit for key ${msg.key} on memcache"
}
inProgressRequest = InProgressGetRequest(msg.key, ctx)
}
BinaryMemcacheResponseStatus.KEY_ENOENT -> {
log.debug(ctx) {
"Cache miss for key ${msg.key} on memcache"
}
sendMessageAndFlush(ctx, CacheValueNotFoundResponse())
}
}
}
override fun contentReceived(content: MemcacheContent) {
log.trace(ctx) {
"${if (content is LastMemcacheContent) "Last chunk" else "Chunk"} of ${
content.content().readableBytes()
} bytes received from memcache for key ${msg.key}"
}
(inProgressRequest as? InProgressGetRequest)?.let { inProgressGetRequest ->
inProgressGetRequest.write(content.content())
if (content is LastMemcacheContent) {
inProgressRequest = null
inProgressGetRequest.commit()
}
}
}
override fun exceptionCaught(ex: Throwable) {
(inProgressRequest as? InProgressGetRequest).let { inProgressGetRequest ->
inProgressGetRequest?.let {
inProgressRequest = null
it.rollback()
}
}
this@MemcacheCacheHandler.exceptionCaught(ctx, ex)
}
}
client.sendRequest(key.retainedDuplicate(), responseHandler).thenAccept { requestHandle ->
log.trace(ctx) {
"Sending GET request for key ${msg.key} to memcache"
}
val request = DefaultBinaryMemcacheRequest(key).apply {
setOpcode(BinaryMemcacheOpcodes.GET)
}
requestHandle.sendRequest(request)
requestHandle.sendContent(LastMemcacheContent.EMPTY_LAST_CONTENT)
}
}
private fun handlePutRequest(ctx: ChannelHandlerContext, msg: CachePutRequest) {
@@ -261,6 +321,7 @@ class MemcacheCacheHandler(
}
sendMessageAndFlush(ctx, CachePutResponse(msg.key))
}
else -> this@MemcacheCacheHandler.exceptionCaught(ctx, MemcacheException(status))
}
}
@@ -282,13 +343,14 @@ class MemcacheCacheHandler(
private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) {
val request = inProgressRequest
when(request) {
when (request) {
is InProgressPutRequest -> {
log.trace(ctx) {
"Received chunk of ${msg.content().readableBytes()} bytes for memcache"
}
request.write(msg.content())
}
is InProgressGetRequest -> {
msg.release()
}
@@ -297,7 +359,7 @@ class MemcacheCacheHandler(
private fun handleLastCacheContent(ctx: ChannelHandlerContext, msg: LastCacheContent) {
val request = inProgressRequest
when(request) {
when (request) {
is InProgressPutRequest -> {
inProgressRequest = null
log.trace(ctx) {
@@ -314,7 +376,7 @@ class MemcacheCacheHandler(
"Trying to send SET request to memcache"
}
request.requestController.whenComplete { requestController, ex ->
if(ex == null) {
if (ex == null) {
log.trace(ctx) {
"Sending SET request to memcache"
}
@@ -332,7 +394,7 @@ class MemcacheCacheHandler(
while (true) {
val read = source.read(bb)
bb.limit()
if(read >= 0 && bb.position() < chunkSize && bb.hasRemaining()) {
if (read >= 0 && bb.position() < chunkSize && bb.hasRemaining()) {
continue
}
val chunk = ctx.alloc().buffer(chunkSize)
@@ -342,7 +404,7 @@ class MemcacheCacheHandler(
log.trace(ctx) {
"Sending ${chunk.readableBytes()} bytes chunk to memcache"
}
if(read < 0) {
if (read < 0) {
requestController.sendContent(DefaultLastMemcacheContent(chunk))
break
} else {
@@ -355,73 +417,12 @@ class MemcacheCacheHandler(
}
}
}
is InProgressGetRequest -> {
log.debug(ctx) {
"Fetching ${request.key} from memcache"
}
val key = ctx.alloc().buffer().also {
it.writeBytes(processCacheKey(request.key, digestAlgorithm))
}
val responseHandler = object : MemcacheResponseHandler {
override fun responseReceived(response: BinaryMemcacheResponse) {
val status = response.status()
when (status) {
BinaryMemcacheResponseStatus.SUCCESS -> {
log.debug(ctx) {
"Cache hit for key ${request.key} on memcache"
}
inProgressRequest = InProgressGetRequest(request.key, ctx)
}
BinaryMemcacheResponseStatus.KEY_ENOENT -> {
log.debug(ctx) {
"Cache miss for key ${request.key} on memcache"
}
sendMessageAndFlush(ctx, CacheValueNotFoundResponse())
}
}
}
override fun contentReceived(content: MemcacheContent) {
log.trace(ctx) {
"${if(content is LastMemcacheContent) "Last chunk" else "Chunk"} of ${content.content().readableBytes()} bytes received from memcache for key ${request.key}"
}
(inProgressRequest as? InProgressGetRequest).let { inProgressGetRequest ->
inProgressGetRequest?.write(content.content())
if (content is LastMemcacheContent) {
inProgressRequest = null
inProgressGetRequest?.commit()
}
}
}
override fun exceptionCaught(ex: Throwable) {
(inProgressRequest as? InProgressGetRequest).let { inProgressGetRequest ->
inProgressGetRequest?.let {
inProgressRequest = null
it.rollback()
}
}
this@MemcacheCacheHandler.exceptionCaught(ctx, ex)
}
}
client.sendRequest(key.retainedDuplicate(), responseHandler).thenAccept { requestHandle ->
log.trace(ctx) {
"Sending GET request for key ${request.key} to memcache"
}
val request = DefaultBinaryMemcacheRequest(key).apply {
setOpcode(BinaryMemcacheOpcodes.GET)
}
requestHandle.sendRequest(request)
requestHandle.sendContent(LastMemcacheContent.EMPTY_LAST_CONTENT)
}
}
}
}
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) {
val request = inProgressRequest
when(request) {
when (request) {
is InProgressPutRequest -> {
inProgressRequest = null
request.requestController.thenAccept { controller ->
@@ -429,6 +430,7 @@ class MemcacheCacheHandler(
}
request.rollback()
}
is InProgressGetRequest -> {
inProgressRequest = null
request.rollback()

View File

@@ -3,21 +3,19 @@ import net.woggioni.rbcs.server.cache.FileSystemCacheProvider;
import net.woggioni.rbcs.server.cache.InMemoryCacheProvider;
module net.woggioni.rbcs.server {
requires java.sql;
requires java.xml;
requires java.logging;
requires java.naming;
requires kotlin.stdlib;
requires io.netty.buffer;
requires io.netty.transport;
requires io.netty.codec.http;
requires io.netty.common;
requires io.netty.handler;
requires io.netty.codec;
requires org.slf4j;
requires net.woggioni.jwo;
requires net.woggioni.rbcs.common;
requires net.woggioni.rbcs.api;
requires io.netty.codec.compression;
requires io.netty.transport;
requires io.netty.buffer;
requires io.netty.common;
requires org.slf4j;
exports net.woggioni.rbcs.server;

View File

@@ -11,7 +11,8 @@ import io.netty.channel.ChannelInboundHandlerAdapter
import io.netty.channel.ChannelInitializer
import io.netty.channel.ChannelOption
import io.netty.channel.ChannelPromise
import io.netty.channel.nio.NioEventLoopGroup
import io.netty.channel.MultiThreadIoEventLoopGroup
import io.netty.channel.nio.NioIoHandler
import io.netty.channel.socket.DatagramChannel
import io.netty.channel.socket.ServerSocketChannel
import io.netty.channel.socket.SocketChannel
@@ -34,8 +35,25 @@ import io.netty.handler.timeout.IdleState
import io.netty.handler.timeout.IdleStateEvent
import io.netty.handler.timeout.IdleStateHandler
import io.netty.util.AttributeKey
import io.netty.util.concurrent.DefaultEventExecutorGroup
import io.netty.util.concurrent.EventExecutorGroup
import java.io.OutputStream
import java.net.InetSocketAddress
import java.nio.file.Files
import java.nio.file.Path
import java.security.PrivateKey
import java.security.cert.X509Certificate
import java.time.Duration
import java.time.Instant
import java.util.Arrays
import java.util.Base64
import java.util.concurrent.CompletableFuture
import java.util.concurrent.Future
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import java.util.regex.Matcher
import java.util.regex.Pattern
import javax.naming.ldap.LdapName
import javax.net.ssl.SSLPeerUnverifiedException
import net.woggioni.rbcs.api.AsyncCloseable
import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.api.exception.ConfigurationException
@@ -59,24 +77,6 @@ import net.woggioni.rbcs.server.handler.MaxRequestSizeHandler
import net.woggioni.rbcs.server.handler.ServerHandler
import net.woggioni.rbcs.server.throttling.BucketManager
import net.woggioni.rbcs.server.throttling.ThrottlingHandler
import java.io.OutputStream
import java.net.InetSocketAddress
import java.nio.file.Files
import java.nio.file.Path
import java.security.PrivateKey
import java.security.cert.X509Certificate
import java.time.Duration
import java.time.Instant
import java.util.Arrays
import java.util.Base64
import java.util.concurrent.CompletableFuture
import java.util.concurrent.Future
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import java.util.regex.Matcher
import java.util.regex.Pattern
import javax.naming.ldap.LdapName
import javax.net.ssl.SSLPeerUnverifiedException
class RemoteBuildCacheServer(private val cfg: Configuration) {
@@ -208,7 +208,6 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
private val cfg: Configuration,
private val channelFactory : ChannelFactory<SocketChannel>,
private val datagramChannelFactory : ChannelFactory<DatagramChannel>,
private val eventExecutorGroup: EventExecutorGroup
) : ChannelInitializer<Channel>(), AsyncCloseable {
companion object {
@@ -360,7 +359,7 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
cacheHandlerFactory.newHandler(cfg, ch.eventLoop(), channelFactory, datagramChannelFactory)
}
}
pipeline.addLast(eventExecutorGroup, ServerHandler.NAME, serverHandler)
pipeline.addLast(ServerHandler.NAME, serverHandler)
pipeline.addLast(ExceptionHandler.NAME, ExceptionHandler)
pipeline.addLast(BlackHoleRequestHandler.NAME, BlackHoleRequestHandler())
}
@@ -441,20 +440,13 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
fun run(): ServerHandle {
// Create the multithreaded event loops for the server
val bossGroup = NioEventLoopGroup(1)
val bossGroup = MultiThreadIoEventLoopGroup(1, NioIoHandler.newFactory())
val channelFactory = ChannelFactory<SocketChannel> { NioSocketChannel() }
val datagramChannelFactory = ChannelFactory<DatagramChannel> { NioDatagramChannel() }
val serverChannelFactory = ChannelFactory<ServerSocketChannel> { NioServerSocketChannel() }
val workerGroup = NioEventLoopGroup(0)
val eventExecutorGroup = run {
val threadFactory = if (cfg.eventExecutor.isUseVirtualThreads) {
Thread.ofVirtual().factory()
} else {
null
}
DefaultEventExecutorGroup(Runtime.getRuntime().availableProcessors(), threadFactory)
}
val serverInitializer = ServerInitializer(cfg, channelFactory, datagramChannelFactory, workerGroup)
val workerGroup = MultiThreadIoEventLoopGroup(0, NioIoHandler.newFactory())
val serverInitializer = ServerInitializer(cfg, channelFactory, datagramChannelFactory)
val bootstrap = ServerBootstrap().apply {
// Configure the server
group(bossGroup, workerGroup)
@@ -475,7 +467,7 @@ class RemoteBuildCacheServer(private val cfg: Configuration) {
return ServerHandle(
httpChannel.closeFuture(),
bossGroup,
setOf(workerGroup, eventExecutorGroup),
setOf(workerGroup),
serverInitializer
)
}

View File

@@ -4,7 +4,6 @@ import io.netty.channel.ChannelFactory
import io.netty.channel.EventLoopGroup
import io.netty.channel.socket.DatagramChannel
import io.netty.channel.socket.SocketChannel
import io.netty.util.concurrent.Future
import net.woggioni.rbcs.api.CacheHandlerFactory
import net.woggioni.rbcs.api.Configuration
import net.woggioni.rbcs.common.RBCS

View File

@@ -2,19 +2,11 @@ package net.woggioni.rbcs.server.cache
import io.netty.buffer.ByteBuf
import io.netty.channel.ChannelHandlerContext
import io.netty.channel.SimpleChannelInboundHandler
import net.woggioni.rbcs.api.CacheHandler
import net.woggioni.rbcs.api.message.CacheMessage
import net.woggioni.rbcs.api.message.CacheMessage.CacheContent
import net.woggioni.rbcs.api.message.CacheMessage.CacheGetRequest
import net.woggioni.rbcs.api.message.CacheMessage.CachePutRequest
import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse
import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse
import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse
import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent
import net.woggioni.rbcs.api.message.CacheMessage.*
import net.woggioni.rbcs.common.ByteBufOutputStream
import net.woggioni.rbcs.common.RBCS.processCacheKey
import net.woggioni.rbcs.common.trace
import java.util.zip.Deflater
import java.util.zip.DeflaterOutputStream
import java.util.zip.InflaterOutputStream
@@ -43,18 +35,14 @@ class InMemoryCacheHandler(
private inner class InProgressPlainPutRequest(ctx: ChannelHandlerContext, override val request: CachePutRequest) :
InProgressPutRequest {
override val buf = ctx.alloc().compositeBuffer()
private val stream = ByteBufOutputStream(buf).let {
if (compressionEnabled) {
DeflaterOutputStream(it, Deflater(compressionLevel))
} else {
it
}
}
override val buf = ctx.alloc().compositeHeapBuffer()
override fun append(buf: ByteBuf) {
this.buf.addComponent(true, buf.retain())
if(buf.isDirect) {
this.buf.writeBytes(buf)
} else {
this.buf.addComponent(true, buf.retain())
}
}
override fun close() {

View File

@@ -1,5 +1,14 @@
package net.woggioni.rbcs.server.test.utils;
import java.math.BigInteger;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.PrivateKey;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Date;
import org.bouncycastle.asn1.DERSequence;
import org.bouncycastle.asn1.x500.X500Name;
import org.bouncycastle.asn1.x509.BasicConstraints;
@@ -15,16 +24,6 @@ import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
import org.bouncycastle.operator.ContentSigner;
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
import java.math.BigInteger;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.PrivateKey;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Date;
public class CertificateUtils {
public record X509Credentials(

View File

@@ -53,7 +53,9 @@
<!-- <Connector port="8080" protocol="HTTP/1.1" executor="tomcatThreadPool"-->
<!-- connectionTimeout="20000"-->
<!-- redirectPort="8443" />-->
<Connector port="8080" protocol="HTTP/1.1"
<Connector port="8080" protocol="org.apache.coyote.http11.Http11NioProtocol"
maxKeepAliveRequests="1000000"
keepAliveTimeout="-1"
connectionTimeout="20000"
redirectPort="8443" />
<!-- A "Connector" using the shared thread pool-->