Generalize OTEL API and add memcache tracing support

- Rename RedisSpan -> SpanHandle for generic span handling
- Generalize TelemetryController methods: startSpan/endSpan with dbSystem param
- Rename RedisOtelSpan -> OtelSpanHandle in rbcs-server-otel
- Update Redis cache handler to use new generic API
- Add OpenTelemetry tracing for memcache GET and SET commands
- Add channel property to MemcacheRequestController for server address attribution
- Add uses TelemetryController directive in memcache module-info

Memcache spans follow the same pattern as Redis:
db.system=memcache, db.operation=GET|SET, server.address, server.port

rbcs-api/src/main/java/net/woggioni/rbcs/api/SpanHandle.java

@@ -0,0 +1,10 @@
+package net.woggioni.rbcs.api;
+
+import org.jetbrains.annotations.NotNull;
+
+public interface SpanHandle {
+
+    void setAttribute(@NotNull String key, @NotNull String value);
+
+    void setAttribute(@NotNull String key, long value);
+}

rbcs-api/src/main/java/net/woggioni/rbcs/api/TelemetryController.java

@@ -2,8 +2,15 @@ package net.woggioni.rbcs.api;
 
 import io.netty.channel.ChannelHandler;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 public interface TelemetryController {
     void initialize();
     @NotNull ChannelHandler createHandler();
+
+    @Nullable SpanHandle startSpan(@NotNull String command, @NotNull String key, @NotNull String dbSystem);
+
+    void endSpan(@Nullable SpanHandle span);
+
+    void endSpan(@Nullable SpanHandle span, @NotNull Throwable error);
 }

rbcs-server-memcache/src/main/java/module-info.java

@@ -1,4 +1,5 @@
 import net.woggioni.rbcs.api.CacheProvider;
+import net.woggioni.rbcs.api.TelemetryController;
 
 module net.woggioni.rbcs.server.memcache {
     requires net.woggioni.rbcs.common;
@@ -16,5 +17,7 @@ module net.woggioni.rbcs.server.memcache {
 
     provides CacheProvider with net.woggioni.rbcs.server.memcache.MemcacheCacheProvider;
 
+    uses TelemetryController;
+
     opens net.woggioni.rbcs.server.memcache.schema;
 }

rbcs-server-memcache/src/main/kotlin/net/woggioni/rbcs/server/memcache/MemcacheCacheHandler.kt

@@ -22,9 +22,11 @@ import java.nio.channels.FileChannel
 import java.nio.channels.ReadableByteChannel
 import java.nio.file.Files
 import java.nio.file.StandardOpenOption
+import java.net.InetSocketAddress
 import java.time.Duration
 import java.time.Instant
 import java.util.concurrent.CompletableFuture
+import java.util.concurrent.atomic.AtomicReference
 import java.util.zip.Deflater
 import java.util.zip.DeflaterOutputStream
 import java.util.zip.InflaterOutputStream
@@ -39,8 +41,11 @@ import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse
 import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse
 import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse
 import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent
+import net.woggioni.rbcs.api.SpanHandle
+import net.woggioni.rbcs.api.TelemetryController
 import net.woggioni.rbcs.common.ByteBufInputStream
 import net.woggioni.rbcs.common.ByteBufOutputStream
+import net.woggioni.rbcs.common.RBCS.loadService
 import net.woggioni.rbcs.common.RBCS.processCacheKey
 import net.woggioni.rbcs.common.RBCS.toIntOrNull
 import net.woggioni.rbcs.common.createLogger
@@ -70,6 +75,10 @@ class MemcacheCacheHandler(
         }
     }
 
+    private val telemetryController by lazy {
+        loadService(TelemetryController::class.java).firstOrNull()
+    }
+
     private interface InProgressRequest {
 
     }
@@ -153,7 +162,9 @@ class MemcacheCacheHandler(
         metadata: CacheValueMetadata,
         val digest: ByteBuf,
         val requestController: CompletableFuture<MemcacheRequestController>,
-        private val alloc: ByteBufAllocator
+        private val alloc: ByteBufAllocator,
+        val entryKey: String,
+        val memcacheSpanRef: AtomicReference<SpanHandle?>,
     ) : InProgressRequest {
         private var totalSize = 0
         private var tmpFile: FileChannel? = null
@@ -251,6 +262,7 @@ class MemcacheCacheHandler(
         val key = ctx.alloc().buffer().also {
             it.writeBytes(processCacheKey(msg.key, keyPrefix, digestAlgorithm))
         }
+        val memcacheSpan = telemetryController?.startSpan("GET", msg.key, "memcache")
         val responseHandler = object : MemcacheResponseHandler {
             override fun responseReceived(response: BinaryMemcacheResponse) {
                 val status = response.status()
@@ -266,8 +278,15 @@ class MemcacheCacheHandler(
                         log.debug(ctx) {
                             "Cache miss for key ${msg.key} on memcache"
                         }
+                        telemetryController?.endSpan(memcacheSpan)
                         sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key))
                     }
+
+                    else -> {
+                        val ex = MemcacheException(status)
+                        telemetryController?.endSpan(memcacheSpan, ex)
+                        this@MemcacheCacheHandler.exceptionCaught(ctx, ex)
+                    }
                 }
             }
 
@@ -282,11 +301,13 @@ class MemcacheCacheHandler(
                     if (content is LastMemcacheContent) {
                         inProgressRequest = null
                         inProgressGetRequest.commit()
+                        telemetryController?.endSpan(memcacheSpan)
                     }
                 }
             }
 
             override fun exceptionCaught(ex: Throwable) {
+                telemetryController?.endSpan(memcacheSpan, ex)
                 (inProgressRequest as? InProgressGetRequest).let { inProgressGetRequest ->
                     inProgressGetRequest?.let {
                         inProgressRequest = null
@@ -297,6 +318,11 @@ class MemcacheCacheHandler(
             }
         }
         client.sendRequest(key.retainedDuplicate(), responseHandler).thenAccept { requestHandle ->
+            val remoteAddr = requestHandle.channel.remoteAddress()
+            if (remoteAddr is InetSocketAddress) {
+                remoteAddr.hostString?.let { memcacheSpan?.setAttribute("server.address", it) }
+                memcacheSpan?.setAttribute("server.port", remoteAddr.port.toLong())
+            }
             log.trace(ctx) {
                 "Sending GET request for key ${msg.key} to memcache"
             }
@@ -312,6 +338,7 @@ class MemcacheCacheHandler(
         val key = ctx.alloc().buffer().also {
             it.writeBytes(processCacheKey(msg.key, keyPrefix, digestAlgorithm))
         }
+        val memcacheSpanRef = AtomicReference<SpanHandle?>(null)
         val responseHandler = object : MemcacheResponseHandler {
             override fun responseReceived(response: BinaryMemcacheResponse) {
                 val status = response.status()
@@ -320,16 +347,22 @@ class MemcacheCacheHandler(
                         log.debug(ctx) {
                             "Inserted key ${msg.key} into memcache"
                         }
+                        telemetryController?.endSpan(memcacheSpanRef.get())
                         sendMessageAndFlush(ctx, CachePutResponse(msg.key))
                     }
 
-                    else -> this@MemcacheCacheHandler.exceptionCaught(ctx, MemcacheException(status))
+                    else -> {
+                        val ex = MemcacheException(status)
+                        telemetryController?.endSpan(memcacheSpanRef.get(), ex)
+                        this@MemcacheCacheHandler.exceptionCaught(ctx, ex)
+                    }
                 }
             }
 
             override fun contentReceived(content: MemcacheContent) {}
 
             override fun exceptionCaught(ex: Throwable) {
+                telemetryController?.endSpan(memcacheSpanRef.get(), ex)
                 this@MemcacheCacheHandler.exceptionCaught(ctx, ex)
             }
         }
@@ -339,7 +372,7 @@ class MemcacheCacheHandler(
                 this@MemcacheCacheHandler.exceptionCaught(ctx, ex)
             }
         }
-        inProgressRequest = InProgressPutRequest(ctx.channel(), msg.metadata, key, requestController, ctx.alloc())
+        inProgressRequest = InProgressPutRequest(ctx.channel(), msg.metadata, key, requestController, ctx.alloc(), msg.key, memcacheSpanRef)
     }
 
     private fun handleCacheContent(ctx: ChannelHandlerContext, msg: CacheContent) {
@@ -362,22 +395,30 @@ class MemcacheCacheHandler(
         val request = inProgressRequest
         when (request) {
             is InProgressPutRequest -> {
+                val putRequest = request
                 inProgressRequest = null
                 log.trace(ctx) {
                     "Received last chunk of ${msg.content().readableBytes()} bytes for memcache"
                 }
-                request.write(msg.content())
+                putRequest.write(msg.content())
-                val key = request.digest.retainedDuplicate()
+                val memcacheSpan = telemetryController?.startSpan("SET", putRequest.entryKey, "memcache")
-                val (payloadSize, payloadSource) = request.commit()
+                putRequest.memcacheSpanRef.set(memcacheSpan)
+                val key = putRequest.digest.retainedDuplicate()
+                val (payloadSize, payloadSource) = putRequest.commit()
                 val extras = ctx.alloc().buffer(8, 8)
                 extras.writeInt(0)
                 extras.writeInt(encodeExpiry(maxAge))
-                val totalBodyLength = request.digest.readableBytes() + extras.readableBytes() + payloadSize
+                val totalBodyLength = putRequest.digest.readableBytes() + extras.readableBytes() + payloadSize
                 log.trace(ctx) {
                     "Trying to send SET request to memcache"
                 }
-                request.requestController.whenComplete { requestController, ex ->
+                putRequest.requestController.whenComplete { requestController, ex ->
                     if (ex == null) {
+                        val remoteAddr = requestController.channel.remoteAddress()
+                        if (remoteAddr is InetSocketAddress) {
+                            remoteAddr.hostString?.let { memcacheSpan?.setAttribute("server.address", it) }
+                            memcacheSpan?.setAttribute("server.port", remoteAddr.port.toLong())
+                        }
                         log.trace(ctx) {
                             "Sending SET request to memcache"
                         }

rbcs-server-memcache/src/main/kotlin/net/woggioni/rbcs/server/memcache/client/MemcacheClient.kt

@@ -147,6 +147,8 @@ class MemcacheClient(
 
                     channel.pipeline().addLast(handler)
                     response.complete(object : MemcacheRequestController {
+                        override val channel: Channel = channel
+
                         private var channelReleased = false
 
                         override fun sendRequest(request: BinaryMemcacheRequest) {

rbcs-server-memcache/src/main/kotlin/net/woggioni/rbcs/server/memcache/client/MemcacheRequestController.kt

@@ -1,10 +1,13 @@
 package net.woggioni.rbcs.server.memcache.client
 
+import io.netty.channel.Channel
 import io.netty.handler.codec.memcache.MemcacheContent
 import io.netty.handler.codec.memcache.binary.BinaryMemcacheRequest
 
 interface MemcacheRequestController {
 
+    val channel: Channel
+
     fun sendRequest(request : BinaryMemcacheRequest)
 
     fun sendContent(content : MemcacheContent)

rbcs-server-otel/src/main/kotlin/net/woggioni/rbcs/server/otel/OtelController.kt

@@ -2,10 +2,13 @@ package net.woggioni.rbcs.server.otel
 
 import io.netty.channel.ChannelHandler
 import io.opentelemetry.api.GlobalOpenTelemetry
+import io.opentelemetry.api.trace.SpanKind
+import io.opentelemetry.api.trace.StatusCode
 import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender
 import io.opentelemetry.instrumentation.netty.v4_1.NettyServerTelemetry
 import io.opentelemetry.instrumentation.runtimetelemetry.RuntimeTelemetry
 import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk
+import net.woggioni.rbcs.api.SpanHandle
 import net.woggioni.rbcs.api.TelemetryController
 import net.woggioni.rbcs.common.createLogger
 import net.woggioni.rbcs.common.info
@@ -14,6 +17,10 @@ class OtelController : TelemetryController {
 
     private val log = createLogger<OtelController>()
 
+    private val tracer by lazy {
+        GlobalOpenTelemetry.getTracer("net.woggioni.rbcs.server.redis", "0.5.0")
+    }
+
     override fun initialize() {
         log.info { "Initializing OpenTelemetry SDK with auto-configuration" }
 
@@ -36,4 +43,24 @@ class OtelController : TelemetryController {
     override fun createHandler(): ChannelHandler {
         return NettyServerTelemetry.create(GlobalOpenTelemetry.get()).createCombinedHandler()
     }
+
+    override fun startSpan(command: String, key: String, dbSystem: String): SpanHandle? {
+        val span = tracer.spanBuilder(command)
+            .setSpanKind(SpanKind.CLIENT)
+            .setAttribute("db.system", dbSystem)
+            .setAttribute("db.operation", command)
+            .startSpan()
+        return OtelSpanHandle(span)
+    }
+
+    override fun endSpan(span: SpanHandle?) {
+        (span as? OtelSpanHandle)?.delegate?.end()
+    }
+
+    override fun endSpan(span: SpanHandle?, error: Throwable) {
+        val s = (span as? OtelSpanHandle)?.delegate ?: return
+        s.recordException(error)
+        s.setStatus(StatusCode.ERROR)
+        s.end()
+    }
 }

rbcs-server-otel/src/main/kotlin/net/woggioni/rbcs/server/otel/OtelSpanHandle.kt

@@ -0,0 +1,17 @@
+package net.woggioni.rbcs.server.otel
+
+import io.opentelemetry.api.trace.Span
+import net.woggioni.rbcs.api.SpanHandle
+
+internal class OtelSpanHandle(
+    val delegate: Span,
+) : SpanHandle {
+
+    override fun setAttribute(key: String, value: String) {
+        delegate.setAttribute(key, value)
+    }
+
+    override fun setAttribute(key: String, value: Long) {
+        delegate.setAttribute(key, value)
+    }
+}

rbcs-server-redis/src/main/java/module-info.java

@@ -1,4 +1,5 @@
 import net.woggioni.rbcs.api.CacheProvider;
+import net.woggioni.rbcs.api.TelemetryController;
 
 module net.woggioni.rbcs.server.redis {
     requires net.woggioni.rbcs.common;
@@ -16,5 +17,7 @@ module net.woggioni.rbcs.server.redis {
 
     provides CacheProvider with net.woggioni.rbcs.server.redis.RedisCacheProvider;
 
+    uses TelemetryController;
+
     opens net.woggioni.rbcs.server.redis.schema;
 }

rbcs-server-redis/src/main/kotlin/net/woggioni/rbcs/server/redis/RedisCacheHandler.kt

@@ -14,6 +14,7 @@ import io.netty.handler.codec.redis.SimpleStringRedisMessage
 import java.io.ByteArrayOutputStream
 import java.io.ObjectInputStream
 import java.io.ObjectOutputStream
+import java.net.InetSocketAddress
 import java.nio.ByteBuffer
 import java.nio.channels.Channels
 import java.nio.channels.FileChannel
@@ -37,8 +38,11 @@ import net.woggioni.rbcs.api.message.CacheMessage.CachePutResponse
 import net.woggioni.rbcs.api.message.CacheMessage.CacheValueFoundResponse
 import net.woggioni.rbcs.api.message.CacheMessage.CacheValueNotFoundResponse
 import net.woggioni.rbcs.api.message.CacheMessage.LastCacheContent
+import net.woggioni.rbcs.api.SpanHandle
+import net.woggioni.rbcs.api.TelemetryController
 import net.woggioni.rbcs.common.ByteBufInputStream
 import net.woggioni.rbcs.common.ByteBufOutputStream
+import net.woggioni.rbcs.common.RBCS.loadService
 import net.woggioni.rbcs.common.RBCS.processCacheKey
 import net.woggioni.rbcs.common.RBCS.toIntOrNull
 import net.woggioni.rbcs.common.createLogger
@@ -62,6 +66,10 @@ class RedisCacheHandler(
         private val log = createLogger<RedisCacheHandler>()
     }
 
+    private val telemetryController by lazy {
+        loadService(TelemetryController::class.java).firstOrNull()
+    }
+
     private interface InProgressRequest
 
     private inner class InProgressGetRequest(
@@ -244,8 +252,10 @@ class RedisCacheHandler(
         }
         val keyBytes = processCacheKey(msg.key, keyPrefix, digestAlgorithm)
         val keyString = String(keyBytes, StandardCharsets.UTF_8)
+        val redisSpan = telemetryController?.startSpan("GET", keyString, "redis")
         val responseHandler = object : RedisResponseHandler {
             override fun responseReceived(response: RedisMessage) {
+                try {
                     when (response) {
                         is FullBulkStringRedisMessage -> {
                             if (response === FullBulkStringRedisMessage.NULL_INSTANCE || response.content().readableBytes() == 0) {
@@ -265,9 +275,9 @@ class RedisCacheHandler(
                         }
 
                         is ErrorRedisMessage -> {
-                        this@RedisCacheHandler.exceptionCaught(
+                            val ex = RedisException("Redis error for GET ${msg.key}: ${response.content()}")
-                            ctx, RedisException("Redis error for GET ${msg.key}: ${response.content()}")
+                            telemetryController?.endSpan(redisSpan, ex)
-                        )
+                            this@RedisCacheHandler.exceptionCaught(ctx, ex)
                         }
 
                         else -> {
@@ -277,13 +287,22 @@ class RedisCacheHandler(
                             sendMessageAndFlush(ctx, CacheValueNotFoundResponse(msg.key))
                         }
                     }
+                } finally {
+                    telemetryController?.endSpan(redisSpan)
+                }
             }
 
             override fun exceptionCaught(ex: Throwable) {
+                telemetryController?.endSpan(redisSpan, ex)
                 this@RedisCacheHandler.exceptionCaught(ctx, ex)
             }
         }
         client.sendCommand(keyBytes, ctx.alloc(), responseHandler).thenAccept { channel ->
+            val remoteAddr = channel.remoteAddress()
+            if (remoteAddr is InetSocketAddress) {
+                remoteAddr.hostString?.let { redisSpan?.setAttribute("server.address", it) }
+                redisSpan?.setAttribute("server.port", remoteAddr.port.toLong())
+            }
             log.trace(ctx) {
                 "Sending GET request for key ${msg.key} to Redis"
             }
@@ -344,8 +363,11 @@ class RedisCacheHandler(
 
                 val expirySeconds = maxAge.toSeconds().toString()
 
+                val redisSpan = telemetryController?.startSpan("SET", request.keyString, "redis")
+
                 val responseHandler = object : RedisResponseHandler {
                     override fun responseReceived(response: RedisMessage) {
+                        try {
                             when (response) {
                                 is SimpleStringRedisMessage -> {
                                     log.debug(ctx) {
@@ -355,26 +377,35 @@ class RedisCacheHandler(
                                 }
 
                                 is ErrorRedisMessage -> {
-                                this@RedisCacheHandler.exceptionCaught(
+                                    val ex = RedisException("Redis error for SET ${request.keyString}: ${response.content()}")
-                                    ctx, RedisException("Redis error for SET ${request.keyString}: ${response.content()}")
+                                    telemetryController?.endSpan(redisSpan, ex)
-                                )
+                                    this@RedisCacheHandler.exceptionCaught(ctx, ex)
                                 }
 
                                 else -> {
-                                this@RedisCacheHandler.exceptionCaught(
+                                    val ex = RedisException("Unexpected response for SET ${request.keyString}: ${response.javaClass.name}")
-                                    ctx, RedisException("Unexpected response for SET ${request.keyString}: ${response.javaClass.name}")
+                                    telemetryController?.endSpan(redisSpan, ex)
-                                )
+                                    this@RedisCacheHandler.exceptionCaught(ctx, ex)
                                 }
                             }
+                        } finally {
+                            telemetryController?.endSpan(redisSpan)
+                        }
                     }
 
                     override fun exceptionCaught(ex: Throwable) {
+                        telemetryController?.endSpan(redisSpan, ex)
                         this@RedisCacheHandler.exceptionCaught(ctx, ex)
                     }
                 }
 
                 // Use a ByteBuf key for server selection
                 client.sendCommand(keyBytes, ctx.alloc(), responseHandler).thenAccept { channel ->
+                    val remoteAddr = channel.remoteAddress()
+                    if (remoteAddr is InetSocketAddress) {
+                        remoteAddr.hostString?.let { redisSpan?.setAttribute("server.address", it) }
+                        redisSpan?.setAttribute("server.port", remoteAddr.port.toLong())
+                    }
                     log.trace(ctx) {
                         "Sending SET request to Redis"
                     }