Implement forward proxy with OpenTelemetry observability

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "hermes"
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.85"
+description = "A minimal HTTP forward proxy with OpenTelemetry observability"
+license = "Apache-2.0"
+
+[dependencies]
+bytes = "1.11"
+http = "1.4"
+http-body-util = "0.1.5"
+hyper = { version = "1.12", features = ["client", "http1", "server"] }
+hyper-util = { version = "0.1.21", features = ["client-legacy", "http1", "tokio"] }
+opentelemetry = "0.33.1"
+opentelemetry-appender-tracing = "0.33.1"
+opentelemetry-otlp = "0.33.1"
+opentelemetry_sdk = "0.33.1"
+thiserror = "2"
+tokio = { version = "1", features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] }
+tracing = "0.1"
+tracing-opentelemetry = "0.34"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
+
+[lints.rust]
+unsafe_code = "forbid"
+
+[lints.clippy]
+all = "warn"

README.md

@@ -0,0 +1,115 @@
+# Hermes
+
+Hermes is a minimal HTTP/1.1 forward proxy written in Rust. It supports streaming plain HTTP requests and HTTPS tunneling with `CONNECT`, with OpenTelemetry traces, metrics, and logs exported over OTLP/HTTP.
+
+## Features
+
+- Absolute-form HTTP forwarding (`GET http://host/path`)
+- HTTPS and arbitrary TCP tunneling with `CONNECT host:port`
+- Streaming request and response bodies
+- Upstream connection pooling and configurable timeouts
+- Removal of standard and `Connection`-nominated hop-by-hop headers
+- W3C Trace Context extraction and injection for plain HTTP traffic
+- Structured, trace-correlated logs on stdout and through OTLP
+- Bounded-cardinality HTTP and tunnel metrics
+- Localhost-only binding by default
+
+TLS is not terminated or inspected. HTTPS remains end-to-end encrypted between the client and destination.
+
+## Build and run
+
+```sh
+cargo build
+cargo run
+```
+
+The proxy listens on `127.0.0.1:8080` by default.
+
+```sh
+# Plain HTTP
+curl --proxy http://127.0.0.1:8080 http://example.com/
+
+# HTTPS via CONNECT
+curl --proxy http://127.0.0.1:8080 https://example.com/
+```
+
+### Proxy configuration
+
+| Environment variable | Default | Description |
+| --- | --- | --- |
+| `HERMES_LISTEN_ADDR` | `127.0.0.1:8080` | Downstream listen address |
+| `HERMES_CONNECT_TIMEOUT_MS` | `10000` | TCP connection timeout for `CONNECT` and the HTTP connector |
+| `HERMES_UPSTREAM_TIMEOUT_MS` | `30000` | Time limit for receiving upstream HTTP response headers |
+| `RUST_LOG` | `info` | `tracing` filter, for example `hermes=debug` |
+
+Do not expose Hermes on a public interface without adding appropriate authentication and destination access controls. The default configuration intentionally binds to localhost.
+
+## OpenTelemetry
+
+Hermes exports traces, metrics, and logs independently to an OpenTelemetry Collector using OTLP/HTTP (protobuf). Exporters use the standard OpenTelemetry environment variables.
+
+```sh
+export OTEL_SERVICE_NAME=hermes
+export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318
+cargo run
+```
+
+Signal-specific settings such as `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`, and `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` are also supported by the exporter. Standard OTLP headers, timeouts, and trace sampling environment variables can be used.
+
+Traces and logs are batched, metrics are exported periodically, and all providers are flushed during shutdown. An unavailable collector does not prevent the proxy from forwarding traffic, although exporter errors may appear in stdout logs.
+
+### Local collector
+
+[`examples/otel-collector.yaml`](examples/otel-collector.yaml) receives OTLP/HTTP and prints telemetry to the collector log:
+
+```sh
+docker run --rm \
+  -p 4318:4318 \
+  -v "$PWD/examples/otel-collector.yaml:/etc/otelcol/config.yaml:ro" \
+  otel/opentelemetry-collector:latest
+```
+
+In another terminal, run Hermes and send a request through it. The collector will print the exported resource spans, metrics, and log records.
+
+### Traces
+
+- `proxy.request`: downstream plain HTTP server operation
+- `proxy.upstream`: upstream HTTP client operation
+- `proxy.connect`: `CONNECT` handshake and tunnel lifetime
+- `proxy.upstream_connect`: upstream TCP connection establishment
+
+For plain HTTP, incoming W3C `traceparent` and `tracestate` headers establish the parent context, and a new context is injected upstream. Trace context cannot be propagated inside an encrypted `CONNECT` tunnel.
+
+### Metrics
+
+Hermes uses standard HTTP instrument names where applicable and adds proxy-specific tunnel instruments:
+
+- `hermes.proxy.requests`
+- `http.server.request.duration`
+- `http.server.active_requests`
+- `http.client.request.duration`
+- `hermes.proxy.errors`
+- `hermes.proxy.tunnels.active`
+- `hermes.proxy.tunnel.duration`
+- `hermes.proxy.tunnel.bytes`
+
+Metric attributes are deliberately bounded. Raw URLs, destination hostnames, and client addresses are not metric attributes.
+
+### Logs
+
+Startup, shutdown, request failures, and tunnel completion are emitted as structured `tracing` events. Logs emitted inside a trace span include its trace and span IDs in OTLP. Headers, bodies, credentials, and query strings are not logged by the proxy.
+
+## Limitations
+
+- HTTP/1.1 downstream and upstream only; no HTTP/2 or gRPC proxying
+- No TLS interception, caching, authentication, ACLs, or rate limiting
+- No HTTP Upgrade forwarding other than `CONNECT`
+- No dynamic configuration or Envoy xDS support
+
+## Development
+
+```sh
+cargo fmt --all -- --check
+cargo test
+cargo clippy --all-targets --all-features -- -D warnings
+```

examples/otel-collector.yaml

@@ -0,0 +1,21 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:4318
+
+exporters:
+  debug:
+    verbosity: detailed
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [debug]
+    metrics:
+      receivers: [otlp]
+      exporters: [debug]
+    logs:
+      receivers: [otlp]
+      exporters: [debug]

src/config.rs

@@ -0,0 +1,61 @@
+use std::{env, net::SocketAddr, num::ParseIntError, time::Duration};
+
+use thiserror::Error;
+
+const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:8080";
+const DEFAULT_CONNECT_TIMEOUT_MS: u64 = 10_000;
+const DEFAULT_UPSTREAM_TIMEOUT_MS: u64 = 30_000;
+
+#[derive(Clone, Debug)]
+pub struct Config {
+    pub listen_addr: SocketAddr,
+    pub connect_timeout: Duration,
+    pub upstream_timeout: Duration,
+}
+
+impl Config {
+    pub fn from_env() -> Result<Self, ConfigError> {
+        let listen_addr = env::var("HERMES_LISTEN_ADDR")
+            .unwrap_or_else(|_| DEFAULT_LISTEN_ADDR.to_owned())
+            .parse()?;
+        let connect_timeout =
+            duration_from_env("HERMES_CONNECT_TIMEOUT_MS", DEFAULT_CONNECT_TIMEOUT_MS)?;
+        let upstream_timeout =
+            duration_from_env("HERMES_UPSTREAM_TIMEOUT_MS", DEFAULT_UPSTREAM_TIMEOUT_MS)?;
+
+        Ok(Self {
+            listen_addr,
+            connect_timeout,
+            upstream_timeout,
+        })
+    }
+}
+
+fn duration_from_env(name: &'static str, default_ms: u64) -> Result<Duration, ConfigError> {
+    let milliseconds = match env::var(name) {
+        Ok(value) => value
+            .parse()
+            .map_err(|source| ConfigError::InvalidDuration { name, source })?,
+        Err(_) => default_ms,
+    };
+
+    if milliseconds == 0 {
+        return Err(ConfigError::ZeroDuration(name));
+    }
+
+    Ok(Duration::from_millis(milliseconds))
+}
+
+#[derive(Debug, Error)]
+pub enum ConfigError {
+    #[error("invalid HERMES_LISTEN_ADDR: {0}")]
+    InvalidListenAddr(#[from] std::net::AddrParseError),
+    #[error("invalid {name}: {source}")]
+    InvalidDuration {
+        name: &'static str,
+        #[source]
+        source: ParseIntError,
+    },
+    #[error("{0} must be greater than zero")]
+    ZeroDuration(&'static str),
+}

src/lib.rs

@@ -0,0 +1,11 @@
+pub mod config;
+pub mod telemetry;
+
+mod metrics;
+mod proxy;
+
+use std::error::Error;
+
+pub use proxy::run;
+
+pub type BoxError = Box<dyn Error + Send + Sync + 'static>;

src/main.rs

@@ -0,0 +1,15 @@
+use hermes::{BoxError, config::Config, telemetry};
+
+#[tokio::main]
+async fn main() -> Result<(), BoxError> {
+    let telemetry = telemetry::init()?;
+
+    let result: Result<(), BoxError> = async {
+        let config = Config::from_env()?;
+        hermes::run(config).await
+    }
+    .await;
+
+    telemetry.shutdown();
+    result
+}

src/metrics.rs

@@ -0,0 +1,173 @@
+use std::time::Duration;
+
+use opentelemetry::{
+    KeyValue,
+    metrics::{Counter, Histogram, Meter, UpDownCounter},
+};
+
+#[derive(Clone)]
+pub(crate) struct Metrics {
+    requests: Counter<u64>,
+    request_duration: Histogram<f64>,
+    active_requests: UpDownCounter<i64>,
+    upstream_duration: Histogram<f64>,
+    errors: Counter<u64>,
+    active_tunnels: UpDownCounter<i64>,
+    tunnel_duration: Histogram<f64>,
+    tunnel_bytes: Counter<u64>,
+}
+
+impl Metrics {
+    pub fn new(meter: Meter) -> Self {
+        Self {
+            requests: meter
+                .u64_counter("hermes.proxy.requests")
+                .with_description("Number of HTTP requests and CONNECT handshakes handled")
+                .with_unit("{request}")
+                .build(),
+            request_duration: meter
+                .f64_histogram("http.server.request.duration")
+                .with_description("Duration of downstream HTTP request handling")
+                .with_unit("s")
+                .build(),
+            active_requests: meter
+                .i64_up_down_counter("http.server.active_requests")
+                .with_description("Number of HTTP requests currently being handled")
+                .with_unit("{request}")
+                .build(),
+            upstream_duration: meter
+                .f64_histogram("http.client.request.duration")
+                .with_description("Duration of upstream HTTP requests through response headers")
+                .with_unit("s")
+                .build(),
+            errors: meter
+                .u64_counter("hermes.proxy.errors")
+                .with_description("Number of proxy failures")
+                .with_unit("{error}")
+                .build(),
+            active_tunnels: meter
+                .i64_up_down_counter("hermes.proxy.tunnels.active")
+                .with_description("Number of active CONNECT tunnels")
+                .with_unit("{tunnel}")
+                .build(),
+            tunnel_duration: meter
+                .f64_histogram("hermes.proxy.tunnel.duration")
+                .with_description("Duration of completed CONNECT tunnels")
+                .with_unit("s")
+                .build(),
+            tunnel_bytes: meter
+                .u64_counter("hermes.proxy.tunnel.bytes")
+                .with_description("Bytes relayed through completed CONNECT tunnels")
+                .with_unit("By")
+                .build(),
+        }
+    }
+
+    pub fn request_started(&self) -> ActiveRequest {
+        self.active_requests.add(1, &[]);
+        ActiveRequest {
+            metrics: self.clone(),
+        }
+    }
+
+    pub fn request_finished(
+        &self,
+        method: &str,
+        status: u16,
+        error_type: Option<&str>,
+        duration: Duration,
+    ) {
+        let mut attributes = vec![
+            KeyValue::new("http.request.method", method.to_owned()),
+            KeyValue::new("http.response.status_code", i64::from(status)),
+        ];
+        if let Some(error_type) = error_type {
+            attributes.push(KeyValue::new("error.type", error_type.to_owned()));
+        }
+
+        self.requests.add(1, &attributes);
+        self.request_duration
+            .record(duration.as_secs_f64(), &attributes);
+    }
+
+    pub fn upstream_finished(
+        &self,
+        method: &str,
+        status: Option<u16>,
+        error_type: Option<&str>,
+        duration: Duration,
+    ) {
+        let mut attributes = vec![KeyValue::new("http.request.method", method.to_owned())];
+        if let Some(status) = status {
+            attributes.push(KeyValue::new(
+                "http.response.status_code",
+                i64::from(status),
+            ));
+        }
+        if let Some(error_type) = error_type {
+            attributes.push(KeyValue::new("error.type", error_type.to_owned()));
+        }
+
+        self.upstream_duration
+            .record(duration.as_secs_f64(), &attributes);
+    }
+
+    pub fn error(&self, phase: &str, error_type: &str) {
+        self.errors.add(
+            1,
+            &[
+                KeyValue::new("proxy.phase", phase.to_owned()),
+                KeyValue::new("error.type", error_type.to_owned()),
+            ],
+        );
+    }
+
+    pub fn tunnel_started(&self) -> ActiveTunnel {
+        self.active_tunnels.add(1, &[]);
+        ActiveTunnel {
+            metrics: self.clone(),
+        }
+    }
+
+    pub fn tunnel_finished(
+        &self,
+        duration: Duration,
+        client_to_upstream: u64,
+        upstream_to_client: u64,
+        error_type: Option<&str>,
+    ) {
+        let attributes = error_type
+            .map(|value| vec![KeyValue::new("error.type", value.to_owned())])
+            .unwrap_or_default();
+        self.tunnel_duration
+            .record(duration.as_secs_f64(), &attributes);
+        self.tunnel_bytes.add(
+            client_to_upstream,
+            &[KeyValue::new("direction", "client_to_upstream")],
+        );
+        self.tunnel_bytes.add(
+            upstream_to_client,
+            &[KeyValue::new("direction", "upstream_to_client")],
+        );
+    }
+}
+
+pub(crate) struct ActiveRequest {
+    metrics: Metrics,
+}
+
+impl Drop for ActiveRequest {
+    fn drop(&mut self) {
+        self.metrics.active_requests.add(-1, &[]);
+    }
+}
+
+pub(crate) struct ActiveTunnel {
+    metrics: Metrics,
+}
+
+impl Drop for ActiveTunnel {
+    fn drop(&mut self) {
+        self.metrics.active_tunnels.add(-1, &[]);
+    }
+}

src/proxy.rs

@@ -0,0 +1,680 @@
+use std::{convert::Infallible, net::SocketAddr, time::Instant};
+
+use bytes::Bytes;
+use http::{HeaderValue, Method, Request, Response, StatusCode, Uri, header::HOST, uri::Authority};
+use http_body_util::{BodyExt, Empty, Full, combinators::UnsyncBoxBody};
+use hyper::{body::Incoming, server::conn::http1, service::service_fn};
+use hyper_util::{
+    client::legacy::{Client, connect::HttpConnector},
+    rt::{TokioExecutor, TokioIo},
+};
+use opentelemetry::global;
+use tokio::{
+    net::{TcpListener, TcpStream},
+    task::JoinSet,
+    time::timeout,
+};
+use tracing::{Instrument, Span, debug, error, field, info, info_span};
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+use crate::{
+    BoxError,
+    config::Config,
+    metrics::Metrics,
+    telemetry::{HeaderExtractor, HeaderInjector, remove_hop_by_hop_headers},
+};
+
+type ProxyBody = UnsyncBoxBody<Bytes, BoxError>;
+type ProxyResponse = Response<ProxyBody>;
+type HttpClient = Client<HttpConnector, Incoming>;
+
+#[derive(Clone)]
+struct ProxyState {
+    client: HttpClient,
+    metrics: Metrics,
+    connect_timeout: std::time::Duration,
+    upstream_timeout: std::time::Duration,
+}
+
+impl ProxyState {
+    fn new(config: &Config) -> Self {
+        let mut connector = HttpConnector::new();
+        connector.enforce_http(true);
+        connector.set_connect_timeout(Some(config.connect_timeout));
+
+        Self {
+            client: Client::builder(TokioExecutor::new()).build(connector),
+            metrics: Metrics::new(global::meter("hermes")),
+            connect_timeout: config.connect_timeout,
+            upstream_timeout: config.upstream_timeout,
+        }
+    }
+}
+
+pub async fn run(config: Config) -> Result<(), BoxError> {
+    let listener = TcpListener::bind(config.listen_addr).await?;
+    let state = ProxyState::new(&config);
+    let mut connections = JoinSet::new();
+    let shutdown = tokio::signal::ctrl_c();
+    tokio::pin!(shutdown);
+
+    info!(listen.address = %config.listen_addr, "proxy listening");
+
+    loop {
+        tokio::select! {
+            _ = &mut shutdown => {
+                info!("shutdown signal received");
+                break;
+            }
+            accepted = listener.accept() => {
+                match accepted {
+                    Ok((stream, peer)) => {
+                        let state = state.clone();
+                        connections.spawn(async move {
+                            serve_connection(stream, peer, state).await;
+                        });
+                    }
+                    Err(error) => {
+                        error!(error.message = %error, "failed to accept downstream connection");
+                    }
+                }
+            }
+            completed = connections.join_next(), if !connections.is_empty() => {
+                if let Some(Err(error)) = completed {
+                    error!(error.message = %error, "connection task failed");
+                }
+            }
+        }
+    }
+
+    info!("proxy listener stopped");
+    Ok(())
+}
+
+async fn serve_connection(stream: TcpStream, peer: SocketAddr, state: ProxyState) {
+    let service = service_fn(move |request| {
+        let state = state.clone();
+        async move { Ok::<_, Infallible>(handle_request(request, state).await) }
+    });
+
+    let connection = http1::Builder::new()
+        .serve_connection(TokioIo::new(stream), service)
+        .with_upgrades();
+
+    if let Err(error) = connection.await {
+        debug!(
+            client.address = %peer,
+            error.message = %error,
+            "downstream connection ended with an error"
+        );
+    }
+}
+
+async fn handle_request(request: Request<Incoming>, state: ProxyState) -> ProxyResponse {
+    if request.method() == Method::CONNECT {
+        handle_connect(request, state).await
+    } else {
+        handle_http(request, state).await
+    }
+}
+
+async fn handle_http(request: Request<Incoming>, state: ProxyState) -> ProxyResponse {
+    let method = request.method().to_string();
+    let uri = request.uri().clone();
+    let parent = global::get_text_map_propagator(|propagator| {
+        propagator.extract(&HeaderExtractor(request.headers()))
+    });
+    let span = info_span!(
+        "proxy.request",
+        otel.name = %format!("HTTP {method}"),
+        otel.kind = "server",
+        http.request.method = %method,
+        url.scheme = %uri.scheme_str().unwrap_or_default(),
+        url.path = %uri.path(),
+        server.address = %uri.authority().map(Authority::host).unwrap_or_default(),
+        http.response.status_code = field::Empty,
+        error.type = field::Empty,
+        otel.status_code = field::Empty,
+    );
+    let _ = span.set_parent(parent);
+
+    async move {
+        let _active = state.metrics.request_started();
+        let started = Instant::now();
+        let result = forward_http(request, &state).await;
+        finish_request(result, &method, started, &state.metrics)
+    }
+    .instrument(span)
+    .await
+}
+
+async fn forward_http(
+    mut request: Request<Incoming>,
+    state: &ProxyState,
+) -> Result<ProxyResponse, ProxyError> {
+    let authority = validate_http_target(request.uri())?;
+    let host = HeaderValue::from_str(authority.as_str())
+        .map_err(|_| ProxyError::bad_request("invalid_target", "invalid upstream Host header"))?;
+
+    remove_hop_by_hop_headers(request.headers_mut());
+    request.headers_mut().insert(HOST, host);
+
+    let response = send_upstream(request, state).await?;
+    let (parts, body) = response.into_parts();
+    let mut response = Response::from_parts(parts, box_incoming_body(body));
+    remove_hop_by_hop_headers(response.headers_mut());
+    Ok(response)
+}
+
+async fn send_upstream(
+    mut request: Request<Incoming>,
+    state: &ProxyState,
+) -> Result<Response<Incoming>, ProxyError> {
+    let method = request.method().to_string();
+    let host = request
+        .uri()
+        .authority()
+        .map(Authority::host)
+        .unwrap_or_default()
+        .to_owned();
+    let port = request.uri().port_u16().unwrap_or(80);
+    let span = info_span!(
+        "proxy.upstream",
+        otel.name = %format!("HTTP {method}"),
+        otel.kind = "client",
+        http.request.method = %method,
+        server.address = %host,
+        server.port = port,
+        http.response.status_code = field::Empty,
+        error.type = field::Empty,
+        otel.status_code = field::Empty,
+    );
+
+    async move {
+        let context = Span::current().context();
+        global::get_text_map_propagator(|propagator| {
+            propagator.inject_context(&context, &mut HeaderInjector(request.headers_mut()));
+        });
+
+        let started = Instant::now();
+        let result = match timeout(state.upstream_timeout, state.client.request(request)).await {
+            Ok(Ok(response)) => Ok(response),
+            Ok(Err(error)) => Err(ProxyError::new(
+                StatusCode::BAD_GATEWAY,
+                "upstream_request",
+                "upstream",
+                error.to_string(),
+            )),
+            Err(_) => Err(ProxyError::new(
+                StatusCode::GATEWAY_TIMEOUT,
+                "upstream_timeout",
+                "upstream",
+                "upstream request timed out".to_owned(),
+            )),
+        };
+
+        let (status, error_type) = match &result {
+            Ok(response) => {
+                let status = response.status();
+                Span::current().record("http.response.status_code", status.as_u16());
+                if status.is_server_error() {
+                    Span::current().record("otel.status_code", "ERROR");
+                }
+                (Some(status.as_u16()), None)
+            }
+            Err(error) => {
+                Span::current().record("error.type", error.error_type);
+                Span::current().record("otel.status_code", "ERROR");
+                (None, Some(error.error_type))
+            }
+        };
+        state
+            .metrics
+            .upstream_finished(&method, status, error_type, started.elapsed());
+
+        result
+    }
+    .instrument(span)
+    .await
+}
+
+async fn handle_connect(request: Request<Incoming>, state: ProxyState) -> ProxyResponse {
+    let authority = request.uri().authority().cloned();
+    let parent = global::get_text_map_propagator(|propagator| {
+        propagator.extract(&HeaderExtractor(request.headers()))
+    });
+    let span = info_span!(
+        "proxy.connect",
+        otel.name = "CONNECT",
+        otel.kind = "server",
+        http.request.method = "CONNECT",
+        server.address = %authority.as_ref().map(Authority::host).unwrap_or_default(),
+        server.port = authority.as_ref().and_then(Authority::port_u16).unwrap_or_default(),
+        http.response.status_code = field::Empty,
+        error.type = field::Empty,
+        tunnel.bytes.client_to_upstream = field::Empty,
+        tunnel.bytes.upstream_to_client = field::Empty,
+        otel.status_code = field::Empty,
+    );
+    let _ = span.set_parent(parent);
+
+    async move {
+        let _active = state.metrics.request_started();
+        let started = Instant::now();
+        let result = prepare_connect(request, &state).await;
+        finish_request(result, "CONNECT", started, &state.metrics)
+    }
+    .instrument(span)
+    .await
+}
+
+async fn prepare_connect(
+    request: Request<Incoming>,
+    state: &ProxyState,
+) -> Result<ProxyResponse, ProxyError> {
+    let authority = validate_connect_target(request.uri())?;
+    let host = authority.host().to_owned();
+    let port = authority.port_u16().unwrap_or_default();
+    let connect_span = info_span!(
+        "proxy.upstream_connect",
+        otel.name = "TCP CONNECT",
+        otel.kind = "client",
+        server.address = %host,
+        server.port = port,
+        error.type = field::Empty,
+        otel.status_code = field::Empty,
+    );
+
+    let upstream = async {
+        let result = match timeout(
+            state.connect_timeout,
+            TcpStream::connect(authority.as_str()),
+        )
+        .await
+        {
+            Ok(Ok(stream)) => Ok(stream),
+            Ok(Err(error)) => Err(ProxyError::new(
+                StatusCode::BAD_GATEWAY,
+                "upstream_connect",
+                "upstream",
+                error.to_string(),
+            )),
+            Err(_) => Err(ProxyError::new(
+                StatusCode::GATEWAY_TIMEOUT,
+                "upstream_timeout",
+                "upstream",
+                "upstream connection timed out".to_owned(),
+            )),
+        };
+
+        if let Err(error) = &result {
+            Span::current().record("error.type", error.error_type);
+            Span::current().record("otel.status_code", "ERROR");
+        }
+        result
+    }
+    .instrument(connect_span)
+    .await?;
+
+    let tunnel_span = Span::current();
+    let metrics = state.metrics.clone();
+    tokio::spawn(run_tunnel(request, upstream, metrics).instrument(tunnel_span));
+
+    Ok(Response::new(empty_body()))
+}
+
+async fn run_tunnel(request: Request<Incoming>, mut upstream: TcpStream, metrics: Metrics) {
+    let upgraded = match hyper::upgrade::on(request).await {
+        Ok(upgraded) => upgraded,
+        Err(error) => {
+            Span::current().record("error.type", "downstream_upgrade");
+            Span::current().record("otel.status_code", "ERROR");
+            metrics.error("downstream", "downstream_upgrade");
+            error!(error.message = %error, "failed to upgrade CONNECT request");
+            return;
+        }
+    };
+
+    let started = Instant::now();
+    let _active = metrics.tunnel_started();
+    let mut downstream = TokioIo::new(upgraded);
+
+    match tokio::io::copy_bidirectional(&mut downstream, &mut upstream).await {
+        Ok((client_to_upstream, upstream_to_client)) => {
+            Span::current().record(
+                "tunnel.bytes.client_to_upstream",
+                i64::try_from(client_to_upstream).unwrap_or(i64::MAX),
+            );
+            Span::current().record(
+                "tunnel.bytes.upstream_to_client",
+                i64::try_from(upstream_to_client).unwrap_or(i64::MAX),
+            );
+            metrics.tunnel_finished(
+                started.elapsed(),
+                client_to_upstream,
+                upstream_to_client,
+                None,
+            );
+            info!(
+                tunnel.bytes.client_to_upstream = client_to_upstream,
+                tunnel.bytes.upstream_to_client = upstream_to_client,
+                tunnel.duration_ms = started.elapsed().as_millis(),
+                "CONNECT tunnel closed"
+            );
+        }
+        Err(error) => {
+            Span::current().record("error.type", "tunnel_io");
+            Span::current().record("otel.status_code", "ERROR");
+            metrics.error("tunnel", "tunnel_io");
+            metrics.tunnel_finished(started.elapsed(), 0, 0, Some("tunnel_io"));
+            error!(error.message = %error, "CONNECT tunnel failed");
+        }
+    }
+}
+
+fn finish_request(
+    result: Result<ProxyResponse, ProxyError>,
+    method: &str,
+    started: Instant,
+    metrics: &Metrics,
+) -> ProxyResponse {
+    let mut error_type = None;
+    let response = match result {
+        Ok(response) => response,
+        Err(error) => {
+            error_type = Some(error.error_type);
+            Span::current().record("error.type", error.error_type);
+            Span::current().record("otel.status_code", "ERROR");
+            metrics.error(error.phase, error.error_type);
+            error!(
+                error.type = error.error_type,
+                proxy.phase = error.phase,
+                error.message = %error.message,
+                "proxy request failed"
+            );
+            error_response(error.status)
+        }
+    };
+
+    let status = response.status();
+    Span::current().record("http.response.status_code", status.as_u16());
+    if status.is_server_error() {
+        Span::current().record("otel.status_code", "ERROR");
+    }
+    let duration = started.elapsed();
+    metrics.request_finished(method, status.as_u16(), error_type, duration);
+    info!(
+        http.request.method = method,
+        http.response.status_code = status.as_u16(),
+        request.duration_ms = duration.as_millis(),
+        "proxy request completed"
+    );
+    response
+}
+
+fn validate_http_target(uri: &Uri) -> Result<Authority, ProxyError> {
+    if uri.scheme_str() != Some("http") {
+        return Err(ProxyError::bad_request(
+            "unsupported_scheme",
+            "only absolute http:// request targets are supported",
+        ));
+    }
+
+    let authority = uri.authority().ok_or_else(|| {
+        ProxyError::bad_request("invalid_target", "absolute request target has no authority")
+    })?;
+    if authority.as_str().contains('@') {
+        return Err(ProxyError::bad_request(
+            "invalid_target",
+            "userinfo is not allowed in request targets",
+        ));
+    }
+
+    Ok(authority.clone())
+}
+
+fn validate_connect_target(uri: &Uri) -> Result<Authority, ProxyError> {
+    if uri.scheme().is_some() || !uri.path().is_empty() {
+        return Err(ProxyError::bad_request(
+            "invalid_target",
+            "CONNECT target must use authority-form",
+        ));
+    }
+
+    let authority = uri.authority().ok_or_else(|| {
+        ProxyError::bad_request("invalid_target", "CONNECT target has no authority")
+    })?;
+    if authority.port_u16().is_none() || authority.as_str().contains('@') {
+        return Err(ProxyError::bad_request(
+            "invalid_target",
+            "CONNECT target must contain a valid host and explicit port",
+        ));
+    }
+
+    Ok(authority.clone())
+}
+
+fn error_response(status: StatusCode) -> ProxyResponse {
+    let mut response = Response::new(full_body(format!(
+        "{}\n",
+        status.canonical_reason().unwrap_or("Proxy Error")
+    )));
+    *response.status_mut() = status;
+    response
+}
+
+fn box_incoming_body(body: Incoming) -> ProxyBody {
+    body.map_err(|error| -> BoxError { Box::new(error) })
+        .boxed_unsync()
+}
+
+fn full_body(value: impl Into<Bytes>) -> ProxyBody {
+    Full::new(value.into())
+        .map_err(|never| match never {})
+        .boxed_unsync()
+}
+
+fn empty_body() -> ProxyBody {
+    Empty::new().map_err(|never| match never {}).boxed_unsync()
+}
+
+#[derive(Debug)]
+struct ProxyError {
+    status: StatusCode,
+    error_type: &'static str,
+    phase: &'static str,
+    message: String,
+}
+
+impl ProxyError {
+    fn new(
+        status: StatusCode,
+        error_type: &'static str,
+        phase: &'static str,
+        message: String,
+    ) -> Self {
+        Self {
+            status,
+            error_type,
+            phase,
+            message,
+        }
+    }
+
+    fn bad_request(error_type: &'static str, message: &'static str) -> Self {
+        Self::new(
+            StatusCode::BAD_REQUEST,
+            error_type,
+            "request_validation",
+            message.to_owned(),
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{
+        sync::{Arc, Mutex},
+        time::Duration,
+    };
+
+    use hyper::client::conn::http1 as client_http1;
+    use tokio::{
+        io::{AsyncReadExt, AsyncWriteExt},
+        task::JoinHandle,
+    };
+
+    use super::*;
+
+    #[test]
+    fn validates_absolute_http_targets() {
+        let uri = Uri::from_static("http://example.com/path?q=1");
+        assert_eq!(validate_http_target(&uri).unwrap(), "example.com");
+
+        assert!(validate_http_target(&Uri::from_static("/path")).is_err());
+        assert!(validate_http_target(&Uri::from_static("https://example.com/")).is_err());
+        assert!(validate_http_target(&Uri::from_static("http://user@example.com/")).is_err());
+    }
+
+    #[test]
+    fn validates_connect_authority() {
+        let uri = Uri::from_static("example.com:443");
+        assert_eq!(validate_connect_target(&uri).unwrap(), "example.com:443");
+
+        assert!(validate_connect_target(&Uri::from_static("example.com")).is_err());
+        assert!(validate_connect_target(&Uri::from_static("http://example.com:443")).is_err());
+    }
+
+    #[tokio::test]
+    async fn forwards_http_and_rewrites_connection_headers() {
+        #[derive(Default)]
+        struct Observed {
+            target: String,
+            host: String,
+            forwarded_proxy_authorization: bool,
+        }
+
+        let observed = Arc::new(Mutex::new(Observed::default()));
+        let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let upstream_addr = upstream_listener.local_addr().unwrap();
+        let upstream_observed = observed.clone();
+        let upstream_task = tokio::spawn(async move {
+            let (stream, _) = upstream_listener.accept().await.unwrap();
+            let service = service_fn(move |request: Request<Incoming>| {
+                let mut observed = upstream_observed.lock().unwrap();
+                observed.target = request.uri().to_string();
+                observed.host = request
+                    .headers()
+                    .get(HOST)
+                    .and_then(|value| value.to_str().ok())
+                    .unwrap_or_default()
+                    .to_owned();
+                observed.forwarded_proxy_authorization =
+                    request.headers().contains_key("proxy-authorization");
+                drop(observed);
+
+                async move {
+                    Ok::<_, Infallible>(Response::new(Full::new(Bytes::from_static(b"proxied"))))
+                }
+            });
+            http1::Builder::new()
+                .serve_connection(TokioIo::new(stream), service)
+                .await
+                .unwrap();
+        });
+
+        let (proxy_addr, proxy_task) = spawn_proxy().await;
+        let (mut sender, connection_task) = connect_client(proxy_addr).await;
+        let request = Request::builder()
+            .method(Method::GET)
+            .uri(format!("http://{upstream_addr}/hello?q=1"))
+            .header(HOST, "wrong.invalid")
+            .header("proxy-authorization", "must-not-be-forwarded")
+            .body(Full::new(Bytes::new()))
+            .unwrap();
+
+        let response = sender.send_request(request).await.unwrap();
+        assert_eq!(response.status(), StatusCode::OK);
+        let body = response.into_body().collect().await.unwrap().to_bytes();
+        assert_eq!(body, Bytes::from_static(b"proxied"));
+
+        upstream_task.abort();
+        let observed = observed.lock().unwrap();
+        assert_eq!(observed.target, "/hello?q=1");
+        assert_eq!(observed.host, upstream_addr.to_string());
+        assert!(!observed.forwarded_proxy_authorization);
+        drop(observed);
+
+        proxy_task.abort();
+        connection_task.abort();
+    }
+
+    #[tokio::test]
+    async fn relays_bytes_through_connect_tunnel() {
+        let echo_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let echo_addr = echo_listener.local_addr().unwrap();
+        let echo_task = tokio::spawn(async move {
+            let (mut stream, _) = echo_listener.accept().await.unwrap();
+            let mut request = [0_u8; 4];
+            stream.read_exact(&mut request).await.unwrap();
+            assert_eq!(&request, b"ping");
+            stream.write_all(b"pong").await.unwrap();
+            stream.shutdown().await.unwrap();
+        });
+
+        let (proxy_addr, proxy_task) = spawn_proxy().await;
+        let (mut sender, connection_task) = connect_client(proxy_addr).await;
+        let request = Request::builder()
+            .method(Method::CONNECT)
+            .uri(echo_addr.to_string())
+            .body(Full::new(Bytes::new()))
+            .unwrap();
+
+        let response = sender.send_request(request).await.unwrap();
+        assert_eq!(response.status(), StatusCode::OK);
+
+        let upgraded = hyper::upgrade::on(response).await.unwrap();
+        let mut tunnel = TokioIo::new(upgraded);
+        tunnel.write_all(b"ping").await.unwrap();
+        let mut response = [0_u8; 4];
+        tunnel.read_exact(&mut response).await.unwrap();
+        assert_eq!(&response, b"pong");
+
+        echo_task.await.unwrap();
+        proxy_task.abort();
+        connection_task.abort();
+    }
+
+    fn test_state() -> ProxyState {
+        ProxyState::new(&Config {
+            listen_addr: "127.0.0.1:0".parse().unwrap(),
+            connect_timeout: Duration::from_secs(2),
+            upstream_timeout: Duration::from_secs(2),
+        })
+    }
+
+    async fn spawn_proxy() -> (SocketAddr, JoinHandle<()>) {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let state = test_state();
+        let task = tokio::spawn(async move {
+            while let Ok((stream, peer)) = listener.accept().await {
+                let state = state.clone();
+                tokio::spawn(serve_connection(stream, peer, state));
+            }
+        });
+        (address, task)
+    }
+
+    async fn connect_client(
+        address: SocketAddr,
+    ) -> (
+        client_http1::SendRequest<Full<Bytes>>,
+        JoinHandle<Result<(), hyper::Error>>,
+    ) {
+        let stream = TcpStream::connect(address).await.unwrap();
+        let (sender, connection) = client_http1::handshake(TokioIo::new(stream)).await.unwrap();
+        let connection_task = tokio::spawn(connection.with_upgrades());
+        (sender, connection_task)
+    }
+}

src/telemetry.rs

@@ -0,0 +1,180 @@
+use std::env;
+
+use http::{HeaderMap, HeaderName, HeaderValue, header};
+use opentelemetry::{
+    global,
+    propagation::{Extractor, Injector},
+};
+use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
+use opentelemetry_otlp::{LogExporter, MetricExporter, Protocol, SpanExporter, WithExportConfig};
+use opentelemetry_sdk::{
+    Resource, logs::SdkLoggerProvider, metrics::SdkMeterProvider,
+    propagation::TraceContextPropagator, trace::SdkTracerProvider,
+};
+use tracing_subscriber::{EnvFilter, Layer, layer::SubscriberExt, util::SubscriberInitExt};
+
+use crate::BoxError;
+
+pub struct TelemetryGuard {
+    tracer_provider: SdkTracerProvider,
+    logger_provider: SdkLoggerProvider,
+    meter_provider: SdkMeterProvider,
+}
+
+impl TelemetryGuard {
+    pub fn shutdown(self) {
+        if let Err(error) = self.tracer_provider.shutdown() {
+            eprintln!("failed to shut down trace provider: {error}");
+        }
+        if let Err(error) = self.logger_provider.shutdown() {
+            eprintln!("failed to shut down log provider: {error}");
+        }
+        if let Err(error) = self.meter_provider.shutdown() {
+            eprintln!("failed to shut down meter provider: {error}");
+        }
+    }
+}
+
+pub fn init() -> Result<TelemetryGuard, BoxError> {
+    global::set_text_map_propagator(TraceContextPropagator::new());
+
+    let service_name = env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| "hermes".to_owned());
+    let resource = Resource::builder().with_service_name(service_name).build();
+
+    let metric_exporter = MetricExporter::builder()
+        .with_http()
+        .with_protocol(Protocol::HttpBinary)
+        .build()?;
+    let meter_provider = SdkMeterProvider::builder()
+        .with_periodic_exporter(metric_exporter)
+        .with_resource(resource.clone())
+        .build();
+    global::set_meter_provider(meter_provider.clone());
+
+    let log_exporter = LogExporter::builder()
+        .with_http()
+        .with_protocol(Protocol::HttpBinary)
+        .build()?;
+    let logger_provider = SdkLoggerProvider::builder()
+        .with_batch_exporter(log_exporter)
+        .with_resource(resource.clone())
+        .build();
+
+    let span_exporter = SpanExporter::builder()
+        .with_http()
+        .with_protocol(Protocol::HttpBinary)
+        .build()?;
+    let tracer_provider = SdkTracerProvider::builder()
+        .with_batch_exporter(span_exporter)
+        .with_resource(resource)
+        .build();
+    global::set_tracer_provider(tracer_provider.clone());
+
+    let fmt_filter = env_filter();
+    let otel_log_filter = exporter_log_filter(env_filter());
+    let tracer = global::tracer("hermes");
+
+    tracing_subscriber::registry()
+        .with(tracing_opentelemetry::layer().with_tracer(tracer))
+        .with(OpenTelemetryTracingBridge::new(&logger_provider).with_filter(otel_log_filter))
+        .with(tracing_subscriber::fmt::layer().with_filter(fmt_filter))
+        .try_init()?;
+
+    Ok(TelemetryGuard {
+        tracer_provider,
+        logger_provider,
+        meter_provider,
+    })
+}
+
+fn env_filter() -> EnvFilter {
+    EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))
+}
+
+fn exporter_log_filter(filter: EnvFilter) -> EnvFilter {
+    filter
+        .add_directive("hyper=off".parse().expect("valid directive"))
+        .add_directive("h2=off".parse().expect("valid directive"))
+        .add_directive("reqwest=off".parse().expect("valid directive"))
+        .add_directive("tonic=off".parse().expect("valid directive"))
+        .add_directive("opentelemetry=off".parse().expect("valid directive"))
+        .add_directive("opentelemetry_sdk=off".parse().expect("valid directive"))
+        .add_directive("opentelemetry_otlp=off".parse().expect("valid directive"))
+}
+
+pub(crate) struct HeaderExtractor<'a>(pub &'a HeaderMap);
+
+impl Extractor for HeaderExtractor<'_> {
+    fn get(&self, key: &str) -> Option<&str> {
+        self.0.get(key).and_then(|value| value.to_str().ok())
+    }
+
+    fn keys(&self) -> Vec<&str> {
+        self.0.keys().map(HeaderName::as_str).collect()
+    }
+}
+
+pub(crate) struct HeaderInjector<'a>(pub &'a mut HeaderMap);
+
+impl Injector for HeaderInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        if let (Ok(name), Ok(value)) = (
+            HeaderName::from_bytes(key.as_bytes()),
+            HeaderValue::from_str(&value),
+        ) {
+            self.0.insert(name, value);
+        }
+    }
+}
+
+pub(crate) fn remove_hop_by_hop_headers(headers: &mut HeaderMap) {
+    let connection_headers = headers
+        .get_all(header::CONNECTION)
+        .iter()
+        .filter_map(|value| value.to_str().ok())
+        .flat_map(|value| value.split(','))
+        .filter_map(|token| HeaderName::from_bytes(token.trim().as_bytes()).ok())
+        .collect::<Vec<_>>();
+
+    for name in connection_headers {
+        headers.remove(name);
+    }
+
+    for name in [
+        "connection",
+        "keep-alive",
+        "proxy-authenticate",
+        "proxy-authorization",
+        "proxy-connection",
+        "te",
+        "trailer",
+        "transfer-encoding",
+        "upgrade",
+    ] {
+        headers.remove(name);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn removes_standard_and_connection_nominated_headers() {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            header::CONNECTION,
+            HeaderValue::from_static("x-remove, keep-alive"),
+        );
+        headers.insert("x-remove", HeaderValue::from_static("secret"));
+        headers.insert("proxy-authorization", HeaderValue::from_static("secret"));
+        headers.insert("x-keep", HeaderValue::from_static("value"));
+
+        remove_hop_by_hop_headers(&mut headers);
+
+        assert!(!headers.contains_key("connection"));
+        assert!(!headers.contains_key("x-remove"));
+        assert!(!headers.contains_key("proxy-authorization"));
+        assert_eq!(headers["x-keep"], "value");
+    }
+}