From 50ab02aef26b9a16a803f58e7b91a3d4e6a03ba9 Mon Sep 17 00:00:00 2001 From: Walter Oggioni Date: Tue, 30 Jun 2026 14:19:57 +0800 Subject: [PATCH] added otel-lgtm image --- .gitea/workflows/build-otel-lgtm.yaml | 38 + otel-lgtm/Dockerfile | 84 + otel-lgtm/README.md | 108 + .../config/grafana-dashboard-jvm-metrics.json | 928 ++++ ...grafana-dashboard-red-metrics-classic.json | 584 +++ .../grafana-dashboard-red-metrics-native.json | 514 ++ otel-lgtm/config/grafana-dashboards.yaml | 23 + otel-lgtm/config/grafana-datasources.yaml | 52 + otel-lgtm/config/grafana.ini | 2160 +++++++++ otel-lgtm/config/loki.yaml | 45 + otel-lgtm/config/mimir.yaml | 52 + ...y-collector-hostmetrics-node-exporter.json | 4116 +++++++++++++++++ otel-lgtm/config/otelcol.yaml | 43 + otel-lgtm/config/supervisord.conf | 58 + otel-lgtm/config/tempo.yaml | 51 + 15 files changed, 8856 insertions(+) create mode 100644 .gitea/workflows/build-otel-lgtm.yaml create mode 100644 otel-lgtm/Dockerfile create mode 100644 otel-lgtm/README.md create mode 100644 otel-lgtm/config/grafana-dashboard-jvm-metrics.json create mode 100644 otel-lgtm/config/grafana-dashboard-red-metrics-classic.json create mode 100644 otel-lgtm/config/grafana-dashboard-red-metrics-native.json create mode 100644 otel-lgtm/config/grafana-dashboards.yaml create mode 100644 otel-lgtm/config/grafana-datasources.yaml create mode 100644 otel-lgtm/config/grafana.ini create mode 100644 otel-lgtm/config/loki.yaml create mode 100644 otel-lgtm/config/mimir.yaml create mode 100644 otel-lgtm/config/opentelemetry-collector-hostmetrics-node-exporter.json create mode 100644 otel-lgtm/config/otelcol.yaml create mode 100644 otel-lgtm/config/supervisord.conf create mode 100644 otel-lgtm/config/tempo.yaml diff --git a/.gitea/workflows/build-otel-lgtm.yaml b/.gitea/workflows/build-otel-lgtm.yaml new file mode 100644 index 0000000..5b44845 --- /dev/null +++ b/.gitea/workflows/build-otel-lgtm.yaml @@ -0,0 +1,38 @@ +name: CI +on: + push: + branches: [ master ] + paths: + - 'otel-lgtm/**' + - '.gitea/workflows/build-otel-lgtm.yaml' +jobs: + "Build otel-lgtm docker images": + runs-on: hostinger + steps: + - + name: Login to Gitea container registry + uses: docker/login-action@v3 + with: + registry: gitea.woggioni.net + username: woggioni + password: ${{ secrets.PUBLISHER_TOKEN }} + - + name: Build and push otel-lgtm vanilla images + uses: docker/build-push-action@v6 + with: + builder: multiplatform-builder + context: "{{defaultContext}}:otel-lgtm" + platforms: | + linux/amd64 + linux/arm64 + push: true + pull: true + ssh: default=/var/lib/gitea-runner/.ssh/id_ed25519 + tags: | + "gitea.woggioni.net/woggioni/otel-lgtm:latest" + "gitea.woggioni.net/woggioni/otel-lgtm:0.0.1" + secrets: | + GIT_AUTH_TOKEN.github.com=${{ secrets.GH_ACCESS_TOKEN }} + build-args: | + ALPINE_VERSION=3.24 + OTELCOL_VERSION=0.154.0 diff --git a/otel-lgtm/Dockerfile b/otel-lgtm/Dockerfile new file mode 100644 index 0000000..9b8d352 --- /dev/null +++ b/otel-lgtm/Dockerfile @@ -0,0 +1,84 @@ +# syntax=docker/dockerfile:1 + +ARG ALPINE_VERSION=3.24 +ARG OTELCOL_VERSION=0.154.0 + +FROM alpine:${ALPINE_VERSION} AS base +FROM base AS downloader +ARG OTELCOL_VERSION +WORKDIR /downloads + +RUN --mount=type=cache,target=/var/cache/apk \ + apk add curl tar ca-certificates + +RUN ARCH=$(uname -m) && \ + case "$ARCH" in \ + x86_64) TARGETARCH=amd64 ;; \ + aarch64) TARGETARCH=arm64 ;; \ + *) echo "Unsupported architecture: $ARCH"; exit 1 ;; \ + esac && \ + curl -fLo otelcol.tar.gz \ + "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTELCOL_VERSION}/otelcol-contrib_${OTELCOL_VERSION}_linux_${TARGETARCH}.tar.gz" && \ + tar xzf otelcol.tar.gz && \ + chmod +x otelcol-contrib + +FROM base + +# Install Grafana stack and supervisord from Alpine repositories. +RUN --mount=type=cache,target=/var/cache/apk \ + apk add \ + ca-certificates \ + supervisor \ + grafana \ + mimir \ + loki \ + tempo + +# Copy the OTel Collector binary from the downloader stage. +COPY --from=downloader /downloads/otelcol-contrib /usr/local/bin/otelcol-contrib + +# Create data, Grafana, and supervisor directories. +RUN mkdir -p /data/grafana && \ + mkdir -p /data/grafana/logs && \ + mkdir -p /data/grafana/plugins && \ + mkdir -p /data/mimir && \ + mkdir -p /data/loki && \ + mkdir -p /data/tempo && \ + mkdir -p /etc/grafana/provisioning/datasources && \ + mkdir -p /etc/grafana/provisioning/dashboards && \ + mkdir -p /etc/grafana/provisioning/plugins && \ + mkdir -p /etc/grafana/provisioning/alerting && \ + mkdir -p /var/log/supervisor && \ + mkdir -p /var/run + +# Copy configuration files. +COPY config/grafana.ini /etc/grafana.ini +COPY config/grafana-datasources.yaml /etc/grafana/provisioning/datasources/datasources.yaml +COPY config/grafana-dashboards.yaml /etc/grafana/provisioning/dashboards/grafana-dashboards.yaml +COPY config/grafana-dashboard-jvm-metrics.json /usr/share/grafana/grafana-dashboard-jvm-metrics.json +COPY config/grafana-dashboard-red-metrics-classic.json /usr/share/grafana/grafana-dashboard-red-metrics-classic.json +COPY config/grafana-dashboard-red-metrics-native.json /usr/share/grafana/grafana-dashboard-red-metrics-native.json +COPY config/opentelemetry-collector-hostmetrics-node-exporter.json /usr/share/grafana/opentelemetry-collector-hostmetrics-node-exporter.json + +RUN mkdir -p /var/lib/grafana/plugins +RUN grafana cli --config /etc/grafana.ini --homepath /usr/share/grafana --pluginsDir /var/lib/grafana/plugins plugins install grafana-exploretraces-app +RUN grafana cli --config /etc/grafana.ini --homepath /usr/share/grafana --pluginsDir /var/lib/grafana/plugins plugins install grafana-lokiexplore-app +RUN grafana cli --config /etc/grafana.ini --homepath /usr/share/grafana --pluginsDir /var/lib/grafana/plugins plugins install grafana-metricsdrilldown-app +RUN grafana cli --config /etc/grafana.ini --homepath /usr/share/grafana --pluginsDir /var/lib/grafana/plugins plugins install grafana-pyroscope-app + +COPY config/mimir.yaml /etc/mimir/mimir.yaml +COPY config/loki.yaml /etc/loki/loki.yaml +COPY config/tempo.yaml /etc/tempo/tempo.yaml +COPY config/otelcol.yaml /etc/otelcol/otelcol.yaml +COPY config/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +# Grafana anonymous admin access, provisioning path, and disable plugin pre-install. +VOLUME ["/data"] + +EXPOSE 3000 4318 + +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD wget -qO- http://127.0.0.1:3000/api/health >/dev/null 2>&1 && \ + wget -qO- http://127.0.0.1:13133/ready >/dev/null 2>&1 + +CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] diff --git a/otel-lgtm/README.md b/otel-lgtm/README.md new file mode 100644 index 0000000..bcea549 --- /dev/null +++ b/otel-lgtm/README.md @@ -0,0 +1,108 @@ +# OTel LGTM (Alpine + supervisord) + +A single-container observability backend running on Alpine Linux with supervisord. + +Components: + +- **Grafana** — visualization UI +- **Mimir** — metrics storage +- **Loki** — log storage +- **Tempo** — trace storage +- **OpenTelemetry Collector (contrib)** — OTLP receiver and router + +## Ports + +| Port | Service | Description | +|------|---------|-------------| +| 3000 | Grafana | Web UI | +| 4318 | OTel Collector | OTLP/HTTP endpoint | + +Only these two ports are exposed. All other services communicate internally on `127.0.0.1`. + +## Build + +```bash +docker build -t otel-lgtm . +``` + +> The Dockerfile uses BuildKit cache mounts for `apk`. Make sure BuildKit is enabled (default in recent Docker Desktop / `docker buildx`). + +### Build args + +| Build arg | Default | Description | +|-----------|---------|-------------| +| `ALPINE_VERSION` | `3.24` | Alpine Linux base image version | +| `OTELCOL_VERSION` | `0.154.0` | OpenTelemetry Collector Contrib version | + +Grafana, Mimir, Loki, and Tempo versions are determined by the chosen Alpine repository. + +Example: + +```bash +docker build --build-arg OTELCOL_VERSION=0.155.0 -t otel-lgtm . +``` + +### Multi-architecture + +The Dockerfile detects the build architecture at build time, so it works on both `amd64` and `arm64`. To build a true multi-arch image in one command, use Docker BuildKit / `docker buildx`: + +```bash +docker buildx build --platform linux/amd64,linux/arm64 -t otel-lgtm . +``` + +## Run + +```bash +docker run -p 3000:3000 -p 4318:4318 -v lgtm-data:/data otel-lgtm +``` + +Grafana is available at http://localhost:3000 with anonymous admin access enabled. + +Send telemetry to the collector at `http://localhost:4318` (OTLP/HTTP). + +## Data persistence + +All persistent data is written under `/data`: + +| Path | Service | +|------|---------| +| `/data/grafana` | Grafana data, logs, and plugins | +| `/data/mimir` | Mimir metrics storage | +| `/data/loki` | Loki log storage | +| `/data/tempo` | Tempo trace storage | + +Mount a volume there to persist data across container restarts. + +## Architecture + +``` +Application ──OTLP/HTTP──> OTel Collector :4318 + │ + ├─ metrics ──OTLP/HTTP──> Mimir :9009 + ├─ logs ─────OTLP/HTTP──> Loki :3100 + └─ traces ───OTLP/HTTP──> Tempo :3200 + +Grafana :3000 queries Mimir, Loki, and Tempo. +``` + +## Read-only root filesystem + +The image can be run with an immutable root filesystem (`--read-only`). A few runtime-only directories must be mounted as writable `tmpfs`: + +```bash +docker run --read-only \ + -p 3000:3000 -p 4318:4318 \ + -v lgtm-data:/data \ + --tmpfs /tmp:rw,noexec,nosuid,size=100m \ + --tmpfs /run:rw,noexec,nosuid,size=10m \ + --tmpfs /var/log/supervisor:rw,noexec,nosuid,size=50m \ + otel-lgtm +``` + +Persistent data continues to live on the `/data` volume, while temporary/runtime state stays in memory. + +## Notes + +- This image is intended for development, demo, and testing environments. +- Grafana authentication is disabled (`anonymous` admin). +- Grafana’s default async plugin pre-installation is disabled to avoid relying on internet access at startup. diff --git a/otel-lgtm/config/grafana-dashboard-jvm-metrics.json b/otel-lgtm/config/grafana-dashboard-jvm-metrics.json new file mode 100644 index 0000000..212e11f --- /dev/null +++ b/otel-lgtm/config/grafana-dashboard-jvm-metrics.json @@ -0,0 +1,928 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Dashboard for JVM metrics with OpenTelemetry instrumentation", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 18812, + "graphTooltip": 0, + "links": [ + { + "asDropdown": false, + "icon": "info", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Semantic Conventions: 1.20.0 - 1.22.0", + "tooltip": "multiple versions of the semantic conventions are supported using 'or' and regex queries", + "type": "link", + "url": "https://github.com/open-telemetry/semantic-conventions/blob/main/schemas/1.20.0" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "HTTP server request rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (instance) (rate({__name__=~\"http_server_request_duration_seconds_count|http_server_request_duration_count|http_server_duration_seconds_count|http_server_duration_count|http_server_duration_milliseconds_count|http_server_requests_milliseconds_count|http_server_requests_count|http_server_requests_seconds_count\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "HTTP server error ratio - ratio of requests that return 5xx", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 4, + "y": 0 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(sum by (instance)(rate({__name__=~\"http_server_request_duration_seconds_count|http_server_request_duration_count\", job=~\"$job\", instance=~\"$instance\", http_response_status_code=~\"5.*\"}[$__rate_interval]))) / on (instance) (sum by (instance)(rate({__name__=~\"http_server_request_duration_seconds_count|http_server_request_duration_count\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))\nor\n(sum by (instance)(rate({__name__=~\"http_server_duration_milliseconds_count|http_server_duration_seconds_count|http_server_duration_count\", job=~\"$job\", instance=~\"$instance\", http_status_code=~\"5.*\"}[$__rate_interval]))) / on (instance) (sum by (instance)(rate({__name__=~\"http_server_duration_milliseconds_count|http_server_duration_seconds_count|http_server_duration_count\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))\nor\n(sum by (instance)(rate({__name__=~\"http_server_requests_milliseconds_count|http_server_requests_count|http_server_requests_seconds_count\", job=~\"$job\", instance=~\"$instance\", outcome=\"SERVER_ERROR\"}[$__rate_interval]))) / on (instance) (sum by (instance)(rate({__name__=~\"http_server_requests_milliseconds_count|http_server_requests_count|http_server_requests_seconds_count\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))\n", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Error %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "95th percentile of HTTP server request duration in seconds", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 9, + "y": 0 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(histogram_quantile(0.95, sum by(le, instance) (rate({__name__=~\"http_server_request_duration_seconds_bucket|http_server_request_duration_bucket|http_server_duration_seconds_bucket|http_server_duration_bucket|http_server_requests_seconds_bucket\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))))\nor\n(histogram_quantile(0.95, sum by(le, instance) (rate({__name__=~\"http_server_duration_milliseconds_bucket|http_server_requests_milliseconds_bucket|http_server_requests_bucket\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))) / 1000)\n", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (95%)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "CPU utilization for the whole system", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 0, + "y": 8 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "{__name__=~\"jvm_cpu_recent_utilization|jvm_cpu_recent_utilization_ratio|process_runtime_jvm_system_cpu_utilization|process_runtime_jvm_system_cpu_utilization_ratio|system_cpu_usage\", job=~\"$job\", instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Used heap memory / heap memory limit ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 7, + "y": 8 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(sum by (instance) ({__name__=~\"jvm_memory_used|jvm_memory_used_bytes\",job=~\"$job\",jvm_memory_type=\"heap\", instance=~\"$instance\"}) / on(instance) sum by (instance) ({__name__=~\"jvm_memory_limit|jvm_memory_limit_bytes\",job=~\"$job\",jvm_memory_type=\"heap\", instance=~\"$instance\"}))\nor\n(sum by (instance) ({__name__=~\"process_runtime_jvm_memory_usage|process_runtime_jvm_memory_usage_bytes\",job=~\"$job\",type=\"heap\", instance=~\"$instance\"}) / on(instance) sum by (instance) ({__name__=~\"process_runtime_jvm_memory_limit|process_runtime_jvm_memory_limit_bytes\",job=~\"$job\",type=\"heap\", instance=~\"$instance\"}))\nor\n(sum by (instance) ({__name__=~\"jvm_memory_used|jvm_memory_used_bytes\",job=~\"$job\",area=\"heap\", instance=~\"$instance\"}) / on(instance) sum by (instance) ({__name__=~\"jvm_memory_max|jvm_memory_max_bytes\",job=~\"$job\",area=\"heap\", instance=~\"$instance\"}))", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Heap Memory utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of time spend for garbage collection pauses", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 15 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(sum by(instance) (rate({__name__=~\"jvm_gc_duration_sum|jvm_gc_duration_seconds_sum|process_runtime_jvm_gc_duration_sum|process_runtime_jvm_gc_duration_seconds_sum|jvm_gc_pause_sum|jvm_gc_pause_seconds_sum\",job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))\nor\n(sum by(instance) (rate(jvm_gc_pause_milliseconds_sum{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) / 1000)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Garbage Collection", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of currently loaded classes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 4, + "y": 15 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "{__name__=~\"jvm_class_count|process_runtime_jvm_classes_current_loaded|jvm_classes_loaded\",job=~\"$job\", instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Classes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of currently executing (also called \"live\") threads", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 9, + "y": 15 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum({__name__=~\"jvm_thread_count|process_runtime_jvm_threads_count|jvm_threads_live\",job=~\"$job\", instance=~\"$instance\"}) by (instance)", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Threads", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": ["JVM", "open-telemetry", "Java", "otel", "opentelemetry", "otlp"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "prometheus" + }, + "description": "Choose a Prometheus data source", + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({__name__=~\"jvm_class_count|process_runtime_jvm_classes_current_loaded|jvm_classes_loaded\"},job)", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values({__name__=~\"jvm_class_count|process_runtime_jvm_classes_current_loaded|jvm_classes_loaded\"},job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({__name__=~\"(jvm_class_count|process_runtime_jvm_classes_current_loaded|jvm_classes_loaded)\", job=~\"$job\"},instance)", + "description": "The instance of the application, e.g. pod1", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values({__name__=~\"(jvm_class_count|process_runtime_jvm_classes_current_loaded|jvm_classes_loaded)\", job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "", + "title": "JVM Overview (OpenTelemetry)", + "uid": "b91844d7-121e-4d0a-93b8-a9c1a05703b3", + "version": 1, + "weekStart": "" +} diff --git a/otel-lgtm/config/grafana-dashboard-red-metrics-classic.json b/otel-lgtm/config/grafana-dashboard-red-metrics-classic.json new file mode 100644 index 0000000..5b588b7 --- /dev/null +++ b/otel-lgtm/config/grafana-dashboard-red-metrics-classic.json @@ -0,0 +1,584 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "request rate, error rate, duration", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "description": "", + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# RED Metrics: (r)equest rate, (e)rror rate, (d)uration\n\nThis dashboard uses explicit bucket histograms and the stable [OpenTelemetry metrics semantic conventions](https://opentelemetry.io/docs/specs/semconv/general/metrics/).\nTo enable this for the [OpenTelemetry Java instrumentation agent](https://github.com/open-telemetry/opentelemetry-java-instrumentation/), set the following environment variables:\n\n```\nexport OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION=explicit_bucket_histogram\n```", + "mode": "markdown" + }, + "pluginVersion": "10.3.3", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=~\"http_server_request_duration_seconds_count|http_server_request_duration_count|http_server_duration_milliseconds_count|http_server_duration_seconds_count|http_server_duration_count\", job=~\"$job\", instance=~\"$instance\"}[5m]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate({__name__=~\"http_server_request_duration_seconds_count|http_server_request_duration_count|http_server_duration_milliseconds_count|http_server_duration_seconds_count|http_server_duration_count\", job=~\"$job\", instance=~\"$instance\", http_response_status_code=~\"5..\"}[5m])) / sum(rate({__name__=~\"http_server_request_duration_seconds_count|http_server_request_duration_count|http_server_duration_milliseconds_count|http_server_duration_seconds_count|http_server_duration_count\", job=~\"$job\", instance=~\"$instance\"}[5m]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 3, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (le) (rate({__name__=~\"http_server_request_duration_seconds_bucket|http_server_request_duration_bucket|http_server_duration_milliseconds_bucket|http_server_duration_seconds_bucket|http_server_duration_bucket\", job=~\"$job\", instance=~\"$instance\"}[5m]))", + "format": "heatmap", + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration histogram (s)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 5, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (le) (rate({__name__=~\"http_server_request_duration_seconds_bucket|http_server_request_duration_bucket|http_server_duration_milliseconds_bucket|http_server_duration_seconds_bucket|http_server_duration_bucket\", job=~\"$job\", instance=~\"$instance\"}[5m]))", + "format": "heatmap", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Duration Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"http_server_request_duration_seconds_bucket|http_server_request_duration_bucket|http_server_duration_milliseconds_bucket|http_server_duration_seconds_bucket|http_server_duration_bucket\", job=~\"$job\", instance=~\"$instance\"}[5m])))", + "format": "time_series", + "instant": false, + "legendFormat": "95th", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.5, sum by (le) (rate({__name__=~\"http_server_request_duration_seconds_bucket|http_server_request_duration_bucket|http_server_duration_milliseconds_bucket|http_server_duration_seconds_bucket|http_server_duration_bucket\", job=~\"$job\", instance=~\"$instance\"}[5m])))", + "hide": false, + "legendFormat": "50th", + "range": true, + "refId": "B" + } + ], + "title": "Duration percentiles", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "allValue": ".+", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(job)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({job=~\"$job\"},instance)", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values({job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "RED Metrics (classic histogram)", + "uid": "f543a537-cb96-470d-a349-660ad1513136", + "version": 1, + "weekStart": "" +} diff --git a/otel-lgtm/config/grafana-dashboard-red-metrics-native.json b/otel-lgtm/config/grafana-dashboard-red-metrics-native.json new file mode 100644 index 0000000..f2c520e --- /dev/null +++ b/otel-lgtm/config/grafana-dashboard-red-metrics-native.json @@ -0,0 +1,514 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "request rate, error rate, duration", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "description": "", + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# RED Metrics: (r)equest rate, (e)rror rate, (d)uration\n\nThis dashboard uses exponential histograms and the stable [OpenTelemetry metrics semantic conventions](https://opentelemetry.io/docs/specs/semconv/general/metrics/).\nTo enable this for the [OpenTelemetry Java instrumentation agent](https://github.com/open-telemetry/opentelemetry-java-instrumentation/), set the following environment variables:\n\n```\nexport OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION=base2_exponential_bucket_histogram\n```", + "mode": "markdown" + }, + "pluginVersion": "10.3.3", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(histogram_count(rate({__name__=~\"http_server_request_duration_seconds|http_server_request_duration|http_server_duration_milliseconds|http_server_duration_seconds|http_server_duration\", job=~\"$job\", instance=~\"$instance\"}[5m])))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(histogram_count(rate({__name__=~\"http_server_request_duration_seconds|http_server_request_duration|http_server_duration_milliseconds|http_server_duration_seconds|http_server_duration\", job=~\"$job\", instance=~\"$instance\", http_response_status_code=~\"5..\"}[5m]))) / sum(histogram_count(rate({__name__=~\"http_server_request_duration_seconds|http_server_request_duration|http_server_duration_milliseconds|http_server_duration_seconds|http_server_duration\", job=~\"$job\", instance=~\"$instance\"}[5m])))\n\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 5, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate({__name__=~\"http_server_request_duration_seconds|http_server_request_duration|http_server_duration_milliseconds|http_server_duration_seconds|http_server_duration\",job=~\"$job\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Duration Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate({__name__=~\"http_server_request_duration_seconds|http_server_request_duration|http_server_duration_milliseconds|http_server_duration_seconds|http_server_duration\",job=~\"$job\", instance=~\"$instance\"}[5m])))", + "format": "time_series", + "instant": false, + "legendFormat": "95th", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.5, sum(rate({__name__=~\"http_server_request_duration_seconds|http_server_request_duration|http_server_duration_milliseconds|http_server_duration_seconds|http_server_duration\",job=~\"$job\", instance=~\"$instance\"}[5m])))", + "hide": false, + "legendFormat": "50th", + "range": true, + "refId": "B" + } + ], + "title": "Duration percentiles", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "allValue": ".+", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(job)", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({job=~\"$job\"},instance)", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values({job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "RED Metrics (native histogram)", + "uid": "f543a537-cb96-470d-a349-660ad1513135", + "version": 1, + "weekStart": "" +} diff --git a/otel-lgtm/config/grafana-dashboards.yaml b/otel-lgtm/config/grafana-dashboards.yaml new file mode 100644 index 0000000..659f8a0 --- /dev/null +++ b/otel-lgtm/config/grafana-dashboards.yaml @@ -0,0 +1,23 @@ +apiVersion: 1 + +providers: + - name: "RED Metrics (classic histogram)" + type: file + options: + path: /usr/share/grafana/grafana-dashboard-red-metrics-classic.json + foldersFromFilesStructure: false + - name: "RED Metrics (exponential/native histogram)" + type: file + options: + path: /usr/share/grafana/grafana-dashboard-red-metrics-native.json + foldersFromFilesStructure: false + - name: "JVM Metrics" + type: file + options: + path: /usr/share/grafana/grafana-dashboard-jvm-metrics.json + foldersFromFilesStructure: false + - name: "OpenTelemetry Collector HostMetrics (Node Exporter)" + type: file + options: + path: /usr/share/grafana/opentelemetry-collector-hostmetrics-node-exporter.json + foldersFromFilesStructure: false diff --git a/otel-lgtm/config/grafana-datasources.yaml b/otel-lgtm/config/grafana-datasources.yaml new file mode 100644 index 0000000..85142d7 --- /dev/null +++ b/otel-lgtm/config/grafana-datasources.yaml @@ -0,0 +1,52 @@ +apiVersion: 1 + +datasources: + - name: Mimir + type: prometheus + uid: mimir + url: http://127.0.0.1:9009/prometheus + access: proxy + isDefault: true + editable: true + jsonData: + timeInterval: 60s + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + urlDisplayLabel: "Trace: $${__value.raw}" + + - name: Tempo + type: tempo + uid: tempo + url: http://127.0.0.1:3200 + editable: true + jsonData: + tracesToLogsV2: + customQuery: true + datasourceUid: "loki" + query: '{$${__tags}} | trace_id = "$${__trace.traceId}"' + tags: + - key: "service.name" + value: "service_name" + + serviceMap: + datasourceUid: "mimir" + search: + hide: false + nodeGraph: + enabled: true + lokiSearch: + datasourceUid: "loki" + - name: Loki + type: loki + uid: loki + url: http://127.0.0.1:3100 + editable: true + jsonData: + derivedFields: + - name: "trace_id" + matcherType: "label" + matcherRegex: "trace_id" + url: "$${__value.raw}" + datasourceUid: "tempo" + urlDisplayLabel: "Trace: $${__value.raw}" diff --git a/otel-lgtm/config/grafana.ini b/otel-lgtm/config/grafana.ini new file mode 100644 index 0000000..8d8bba5 --- /dev/null +++ b/otel-lgtm/config/grafana.ini @@ -0,0 +1,2160 @@ +##################### Grafana Configuration Example ##################### +# +# Everything has defaults so you only need to uncomment things you want to +# change + +# possible values : production, development +;app_mode = production + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +;instance_name = ${HOSTNAME} + +#################################### Paths #################################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +data = /data/grafana/data + +# Temporary files in `data` directory older than given duration will be removed +;temp_data_lifetime = 24h + +# Directory where grafana can store logs +logs = /data/grafana/logs + +# Directory where grafana will automatically scan and look for plugins +plugins = /var/lib/grafana/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +provisioning = /etc/grafana/provisioning + +# Directories that are permitted to contain local repositories. +# This is a list. Each entry is delimited by a pipe (|). No leading or trailing spaces are supported. +# These do not need to be absolute paths, in which case they'll be relative to the path where you are running Grafana. +# Empty entries will return an error, unless the string is just a single pipe. +# Example: permitted_provisioning_paths = /tmp|/etc/grafana/repositories|conf/provisioning +;permitted_provisioning_paths = devenv/dev-dashboards|conf/provisioning + +#################################### Server #################################### +[server] +# Protocol (http, https, h2, socket) +;protocol = http + +# Minimum TLS version allowed. By default, this value is empty. Accepted values are: TLS1.2, TLS1.3. If nothing is set TLS1.2 would be taken +;min_tls_version = "" + +# The ip address to bind to, empty will bind to all interfaces +;http_addr = + +# The http port to use +;http_port = 3000 + +# The public facing domain name used to access grafana from a browser +;domain = localhost + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +;enforce_domain = false + +# The full public facing url you use in browser, used for redirects and emails +# If you use reverse proxy and sub path specify full url (with sub path) +;root_url = %(protocol)s://%(domain)s:%(http_port)s/ + +# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. +;serve_from_sub_path = false + +# Log web requests +;router_logging = false + +# the path relative working path +;static_root_path = public + +# enable gzip +;enable_gzip = false + +# https certs & key file +;cert_file = +;cert_key = + +# optional password to be used to decrypt key file +;cert_pass = + +# Certificates file watch interval +;certs_watch_interval = + +# Unix socket gid +# Changing the gid of a file without privileges requires that the target group is in the group of the process and that the process is the file owner +# It is recommended to set the gid as http server user gid +# Not set when the value is -1 +;socket_gid = + +# Unix socket mode +;socket_mode = + +# Unix socket path +;socket = + +# CDN Url +;cdn_url = + +# Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections. +# `0` means there is no timeout for reading the request. +;read_timeout = 0 + +# This setting enables you to specify additional headers that the server adds to HTTP(S) responses. +[server.custom_response_headers] +#exampleHeader1 = exampleValue1 +#exampleHeader2 = exampleValue2 + +[environment] +# Sets whether the local file system is available for Grafana to use. Default is true for backward compatibility. +;local_file_system_available = true + +#################################### GRPC Server ######################### +;[grpc_server] +;network = "tcp" +;address = "127.0.0.1:10000" +;use_tls = false +;cert_file = +;cert_key = +;max_recv_msg_size = +;max_send_msg_size = +# this will log the request and response for each unary gRPC call +;enable_logging = false +;max_connection_age = +;max_connection_age_grace = +;max_connection_idle = +;keepalive_time = +;keepalive_timeout = +;keepalive_min_time = + +#################################### Database #################################### +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url properties. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +;type = sqlite3 +;host = 127.0.0.1:3306 +;name = grafana +;user = root +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +;password = +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +;url = + +# Set to true or false to enable or disable high availability mode. +# When it's set to false some functions will be simplified and only run in-process +# instead of relying on the database. +# +# Only set it to false if you run only a single instance of Grafana. +;high_availability = true + +# Max idle conn setting default is 2 +;max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +;max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +;conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +;log_queries = + +# For "postgres", use either "disable", "require" or "verify-full" +# For "mysql", use either "true", "false", or "skip-verify". +;ssl_mode = disable + +# For "postgres", use either "1" to enable or "0" to disable SNI +;ssl_sni = + +# Database drivers may support different transaction isolation levels. +# Currently, only "mysql" driver supports isolation levels. +# If the value is empty - driver's default isolation level is applied. +# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE". +;isolation_level = + +;ca_cert_path = +;client_key_path = +;client_cert_path = +;server_cert_name = + +# For "sqlite3" only, path relative to data_path setting +;path = grafana.db + +# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) +;cache_mode = private + +# For "sqlite3" only. Enable/disable Write-Ahead Logging, https://sqlite.org/wal.html. Default is false. +;wal = false + +# For "mysql" and "postgres" only. Lock the database for the migrations, default is true. +;migration_locking = true + +# For "mysql" and "postgres" only. How many seconds to wait before failing to lock the database for the migrations, default is 0. +;locking_attempt_timeout_sec = 0 + +# For "sqlite" only. How many times to retry query in case of database is locked failures. Default is 0 (disabled). +;query_retries = 0 + +# For "sqlite" only. How many times to retry transaction in case of database is locked failures. Default is 5. +;transaction_retries = 5 + +# Set to true to add metrics and tracing for database queries. +;instrument_queries = false + +# Set to true to delete auto-generated primary keys during migrations. +# This is useful when databases have auto-generated primary keys enabled. +;delete_auto_gen_ids = false + +# Set to true to skip dashboard UID migrations on startup. +# Improves startup performance for instances with large numbers of annotations who do not plan to downgrade Grafana. +;skip_dashboard_uid_migration_on_startup = false + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +;type = database + +# cache connectionstring options +# database: will use Grafana primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,username=grafana,password=grafanaRocks,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. +# memcache: 127.0.0.1:11211 +;connstr = + +# prefix prepended to all the keys in the remote cache +; prefix = + +# This enables encryption of values stored in the remote cache +;encryption = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +;logging = false + +# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds. +# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +;timeout = 30 + +# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds. +;dialTimeout = 10 + +# How many seconds the data proxy waits before sending a keepalive probe request. +;keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +;tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +;expect_continue_timeout_seconds = 1 + +# Optionally limits the total number of connections per host, including connections in the dialing, +# active, and idle states. On limit violation, dials will block. +# A value of zero (0) means no limit. +;max_conns_per_host = 0 + +# The maximum number of idle connections that Grafana will keep alive. +;max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +;idle_conn_timeout_seconds = 90 + +# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. +;send_user_header = false + +# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. +;response_limit = 0 + +# Limits the number of rows that Grafana will process from SQL data sources. +;row_limit = 1000000 + +# Sets a custom value for the `User-Agent` header for outgoing data proxy requests. If empty, the default value is `Grafana/` (for example `Grafana/9.0.0`). +;user_agent = + +#################################### Analytics #################################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +;reporting_enabled = true + +# The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs +;reporting_distributor = grafana-labs + +# Set to false to disable all checks to https://grafana.com +# for new versions of grafana. The check is used +# in some UI views to notify that a grafana update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com/api/grafana/versions/stable to get the latest version. +;check_for_updates = true + +# Set to false to disable all checks to https://grafana.com +# for new versions of plugins. The check is used +# in some UI views to notify that a plugin update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com to get the latest versions. +;check_for_plugin_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +;google_analytics_ua_id = + +# Google Analytics 4 tracking code, only enabled if you specify an id here +;google_analytics_4_id = + +# When Google Analytics 4 Enhanced event measurement is enabled, we will try to avoid sending duplicate events and let Google Analytics 4 detect navigation changes, etc. +;google_analytics_4_send_manual_page_views = false + +# Google Tag Manager ID, only enabled if you specify an id here +;google_tag_manager_id = + +# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set +;rudderstack_write_key = + +# Rudderstack data plane url, enabled only if rudderstack_write_key is also set +;rudderstack_data_plane_url = + +# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set +;rudderstack_sdk_url = + +# Rudderstack v3 SDK, optional, defaults to false. If set, Rudderstack v3 SDK will be used instead of v1 +;rudderstack_v3_sdk_url = + +# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config +;rudderstack_config_url = + +# Rudderstack Integrations URL, optional. Only valid if you pass the SDK version 1.1 or higher +;rudderstack_integrations_url = + +# Intercom secret, optional, used to hash user_id before passing to Intercom via Rudderstack +;intercom_secret = + +# Application Insights connection string. Specify an URL string to enable this feature. +;application_insights_connection_string = + +# Optional. Specifies an Application Insights endpoint URL where the endpoint string is wrapped in backticks ``. +;application_insights_endpoint_url = + +# Optional, defaults to true. Configure automatic route tracking for single page applications in Application Insights. +# See https://learn.microsoft.com/en-us/azure/azure-monitor/app/application-insights-faq#is-there-a-way-to-see-fewer-events-per-transaction-when-i-use-the-application-insights-javascript-sdk +;application_insights_auto_route_tracking = + +# Controls if the UI contains any links to user feedback forms +;feedback_links_enabled = true + +# Static context that is being added to analytics events +;reporting_static_context = grafanaInstance=12, os=linux + +# Logs interaction events to the browser javascript console, intended for development only +;browser_console_reporter = false + +#################################### Security #################################### +[security] +# disable creation of admin user on first start of grafana +;disable_initial_admin_creation = false + +# default admin user, created on startup +;admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +;admin_password = admin + +# default admin email, created on startup +;admin_email = admin@localhost + +# IMPORTANT: Set this to a unique, random string in production. +# Used for signing. If you change it later you may need to re-encode secrets. +# Example: openssl rand -hex 32 +;secret_key = CHANGE_ME_TO_A_RANDOM_SECRET + +# current key provider used for envelope encryption, default to static value specified by secret_key +;encryption_provider = secretKey.v1 + +# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 +;available_encryption_providers = + +# disable gravatar profile images +;disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +;data_source_proxy_whitelist = + +# disable protection against brute force login attempts +;disable_brute_force_login_protection = false + +# max number of failed login attempts before user gets locked +;brute_force_login_protection_max_attempts = 5 + +# disable protection against brute force login attempts by username +;disable_username_login_protection = false + +# disable protection against brute force login attempts by IP address +; disable_ip_address_login_protection = true + +# set to true if you host Grafana behind HTTPS. default is false. +;cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +;cookie_samesite = lax + +# set to true if you want to allow browsers to render Grafana in a ,