Tempo metric generator for span info

Grafana Tempo
1

I don’t see traces_spanmetrics_latency in http://localhost:3100/metrics port forwarding to kubectl port-forward tempo-distributed-metrics-generator-66c49fb6c8-sw54d 3100:3100

➜ ~ kubectl get configmap tempo-distributed-config -n monitoring -o yaml
apiVersion: v1
data:
tempo-query.yaml: |
backend: 127.0.0.1:3100
tempo.yaml: |2

distributor:
  receivers:
    otlp:
      protocols:
        grpc: null
        http: null
  ring:
    kvstore:
      store: memberlist
ingester:
  lifecycler:
    ring:
      kvstore:
        store: memberlist
      replication_factor: 1
memberlist:
  join_members:
  - dns+tempo-distributed-gossip-ring:7946
metrics_generator:
  processor:
    span_metrics:
      dimensions:
      - service
      - span_name
      - span_kind
      - status_code
      - status_message
      histogram_buckets:
      - 0.005
      - 0.01
      - 0.025
      - 0.05
      - 0.1
      - 0.25
      - 0.5
      - 1
      - 2.5
      - 5
  storage:
    path: /var/tempo/wal
querier:
  frontend_worker:
    frontend_address: tempo-distributed-query-frontend.monitoring.svc.cluster.local:9095
server:
  http_listen_port: 3100
storage:
  trace:
    backend: gcs
    gcs:
      bucket_name: tempo-traces-staging

kind: ConfigMap
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{“apiVersion”:“v1”,“data”:{“tempo-query.yaml”:“backend: 127.0.0.1:3100\n”,“tempo.yaml”:“\ndistributor:\n receivers:\n otlp:\n protocols:\n grpc: null\n http: null\n ring:\n kvstore:\n store: memberlist\ningester:\n lifecycler:\n ring:\n kvstore:\n store: memberlist\n replication_factor: 1\nmemberlist:\n join_members:\n - dns+tempo-distributed-gossip-ring:7946\nmetrics_generator:\n processor:\n span_metrics:\n dimensions:\n - service\n - span_name\n - span_kind\n - status_code\n - status_message\n histogram_buckets:\n - 0.005\n - 0.01\n - 0.025\n - 0.05\n - 0.1\n - 0.25\n - 0.5\n - 1\n - 2.5\n - 5\n storage:\n path: /var/tempo/wal\nquerier:\n frontend_worker:\n frontend_address: tempo-distributed-query-frontend.monitoring.svc.cluster.local:9095\nserver:\n http_listen_port: 3100\nstorage:\n trace:\n backend: gcs\n gcs:\n bucket_name: tempo-traces-staging\n”},“kind”:“ConfigMap”,“metadata”:{“annotations”:{},“labels”:{“app.kubernetes.io/instance":“tempo-distributed”,“app.kubernetes.io/managed-by”:“Helm”,“app.kubernetes.io/name”:“tempo”,“app.kubernetes.io/version”:“2.6.0”,“argocd.argoproj.io/instance”:“tempo”,“helm.sh/chart”:“tempo-distributed-1.18.0”},“name”:“tempo-distributed-config”,“namespace”:"monitoring”}}
creationTimestamp: “2024-11-06T12:04:50Z”
labels:
app.kubernetes.io/instance: tempo-distributed
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: tempo
app.kubernetes.io/version: 2.6.0
argocd.argoproj.io/instance: tempo
helm.sh/chart: tempo-distributed-1.18.0
name: tempo-distributed-config
namespace: monitoring
resourceVersion: “749310804”
uid: 20ceca73-7c81-4411-b8aa-337e732e365c
➜ ~

the documentation I looked at: