How to collect pod logs using grafana alloy and send it to loki

Grafana Alloy
1

I have a full stack app deployed in my kind cluster and I have attached all the files that are used for configuring grafana, loki and grafana-alloy. My issue is that the pod logs are not getting discovered.

grafana-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: grafana
  labels:
    app: grafana
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      labels:
        app: grafana
    spec:
      containers:
        - name: grafana
          image: grafana/grafana:latest
          ports:
            - containerPort: 3000
          env:
            - name: GF_SERVER_ROOT_URL
              value: "%(protocol)s://%(domain)s/grafana/"

---
apiVersion: v1
kind: Service
metadata:
  name: grafana
spec:
  type: ClusterIP
  ports:
    - port: 3000
      targetPort: 3000
      name: http
  selector:
    app: grafana

loki-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: loki-config
  namespace: default
data:
  loki-config.yaml: |
    auth_enabled: false
    server:
      http_listen_port: 3100
    ingester:
      wal:
        enabled: true
        dir: /loki/wal
      lifecycler:
        ring:
          kvstore:
            store: inmemory
          replication_factor: 1
      chunk_idle_period: 3m
      max_chunk_age: 1h
    schema_config:
      configs:
      - from: 2022-01-01
        store: boltdb-shipper
        object_store: filesystem
        schema: v11
        index:
          prefix: index_
          period: 24h
    compactor:
      shared_store: filesystem
      working_directory: /loki/compactor
    storage_config:
      boltdb_shipper:
        active_index_directory: /loki/index
        cache_location: /loki/boltdb-cache
        shared_store: filesystem
      filesystem:
        directory: /loki/chunks
    limits_config:
      reject_old_samples: true
      reject_old_samples_max_age: 168h

loki-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: loki
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: loki
  template:
    metadata:
      labels:
        app: loki
    spec:
      containers:
      - name: loki
        image: grafana/loki:2.9.0
        ports:
        - containerPort: 3100
        args:
        - -config.file=/etc/loki/loki-config.yaml
        volumeMounts:
        - name: config
          mountPath: /etc/loki
        - name: wal
          mountPath: /loki/wal
        - name: chunks
          mountPath: /loki/chunks
        - name: index
          mountPath: /loki/index
        - name: cache
          mountPath: /loki/boltdb-cache
        - name: compactor
          mountPath: /loki/compactor

      volumes:
      - name: config
        configMap:
          name: loki-config
      - name: wal
        emptyDir: {}
      - name: chunks
        emptyDir: {}
      - name: index
        emptyDir: {}
      - name: cache
        emptyDir: {}
      - name: compactor
        emptyDir: {}

---
apiVersion: v1
kind: Service
metadata:
  name: loki
  namespace: default
spec:
  selector:
    app: loki
  ports:
  - name: http
    port: 3100
    targetPort: 3100

alloy-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: alloy-config
  labels:
     app: alloy
data:
  alloy-config.alloy: |
  apiVersion: v1
  kind: ConfigMap
  metadata:
    name: alloy-config
    labels:
       app: alloy
  data:
    alloy-config.alloy: |
      discovery.kubernetes "pods" {
    role = "pod"
  }
  
  loki.source.kubernetes "pods" {
    targets    = discovery.kubernetes.pods.targets
    forward_to = [loki.write.local.receiver]
  }
  
  loki.write "local" {
    endpoint {
      url = "http://address:port/loki/api/v1/push"
      tenant_id = "local"
    }
  }

alloy-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: grafana-alloy
  labels:
    app: alloy
spec:
  replicas: 1
  selector:
    matchLabels:
      app: alloy
  template:
    metadata:
      labels:
        app: alloy
    spec:
      containers:
        - name: alloy
          image: grafana/alloy:latest
          args:
            - run
            - /etc/alloy/alloy-config.alloy
          volumeMounts:
            - name: config
              mountPath: /etc/alloy
            - name: varlog
              mountPath: /var/log
              readOnly: true
            - name: pods
              mountPath: /var/log/pods
              readOnly: true
            - name: containers
              mountPath: /var/lib/docker/containers
              readOnly: true
            - name: kubelet
              mountPath: /var/lib/kubelet
              readOnly: true
            - name: containers-log
              mountPath: /var/log/containers
              readOnly: true

      volumes:
        - name: config
          configMap:
            name: alloy-config
        - name: varlog
          hostPath:
            path: /var/log
            type: Directory
        - name: pods
          hostPath:
            path: /var/log/pods
            type: DirectoryOrCreate
        - name: containers
          hostPath:
            path: /var/lib/docker/containers
            type: DirectoryOrCreate
        - name: kubelet
          hostPath:
            path: /var/lib/kubelet
            type: DirectoryOrCreate
        - name: containers-log
          hostPath:
            path: /var/log/containers
            type: Directory

Whereas the logs that are present in my system works

local.file_match "tmp" {
  path_targets = [{"__path__" = "/var/log/**/*.log"}]
}

loki.source.file "files" {
  targets    = local.file_match.tmp.targets
  forward_to = [loki.write.loki_write.receiver]
}

loki.write "loki_write" {
  endpoint {
    url = "http://address:port/loki/api/v1/push"
  }
}

I have checked the grafana-alloy logs but I couldn’t see any errors there.

2

I usually mount the docker or podman logs into Alloy container and treat them as file targets. Try that and see if it might be easier for you.

3

I tried mounting it and in the grafana UI I only saw filename as label but generally I need to see different labels like pods, containers, etc.,

4

There is an example here on how to get pod logs using loki.source.kubernetes: Collect Kubernetes logs and forward them to Loki | Grafana Alloy documentation

There is also a well-maintained helm chart for K8s monitoring (probably overkill if you just need logs), but it’s still a good source of examples. See k8s-monitoring-helm/charts/k8s-monitoring-v1/templates/alloy_config/_pod_logs_discovery.alloy.txt at 6ebce4adf930d2f39f0e37e575f69877ada20227 · grafana/k8s-monitoring-helm · GitHub for how it’s configured, and this one uses loki.source.file with mounted container logs.

1 Like