Information is missing when collecting logs

Hello,

I’m experimenting with loki and I think it is great. However, I have some questions.

It turns out that promtail does not retrieve all the information including some containers and namespace. I’m in a kubernetes environment and have deployed loki/promtail via their helm charts.

Here is my promtail config file :

# -- Overrides the chart's name
nameOverride: promtail

# -- Overrides the chart's computed fullname
fullnameOverride: null

initContainer:
  # -- Specifies whether the init container for setting inotify max user instances is to be enabled
  enabled: false
  image:
    # -- The Docker registry for the init container
    registry: docker.io
    # -- Docker image repository for the init container
    repository: busybox
    # -- Docker tag for the init container
    tag: 1.33
    # -- Docker image pull policy for the init container image
    pullPolicy: IfNotPresent
  # -- The inotify max user instances to configure
  fsInotifyMaxUserInstances: 128

image:
  # -- The Docker registry
  registry: docker.io
  # -- Docker image repository
  repository: grafana/promtail
  # -- Overrides the image tag whose default is the chart's appVersion
  tag: null
  # -- Docker image pull policy
  pullPolicy: IfNotPresent

# -- Image pull secrets for Docker images
imagePullSecrets: []

# -- Annotations for the SaemonSet
annotations: {}

# -- The update strategy for the DaemonSet
updateStrategy: {}

# -- Pod labels
podLabels: {}

# -- Pod annotations
podAnnotations:
  prometheus.io/scrape: "true"
  prometheus.io/port: "http-metrics"

# -- The name of the PriorityClass
priorityClassName: null

# -- Liveness probe
livenessProbe: {}

# -- Readiness probe
# @default -- See `values.yaml`
readinessProbe:
  failureThreshold: 5
  httpGet:
    path: /ready
    port: http-metrics
  initialDelaySeconds: 10
  periodSeconds: 10
  successThreshold: 1
  timeoutSeconds: 1

# -- Resource requests and limits
resources:
  limits:
    cpu: 1
    memory: 2048Mi
  requests:
    cpu: 100m
    memory: 128Mi

# -- The security context for pods
podSecurityContext:
  runAsUser: 0
  runAsGroup: 0

# -- The security context for containers
containerSecurityContext:
  readOnlyRootFilesystem: true
  capabilities:
    drop:
      - ALL
  allowPrivilegeEscalation: false

rbac:
  # -- Specifies whether RBAC resources are to be created
  create: true
  # -- Specifies whether a PodSecurityPolicy is to be created
  pspEnabled: true

serviceAccount:
  # -- Specifies whether a ServiceAccount should be created
  create: true
  # -- The name of the ServiceAccount to use.
  # If not set and `create` is true, a name is generated using the fullname template
  name: null
  # -- Image pull secrets for the service account
  imagePullSecrets: []
  # -- Annotations for the service account
  annotations: {}

# -- Node selector for pods
nodeSelector: {}

# -- Affinity configuration for pods
affinity: {}

# -- Tolerations for pods. By default, pods will be scheduled on master/control-plane nodes.
tolerations:
  - key: node-role.kubernetes.io/master
    operator: Exists
    effect: NoSchedule
  - key: node-role.kubernetes.io/control-plane
    operator: Exists
    effect: NoSchedule

# -- Default volumes that are mounted into pods. In most cases, these should not be changed.
# Use `extraVolumes`/`extraVolumeMounts` for additional custom volumes.
# @default -- See `values.yaml`
defaultVolumes:
  - name: containers
    hostPath:
      path: /var/lib/docker/containers
  - name: pods
    hostPath:
      path: /var/log/pods

# -- Default volume mounts. Corresponds to `volumes`.
# @default -- See `values.yaml`
defaultVolumeMounts:
  - name: containers
    mountPath: /var/lib/docker/containers
    readOnly: true
  - name: pods
    mountPath: /var/log/pods
    readOnly: true

# Extra volumes to be added in addition to those specified under `defaultVolumes`.
extraVolumes:
  - name: journal
    hostPath:
      path: /var/log/journal

# Extra volume mounts together. Corresponds to `extraVolumes`.
extraVolumeMounts:
  - name: journal
    mountPath: /var/log/journal
    readOnly: true

# Extra args for the Promtail container.
extraArgs: []
# -- Example:
# -- extraArgs:
# --   - -client.external-labels=hostname=$(HOSTNAME)

# -- Extra environment variables
extraEnv: []

# -- Extra environment variables from secrets or configmaps
extraEnvFrom: []

# ServiceMonitor configuration
serviceMonitor:
  # -- If enabled, ServiceMonitor resources for Prometheus Operator are created
  enabled: false
  # -- Alternative namespace for ServiceMonitor resources
  namespace: null
  # -- Namespace selector for ServiceMonitor resources
  namespaceSelector: {}
  # -- ServiceMonitor annotations
  annotations: {}
  # -- Additional ServiceMonitor labels
  labels: {}
  # -- ServiceMonitor scrape interval
  interval: null
  # -- ServiceMonitor scrape timeout in Go duration format (e.g. 15s)
  scrapeTimeout: null

# -- Configure additional ports and services. For each configured port, a corresponding service is created.
# See values.yaml for details
extraPorts: {}
#  syslog:
#    name: tcp-syslog
#    containerPort: 1514
#    protocol: TCP
#    service:
#      type: ClusterIP
#      clusterIP: null
#      port: 1514
#      externalIPs: []
#      nodePort: null
#      annotations: {}
#      labels: {}
#      loadBalancerIP: null
#      loadBalancerSourceRanges: []
#      externalTrafficPolicy: null

# -- PodSecurityPolicy configuration.
# @default -- See `values.yaml`
podSecurityPolicy:
  privileged: true
  allowPrivilegeEscalation: true
  volumes:
    - 'secret'
    - 'hostPath'
    - 'downwardAPI'
  hostNetwork: false
  hostIPC: false
  hostPID: false
  runAsUser:
    rule: 'RunAsAny'
  seLinux:
    rule: 'RunAsAny'
  supplementalGroups:
    rule: 'RunAsAny'
  fsGroup:
    rule: 'RunAsAny'
  readOnlyRootFilesystem: true
  requiredDropCapabilities:
    - ALL

# -- Section for crafting Promtails config file. The only directly relevant value is `config.file`
# which is a templated string that references the other values and snippets below this key.
# @default -- See `values.yaml`
config:
  # -- The log level of the Promtail server
  # Must be reference in `config.file` to configure `server.log_level`
  # See default config in `values.yaml`
  logLevel: info
  # -- The port of the Promtail server
  # Must be reference in `config.file` to configure `server.http_listen_port`
  # See default config in `values.yaml`
  serverPort: 3101
  # -- The Loki address to post logs to.
  # Must be reference in `config.file` to configure `client.url`.
  # See default config in `values.yaml`
  lokiAddress: https://gateway-loki.monitoring.x/loki/api/v1/push
  # -- A section of reusable snippets that can be reference in `config.file`.
  # Custom snippets may be added in order to reduce redundancy.
  # This is especially helpful when multiple `kubernetes_sd_configs` are use which usually have large parts in common.
  # @default -- See `values.yaml`
  snippets:
    pipelineStages:
      - docker: {}
    common:
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_node_name
        target_label: node_name
      - action: replace
        source_labels:
          - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        replacement: $1
        separator: /
        source_labels:
          - namespace
          - app
        target_label: job
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_container_name
        target_label: container
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
          - __meta_kubernetes_pod_uid
          - __meta_kubernetes_pod_container_name
        target_label: __path__
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        regex: true/(.*)
        separator: /
        source_labels:
          - __meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash
          - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
          - __meta_kubernetes_pod_container_name
        target_label: __path__

    # If set to true, adds an additional label for the scrape job.
    # This helps debug the Promtail config.
    addScrapeJobLabel: false

    # -- You can put here any keys that will be directly added to the config file's 'client' block.
    # @default -- empty
    extraClientConfigs: |
      basic_auth:
        username: x
        password: x

    # -- You can put here any additional scrape configs you want to add to the config file.
    # @default -- empty
    extraScrapeConfigs: |
      # Add an additional scrape config for syslog
      - job_name: journal
        journal:
          path: /var/log/journal
          max_age: 12h
          labels:
            job: systemd-journal
        relabel_configs:
          - source_labels:
              - '__journal__systemd_unit'
            target_label: 'unit'
          - source_labels:
              - '__journal__hostname'
            target_label: 'hostname'

    # -- You can put here any additional relabel_configs to "kubernetes-pods" job
    extraRelabelConfigs: []

    scrapeConfigs: |
      # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
      - job_name: kubernetes-pods
        pipeline_stages:
          {{- toYaml .Values.config.snippets.pipelineStages | nindent 4 }}
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels:
              - __meta_kubernetes_pod_controller_name
            regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})?
            action: replace
            target_label: __tmp_controller_name
          - source_labels:
              - __meta_kubernetes_pod_label_app_kubernetes_io_name
              - __meta_kubernetes_pod_label_app
              - __tmp_controller_name
              - __meta_kubernetes_pod_name
            regex: ^;*([^;]+)(;.*)?$
            action: replace
            target_label: app
          - source_labels:
              - __meta_kubernetes_pod_label_app_kubernetes_io_component
              - __meta_kubernetes_pod_label_component
            regex: ^;*([^;]+)(;.*)?$
            action: replace
            target_label: component
          {{- if .Values.config.snippets.addScrapeJobLabel }}
          - replacement: kubernetes-pods
            target_label: scrape_job
          {{- end }}
          {{- toYaml .Values.config.snippets.common | nindent 4 }}
          {{- with .Values.config.snippets.extraRelabelConfigs }}
          {{- toYaml . | nindent 4 }}
          {{- end }}
  # -- Config file contents for Promtail.
  # Must be configured as string.
  # It is templated so it can be assembled from reusable snippets in order to avoid redundancy.
  # @default -- See `values.yaml`
  file: |
    server:
      log_level: {{ .Values.config.logLevel }}
      http_listen_port: {{ .Values.config.serverPort }}
    client:
      url: {{ tpl .Values.config.lokiAddress . }}
      {{- tpl .Values.config.snippets.extraClientConfigs . | nindent 2 }}
    positions:
      filename: /run/promtail/positions.yaml
    scrape_configs:
      {{- tpl .Values.config.snippets.scrapeConfigs . | nindent 2 }}
      {{- tpl .Values.config.snippets.extraScrapeConfigs . | nindent 2 }}
networkPolicy:
  # -- Specifies whether Network Policies should be created
  enabled: false
  metrics:
    # -- Specifies the Pods which are allowed to access the metrics port.
    # As this is cross-namespace communication, you also neeed the namespaceSelector.
    podSelector: {}
    # -- Specifies the namespaces which are allowed to access the metrics port
    namespaceSelector: {}
    # -- Specifies specific network CIDRs which are allowed to access the metrics port.
    # In case you use namespaceSelector, you also have to specify your kubelet networks here.
    # The metrics ports are also used for probes.
    cidrs: []
  k8sApi:
    # -- Specify the k8s API endpoint port
    port: 8443
    # -- Specifies specific network CIDRs you want to limit access to
    cidrs: []

When I go to the promtail web interface, I can see that I’m missing some things.

Have I set up promtail incorrectly? Should I play with the scrape regexes?

Hello,

Is there a more specific configuration to do on promtail? Are there any best practices to follow for my deployments concerning how them sends logs?

Typically, I can’t get the logs from some containers that have the following log format :

{"log":"{\"level\":30,\"time\":1633482819476,\"pid\":45,\"hostname\":\"x-x\",\"req\":{\"id\":x,\"method\":\"GET\",\"url\":\"/\",\"query\":{},\"params\":{},\"headers\":{\"host\":\"x.app\",\"x-request-id\":\"x\",\"x-real-ip\":\"x\",\"x-forwarded-for\":\"x\",\"x-forwarded-host\":\"x.app\",\"x-forwarded-port\":\"443\",\"x-forwarded-proto\":\"https\",\"x-original-uri\":\"/\",\"x-scheme\":\"https\",\"x-original-forwarded-for\":\"x\",\"cdn-loop\":\"cloudflare\",\"cf-connecting-ip\":\"x\",\"accept-encoding\":\"gzip\",\"cf-ipcountry\":\"FR\",\"cf-ray\":\"x\",\"cf-visitor\":\"{\\\"scheme\\\":\\\"https\\\"}\",\"accept\":\"*/*\",\"user-agent\":\"Uptime-Kuma/1.7.3\"},\"remoteAddress\":\"::ffff:x\",\"remotePort\":x},\"res\":{\"statusCode\":200,\"headers\":{\"content-security-policy\":\"default-src 'self';base-uri 'self';block-all-mixed-content;font-src 'self' https: data:;frame-ancestors 'self';img-src 'self' data:;object-src 'none';script-src 'self';script-src-attr 'none';style-src 'self' https: 'unsafe-inline';upgrade-insecure-requests\",\"x-dns-prefetch-control\":\"off\",\"expect-ct\":\"max-age=0\",\"x-frame-options\":\"SAMEORIGIN\",\"strict-transport-security\":\"max-age=15552000; includeSubDomains\",\"x-download-options\":\"noopen\",\"x-content-type-options\":\"nosniff\",\"x-permitted-cross-domain-policies\":\"none\",\"referrer-policy\":\"no-referrer\",\"x-xss-protection\":\"0\",\"access-control-allow-origin\":\"*\",\"content-type\":\"application/json; charset=utf-8\",\"content-length\":\"82\",\"etag\":\"W/\\\"x\\\"\"}},\"responseTime\":1,\"msg\":\"request completed\"}\r\n","stream":"stdout","time":"2021-10-06T01:13:39.476875912Z"}

Someone can help me ?

In gui of promtail, i found this :

Dropped: ignoring target, already exists
Dropped: dropping target, no labels

However, I can’t understand why I have this because each of my applications have labels and annotations

I update my promtail config file :

# -- Overrides the chart's name
nameOverride: promtail

# -- Overrides the chart's computed fullname
fullnameOverride: null

initContainer:
  # -- Specifies whether the init container for setting inotify max user instances is to be enabled
  enabled: false
  image:
    # -- The Docker registry for the init container
    registry: docker.io
    # -- Docker image repository for the init container
    repository: busybox
    # -- Docker tag for the init container
    tag: 1.33
    # -- Docker image pull policy for the init container image
    pullPolicy: IfNotPresent
  # -- The inotify max user instances to configure
  fsInotifyMaxUserInstances: 128

image:
  # -- The Docker registry
  registry: docker.io
  # -- Docker image repository
  repository: grafana/promtail
  # -- Overrides the image tag whose default is the chart's appVersion
  tag: null
  # -- Docker image pull policy
  pullPolicy: Always

# -- Image pull secrets for Docker images
imagePullSecrets: []

# -- Annotations for the SaemonSet
annotations: {}

# -- The update strategy for the DaemonSet
updateStrategy: {}

# -- Pod labels
podLabels: {}

# -- Pod annotations
podAnnotations:
  prometheus.io/scrape: "true"
  prometheus.io/port: "http-metrics"

# -- The name of the PriorityClass
priorityClassName: null

# -- Liveness probe
livenessProbe: {}

# -- Readiness probe
# @default -- See `values.yaml`
readinessProbe:
  failureThreshold: 5
  httpGet:
    path: /ready
    port: http-metrics
  initialDelaySeconds: 10
  periodSeconds: 10
  successThreshold: 1
  timeoutSeconds: 1

# -- Resource requests and limits
resources:
  limits:
    cpu: 1
    memory: 2048Mi
  requests:
    cpu: 100m
    memory: 128Mi

# -- The security context for pods
podSecurityContext:
  runAsUser: 0
  runAsGroup: 0

# -- The security context for containers
containerSecurityContext:
  readOnlyRootFilesystem: true
  capabilities:
    drop:
      - ALL
  allowPrivilegeEscalation: false

rbac:
  # -- Specifies whether RBAC resources are to be created
  create: true
  # -- Specifies whether a PodSecurityPolicy is to be created
  pspEnabled: true

serviceAccount:
  # -- Specifies whether a ServiceAccount should be created
  create: true
  # -- The name of the ServiceAccount to use.
  # If not set and `create` is true, a name is generated using the fullname template
  name: null
  # -- Image pull secrets for the service account
  imagePullSecrets: []
  # -- Annotations for the service account
  annotations: {}

# -- Node selector for pods
nodeSelector: {}

# -- Affinity configuration for pods
affinity: {}

# -- Tolerations for pods. By default, pods will be scheduled on master/control-plane nodes.
tolerations:
  - key: node-role.kubernetes.io/master
    operator: Exists
    effect: NoSchedule
  - key: node-role.kubernetes.io/control-plane
    operator: Exists
    effect: NoSchedule

# -- Default volumes that are mounted into pods. In most cases, these should not be changed.
# Use `extraVolumes`/`extraVolumeMounts` for additional custom volumes.
# @default -- See `values.yaml`
defaultVolumes:
  - name: containers
    hostPath:
      path: /var/lib/docker/containers
  - name: pods
    hostPath:
      path: /var/log/pods

# -- Default volume mounts. Corresponds to `volumes`.
# @default -- See `values.yaml`
defaultVolumeMounts:
  - name: containers
    mountPath: /var/lib/docker/containers
    readOnly: true
  - name: pods
    mountPath: /var/log/pods
    readOnly: true

# Extra volumes to be added in addition to those specified under `defaultVolumes`.
extraVolumes:
  - name: journal
    hostPath:
      path: /var/log/journal

# Extra volume mounts together. Corresponds to `extraVolumes`.
extraVolumeMounts:
  - name: journal
    mountPath: /var/log/journal
    readOnly: true

# Extra args for the Promtail container.
extraArgs: []
# -- Example:
# -- extraArgs:
# --   - -client.external-labels=hostname=$(HOSTNAME)

# -- Extra environment variables
extraEnv: []

# -- Extra environment variables from secrets or configmaps
extraEnvFrom: []

# ServiceMonitor configuration
serviceMonitor:
  # -- If enabled, ServiceMonitor resources for Prometheus Operator are created
  enabled: false
  # -- Alternative namespace for ServiceMonitor resources
  namespace: null
  # -- Namespace selector for ServiceMonitor resources
  namespaceSelector: {}
  # -- ServiceMonitor annotations
  annotations: {}
  # -- Additional ServiceMonitor labels
  labels: {}
  # -- ServiceMonitor scrape interval
  interval: null
  # -- ServiceMonitor scrape timeout in Go duration format (e.g. 15s)
  scrapeTimeout: null

# -- Configure additional ports and services. For each configured port, a corresponding service is created.
# See values.yaml for details
extraPorts: {}
#  syslog:
#    name: tcp-syslog
#    containerPort: 1514
#    protocol: TCP
#    service:
#      type: ClusterIP
#      clusterIP: null
#      port: 1514
#      externalIPs: []
#      nodePort: null
#      annotations: {}
#      labels: {}
#      loadBalancerIP: null
#      loadBalancerSourceRanges: []
#      externalTrafficPolicy: null

# -- PodSecurityPolicy configuration.
# @default -- See `values.yaml`
podSecurityPolicy:
  privileged: true
  allowPrivilegeEscalation: true
  volumes:
    - 'secret'
    - 'hostPath'
    - 'downwardAPI'
  hostNetwork: false
  hostIPC: false
  hostPID: false
  runAsUser:
    rule: 'RunAsAny'
  seLinux:
    rule: 'RunAsAny'
  supplementalGroups:
    rule: 'RunAsAny'
  fsGroup:
    rule: 'RunAsAny'
  readOnlyRootFilesystem: true
  requiredDropCapabilities:
    - ALL

# -- Section for crafting Promtails config file. The only directly relevant value is `config.file`
# which is a templated string that references the other values and snippets below this key.
# @default -- See `values.yaml`
config:
  # -- The log level of the Promtail server
  # Must be reference in `config.file` to configure `server.log_level`
  # See default config in `values.yaml`
  logLevel: info
  # -- The port of the Promtail server
  # Must be reference in `config.file` to configure `server.http_listen_port`
  # See default config in `values.yaml`
  serverPort: 3101
  # -- The Loki address to post logs to.
  # Must be reference in `config.file` to configure `client.url`.
  # See default config in `values.yaml`
  lokiAddress: https://gateway-loki.monitoring.x.net/loki/api/v1/push
  # -- A section of reusable snippets that can be reference in `config.file`.
  # Custom snippets may be added in order to reduce redundancy.
  # This is especially helpful when multiple `kubernetes_sd_configs` are use which usually have large parts in common.
  # @default -- See `values.yaml`
  snippets:
    pipelineStages:
      - docker: {}
    common:
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_node_name
        target_label: node_name
      - action: replace
        source_labels:
          - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        replacement: $1
        separator: /
        source_labels:
          - namespace
          - app
        target_label: job
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
          - __meta_kubernetes_pod_container_name
        target_label: container
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        separator: /
        source_labels:
          - __meta_kubernetes_pod_uid
          - __meta_kubernetes_pod_container_name
        target_label: __path__
      - action: replace
        replacement: /var/log/pods/*$1/*.log
        regex: true/(.*)
        separator: /
        source_labels:
          - __meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash
          - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
          - __meta_kubernetes_pod_container_name
        target_label: __path__

    # If set to true, adds an additional label for the scrape job.
    # This helps debug the Promtail config.
    addScrapeJobLabel: false

    # -- You can put here any keys that will be directly added to the config file's 'client' block.
    # @default -- empty
    extraClientConfigs: |
      basic_auth:
        username: x
        password: x
      backoff_config:
        max_period: 5m
        max_retries: 10
        min_period: 500ms
      batchsize: 1048576
      batchwait: 1s
      external_labels: {}
      timeout: 10s

    # -- You can put here any additional scrape configs you want to add to the config file.
    # @default -- empty
    extraScrapeConfigs: |
      # Add an additional scrape config for syslog
      - job_name: journal
        journal:
          path: /var/log/journal
          max_age: 12h
          labels:
            job: systemd-journal
        relabel_configs:
          - source_labels:
              - '__journal__systemd_unit'
            target_label: 'unit'
          - source_labels:
              - '__journal__hostname'
            target_label: 'hostname'
#      - job_name: system
#        static_configs:
#        - targets:
#            - localhost
#          labels:
#            job: varlogs
#            __path__: /var/log/*.log

    # -- You can put here any additional relabel_configs to "kubernetes-pods" job
    extraRelabelConfigs: []

    scrapeConfigs: |
      # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
      - job_name: kubernetes-pods-name
        pipeline_stages:
          - docker: {}
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        - source_labels:
          - __meta_kubernetes_pod_label_name
          target_label: __service__
        - source_labels:
          - __meta_kubernetes_pod_node_name
          target_label: __host__
        #- action: drop
        #  regex: ""
        #  source_labels:
        #  - __service__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - action: replace
          replacement: $1
          separator: /
          source_labels:
          - __meta_kubernetes_namespace
          - __service__
          target_label: job
        - action: replace
          source_labels:
          - __meta_kubernetes_namespace
          target_label: namespace
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_name
          target_label: pod
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_container_name
          target_label: container
        - replacement: /var/log/pods/*$1/*.log
          separator: /
          source_labels:
          - __meta_kubernetes_pod_uid
          - __meta_kubernetes_pod_container_name
          target_label: __path__
      - job_name: kubernetes-pods-app
        pipeline_stages:
          - docker: {}
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        - action: drop
          regex: .+
          source_labels:
          - __meta_kubernetes_pod_label_name
        - source_labels:
          - __meta_kubernetes_pod_label_app
          target_label: __service__
        - source_labels:
          - __meta_kubernetes_pod_node_name
          target_label: __host__
        #- action: drop
        #  regex: ""
        #  source_labels:
        #  - __service__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - action: replace
          replacement: $1
          separator: /
          source_labels:
          - __meta_kubernetes_namespace
          - __service__
          target_label: job
        - action: replace
          source_labels:
          - __meta_kubernetes_namespace
          target_label: namespace
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_name
          target_label: pod
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_container_name
          target_label: container
        - replacement: /var/log/pods/*$1/*.log
          separator: /
          source_labels:
          - __meta_kubernetes_pod_uid
          - __meta_kubernetes_pod_container_name
          target_label: __path__
      - job_name: kubernetes-pods-direct-controllers
        pipeline_stages:
          - docker: {}
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        - action: drop
          regex: .+
          separator: ''
          source_labels:
          - __meta_kubernetes_pod_label_name
          - __meta_kubernetes_pod_label_app
        - action: drop
          regex: '[0-9a-z-.]+-[0-9a-f]{8,10}'
          source_labels:
          - __meta_kubernetes_pod_controller_name
        - source_labels:
          - __meta_kubernetes_pod_controller_name
          target_label: __service__
        - source_labels:
          - __meta_kubernetes_pod_node_name
          target_label: __host__
        #- action: drop
        #  regex: ""
        #  source_labels:
        #  - __service__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - action: replace
          replacement: $1
          separator: /
          source_labels:
          - __meta_kubernetes_namespace
          - __service__
          target_label: job
        - action: replace
          source_labels:
          - __meta_kubernetes_namespace
          target_label: namespace
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_name
          target_label: pod
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_container_name
          target_label: container
        - replacement: /var/log/pods/*$1/*.log
          separator: /
          source_labels:
          - __meta_kubernetes_pod_uid
          - __meta_kubernetes_pod_container_name
          target_label: __path__
      - job_name: kubernetes-pods-indirect-controller
        pipeline_stages:
          - docker: {}
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        - action: drop
          regex: .+
          separator: ''
          source_labels:
          - __meta_kubernetes_pod_label_name
          - __meta_kubernetes_pod_label_app
        - action: keep
          regex: '[0-9a-z-.]+-[0-9a-f]{8,10}'
          source_labels:
          - __meta_kubernetes_pod_controller_name
        - action: replace
          regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}'
          source_labels:
          - __meta_kubernetes_pod_controller_name
          target_label: __service__
        - source_labels:
          - __meta_kubernetes_pod_node_name
          target_label: __host__
        #- action: drop
        #  regex: ""
        #  source_labels:
        #  - __service__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - action: replace
          replacement: $1
          separator: /
          source_labels:
          - __meta_kubernetes_namespace
          - __service__
          target_label: job
        - action: replace
          source_labels:
          - __meta_kubernetes_namespace
          target_label: namespace
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_name
          target_label: pod
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_container_name
          target_label: container
        - replacement: /var/log/pods/*$1/*.log
          separator: /
          source_labels:
          - __meta_kubernetes_pod_uid
          - __meta_kubernetes_pod_container_name
          target_label: __path__
      - job_name: kubernetes-pods-static
        pipeline_stages:
          - docker: {}
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        - action: drop
          regex: ''
          source_labels:
          - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_label_component
          target_label: __service__
        - source_labels:
          - __meta_kubernetes_pod_node_name
          target_label: __host__
        #- action: drop
        #  regex: ""
        #  source_labels:
        #  - __service__
        - action: labelmap
          regex: __meta_kubernetes_pod_label_(.+)
        - action: replace
          replacement: $1
          separator: /
          source_labels:
          - __meta_kubernetes_namespace
          - __service__
          target_label: job
        - action: replace
          source_labels:
          - __meta_kubernetes_namespace
          target_label: namespace
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_name
          target_label: pod
        - action: replace
          source_labels:
          - __meta_kubernetes_pod_container_name
          target_label: container
        - replacement: /var/log/pods/*$1/*.log
          separator: /
          source_labels:
          - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
          - __meta_kubernetes_pod_container_name
          target_label: __path__

  # -- Config file contents for Promtail.
  # Must be configured as string.
  # It is templated so it can be assembled from reusable snippets in order to avoid redundancy.
  # @default -- See `values.yaml`
  file: |
    server:
      log_level: {{ .Values.config.logLevel }}
      http_listen_port: {{ .Values.config.serverPort }}
    client:
      url: {{ tpl .Values.config.lokiAddress . }}
      {{- tpl .Values.config.snippets.extraClientConfigs . | nindent 2 }}
    positions:
      filename: /run/promtail/positions.yaml
    scrape_configs:
      {{- tpl .Values.config.snippets.scrapeConfigs . | nindent 2 }}
      {{- tpl .Values.config.snippets.extraScrapeConfigs . | nindent 2 }}
    target_config:
      sync_period: 10s
networkPolicy:
  # -- Specifies whether Network Policies should be created
  enabled: false
  metrics:
    # -- Specifies the Pods which are allowed to access the metrics port.
    # As this is cross-namespace communication, you also neeed the namespaceSelector.
    podSelector: {}
    # -- Specifies the namespaces which are allowed to access the metrics port
    namespaceSelector: {}
    # -- Specifies specific network CIDRs which are allowed to access the metrics port.
    # In case you use namespaceSelector, you also have to specify your kubelet networks here.
    # The metrics ports are also used for probes.
    cidrs: []
  k8sApi:
    # -- Specify the k8s API endpoint port
    port: 8443
    # -- Specifies specific network CIDRs you want to limit access to
    cidrs: []