Loki Distributed with S3 backend Timestamp and Grafana connection issues

Setup Loki Distributed using the official helm charts.

https://github.com/grafana/helm-charts/tree/main/charts/loki-distributed

Values.yaml file

nameOverride: null

# -- Overrides the chart's computed full name
fullnameOverride: null

# -- Image pull secrets for Docker images
imagePullSecrets: []

loki:
  # Configures the readiness probe for all of the Loki pods
  readinessProbe:
    httpGet:
      path: /ready
      port: http
    initialDelaySeconds: 30
    timeoutSeconds: 1
  image:
    # -- The Docker registry
    registry: docker.io
    # -- Docker image repository
    repository: grafana/loki
    # -- Overrides the image tag whose default is the chart's appVersion
    tag: null
    # -- Docker image pull policy
    pullPolicy: IfNotPresent
  # -- Common annotations for all pods
  podAnnotations: {}
  # -- The number of old ReplicaSets to retain to allow rollback
  revisionHistoryLimit: 10
  # -- The SecurityContext for Loki pods
  podSecurityContext:
    fsGroup: 10001
    runAsGroup: 10001
    runAsNonRoot: true
    runAsUser: 10001
  # -- The SecurityContext for Loki containers
  containerSecurityContext:
    readOnlyRootFilesystem: true
    capabilities:
      drop:
        - ALL
    allowPrivilegeEscalation: false
  # -- Specify an existing secret containing loki configuration. If non-empty, overrides `loki.config`
  existingSecretForConfig: ""
  # -- Config file contents for Loki
  # @default -- See values.yaml
  config: |
    auth_enabled: false

    server:
      log_level: info
      http_listen_port: 3100

    distributor:
      ring:
        kvstore:
          store: memberlist

    ingester:
      max_transfer_retries: 0
      chunk_idle_period: 1h
      chunk_target_size: 1536000
      max_chunk_age: 1h
      lifecycler:
        join_after: 0s
        ring:
          kvstore:
            store: memberlist
    memberlist:
      join_members:
        - {{ include "loki.fullname" . }}-memberlist

    limits_config:
      ingestion_rate_mb: 10
      ingestion_burst_size_mb: 20
      max_concurrent_tail_requests: 20
      max_cache_freshness_per_query: 10m

    schema_config:
      configs:
        - from: 2020-09-07
          store: boltdb-shipper
          object_store: aws
          schema: v11
          index:
            prefix: loki_index_
            period: 24h

    storage_config:
      aws:
        s3: s3://eu-west-1
        bucketnames: loki-dev-v1
      boltdb_shipper:
        active_index_directory: /var/loki/index
        shared_store: s3
        cache_location: /var/loki/cache
        index_gateway_client:
          server_address: dns:///{{ include "loki.indexGatewayFullname" . }}:9095

    query_range:
      align_queries_with_step: true
      max_retries: 5
      split_queries_by_interval: 15m
      cache_results: true
      results_cache:
        cache:
          enable_fifocache: true
          fifocache:
            max_size_items: 1024
            validity: 24h

    frontend_worker:
      frontend_address: {{ include "loki.queryFrontendFullname" . }}:9095

    frontend:
      log_queries_longer_than: 5s
      compress_responses: true
      tail_proxy_url: http://{{ include "loki.querierFullname" . }}:3100

serviceAccount:
  # -- Specifies whether a ServiceAccount should be created
  create: true
  # -- The name of the ServiceAccount to use.
  # If not set and create is true, a name is generated using the fullname template
  name: null
  # -- Image pull secrets for the service account
  imagePullSecrets: []
  # -- Annotations for the service account
  annotations: {}
  # -- Set this toggle to false to opt out of automounting API credentials for the service account
  automountServiceAccountToken: true

# RBAC configuration
rbac:
  # -- If enabled, a PodSecurityPolicy is created
  pspEnabled: false

# ServiceMonitor configuration
serviceMonitor:
  # -- If enabled, ServiceMonitor resources for Prometheus Operator are created
  enabled: false
  # -- Alternative namespace for ServiceMonitor resources
  namespace: null
  # -- Namespace selector for ServiceMonitor resources
  namespaceSelector: {}
  # -- ServiceMonitor annotations
  annotations: {}
  # -- Additional ServiceMonitor labels
  labels: {}
  # -- ServiceMonitor scrape interval
  interval: null
  # -- ServiceMonitor scrape timeout in Go duration format (e.g. 15s)
  scrapeTimeout: null
  # -- ServiceMonitor relabel configs to apply to samples before scraping
  # https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig
  relabelings: []
  # -- ServiceMonitor will use http by default, but you can pick https as well
  scheme: http
  # -- ServiceMonitor will use these tlsConfig settings to make the health check requests
  tlsConfig: null

# Rules for the Prometheus Operator
prometheusRule:
  # -- If enabled, a PrometheusRule resource for Prometheus Operator is created
  enabled: false
  # -- Alternative namespace for the PrometheusRule resource
  namespace: null
  # -- PrometheusRule annotations
  annotations: {}
  # -- Additional PrometheusRule labels
  labels: {}
  # -- Contents of Prometheus rules file
  groups: []
  # - name: loki-rules
  #   rules:
  #     - record: job:loki_request_duration_seconds_bucket:sum_rate
  #       expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job)
  #     - record: job_route:loki_request_duration_seconds_bucket:sum_rate
  #       expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job, route)
  #     - record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate
  #       expr: sum(rate(container_cpu_usage_seconds_total[1m])) by (node, namespace, pod, container)

# Configuration for the ingester
ingester:
  # -- Number of replicas for the ingester
  replicas: 3
  image:
    # -- The Docker registry for the ingester image. Overrides `loki.image.registry`
    registry: null
    # -- Docker image repository for the ingester image. Overrides `loki.image.repository`
    repository: null
    # -- Docker image tag for the ingester image. Overrides `loki.image.tag`
    tag: null
  # -- The name of the PriorityClass for ingester pods
  priorityClassName: null
  # -- Annotations for ingester pods
  podAnnotations: {}
  # -- Labels for ingestor service
  serviceLabels: {}
  # -- Additional CLI args for the ingester
  extraArgs: []
  # -- Environment variables to add to the ingester pods
  extraEnv: []
  # -- Environment variables from secrets or configmaps to add to the ingester pods
  extraEnvFrom: []
  # -- Volume mounts to add to the ingester pods
  extraVolumeMounts: []
  # -- Volumes to add to the ingester pods
  extraVolumes: []
  # -- Resource requests and limits for the ingester
  resources: {}
  # -- Grace period to allow the ingester to shutdown before it is killed. Especially for the ingestor,
  # this must be increased. It must be long enough so ingesters can be gracefully shutdown flushing/transferring
  # all data and to successfully leave the member ring on shutdown.
  terminationGracePeriodSeconds: 300
  # -- Affinity for ingester pods. Passed through `tpl` and, thus, to be configured as string
  # @default -- Hard node and soft zone anti-affinity
  affinity: |
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchLabels:
              {{- include "loki.ingesterSelectorLabels" . | nindent 10 }}
          topologyKey: kubernetes.io/hostname
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchLabels:
                {{- include "loki.ingesterSelectorLabels" . | nindent 12 }}
            topologyKey: failure-domain.beta.kubernetes.io/zone
  # -- Node selector for ingester pods
  nodeSelector: {}
  # -- Tolerations for ingester pods
  tolerations: []
  persistence:
    # Vaib - This si used by the boltdb shipper to save the indexes before flushing it to s3
    # Vaib - https://github.com/grafana/loki/blob/main/docs/sources/operations/storage/boltdb-shipper.md
    # -- Enable creating PVCs which is required when using boltdb-shipper
    enabled: true
    # -- Size of persistent disk
    size: 50Gi
    # -- Storage class to be used.
    # If defined, storageClassName: <storageClass>.
    # If set to "-", storageClassName: "", which disables dynamic provisioning.
    # If empty or set to null, no storageClassName spec is
    # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
    storageClass: null

# Configuration for the distributor
distributor:
  # -- Number of replicas for the distributor
  replicas: 1
  autoscaling:
    # -- Enable autoscaling for the distributor
    enabled: true
    # -- Minimum autoscaling replicas for the distributor
    minReplicas: 3
    # -- Maximum autoscaling replicas for the distributor
    maxReplicas: 5
    # -- Target CPU utilisation percentage for the distributor
    targetCPUUtilizationPercentage: 60
    # -- Target memory utilisation percentage for the distributor
    targetMemoryUtilizationPercentage:
  image:
    # -- The Docker registry for the distributor image. Overrides `loki.image.registry`
    registry: null
    # -- Docker image repository for the distributor image. Overrides `loki.image.repository`
    repository: null
    # -- Docker image tag for the distributor image. Overrides `loki.image.tag`
    tag: null
  # -- The name of the PriorityClass for distributor pods
  priorityClassName: null
  # -- Annotations for distributor pods
  podAnnotations: {}
  # -- Labels for distributor service
  serviceLabels: {}
  # -- Additional CLI args for the distributor
  extraArgs: []
  # -- Environment variables to add to the distributor pods
  extraEnv: []
  # -- Environment variables from secrets or configmaps to add to the distributor pods
  extraEnvFrom: []
  # -- Volume mounts to add to the distributor pods
  extraVolumeMounts: []
  # -- Volumes to add to the distributor pods
  extraVolumes: []
  # -- Resource requests and limits for the distributor
  resources: {}
  # -- Grace period to allow the distributor to shutdown before it is killed
  terminationGracePeriodSeconds: 30
  # -- Affinity for distributor pods. Passed through `tpl` and, thus, to be configured as string
  # @default -- Hard node and soft zone anti-affinity
  affinity: |
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchLabels:
              {{- include "loki.distributorSelectorLabels" . | nindent 10 }}
          topologyKey: kubernetes.io/hostname
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchLabels:
                {{- include "loki.distributorSelectorLabels" . | nindent 12 }}
            topologyKey: failure-domain.beta.kubernetes.io/zone
  # -- Node selector for distributor pods
  nodeSelector: {}
  # -- Tolerations for distributor pods
  tolerations: []

# Configuration for the querier
# The querier service handles queries using the LogQL query language, fetching logs both from the ingesters and from long-term storage.

querier:
  # -- Number of replicas for the querier
  replicas: 1
  autoscaling:
    # -- Enable autoscaling for the querier, this is only used if `queryIndex.enabled: true`
    enabled: true
    # -- Minimum autoscaling replicas for the querier
    minReplicas: 3
    # -- Maximum autoscaling replicas for the querier
    maxReplicas: 20
    # -- Target CPU utilisation percentage for the querier
    targetCPUUtilizationPercentage: 60
    # -- Target memory utilisation percentage for the querier
    targetMemoryUtilizationPercentage:
  image:
    # -- The Docker registry for the querier image. Overrides `loki.image.registry`
    registry: null
    # -- Docker image repository for the querier image. Overrides `loki.image.repository`
    repository: null
    # -- Docker image tag for the querier image. Overrides `loki.image.tag`
    tag: null
  # -- The name of the PriorityClass for querier pods
  priorityClassName: null
  # -- Annotations for querier pods
  podAnnotations: {}
  # -- Labels for querier service
  serviceLabels: {}
  # -- Additional CLI args for the querier
  extraArgs: []
  # -- Environment variables to add to the querier pods
  extraEnv: []
  # -- Environment variables from secrets or configmaps to add to the querier pods
  extraEnvFrom: []
  # -- Volume mounts to add to the querier pods
  extraVolumeMounts: []
  # -- Volumes to add to the querier pods
  extraVolumes: []
  # -- Resource requests and limits for the querier
  resources: {}
  # -- Grace period to allow the querier to shutdown before it is killed
  terminationGracePeriodSeconds: 30
  # -- Affinity for querier pods. Passed through `tpl` and, thus, to be configured as string
  # @default -- Hard node and soft zone anti-affinity
  affinity: |
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchLabels:
              {{- include "loki.querierSelectorLabels" . | nindent 10 }}
          topologyKey: kubernetes.io/hostname
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchLabels:
                {{- include "loki.querierSelectorLabels" . | nindent 12 }}
            topologyKey: failure-domain.beta.kubernetes.io/zone
  # -- Node selector for querier pods
  nodeSelector: {}
  # -- Tolerations for querier pods
  tolerations: []
  persistence:
    # -- Enable creating PVCs for the querier cache
    enabled: true
    # -- Size of persistent disk
    size: 50Gi
    # -- Storage class to be used.
    # If defined, storageClassName: <storageClass>.
    # If set to "-", storageClassName: "", which disables dynamic provisioning.
    # If empty or set to null, no storageClassName spec is
    # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
    storageClass: null

# Configuration for the query-frontend
queryFrontend:
  # -- Number of replicas for the query-frontend
  replicas: 1
  autoscaling:
    # -- Enable autoscaling for the query-frontend
    enabled: true
    # -- Minimum autoscaling replicas for the query-frontend
    minReplicas: 2
    # -- Maximum autoscaling replicas for the query-frontend
    maxReplicas: 3
    # -- Target CPU utilisation percentage for the query-frontend
    targetCPUUtilizationPercentage: 60
    # -- Target memory utilisation percentage for the query-frontend
    targetMemoryUtilizationPercentage:
  image:
    # -- The Docker registry for the query-frontend image. Overrides `loki.image.registry`
    registry: null
    # -- Docker image repository for the query-frontend image. Overrides `loki.image.repository`
    repository: null
    # -- Docker image tag for the query-frontend image. Overrides `loki.image.tag`
    tag: null
  # -- The name of the PriorityClass for query-frontend pods
  priorityClassName: null
  # -- Annotations for query-frontend pods
  podAnnotations: {}
  # -- Labels for query-frontend service
  serviceLabels: {}
  # -- Additional CLI args for the query-frontend
  extraArgs: []
  # -- Environment variables to add to the query-frontend pods
  extraEnv: []
  # -- Environment variables from secrets or configmaps to add to the query-frontend pods
  extraEnvFrom: []
  # -- Volume mounts to add to the query-frontend pods
  extraVolumeMounts: []
  # -- Volumes to add to the query-frontend pods
  extraVolumes: []
  # -- Resource requests and limits for the query-frontend
  resources: {}
  # -- Grace period to allow the query-frontend to shutdown before it is killed
  terminationGracePeriodSeconds: 30
  # -- Affinity for query-frontend pods. Passed through `tpl` and, thus, to be configured as string
  # @default -- Hard node and soft zone anti-affinity
  affinity: |
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchLabels:
              {{- include "loki.queryFrontendSelectorLabels" . | nindent 10 }}
          topologyKey: kubernetes.io/hostname
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchLabels:
                {{- include "loki.queryFrontendSelectorLabels" . | nindent 12 }}
            topologyKey: failure-domain.beta.kubernetes.io/zone
  # -- Node selector for query-frontend pods
  nodeSelector: {}
  # -- Tolerations for query-frontend pods
  tolerations: []

# Configuration for the table-manager
tableManager:
  # -- Specifies whether the table-manager should be enabled
  enabled: false
  image:
    # -- The Docker registry for the table-manager image. Overrides `loki.image.registry`
    registry: null
    # -- Docker image repository for the table-manager image. Overrides `loki.image.repository`
    repository: null
    # -- Docker image tag for the table-manager image. Overrides `loki.image.tag`
    tag: null
  # -- The name of the PriorityClass for table-manager pods
  priorityClassName: null
  # -- Annotations for table-manager pods
  podAnnotations: {}
  # -- Labels for table-manager service
  serviceLabels: {}
  # -- Additional CLI args for the table-manager
  extraArgs: []
  # -- Environment variables to add to the table-manager pods
  extraEnv: []
  # -- Environment variables from secrets or configmaps to add to the table-manager pods
  extraEnvFrom: []
  # -- Volume mounts to add to the table-manager pods
  extraVolumeMounts: []
  # -- Volumes to add to the table-manager pods
  extraVolumes: []
  # -- Resource requests and limits for the table-manager
  resources: {}
  # -- Grace period to allow the table-manager to shutdown before it is killed
  terminationGracePeriodSeconds: 30
  # -- Affinity for table-manager pods. Passed through `tpl` and, thus, to be configured as string
  # @default -- Hard node and soft zone anti-affinity
  affinity: |
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchLabels:
              {{- include "loki.tableManagerSelectorLabels" . | nindent 10 }}
          topologyKey: kubernetes.io/hostname
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchLabels:
                {{- include "loki.tableManagerSelectorLabels" . | nindent 12 }}
            topologyKey: failure-domain.beta.kubernetes.io/zone
  # -- Node selector for table-manager pods
  nodeSelector: {}
  # -- Tolerations for table-manager pods
  tolerations: []

# Configuration for the gateway
gateway:
  # -- Specifies whether the gateway should be enabled
  enabled: true
  # -- Number of replicas for the gateway
  replicas: 1
  autoscaling:
    # -- Enable autoscaling for the gateway
    enabled: false
    # -- Minimum autoscaling replicas for the gateway
    minReplicas: 1
    # -- Maximum autoscaling replicas for the gateway
    maxReplicas: 3
    # -- Target CPU utilisation percentage for the gateway
    targetCPUUtilizationPercentage: 60
    # -- Target memory utilisation percentage for the gateway
    targetMemoryUtilizationPercentage:
  # -- See `kubectl explain deployment.spec.strategy` for more
  # -- ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
  deploymentStrategy:
    type: RollingUpdate
  image:
    # -- The Docker registry for the gateway image
    registry: docker.io
    # -- The gateway image repository
    repository: nginxinc/nginx-unprivileged
    # -- The gateway image tag
    tag: 1.19-alpine
    # -- The gateway image pull policy
    pullPolicy: IfNotPresent
  # -- The name of the PriorityClass for gateway pods
  priorityClassName: null
  # -- Annotations for gateway pods
  podAnnotations: {}
  # -- Additional CLI args for the gateway
  extraArgs: []
  # -- Environment variables to add to the gateway pods
  extraEnv: []
  # -- Environment variables from secrets or configmaps to add to the gateway pods
  extraEnvFrom: []
  # -- Volumes to add to the gateway pods
  extraVolumes: []
  # -- Volume mounts to add to the gateway pods
  extraVolumeMounts: []
  # -- The SecurityContext for gateway containers
  podSecurityContext:
    fsGroup: 101
    runAsGroup: 101
    runAsNonRoot: true
    runAsUser: 101
  # -- The SecurityContext for gateway containers
  containerSecurityContext:
    readOnlyRootFilesystem: true
    capabilities:
      drop:
        - ALL
    allowPrivilegeEscalation: false
  # -- Resource requests and limits for the gateway
  resources: {}
  # -- Grace period to allow the gateway to shutdown before it is killed
  terminationGracePeriodSeconds: 30
  # -- Affinity for gateway pods. Passed through `tpl` and, thus, to be configured as string
  # @default -- Hard node and soft zone anti-affinity
  affinity: |
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchLabels:
              {{- include "loki.gatewaySelectorLabels" . | nindent 10 }}
          topologyKey: kubernetes.io/hostname
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchLabels:
                {{- include "loki.gatewaySelectorLabels" . | nindent 12 }}
            topologyKey: failure-domain.beta.kubernetes.io/zone
  # -- Node selector for gateway pods
  nodeSelector: {}
  # -- Tolerations for gateway pods
  tolerations: []
  # Gateway service configuration
  service:
    # -- Port of the gateway service
    port: 80
    # -- Type of the gateway service
    type: ClusterIP
    # -- ClusterIP of the gateway service
    clusterIP: null
    # -- Node port if service type is NodePort
    nodePort: null
    # -- Load balancer IPO address if service type is LoadBalancer
    loadBalancerIP: null
    # -- Annotations for the gateway service
    annotations: {}
    # -- Labels for gateway service
    labels: {}
  # Gateway ingress configuration
  ingress:
    # -- Specifies whether an ingress for the gateway should be created
    enabled: false
    # -- Ingress Class Name. MAY be required for Kubernetes versions >= 1.18
    # ingressClassName: nginx
    # -- Annotations for the gateway ingress
    annotations: {}
    # -- Hosts configuration for the gateway ingress
    hosts:
      - host: gateway.loki.example.com
        paths:
          - path: /
            # -- pathType (e.g. ImplementationSpecific, Prefix, .. etc.) might also be required by some Ingress Controllers
            # pathType: Prefix
    # -- TLS configuration for the gateway ingress
    tls:
      - secretName: loki-gateway-tls
        hosts:
          - gateway.loki.example.com
  # Basic auth configuration
  basicAuth:
    # -- Enables basic authentication for the gateway
    enabled: false
    # -- The basic auth username for the gateway
    username: null
    # -- The basic auth password for the gateway
    password: null
    # -- Uses the specified username and password to compute a htpasswd using Sprig's `htpasswd` function.
    # The value is templated using `tpl`. Override this to use a custom htpasswd, e.g. in case the default causes
    # high CPU load.
    htpasswd: >-
      {{ htpasswd (required "'gateway.basicAuth.username' is required" .Values.gateway.basicAuth.username) (required "'gateway.basicAuth.password' is required" .Values.gateway.basicAuth.password) }}
    # -- Existing basic auth secret to use. Must contain '.htpasswd'
    existingSecret: null
  # Configures the readiness probe for the gateway
  readinessProbe:
    httpGet:
      path: /
      port: http
    initialDelaySeconds: 15
    timeoutSeconds: 1
  nginxConfig:
    # -- NGINX log format
    logFormat: |-
      main '$remote_addr - $remote_user [$time_local]  $status '
              '"$request" $body_bytes_sent "$http_referer" '
              '"$http_user_agent" "$http_x_forwarded_for"';
    # -- Allows appending custom configuration to the server block
    serverSnippet: ""
    # -- Allows appending custom configuration to the http block
    httpSnippet: ""
    # -- Config file contents for Nginx. Passed through the `tpl` function to allow templating
    # @default -- See values.yaml
    file: |
      worker_processes  5;  ## Default: 1
      error_log  /dev/stderr;
      pid        /tmp/nginx.pid;
      worker_rlimit_nofile 8192;

      events {
        worker_connections  4096;  ## Default: 1024
      }

      http {
        client_body_temp_path /tmp/client_temp;
        proxy_temp_path       /tmp/proxy_temp_path;
        fastcgi_temp_path     /tmp/fastcgi_temp;
        uwsgi_temp_path       /tmp/uwsgi_temp;
        scgi_temp_path        /tmp/scgi_temp;

        default_type application/octet-stream;
        log_format   {{ .Values.gateway.nginxConfig.logFormat }}
        access_log   /dev/stderr  main;
        sendfile     on;
        tcp_nopush   on;
        resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }};

        {{- with .Values.gateway.nginxConfig.httpSnippet }}
        {{ . | nindent 2 }}
        {{- end }}

        server {
          listen             8080;

          {{- if .Values.gateway.basicAuth.enabled }}
          auth_basic           "Loki";
          auth_basic_user_file /etc/nginx/secrets/.htpasswd;
          {{- end }}

          location = / {
            return 200 'OK';
            auth_basic off;
          }

          location = /api/prom/push {
            proxy_pass       http://{{ include "loki.distributorFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
          }

          location = /api/prom/tail {
            proxy_pass       http://{{ include "loki.querierFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
          }

          location ~ /api/prom/.* {
            proxy_pass       http://{{ include "loki.queryFrontendFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
          }

          location = /loki/api/v1/push {
            proxy_pass       http://{{ include "loki.distributorFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
          }

          location = /loki/api/v1/tail {
            proxy_pass       http://{{ include "loki.querierFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
          }

          location ~ /loki/api/.* {
            proxy_pass       http://{{ include "loki.queryFrontendFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
          }

          {{- with .Values.gateway.nginxConfig.serverSnippet }}
          {{ . | nindent 4 }}
          {{- end }}
        }
      }

Installed Promtail using the official chart:

https://github.com/grafana/helm-charts/tree/main/charts/promtail

Values.yaml file:

scrapeConfigs: |
      # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
      - job_name: kubernetes-pods-app-kubernetes-io-name
        # Kubernets auto discovery pods
        kubernetes_sd_configs:
          - role: pod
        # Replacing kubernetes.io.name with app
        relabel_configs:
          - action: replace
            source_labels:
              - __meta_kubernetes_pod_label_app_kubernetes_io_name
            target_label: app

          - action: replace
            source_labels:
            - __meta_kubernetes_pod_name
            target_label: pod

        # Dropping logs that does not have app label
          - action: drop
            regex: ''
            source_labels:
              - app

        # Replace namespace with namespace
          - action: replace
            source_labels:
            - __meta_kubernetes_namespace
            target_label: namespace

        # Required by promtail config to tail the pod logs
          - action: replace
            replacement: /var/log/pods/*$1/*.log
            separator: /
            source_labels:
            - __meta_kubernetes_pod_uid
            - __meta_kubernetes_pod_container_name
            target_label: __path__

        # Removing helath and ping logs from the application
        pipeline_stages:
          - match:
              selector: '{app=~".*"} |~ ".*/(health|ping).*"'
              action: drop
              drop_counter_reason: promtail_noisy_error

          - match:
              selector: '{namespace="loki"}'
              action: drop
              drop_counter_reason: dont_like_it

          - match:
              selector: '{app=~".*"}'
              stages:
                - template:
                    source: pod
                    template: '{ "pod": {{ `"{{ .Value }}"` }}, "app": {{ `"{{ .app }}"` }} ,"pod_log": {{ `{{ .Entry }}` }} }'
                - output:
                    source: pod
          - labeldrop:
              - filename
              - pod
              - app

  # -- Config file contents for Promtail.
  # Must be configured as string.
  # It is templated so it can be assembled from reusable snippets in order to avoid redundancy.
  # @default -- See `values.yaml`
  file: |
    server:
      log_level: {{ .Values.config.logLevel }}
      http_listen_port: {{ .Values.config.serverPort }}

    client:
      url: {{ tpl .Values.config.lokiAddress . }}
      {{- tpl .Values.config.snippets.extraClientConfigs . | nindent 2 }}

    positions:
      filename: /run/promtail/positions_ash.yaml

There are many issues that I am encountering:

  1. When I am trying to add the Loki source in grafana(gateway endpoint)

Grafana is throwing error:

Loki: Internal Server Error. 500. rpc error: code = Unavailable desc = name resolver error: produced zero addresses

  1. Distributer logs showing:

level=warn ts=2021-10-06T10:41:23.729163833Z caller=grpc_logging.go:38 method=/logproto.Pusher/Push duration=86.664µs err=“rpc error: code = Code(400) desc = entry with timestamp 2021-10-06 10:41:22.689555007 +0000 UTC ignored, reason: ‘entry out of order’ for stream: {namespace=“integration”},\nentry with timestamp 2021-10-06 10:41:22.689562602 +0000 UTC ignored, reason: ‘entry out of order’ for stream: {namespace=“integration”},\nentry with timestamp 2021-10-06 10:41:22.689582704 +0000 UTC ignored, reason: ‘entry out of order’ for stream: {namespace=“integration”},\ntotal ignored: 3 out of 6” msg=“gRPC\n”

I am not sure what I am doing wrong. I tried all possible changes in the values file.

I suspect there is something wrong with the Loki configuration file.

Note: When I try installing the loki and promtail with default values.yaml file it works but I can still see the error.

reason: ‘entry out of order’