Loki is not starting up in eks memberlist-kv error="invalid service state

Grafana Loki
1

loki-0 level=info ts=2024-10-27T22:58:56.010265604Z caller=table_manager.go:167 msg=“handing over indexes to shipper” │
│ loki-0 level=info ts=2024-10-27T22:58:56.013790626Z caller=worker.go:112 msg=“Starting querier worker using query-scheduler and scheduler ring for addresses” │
│ loki-0 level=info ts=2024-10-27T22:58:56.013977286Z caller=modules.go:761 msg=“RulerStorage is nil. Not starting the ruler.” │
│ loki-0 level=info ts=2024-10-27T22:58:56.015950982Z caller=module_service.go:82 msg=initialising module=server │
│ loki-0 level=info ts=2024-10-27T22:58:56.016000741Z caller=module_service.go:82 msg=initialising module=memberlist-kv │
│ loki-0 level=info ts=2024-10-27T22:58:56.01601215Z caller=module_service.go:82 msg=initialising module=query-frontend-tripperware │
│ loki-0 level=error ts=2024-10-27T22:58:56.016152413Z caller=loki.go:386 msg=“module failed” module=memberlist-kv error="invalid service state: Failed, expected: Running, failure: service &{0xc0004b3c20 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true │
│ loki-0 level=error ts=2024-10-27T22:58:56.016202498Z caller=loki.go:386 msg=“module failed” module=ring error="failed to start ring, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid serv │
│ loki-0 level=error ts=2024-10-27T22:58:56.016227064Z caller=loki.go:386 msg=“module failed” module=ingester error="failed to start ingester, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: inva │
│ loki-0 level=error ts=2024-10-27T22:58:56.018400409Z caller=loki.go:386 msg=“module failed” module=store error="failed to start store, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid se │
│ loki-0 level=error ts=2024-10-27T22:58:56.018432078Z caller=loki.go:386 msg=“module failed” module=query-scheduler error="failed to start query-scheduler, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, │
│ loki-0 level=error ts=2024-10-27T22:58:56.018449066Z caller=loki.go:386 msg=“module failed” module=compactor error="failed to start compactor, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: in │
│ loki-0 level=info ts=2024-10-27T22:58:56.017043397Z caller=module_service.go:114 msg=“module stopped” module=query-frontend-tripperware │
│ loki-0 level=error ts=2024-10-27T22:58:56.018491145Z caller=loki.go:386 msg=“module failed” module=distributor error="failed to start distributor, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure │
│ loki-0 level=error ts=2024-10-27T22:58:56.01857273Z caller=loki.go:386 msg=“module failed” module=ingester-querier error=“failed to start ingester-querier, because it depends on module ring, which has failed: context canceled” │
│ loki-0 level=error ts=2024-10-27T22:58:56.018738198Z caller=loki.go:386 msg=“module failed” module=usage-report error=“failed to start usage-report, because it depends on module ring, which has failed: context canceled” │
│ loki-0 level=error ts=2024-10-27T22:58:56.018787686Z caller=loki.go:386 msg=“module failed” module=querier error=“failed to start querier, because it depends on module ingester-querier, which has failed: context canceled” │
│ loki-0 level=error ts=2024-10-27T22:58:56.018836206Z caller=loki.go:386 msg=“module failed” module=query-frontend error=“failed to start query-frontend, because it depends on module query-scheduler, which has failed: context canceled” │
│ loki-0 level=info ts=2024-10-27T22:58:56.019899682Z caller=modules.go:1090 msg=“server stopped” │
│ loki-0 level=info ts=2024-10-27T22:58:56.020047329Z caller=module_service.go:114 msg=“module stopped” module=server │
│ loki-0 level=info ts=2024-10-27T22:58:56.020124422Z caller=loki.go:375 msg=“Loki stopped” │
│ loki-0 failed services │
│ loki-0 github.com/grafana/loki/pkg/loki.(*Loki).Run │
│ loki-0 /src/loki/pkg/loki/loki.go:421 │
│ loki-0 main.main │
│ loki-0 /src/loki/cmd/loki/main.go:105 │
│ loki-0 runtime.main │
│ loki-0 /usr/local/go/src/runtime/proc.go:255 │
│ loki-0 runtime.goexit │
│ loki-0 /usr/local/go/src/runtime/asm_amd64.s:1581 │
│ loki-0 level=error ts=2024-10-27T22:58:56.020208542Z caller=log.go:103 msg=“error running loki” err="failed services\ngithub.com/grafana/loki/pkg/loki.(*Loki).Run\n\t/src/loki/pkg/loki/loki.go:421\nmain.main\n\t/src/loki/cmd/loki/main.go:105\nruntime.main\n\t/usr/lo │
│ Stream closed EOF for logging/loki-0 (loki)

2 
loki:
  enabled: true
  isDefault: true
  url: http://{{(include "loki.serviceName" .)}}:{{ .Values.loki.service.port }}
  readinessProbe:
    httpGet:
      path: /ready
      port: http-metrics
    initialDelaySeconds: 200
  livenessProbe:
    httpGet:
      path: /ready
      port: http-metrics
    initialDelaySeconds: 200
  datasource:
    jsonData: "{}"
    uid: ""
  backend:
    image:
      tag: 3.0.1
  commonConfig:
    memberlist:
      bind_addr: []
    ring:
      instance_interface_names:
        - bond0
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr=$(MY_POD_IP)
#  commonConfig:
#    replication_factor: 1

  distributor:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr=$(MY_POD_IP)
  querier:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr=$(MY_POD_IP)
  gateway:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr=$(MY_POD_IP)
  compactor:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr=$(MY_POD_IP)
  index-gateway:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr=$(MY_POD_IP)
  queryFrontend:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr=$(MY_POD_IP)

  ingester:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr=$(MY_POD_IP)
promtail:
  enabled: true
  config:
    logLevel: info
    serverPort: 3101
    clients:
      - url: http://{{ .Release.Name }}:3100/loki/api/v1/push


grafana:
  enabled: true
  sidecar:
    datasources:
      label: ""
      labelValue: ""
      enabled: true
      maxLines: 1000
  image:
    tag: latest
3

I have looked at almost all blogs and tried updating the configs but nothing works :frowning:

4
  1. Try changing bind_addr to bind_addr: ['0.0.0.0']
  2. Try adding abort_if_cluster_join_fails: false so you can troubleshoot with the pods running.
5

TY!, Iam still getting the error after changing as per your suggestion
ller=module_service.go:82 msg=initialising module=memberlist-kv │ │ loki-0 level=info ts=2024-10-29T03:27:27.147701891Z caller=module_service.go:82 msg=initialising module=query-frontend-tripperware │ │ loki-0 level=error ts=2024-10-29T03:27:27.147745356Z caller=loki.go:517 msg="module failed" module=memberlist-kv error="invalid service state: Failed, expected: Running, failure: service &{0xc000959680 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true │ │ loki-0 level=error ts=2024-10-29T03:27:27.147795449Z caller=loki.go:517 msg="module failed" module=store error="failed to start store, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid se │ │ loki-0 level=info ts=2024-10-29T03:27:27.148694866Z caller=

loki:
  enabled: true
  isDefault: true
  url: http://{{(include "loki.serviceName" .)}}:{{ .Values.loki.service.port }}
  readinessProbe:
    httpGet:
      path: /ready
      port: http-metrics
    initialDelaySeconds: 200
  livenessProbe:
    httpGet:
      path: /ready
      port: http-metrics
    initialDelaySeconds: 200
  datasource:
    jsonData: "{}"
    uid: ""
#  chunk_store_config:
#    max_look_back_period: 336h
  image:
    tag: 2.9.3
    # -- The Docker registry
    registry: docker.io
    # -- Docker image repository
    repository: grafana/loki
    # -- Overrides the image tag whose default is the chart's appVersion
    digest: null
    # -- Docker image pull policy
    pullPolicy: IfNotPresent
  commonConfig:
    memberlist:
      bind_addr: ['0.0.0.0']
      abort_if_cluster_join_fails: false
    ring:
      instance_addr: $(MY_POD_IP)
      instance_interface_names:
        - bond0
      kvstore:
        store: inmemory
        replication_factor: 1
        path_prefix: /tmp/loki
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr='0.0.0.0' # $(MY_POD_IP)*
#  commonConfig:
#    replication_factor: 1
  schema_config:
    configs:
      - from: 2020-05-15
        store: tsdb
        object_store: filesystem
        schema: v13
        index:
          prefix: index_
          period: 24h

  storage_config:
    filesystem:
      directory: /tmp/loki/chunks

  distributor:
    replicas: 1

    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr='0.0.0.0' # $(MY_POD_IP)*
  querier:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr='0.0.0.0' # $(MY_POD_IP)*
  gateway:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr='0.0.0.0' # $(MY_POD_IP)*
  compactor:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr='0.0.0.0' # $(MY_POD_IP)*
  index_gateway:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr='0.0.0.0' # $(MY_POD_IP)*

  frontenc:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr='0.0.0.0' # $(MY_POD_IP)*
  queryScheduler:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr='0.0.0.0' # $(MY_POD_IP)*
  ingester:
    replicas: 1
    extraEnv:
      - name: MY_POD_IP
        valueFrom:
          fieldRef:
            fieldPath: status.podIP
    extraArgs:
      - -memberlist.bind-addr='0.0.0.0' # $(MY_POD_IP)*
promtail:
  enabled: true
  config:
    logLevel: info
    serverPort: 3101
    clients:
      - url: http://{{ .Release.Name }}:3100/loki/api/v1/push


grafana:
  enabled: true
  sidecar:
    datasources:
      label: ""
      labelValue: ""
      enabled: true
      maxLines: 1000
  image:
    tag: latest