Error while querying logs on grafana where loki datasource is used

Grafana Loki
1

Getting this error in loki logs while trying to fetch lables, we are using alloy agents to push logs to loki and loki i s deploeyed as simpleScalable mode in gke cluster and using gcp bucket as backend

error: code = Unimplemented desc = unknown service logproto.Querier

Current loki config file

    auth_enabled: false
    bloom_build:
      builder:
        planner_address: loki-backend-headless.monitoring.svc.cluster.local:9095
      enabled: false
    bloom_gateway:
      client:
        addresses: dnssrvnoa+_grpc._tcp.loki-backend-headless.monitoring.svc.cluster.local
      enabled: false
    common:
      compactor_grpc_address: 'loki-backend.monitoring.svc.cluster.local:9095'
      path_prefix: /var/loki
      replication_factor: 3
      storage:
        gcs:
          bucket_name: loki-poc-gke
          chunk_buffer_size: 0
          enable_http2: true
          request_timeout: 0s
    compactor:
      compaction_interval: 10m
      delete_request_cancel_period: 10m
      delete_request_store: gcs
      delete_request_store_key_prefix: index/
      jobs_config:
        deletion:
          chunk_processing_concurrency: 3
          deletion_manifest_store_prefix: __deletion_manifest__/
          max_retries: 3
          timeout: 15m
      max_compaction_parallelism: 1
      retention_delete_delay: 2h
      retention_delete_worker_count: 150
      retention_enabled: true
      upload_parallelism: 10
      working_directory: /var/loki/compactor
    distributor:
      max_recv_msg_size: 104857600
      push_worker_count: 256
    frontend:
      scheduler_address: ""
      tail_proxy_url: ""
    frontend_worker:
      scheduler_address: ""
    index_gateway:
      mode: simple
    ingester:
      chunk_block_size: 2097152
      chunk_encoding: snappy
      chunk_idle_period: 1h
      chunk_retain_period: 30s
      chunk_target_size: 2097152
      concurrent_flushes: 32
      flush_check_period: 20s
      flush_op_timeout: 10m
      lifecycler:
        address: 0.0.0.0
        final_sleep: 0s
        ring:
          kvstore:
            store: memberlist
          replication_factor: 3
      max_chunk_age: 1h
      wal:
        checkpoint_duration: 5m
        dir: /var/loki/wal
        enabled: true
    limits_config:
      max_cache_freshness_per_query: 10m
      max_query_length: 30d
      max_query_lookback: 30d
      query_timeout: 600s
      reject_old_samples: true
      reject_old_samples_max_age: 168h
      retention_period: 672h
      split_queries_by_interval: 15m
      volume_enabled: true
    memberlist:
      bind_addr:
      - 0.0.0.0
      bind_port: 7946
      dead_node_reclaim_time: 30s
      gossip_interval: 1s
      gossip_nodes: 3
      join_members:
      - loki-memberlist.monitoring.svc.cluster.local
      node_name: ${HOSTNAME}
      retransmit_factor: 4
    pattern_ingester:
      enabled: false
    querier:
      engine:
        max_look_back_period: 0s
      max_concurrent: 20
      query_ingesters_within: 3h
      tail_max_duration: 1h
    query_range:
      align_queries_with_step: true
      parallelise_shardable_queries: true
      results_cache:
        cache:
          embedded_cache:
            enabled: true
            max_size_mb: 100
    runtime_config:
      file: /etc/loki/runtime-config/runtime-config.yaml
    schema_config:
      configs:
      - from: "2024-01-01"
        index:
          period: 24h
          prefix: index_
        object_store: gcs
        schema: v13
        store: tsdb
    server:
      grpc_listen_port: 9095
      grpc_server_max_concurrent_streams: 1000
      grpc_server_max_recv_msg_size: 104857600
      grpc_server_max_send_msg_size: 104857600
      http_listen_port: 3100
      http_server_read_timeout: 600s
      http_server_write_timeout: 600s
    storage_config:
      bloom_shipper:
        working_directory: /var/loki/data/bloomshipper
      boltdb_shipper:
        index_gateway_client:
          server_address: dns+loki-backend-headless.monitoring.svc.cluster.local:9095
      gcs:
        bucket_name: loki-poc-gke
        chunk_buffer_size: 0
        enable_http2: true
        request_timeout: 0s
      hedging:
        at: 250ms
        max_per_second: 20
        up_to: 3
      tsdb_shipper:
        index_gateway_client:
          server_address: dns+loki-backend-headless.monitoring.svc.cluster.local:9095
      use_thanos_objstore: false
    tracing:
      enabled: false
    ui:
      enabled: true
2

That sounds like you are hitting an endpoint that is not the read path. I would double check your Loki endpoint. Send an API call directly and see if it works.