Added self-signed certificated for internal encryption between loki components

I have successfully added self signed certificates for Distributed gateway and Distributor. The connection between the promtail and grafana is working fine. When I have added self-signed certificate to ingester the communication gets failed between distributor and ingester.

I have shared the config secret:

common:
  compactor_address: https://loki-loki-distributed-compactor:3100
auth_enabled: false
chunk_store_config:
  max_look_back_period: 744h
  chunk_cache_config:
    embedded_cache:
      enabled: false
    memcached:
      batch_size: 400
      parallelism: 20960
    memcached_client:
      consistent_hash: true
      addresses: dnssrv+_memcached-client._tcp.loki-loki-distributed-memcached-chunks.loki-test.svc.cluster.local
  write_dedupe_cache_config:
    memcached:
      batch_size: 400
      parallelism: 2096
    memcached_client:
      consistent_hash: true
      addresses: dnssrv+_memcached-client._tcp.loki-loki-distributed-memcached-index-writes.loki-test.svc.cluster.local
compactor:
  compaction_interval: 30m
  retention_delete_delay: 1h
  retention_delete_worker_count: 100
  retention_enabled: true
  shared_store: s3
  working_directory: /var/loki/retention
distributor:
  ring:
    kvstore:
      store: memberlist
querier:
  max_concurrent: 2976
  #multi_tenant_queries_enabled:
frontend:
  compress_responses: true
  max_outstanding_per_tenant: 200000000
  scheduler_address: loki-loki-distributed-query-scheduler:9095
  log_queries_longer_than: 5s
  tail_proxy_url: http://loki-loki-distributed-querier:3100
frontend_worker:
  scheduler_address: loki-loki-distributed-query-scheduler:9095
  parallelism: 2976
  match_max_concurrent: true
ingester:
  autoforget_unhealthy: true
  chunk_block_size: 262144
  max_chunk_age: 168h
  chunk_encoding: snappy
  chunk_idle_period: 30m
  chunk_retain_period: 1m
  lifecycler:
    ring:
      kvstore:
        store: memberlist
      replication_factor: 2
  max_transfer_retries: 0
  wal:
    dir: /var/loki/wal
ingester_client:
  grpc_client_config:
    tls_ca_path: /etc/ssl/certs/ca.crt
    tls_cert_path: /etc/tls/private/tls.crt
    tls_enabled: true
    tls_key_path: /etc/tls/private/tls.key
limits_config:
  enforce_metric_name: false
  ingestion_burst_size_mb: 1000
  ingestion_rate_mb: 200
  ingestion_rate_strategy: local
  max_cache_freshness_per_query: 10m
  reject_old_samples: false
  max_global_streams_per_user: 100000  
  per_stream_rate_limit: 200M
  per_stream_rate_limit_burst: 1000M
  max_query_parallelism: 10240
  max_query_series: 2096
  reject_old_samples_max_age: 168h
  retention_period: 8784h
  split_queries_by_interval: 4h
  query_timeout: 20m
  tsdb_max_query_parallelism: 10240
memberlist:
  join_members:
  - loki-loki-distributed-memberlist
query_range:
  align_queries_with_step: true
  cache_results: true
  max_retries: 50
  results_cache:
    cache:
      enable_fifocache: true
      fifocache:
        max_size_items: 2048
        validity: 72h
      memcached_client:
        host: loki-loki-distributed-memcached-frontend
        service: memcached-client
        consistent_hash: true
        max_idle_conns: 16
        timeout: 15s
        update_interval: 1m
      embedded_cache:
        enabled: false
        ttl: 72h

ruler:
  alertmanager_url: https://alertmanager.xx
  external_url: https://alertmanager.xx
  ring:
    kvstore:
      store: memberlist
  rule_path: /tmp/loki/scratch
  storage:
    local:
      directory: /etc/loki/rules
    type: local
schema_config:
  configs:
  - from: "2023-12-31"
    index:
      period: 24h
      prefix: loki_index_
    object_store: aws
    schema: v11
    store: boltdb-shipper
  - from: "2024-01-01"
    index:
      period: 24h
      prefix: loki_index_
    object_store: aws
    schema: v12
    store: tsdb   
server:
  http_listen_port: 3100
  http_tls_config:
    cert_file: /etc/tls/private/tls.crt
    key_file: /etc/tls/private/tls.key
  http_server_read_timeout: 600s
  http_server_write_timeout: 600s
  grpc_server_max_recv_msg_size: 104857600
  grpc_server_max_send_msg_size: 104857600
  grpc_server_max_concurrent_streams: 0
  grpc_tls_config:
    cert_file: /etc/tls/private/tls.crt
    key_file: /etc/tls/private/tls.key
index_gateway:
  mode: simple
storage_config:
  aws:
    access_key_id: ****************
    bucketnames: ************
    s3: https://*******************
    secret_access_key: **************************
    backoff_config:
      max_retries: 15
  boltdb_shipper:
    active_index_directory: /var/loki/index
    cache_location: /var/loki/cache
    cache_ttl: 1m
    shared_store: s3
    index_gateway_client:
      server_address: dns:///loki-loki-distributed-index-gateway:9095
  tsdb_shipper:
    active_index_directory: /var/loki/tsdb-index
    cache_location: /var/loki/tsdb-cache
    cache_ttl: 1m
    shared_store: s3
    index_gateway_client:
      server_address: dns:///loki-loki-distributed-index-gateway:9095      
  index_queries_cache_config:
    memcached:
      batch_size: 400
      parallelism: 2096
    memcached_client:
      addresses: dnssrv+_memcached-client._tcp.loki-loki-distributed-memcached-index-queries.loki-test.svc.cluster.local
      consistent_hash: true
  filesystem:
    directory: /var/loki/chunks
table_manager:
  retention_deletes_enabled: true
  retention_period: 744h

Error Logs of Ingester:
level=info ts=2024-01-17T12:40:36.366682726Z caller=table_manager.go:134 msg=“uploading tables”

165level=info ts=2024-01-17T12:40:36.377867406Z caller=table_manager.go:166 msg=“handing over indexes to shipper”

166level=info ts=2024-01-17T12:40:36.390038344Z caller=table_manager.go:134 msg=“uploading tables”

167ts=2024-01-17T12:41:01.274357392Z caller=memberlist_logger.go:74 level=info msg=“Suspect loki-loki-distributed-index-gateway-0-b230c3e1 has failed, no acks received”

168ts=2024-01-17T12:41:19.279605735Z caller=memberlist_logger.go:74 level=info msg=“Marking loki-loki-distributed-index-gateway-0-b230c3e1 as failed, suspect timeout reached (2 peer confirmations)”

169level=info ts=2024-01-17T12:41:36.366978192Z caller=table_manager.go:134 msg=“uploading tables”

Error Logs of Distributor:
ts=2024-01-18T09:42:48.029163017Z caller=memberlist_logger.go:74 level=error msg=“Push/Pull with loki-loki-distributed-index-gateway-0-e9c5f185 failed: dial tcp 172.21.162.186:7946: connect: connection refused”

34ts=2024-01-18T09:43:21.747005591Z caller=memberlist_logger.go:74 level=info msg=“Marking loki-loki-distributed-index-gateway-0-e9c5f185 as failed, suspect timeout reached (2 peer confirmations)”

35ts=2024-01-18T10:04:12.27890998Z caller=memberlist_logger.go:74 level=info msg=“Suspect loki-loki-distributed-index-gateway-0-07250412 has failed, no acks received”

36ts=2024-01-18T10:04:32.842492824Z caller=memberlist_logger.go:74 level=info msg=“Marking loki-loki-distributed-index-gateway-0-07250412 as failed, suspect timeout reached (2 peer confirmations)”

37ts=2024-01-18T10:09:22.279632698Z caller=memberlist_logger.go:74 level=info msg=“Suspect loki-loki-distributed-index-gateway-0-cc62b67e has failed, no acks received”