When I query the data of a namespace in the last 12 hours on Grafana, Querier and frontend frequently report errors, TraceID 11b02136a16dbd46 and 5af405c4f43aecef
- Infrastructure: kubernetes
- Deployment tool: helm
- Loki version: v3.1.0
config：
auth_enabled: false
chunk_store_config:
chunk_cache_config:
embedded_cache:
enabled: false
memcached_client:
addresses: dnssrv+_memcached-client._tcp.loki-distributed-memcached-chunks.loki-distributed.svc.cluster.local
consistent_hash: true
common:
compactor_address: http://loki-distributed-compactor:3100
storage:
s3:
access_key_id: loki
bucketnames: loki
endpoint: loki-minio.loki-distributed.svc:9000
insecure: true
s3forcepathstyle: true
secret_access_key: loki
compactor:
working_directory: /var/loki/compactor
distributor:
ring:
kvstore:
store: memberlist
frontend:
compress_responses: true
grpc_client_config:
connect_backoff_max_delay: 10s
connect_timeout: 10s
grpc_compression: gzip
log_queries_longer_than: 5s
tail_proxy_url: http://loki-distributed-querier:3100
frontend_worker:
frontend_address: loki-distributed-query-frontend-headless:9095
ingester:
chunk_block_size: 262144
chunk_encoding: snappy
chunk_idle_period: 30m
chunk_retain_period: 1m
flush_check_period: 5s
flush_op_timeout: 10m
lifecycler:
ring:
kvstore:
store: memberlist
replication_factor: 1
wal:
dir: /var/loki/wal
ingester_client:
grpc_client_config:
grpc_compression: gzip
limits_config:
max_cache_freshness_per_query: 10m
max_query_parallelism: 80
reject_old_samples: true
reject_old_samples_max_age: 168h
split_queries_by_interval: 15m
memberlist:
join_members:
- loki-distributed-memberlist
querier:
max_concurrent: 8
query_range:
align_queries_with_step: true
cache_results: true
max_retries: 5
results_cache:
cache:
memcached_client:
addresses: dnssrv+_memcached-client._tcp.loki-distributed-memcached-frontend.loki-distributed.svc.cluster.local
consistent_hash: true
ruler:
alertmanager_url: https://alertmanager.xx
external_url: https://alertmanager.xx
ring:
kvstore:
store: memberlist
rule_path: /tmp/loki/scratch
storage:
local:
directory: /etc/loki/rules
type: local
runtime_config:
file: /var/loki-distributed-runtime/runtime.yaml
schema_config:
configs:
- from: "2020-09-07"
index:
period: 24h
prefix: index_
object_store: s3
schema: v13
store: tsdb
server:
http_listen_port: 3100
storage_config:
tsdb_shipper:
active_index_directory: /var/loki/index
cache_location: /var/loki/cache
cache_ttl: 168h
table_manager:
retention_deletes_enabled: false
retention_period: 0s
