As I understand, log replication occurs on the distributor level on write.
Does loki have background replication?
I have 3 replicas(also replication_factor: 3). How will loki handle the fall(or manual shutdown) of one of the replicas? Will all logs be available on all 3 replicas eventually?
Loki version: 2.2
Config:
Config
target: all
http_prefix: “”
server:
http_listen_address: “”
http_listen_port: 3100
http_listen_conn_limit: 0
grpc_listen_address: “”
grpc_listen_port: 9095
grpc_listen_conn_limit: 0
http_tls_config:
cert_file: “”
key_file: “”
client_auth_type: “”
client_ca_file: “”
grpc_tls_config:
cert_file: “”
key_file: “”
client_auth_type: “”
client_ca_file: “”
register_instrumentation: true
graceful_shutdown_timeout: 30s
http_server_read_timeout: 30s
http_server_write_timeout: 30s
http_server_idle_timeout: 2m0s
grpc_server_max_recv_msg_size: 4194304
grpc_server_max_send_msg_size: 4194304
grpc_server_max_concurrent_streams: 100
grpc_server_max_connection_idle: 2562047h47m16.854775807s
grpc_server_max_connection_age: 2562047h47m16.854775807s
grpc_server_max_connection_age_grace: 2562047h47m16.854775807s
grpc_server_keepalive_time: 2h0m0s
grpc_server_keepalive_timeout: 20s
grpc_server_min_time_between_pings: 5m0s
grpc_server_ping_without_stream_allowed: false
log_format: logfmt
log_level: info
log_source_ips_enabled: false
log_source_ips_header: “”
log_source_ips_regex: “”
http_path_prefix: “”
distributor:
ring:
kvstore:
store: memberlist
prefix: collectors/
consul:
host: localhost:8500
acl_token: “”
http_client_timeout: 20s
consistent_reads: false
watch_rate_limit: 1
watch_burst_size: 1
etcd:
endpoints:
dial_timeout: 10s
max_retries: 10
tls_enabled: false
tls_cert_path: “”
tls_key_path: “”
tls_ca_path: “”
tls_server_name: “”
tls_insecure_skip_verify: false
multi:
primary: “”
secondary: “”
mirror_enabled: false
mirror_timeout: 2s
heartbeat_period: 5s
heartbeat_timeout: 1m0s
instance_id: mon-loki-0
instance_interface_names:
- eth0
- en0
instance_port: 0
instance_addr: “”
querier:
query_timeout: 1m0s
tail_max_duration: 1h0m0s
engine:
timeout: 5m0s
max_look_back_period: 30s
max_concurrent: 20
ingester_client:
pool_config:
client_cleanup_period: 15s
health_check_ingesters: true
remote_timeout: 5s
grpc_client_config:
max_recv_msg_size: 104857600
max_send_msg_size: 16777216
grpc_compression: “”
rate_limit: 0
rate_limit_burst: 0
backoff_on_ratelimits: false
backoff_config:
min_period: 100ms
max_period: 10s
max_retries: 10
tls_enabled: false
tls_cert_path: “”
tls_key_path: “”
tls_ca_path: “”
tls_server_name: “”
tls_insecure_skip_verify: false
ingester:
lifecycler:
ring:
kvstore:
store: memberlist
prefix: collectors/
consul:
host: localhost:8500
acl_token: “”
http_client_timeout: 20s
consistent_reads: false
watch_rate_limit: 1
watch_burst_size: 1
etcd:
endpoints:
dial_timeout: 10s
max_retries: 10
tls_enabled: false
tls_cert_path: “”
tls_key_path: “”
tls_ca_path: “”
tls_server_name: “”
tls_insecure_skip_verify: false
multi:
primary: “”
secondary: “”
mirror_enabled: false
mirror_timeout: 2s
heartbeat_timeout: 1m0s
replication_factor: 3
zone_awareness_enabled: false
num_tokens: 128
heartbeat_period: 5s
observe_period: 0s
join_after: 0s
min_ready_duration: 1m0s
interface_names:
- eth0
- en0
final_sleep: 30s
tokens_file_path: “”
availability_zone: “”
unregister_on_shutdown: true
address: “”
port: 0
id: mon-loki-0
concurrent_flushes: 16
flush_check_period: 30s
flush_op_timeout: 10s
chunk_retain_period: 1m0s
chunk_idle_period: 3m0s
chunk_block_size: 262144
chunk_target_size: 0
chunk_encoding: gzip
max_chunk_age: 1h0m0s
sync_period: 0s
sync_min_utilization: 0
max_returned_stream_errors: 10
query_store_max_look_back_period: 1h28m0s
wal:
enabled: false
dir: wal
checkpoint_duration: 5m0s
flush_on_shutdown: false
replay_memory_ceiling: 4294967296
storage_config:
engine: chunks
aws:
dynamodb:
dynamodb_url: “”
api_limit: 2
throttle_limit: 10
metrics:
url: “”
target_queue_length: 100000
scale_up_factor: 1.3
ignore_throttle_below: 1
queue_length_query: sum(avg_over_time(cortex_ingester_flush_queue_length{job=“cortex/ingester”}[2m]))
write_throttle_query: sum(rate(cortex_dynamo_throttled_total{operation=“DynamoDB.BatchWriteItem”}[1m]))
by (table) > 0
write_usage_query: sum(rate(cortex_dynamo_consumed_capacity_total{operation=“DynamoDB.BatchWriteItem”}[15m]))
by (table) > 0
read_usage_query: sum(rate(cortex_dynamo_consumed_capacity_total{operation=“DynamoDB.QueryPages”}[1h]))
by (table) > 0
read_error_query: sum(increase(cortex_dynamo_failures_total{operation=“DynamoDB.QueryPages”,error=“ProvisionedThroughputExceededException”}[1m]))
by (table) > 0
chunk_gang_size: 10
chunk_get_max_parallelism: 32
backoff_config:
min_period: 100ms
max_period: 50s
max_retries: 20
s3: “”
s3forcepathstyle: false
bucketnames: “”
endpoint: “”
region: “”
access_key_id: “”
secret_access_key: “”
insecure: false
sse_encryption: false
http_config:
idle_conn_timeout: 1m30s
response_header_timeout: 0s
insecure_skip_verify: false
signature_version: v4
sse:
type: “”
kms_key_id: “”
kms_encryption_context: “”
azure:
environment: AzureGlobal
container_name: cortex
account_name: “”
account_key: “”
download_buffer_size: 512000
upload_buffer_size: 256000
upload_buffer_count: 1
request_timeout: 30s
max_retries: 5
min_retry_delay: 10ms
max_retry_delay: 500ms
bigtable:
project: “”
instance: “”
grpc_client_config:
max_recv_msg_size: 104857600
max_send_msg_size: 16777216
grpc_compression: “”
rate_limit: 0
rate_limit_burst: 0
backoff_on_ratelimits: false
backoff_config:
min_period: 100ms
max_period: 10s
max_retries: 10
tls_enabled: true
tls_cert_path: “”
tls_key_path: “”
tls_ca_path: “”
tls_server_name: “”
tls_insecure_skip_verify: false
table_cache_enabled: true
table_cache_expiration: 30m0s
gcs:
bucket_name: “”
chunk_buffer_size: 0
request_timeout: 0s
cassandra:
addresses: “”
port: 9042
keyspace: “”
consistency: QUORUM
replication_factor: 3
disable_initial_host_lookup: false
SSL: false
host_verification: true
CA_path: “”
tls_cert_path: “”
tls_key_path: “”
auth: false
username: “”
password: “”
password_file: “”
custom_authenticators:
timeout: 2s
connect_timeout: 5s
reconnect_interval: 1s
max_retries: 0
retry_max_backoff: 10s
retry_min_backoff: 100ms
query_concurrency: 0
num_connections: 2
convict_hosts_on_failure: true
table_options: “”
boltdb:
directory: “”
filesystem:
directory: /data/loki/chunks
swift:
auth_version: 0
auth_url: “”
username: “”
user_domain_name: “”
user_domain_id: “”
user_id: “”
password: “”
domain_id: “”
domain_name: “”
project_id: “”
project_name: “”
project_domain_id: “”
project_domain_name: “”
region_name: “”
container_name: “”
max_retries: 3
connect_timeout: 10s
request_timeout: 5s
index_cache_validity: 5m0s
index_queries_cache_config:
enable_fifocache: false
default_validity: 0s
background:
writeback_goroutines: 10
writeback_buffer: 10000
memcached:
expiration: 0s
batch_size: 1024
parallelism: 100
memcached_client:
host: “”
service: memcached
addresses: “”
timeout: 100ms
max_idle_conns: 16
update_interval: 1m0s
consistent_hash: true
circuit_breaker_consecutive_failures: 10
circuit_breaker_timeout: 10s
circuit_breaker_interval: 10s
redis:
endpoint: “”
master_name: “”
timeout: 500ms
expiration: 0s
db: 0
pool_size: 0
password: “”
tls_enabled: false
tls_insecure_skip_verify: false
idle_timeout: 0s
max_connection_age: 0s
fifocache:
max_size_bytes: “”
max_size_items: 0
validity: 0s
size: 0
prefix: store.index-cache-read.
delete_store:
store: “”
requests_table_name: delete_requests
table_provisioning:
enable_ondemand_throughput_mode: false
provisioned_write_throughput: 1
provisioned_read_throughput: 300
write_scale:
enabled: false
role_arn: “”
min_capacity: 3000
max_capacity: 6000
out_cooldown: 1800
in_cooldown: 1800
target: 80
read_scale:
enabled: false
role_arn: “”
min_capacity: 3000
max_capacity: 6000
out_cooldown: 1800
in_cooldown: 1800
target: 80
tags: {}
grpc_store: {}
max_chunk_batch_size: 50
boltdb_shipper:
active_index_directory: /data/loki/boltdb-shipper-active
shared_store: filesystem
cache_location: /data/loki/boltdb-shipper-cache
cache_ttl: 24h0m0s
resync_interval: 5m0s
query_ready_num_days: 0
chunk_store_config:
chunk_cache_config:
enable_fifocache: false
default_validity: 0s
background:
writeback_goroutines: 10
writeback_buffer: 10000
memcached:
expiration: 0s
batch_size: 1024
parallelism: 100
memcached_client:
host: “”
service: memcached
addresses: “”
timeout: 100ms
max_idle_conns: 16
update_interval: 1m0s
consistent_hash: true
circuit_breaker_consecutive_failures: 10
circuit_breaker_timeout: 10s
circuit_breaker_interval: 10s
redis:
endpoint: “”
master_name: “”
timeout: 500ms
expiration: 0s
db: 0
pool_size: 0
password: “”
tls_enabled: false
tls_insecure_skip_verify: false
idle_timeout: 0s
max_connection_age: 0s
fifocache:
max_size_bytes: “”
max_size_items: 0
validity: 0s
size: 0
prefix: store.chunks-cache.
write_dedupe_cache_config:
enable_fifocache: false
default_validity: 0s
background:
writeback_goroutines: 0
writeback_buffer: 0
memcached:
expiration: 0s
batch_size: 0
parallelism: 0
memcached_client:
host: “”
service: “”
addresses: “”
timeout: 0s
max_idle_conns: 0
update_interval: 0s
consistent_hash: false
circuit_breaker_consecutive_failures: 0
circuit_breaker_timeout: 0s
circuit_breaker_interval: 0s
redis:
endpoint: “”
master_name: “”
timeout: 0s
expiration: 0s
db: 0
pool_size: 0
password: “”
tls_enabled: false
tls_insecure_skip_verify: false
idle_timeout: 0s
max_connection_age: 0s
fifocache:
max_size_bytes: “”
max_size_items: 0
validity: 0s
size: 0
prefix: “”
cache_lookups_older_than: 0s
max_look_back_period: 0s
schema_config:
configs:
- from: “2020-10-24”
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 1d
tags: {}
chunks:
prefix: “”
period: 0s
tags: {}
row_shards: 16
limits_config:
ingestion_rate_strategy: local
ingestion_rate_mb: 4
ingestion_burst_size_mb: 6
max_label_name_length: 1024
max_label_value_length: 2048
max_label_names_per_series: 30
reject_old_samples: true
reject_old_samples_max_age: 168h0m0s
creation_grace_period: 10m0s
enforce_metric_name: false
max_line_size: 0
max_streams_per_user: 10000
max_global_streams_per_user: 0
max_chunks_per_query: 2000000
max_query_series: 500
max_query_lookback: 0s
max_query_length: 0s
max_query_parallelism: 14
cardinality_limit: 100000
max_streams_matchers_per_query: 1000
max_concurrent_tail_requests: 10
max_entries_limit_per_query: 5000
max_cache_freshness_per_query: 1m0s
split_queries_by_interval: 0s
ruler_evaluation_delay_duration: 0s
ruler_max_rules_per_rule_group: 0
ruler_max_rule_groups_per_tenant: 0
per_tenant_override_config: “”
per_tenant_override_period: 10s
table_manager:
throughput_updates_disabled: false
retention_deletes_enabled: true
retention_period: 180d
poll_interval: 2m0s
creation_grace_period: 10m0s
index_tables_provisioning:
enable_ondemand_throughput_mode: false
provisioned_write_throughput: 1000
provisioned_read_throughput: 300
write_scale:
enabled: false
role_arn: “”
min_capacity: 3000
max_capacity: 6000
out_cooldown: 1800
in_cooldown: 1800
target: 80
read_scale:
enabled: false
role_arn: “”
min_capacity: 3000
max_capacity: 6000
out_cooldown: 1800
in_cooldown: 1800
target: 80
enable_inactive_throughput_on_demand_mode: false
inactive_write_throughput: 1
inactive_read_throughput: 300
inactive_write_scale:
enabled: false
role_arn: “”
min_capacity: 3000
max_capacity: 6000
out_cooldown: 1800
in_cooldown: 1800
target: 80
inactive_read_scale:
enabled: false
role_arn: “”
min_capacity: 3000
max_capacity: 6000
out_cooldown: 1800
in_cooldown: 1800
target: 80
inactive_write_scale_lastn: 4
inactive_read_scale_lastn: 4
chunk_tables_provisioning:
enable_ondemand_throughput_mode: false
provisioned_write_throughput: 1000
provisioned_read_throughput: 300
write_scale:
enabled: false
role_arn: “”
min_capacity: 3000
max_capacity: 6000
out_cooldown: 1800
in_cooldown: 1800
target: 80
read_scale:
enabled: false
role_arn: “”
min_capacity: 3000
max_capacity: 6000
out_cooldown: 1800
in_cooldown: 1800
target: 80
enable_inactive_throughput_on_demand_mode: false
inactive_write_throughput: 1
inactive_read_throughput: 300
inactive_write_scale:
enabled: false
role_arn: “”
min_capacity: 3000
max_capacity: 6000
out_cooldown: 1800
in_cooldown: 1800
target: 80
inactive_read_scale:
enabled: false
role_arn: “”
min_capacity: 3000
max_capacity: 6000
out_cooldown: 1800
in_cooldown: 1800
target: 80
inactive_write_scale_lastn: 4
inactive_read_scale_lastn: 4
frontend_worker:
frontend_address: “”
scheduler_address: “”
dns_lookup_duration: 10s
parallelism: 10
match_max_concurrent: false
id: “”
grpc_client_config:
max_recv_msg_size: 104857600
max_send_msg_size: 16777216
grpc_compression: “”
rate_limit: 0
rate_limit_burst: 0
backoff_on_ratelimits: false
backoff_config:
min_period: 100ms
max_period: 10s
max_retries: 10
tls_enabled: false
tls_cert_path: “”
tls_key_path: “”
tls_ca_path: “”
tls_server_name: “”
tls_insecure_skip_verify: false
frontend:
log_queries_longer_than: 0s
max_body_size: 10485760
query_stats_enabled: false
max_outstanding_per_tenant: 100
compress_responses: false
downstream_url: “”
tail_proxy_url: “”
ruler:
external_url: “”
ruler_client:
max_recv_msg_size: 104857600
max_send_msg_size: 16777216
grpc_compression: “”
rate_limit: 0
rate_limit_burst: 0
backoff_on_ratelimits: false
backoff_config:
min_period: 100ms
max_period: 10s
max_retries: 10
tls_enabled: false
tls_cert_path: “”
tls_key_path: “”
tls_ca_path: “”
tls_server_name: “”
tls_insecure_skip_verify: false
evaluation_interval: 1m0s
poll_interval: 1m0s
storage:
type: configdb
configdb:
configs_api_url: “”
client_timeout: 5s
tls_cert_path: “”
tls_key_path: “”
tls_ca_path: “”
tls_server_name: “”
tls_insecure_skip_verify: false
azure:
environment: AzureGlobal
container_name: cortex
account_name: “”
account_key: “”
download_buffer_size: 512000
upload_buffer_size: 256000
upload_buffer_count: 1
request_timeout: 30s
max_retries: 5
min_retry_delay: 10ms
max_retry_delay: 500ms
gcs:
bucket_name: “”
chunk_buffer_size: 0
request_timeout: 0s
s3:
s3: “”
s3forcepathstyle: false
bucketnames: “”
endpoint: “”
region: “”
access_key_id: “”
secret_access_key: “”
insecure: false
sse_encryption: false
http_config:
idle_conn_timeout: 1m30s
response_header_timeout: 0s
insecure_skip_verify: false
signature_version: v4
sse:
type: “”
kms_key_id: “”
kms_encryption_context: “”
swift:
auth_version: 0
auth_url: “”
username: “”
user_domain_name: “”
user_domain_id: “”
user_id: “”
password: “”
domain_id: “”
domain_name: “”
project_id: “”
project_name: “”
project_domain_id: “”
project_domain_name: “”
region_name: “”
container_name: “”
max_retries: 3
connect_timeout: 10s
request_timeout: 5s
local:
directory: “”
rule_path: /rules
alertmanager_url: “”
enable_alertmanager_discovery: false
alertmanager_refresh_interval: 1m0s
enable_alertmanager_v2: false
notification_queue_capacity: 10000
notification_timeout: 10s
alertmanager_client:
tls_cert_path: “”
tls_key_path: “”
tls_ca_path: “”
tls_server_name: “”
tls_insecure_skip_verify: false
basic_auth_username: “”
basic_auth_password: “”
for_outage_tolerance: 1h0m0s
for_grace_period: 10m0s
resend_delay: 1m0s
enable_sharding: false
sharding_strategy: default
search_pending_for: 5m0s
ring:
kvstore:
store: memberlist
prefix: rulers/
consul:
host: localhost:8500
acl_token: “”
http_client_timeout: 20s
consistent_reads: false
watch_rate_limit: 1
watch_burst_size: 1
etcd:
endpoints:
dial_timeout: 10s
max_retries: 10
tls_enabled: false
tls_cert_path: “”
tls_key_path: “”
tls_ca_path: “”
tls_server_name: “”
tls_insecure_skip_verify: false
multi:
primary: “”
secondary: “”
mirror_enabled: false
mirror_timeout: 2s
heartbeat_period: 5s
heartbeat_timeout: 1m0s
instance_id: mon-loki-0
instance_interface_names:- eth0
- en0
instance_port: 0
instance_addr: “”
num_tokens: 128
flush_period: 1m0s
enable_api: false
query_range:
split_queries_by_interval: 0s
split_queries_by_day: false
align_queries_with_step: false
results_cache:
cache:
enable_fifocache: false
default_validity: 0s
background:
writeback_goroutines: 10
writeback_buffer: 10000
memcached:
expiration: 0s
batch_size: 1024
parallelism: 100
memcached_client:
host: “”
service: memcached
addresses: “”
timeout: 100ms
max_idle_conns: 16
update_interval: 1m0s
consistent_hash: true
circuit_breaker_consecutive_failures: 10
circuit_breaker_timeout: 10s
circuit_breaker_interval: 10s
redis:
endpoint: “”
master_name: “”
timeout: 500ms
expiration: 0s
db: 0
pool_size: 0
password: “”
tls_enabled: false
tls_insecure_skip_verify: false
idle_timeout: 0s
max_connection_age: 0s
fifocache:
max_size_bytes: “”
max_size_items: 0
validity: 0s
size: 0
prefix: frontend.
compression: “”
cache_results: false
max_retries: 5
parallelise_shardable_queries: false
runtime_config:
period: 10s
file: “”
memberlist:
node_name: “”
randomize_node_name: true
stream_timeout: 0s
retransmit_factor: 0
pull_push_interval: 0s
gossip_interval: 0s
gossip_nodes: 0
gossip_to_dead_nodes_time: 0s
dead_node_reclaim_time: 0s
join_members:
- mon-loki-headless:7946
min_join_backoff: 1s
max_join_backoff: 1m0s
max_join_retries: 10
abort_if_cluster_join_fails: true
rejoin_interval: 0s
left_ingesters_timeout: 5m0s
leave_timeout: 5s
message_history_buffer_bytes: 0
bind_addr:
bind_port: 7946
packet_dial_timeout: 5s
packet_write_timeout: 5s
tracing:
enabled: true
compactor:
working_directory: /data/loki/boltdb-shipper-compactor
shared_store: filesystem
compaction_interval: 2h0m0s