Hi,
I am using the following loki configuration and i am not able to hold data in loki for more than one hour, i have tried many configurations but none seem to help.
I am using the below api for ingesting data into loki:
http://machineip:http_listen_port/loki/api/v1/push
body used with the api for ingesting the data :-
- {“streams”: [{ “stream”: { “job”: “m”, “env”: “nf”, “tier”: “grafana-loki”, “ip”: “11.11.11.11”, “user”: “cad”,“timestamp”: “2023-05-23T05:04:58.846Z”, “action”: “GET”,“logsequence”: “7”, “path”: “/1986.js”, “protocol”: “HTTP/1.1”, “status_code”: “503”, “size”: “932”, “referer”: “Mozilla/5.0”, “useragent”: “(Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6” }, “values”: [ [“1690534804000000000”, “11.11.11.11 - steev [25/Jan/2000:14:00:01 -0500] ‘GET /1986.js HTTP/1.1’ 503 932 ‘-’ ‘Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6’” ] ] }]
Below is the yml configuration i am using:-
auth_enabled: false
server:
http_listen_port: 3200
grpc_listen_port: 9096
grpc_server_max_recv_msg_size: 8388608
grpc_server_max_send_msg_size: 8388608
querier:
engine:
max_look_back_period: 168h
ingester:
wal:
enabled: true
dir: /tmp/wal
lifecycler:
address: 10.10.10.10
ring:
kvstore:
store: inmemory
replication_factor: 1
final_sleep: 0s
chunk_idle_period: 24h # Any chunk not receiving new logs in this time will be flushed
max_chunk_age: 24h # All chunks will be flushed when they hit this age, default is 1h
chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
chunk_retain_period: 5m # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
max_transfer_retries: 0 # Chunk transfers disabled
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
storage_config:
boltdb_shipper:
active_index_directory: /tmp/loki/boltdb-shipper-active
cache_location: /tmp/loki/boltdb-shipper-cache
cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
shared_store: filesystem
filesystem:
directory: /tmp/loki/chunks
compactor:
working_directory: /tmp/loki/boltdb-shipper-compactor
shared_store: filesystem
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
max_streams_per_user: 1000000
max_entries_limit_per_query: 5000000
ingestion_rate_mb: 100
ingestion_burst_size_mb: 20
chunk_store_config:
max_look_back_period: 168h
table_manager:
retention_deletes_enabled: false
retention_period: 168h
Please give me suggestions if i am configuring it properly or if anyone has faced the same problem please help.