I’ve set up Loki and Promtail along with Grafana, and it worked a couple of times. However, now I’m having trouble establishing a connection between Grafana and Loki. According to the logs, Promtail is successfully collecting logs from the specified path, and Loki is running, but it’s not receiving logs from Promtail. Additionally, Grafana is unable to connect to Loki.
Loki Logs:
level=info ts=2024-08-23T14:32:01.3893898Z caller=checkpoint.go:611 msg=“starting checkpoint”
level=info ts=2024-08-23T14:32:01.3922289Z caller=checkpoint.go:336 msg=“attempting checkpoint for” dir=C:\Loki\wal\checkpoint.002356
level=info ts=2024-08-23T14:32:01.4003189Z caller=checkpoint.go:498 msg=“atomic checkpoint finished” old=C:\Loki\wal\checkpoint.002356.tmp new=C:\Loki\wal\checkpoint.002356
ts=2024-08-23T14:32:06.5091245Z caller=spanlogger.go:109 level=info msg=“building table names cache”
ts=2024-08-23T14:32:06.5091245Z caller=spanlogger.go:109 level=info msg=“table names cache built” duration=0s
level=info ts=2024-08-23T14:32:59.8994535Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:33:00.096712Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:33:59.8972353Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:34:00.0982098Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:34:59.8977306Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:35:00.0958497Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:35:39.0776218Z caller=roundtrip.go:343 org_id=fake traceID=637e5ba1c5cf14d7 msg=“executing query” type=labels label= length=10m0s query=
ts=2024-08-23T14:35:39.0826107Z caller=spanlogger.go:109 user=fake level=info org_id=fake traceID=637e5ba1c5cf14d7 caller=metrics.go latency=fast query_type=labels splits=0 start=2024-08-23T19:25:39.063+05:00 end=2024-08-23T19:35:39.063+05:00 start_delta=10m0.0196107s end_delta=19.6107ms length=10m0s duration=2.8008ms status=200 label= query= query_hash=2166136261 total_entries=0 cache_label_results_req=0 cache_label_results_hit=0 cache_label_results_stored=0 cache_label_results_download_time=0s cache_label_results_query_length_served=0s
level=info ts=2024-08-23T14:35:39.0841995Z caller=metrics.go:262 component=frontend org_id=fake traceID=637e5ba1c5cf14d7 latency=fast query_type=labels splits=0 start=2024-08-23T19:25:39.063+05:00 end=2024-08-23T19:35:39.063+05:00 start_delta=10m0.0211995s end_delta=21.1995ms length=10m0s duration=6.0735ms status=200 label= query= query_hash=2166136261 total_entries=0 cache_label_results_req=0 cache_label_results_hit=0 cache_label_results_stored=0 cache_label_results_download_time=0s cache_label_results_query_length_served=0s
level=info ts=2024-08-23T14:35:59.867227Z caller=roundtrip.go:343 org_id=fake traceID=573a7841bf1b5cf5 msg=“executing query” type=labels label= length=10m0s query=
ts=2024-08-23T14:35:59.8718278Z caller=spanlogger.go:109 user=fake level=info org_id=fake traceID=573a7841bf1b5cf5 caller=metrics.go latency=fast query_type=labels splits=0 start=2024-08-23T19:25:59.854+05:00 end=2024-08-23T19:35:59.854+05:00 start_delta=10m0.0178278s end_delta=17.8278ms length=10m0s duration=1.9959ms status=200 label= query= query_hash=2166136261 total_entries=0 cache_label_results_req=0 cache_label_results_hit=0 cache_label_results_stored=0 cache_label_results_download_time=0s cache_label_results_query_length_served=0s
level=info ts=2024-08-23T14:35:59.8735848Z caller=metrics.go:262 component=frontend org_id=fake traceID=573a7841bf1b5cf5 latency=fast query_type=labels splits=0 start=2024-08-23T19:25:59.854+05:00 end=2024-08-23T19:35:59.854+05:00 start_delta=10m0.0195848s end_delta=19.5848ms length=10m0s duration=5.7599ms status=200 label= query= query_hash=2166136261 total_entries=0 cache_label_results_req=0 cache_label_results_hit=0 cache_label_results_stored=0 cache_label_results_download_time=0s cache_label_results_query_length_served=0s
level=info ts=2024-08-23T14:35:59.8993051Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:36:00.0925781Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:36:59.9122155Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:37:00.0086828Z caller=table_manager.go:230 index-store=boltdb-shipper-2023-01-01 msg=“syncing tables”
ts=2024-08-23T14:37:00.0086828Z caller=spanlogger.go:109 level=info msg=“building table names cache”
ts=2024-08-23T14:37:00.0086828Z caller=spanlogger.go:109 level=info msg=“table names cache built” duration=0s
ts=2024-08-23T14:37:00.0086828Z caller=spanlogger.go:109 level=info msg=“building table names cache”
ts=2024-08-23T14:37:00.0086828Z caller=spanlogger.go:109 level=info msg=“table names cache built” duration=0s
level=info ts=2024-08-23T14:37:00.0086828Z caller=table_manager.go:273 index-store=boltdb-shipper-2023-01-01 msg=“query readiness setup completed” duration=0s distinct_users_len=0 distinct_users=
level=info ts=2024-08-23T14:37:00.0897964Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:37:01.3966489Z caller=checkpoint.go:611 msg=“starting checkpoint”
level=info ts=2024-08-23T14:37:01.399893Z caller=checkpoint.go:336 msg=“attempting checkpoint for” dir=C:\Loki\wal\checkpoint.002357
level=info ts=2024-08-23T14:37:01.4052825Z caller=checkpoint.go:498 msg=“atomic checkpoint finished” old=C:\Loki\wal\checkpoint.002357.tmp new=C:\Loki\wal\checkpoint.002357
level=info ts=2024-08-23T14:37:59.9201842Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:38:00.0923366Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:38:59.9038227Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:39:00.0982577Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:39:59.9048775Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:40:00.0940281Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:40:59.9116823Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:41:00.0965228Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:41:59.9114297Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:42:00.0301667Z caller=table_manager.go:230 index-store=boltdb-shipper-2023-01-01 msg=“syncing tables”
ts=2024-08-23T14:42:00.0301667Z caller=spanlogger.go:109 level=info msg=“building table names cache”
ts=2024-08-23T14:42:00.0301667Z caller=spanlogger.go:109 level=info msg=“table names cache built” duration=0s
ts=2024-08-23T14:42:00.0301667Z caller=spanlogger.go:109 level=info msg=“building table names cache”
ts=2024-08-23T14:42:00.0312403Z caller=spanlogger.go:109 level=info msg=“table names cache built” duration=1.0736ms
level=info ts=2024-08-23T14:42:00.0312403Z caller=table_manager.go:273 index-store=boltdb-shipper-2023-01-01 msg=“query readiness setup completed” duration=0s distinct_users_len=0 distinct_users=
level=info ts=2024-08-23T14:42:00.0982975Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:42:01.408897Z caller=checkpoint.go:611 msg=“starting checkpoint”
level=info ts=2024-08-23T14:42:01.408897Z caller=checkpoint.go:336 msg=“attempting checkpoint for” dir=C:\Loki\wal\checkpoint.002358
level=info ts=2024-08-23T14:42:01.408897Z caller=checkpoint.go:498 msg=“atomic checkpoint finished” old=C:\Loki\wal\checkpoint.002358.tmp new=C:\Loki\wal\checkpoint.002358
ts=2024-08-23T14:42:06.524313Z caller=spanlogger.go:109 level=info msg=“building table names cache”
ts=2024-08-23T14:42:06.5246896Z caller=spanlogger.go:109 level=info msg=“table names cache built” duration=125.4µs
level=info ts=2024-08-23T14:42:59.9149552Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:43:00.1001964Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
level=info ts=2024-08-23T14:43:59.9143538Z caller=table_manager.go:136 index-store=boltdb-shipper-2023-01-01 msg=“uploading tables”
level=info ts=2024-08-23T14:44:00.1021686Z caller=table_manager.go:186 index-store=boltdb-shipper-2023-01-01 msg=“handing over indexes to shipper”
Promtail logs:
level=info ts=2024-08-20T14:51:56.4297659Z caller=promtail.go:133 msg=“Reloading configuration file” md5sum=9d4c80cb031b25dd407d1aaaa047055f
level=info ts=2024-08-20T14:51:56.7901498Z caller=server.go:352 msg=“server listening on addresses” http=[::]:9080 grpc=[::]:9081
level=info ts=2024-08-20T14:51:56.8424921Z caller=main.go:173 msg=“Starting Promtail” version=“(version=3.1.1, branch=release-3.1.x, revision=89fe788d)”
level=warn ts=2024-08-20T14:51:56.8513093Z caller=promtail.go:263 msg=“enable watchConfig”
level=info ts=2024-08-20T14:52:01.8444219Z caller=filetargetmanager.go:372 msg=“Adding target” key=“C:\promtail\log\idwarelogs\\.log:{job="idwarelogs"}”
level=info ts=2024-08-20T14:52:02.0307624Z caller=filetarget.go:343 msg=“watching new directory” directory=C:\promtail\log\idwarelogs\Reception
level=info ts=2024-08-20T14:52:02.0307624Z caller=filetarget.go:343 msg=“watching new directory” directory=C:\promtail\log\idwarelogs\UMM
level=info ts=2024-08-20T14:52:02.0470555Z caller=filetarget.go:343 msg=“watching new directory” directory=“C:\promtail\log\idwarelogs\AM Portal”
level=info ts=2024-08-20T14:52:02.047753Z caller=filetarget.go:343 msg=“watching new directory” directory=C:\promtail\log\idwarelogs\OCMS
level=info ts=2024-08-20T14:52:02.0480771Z caller=filetarget.go:343 msg=“watching new directory” directory=C:\promtail\log\idwarelogs\Host
level=info ts=2024-08-20T14:52:02.0487397Z caller=filetarget.go:343 msg=“watching new directory” directory=“C:\promtail\log\idwarelogs\AM Admin”
level=info ts=2024-08-20T14:52:02.049416Z caller=filetarget.go:343 msg=“watching new directory” directory=“C:\promtail\log\idwarelogs\Visitor Portal”
level=info ts=2024-08-20T14:52:02.0500686Z caller=filetarget.go:343 msg=“watching new directory” directory=“C:\promtail\log\idwarelogs\OCMS webservice”
level=info ts=2024-08-20T14:52:02.0506016Z caller=filetarget.go:343 msg=“watching new directory” directory=“C:\promtail\log\idwarelogs\Configuation server”
level=info ts=2024-08-20T14:52:02.1161339Z caller=tailer.go:147 component=tailer msg=“tail routine: started” path=“C:\promtail\log\idwarelogs\AM Admin\AM_Admin.log”
level=info ts=2024-08-20T14:52:02.1825247Z caller=tailer.go:147 component=tailer msg=“tail routine: started” path=“C:\promtail\log\idwarelogs\AM Portal\AM_Admin.log”
ts=2024-08-20T14:52:02.1816589Z caller=log.go:168 level=info msg=“Seeked C:\promtail\log\idwarelogs\AM Admin\AM_Admin.log - &{Offset:26738335 Whence:0}”
ts=2024-08-20T14:52:02.2055394Z caller=log.go:168 level=info msg=“Seeked C:\promtail\log\idwarelogs\AM Portal\AM_Admin.log - &{Offset:6679038 Whence:0}”
ts=2024-08-20T14:52:02.2698671Z caller=log.go:168 level=info msg=“Seeked C:\promtail\log\idwarelogs\Configuation server\AM_Admin.log - &{Offset:21235530 Whence:0}”
level=info ts=2024-08-20T14:52:02.2698671Z caller=tailer.go:147 component=tailer msg=“tail routine: started” path=“C:\promtail\log\idwarelogs\Configuation server\AM_Admin.log”
ts=2024-08-20T14:52:02.3387306Z caller=log.go:168 level=info msg=“Seeked C:\promtail\log\idwarelogs\Host\HOST_SPA.log - &{Offset:141525 Whence:0}”
level=info ts=2024-08-20T14:52:02.3387306Z caller=tailer.go:147 component=tailer msg=“tail routine: started” path=C:\promtail\log\idwarelogs\Host\HOST_SPA.log
ts=2024-08-20T14:52:02.388465Z caller=log.go:168 level=info msg=“Seeked C:\promtail\log\idwarelogs\OCMS\OCMS.log - &{Offset:5732627 Whence:0}”
level=info ts=2024-08-20T14:52:02.388465Z caller=tailer.go:147 component=tailer msg=“tail routine: started” path=C:\promtail\log\idwarelogs\OCMS\OCMS.log
ts=2024-08-20T14:52:02.4419406Z caller=log.go:168 level=info msg=“Seeked C:\promtail\log\idwarelogs\OCMS webservice\OCMS.log - &{Offset:8650695 Whence:0}”
level=info ts=2024-08-20T14:52:02.4419406Z caller=tailer.go:147 component=tailer msg=“tail routine: started” path=“C:\promtail\log\idwarelogs\OCMS webservice\OCMS.log”
ts=2024-08-20T14:52:02.5116199Z caller=log.go:168 level=info msg=“Seeked C:\promtail\log\idwarelogs\Reception\RECEPTION_SPA.log - &{Offset:14708 Whence:0}”
level=info ts=2024-08-20T14:52:02.5116199Z caller=tailer.go:147 component=tailer msg=“tail routine: started” path=C:\promtail\log\idwarelogs\Reception\RECEPTION_SPA.log
ts=2024-08-20T14:52:02.5877533Z caller=log.go:168 level=info msg=“Seeked C:\promtail\log\idwarelogs\UMM\UMM-ADMIN.log - &{Offset:5453404 Whence:0}”
level=info ts=2024-08-20T14:52:02.5877533Z caller=tailer.go:147 component=tailer msg=“tail routine: started” path=C:\promtail\log\idwarelogs\UMM\UMM-ADMIN.log
ts=2024-08-20T14:52:02.7051015Z caller=log.go:168 level=info msg=“Seeked C:\promtail\log\idwarelogs\UMM\UMM_STS.log - &{Offset:8595687 Whence:0}”
level=info ts=2024-08-20T14:52:02.7051015Z caller=tailer.go:147 component=tailer msg=“tail routine: started” path=C:\promtail\log\idwarelogs\UMM\UMM_STS.log
level=info ts=2024-08-20T14:52:02.7304395Z caller=tailer.go:147 component=tailer msg=“tail routine: started” path=“C:\promtail\log\idwarelogs\Visitor Portal\VISITOR_SPA.log”
ts=2024-08-20T14:52:02.7307556Z caller=log.go:168 level=info msg=“Seeked C:\promtail\log\idwarelogs\Visitor Portal\VISITOR_SPA.log - &{Offset:3649 Whence:0}”
Loki Configurations:
auth_enabled: false
server:
http_listen_port: 3100 # Port where Loki will be accessible
grpc_listen_port: 9095 # gRPC port for communication between components (if needed)
common:
path_prefix: C:/Loki # Base path for all Loki data directories
storage:
filesystem:
chunks_directory: C:/Loki/chunks # Directory for storing log chunks
rules_directory: C:/Loki/rules # Directory for storing rules
replication_factor: 1 # Not needed for a single-instance deployment
ring:
kvstore:
store: inmemory # Using in-memory storage for ring; suitable for single-node setup
schema_config:
configs:
- from: 2023-01-01 # Date when this schema version is applied
store: boltdb-shipper # Using BoltDB Shipper
object_store: filesystem # Storing indexes and chunks on the local filesystem
schema: v11 # Schema version
index:
prefix: index_ # Index prefix for files
period: 24h # Indexing period, typically set to daily
storage_config:
boltdb_shipper:
active_index_directory: C:/Loki/index # Directory for active indexes
cache_location: C:/Loki/cache # Directory for BoltDB Shipper cache
filesystem:
directory: C:/Loki/index # Directory for filesystem index storage (used by object_store)
limits_config:
reject_old_samples: true # Reject old log entries beyond the max age
reject_old_samples_max_age: 168h # Maximum age for log entries
allow_structured_metadata: false # Disable structured metadata
table_manager:
retention_deletes_enabled: false # Enable deletion of old data (retention)
retention_period: 0s # Retention period; set to 0s to keep all data indefinitely
Promtail Config:
server:
http_listen_port: 9080
grpc_listen_port: 9081
positions:
filename: C:\promtail\log\promtail.positions.yaml
clients:
- url: http://localhost:3100/loki/api/v1/push
timeout: 30s
scrape_configs:
-
job_name: idware_logs
static_configs:- targets:
- localhost
labels:
job: idwarelogs
path: C:\promtail\log\idwarelogs**.log
- localhost
- targets:
- localhost
labels:
job: idwarelogs
path: C:\promtail\log\idwarelogs**.log
- localhost
pipeline_stages:
- regex:
expression: ‘^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+|(?P[^|]+)|(?P[^|]+)|.*$’ - labels:
loglevel:
appname:
- targets:
Error on Grafana :
Data source connected, but no labels were received. Verify that Loki and Promtail are correctly configured.
Could someone please help me identify where I might be going wrong and advise on how to fix this issue?