I am facing issues one of the ingester is crash and cannot spin up again. The error log is as follows (below after this post).
The first line said that it error recovering from TSDB WAL, how can I gracefully restart this pod? Because when I delete and run it again, it produce the same error log.
Is it safe to delete the PVC then restart the pod again?
corruption in segment /var/loki/index/wal/alibabacloud_2020-05-15/1697908069/00000000 at 20538: unexpected checksum 4aaaa698, expected 7ffa1b2b
error recovering from TSDB WAL
github.com/grafana/loki/pkg/storage/stores/tsdb.recoverHead
/src/loki/pkg/storage/stores/tsdb/head_manager.go:594
github.com/grafana/loki/pkg/storage/stores/tsdb.(*tsdbManager).BuildFromWALs
/src/loki/pkg/storage/stores/tsdb/manager.go:289
github.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).buildTSDBFromWALs
/src/loki/pkg/storage/stores/tsdb/head_manager.go:308
github.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).Start
/src/loki/pkg/storage/stores/tsdb/head_manager.go:274
github.com/grafana/loki/pkg/storage/stores/tsdb.(*store).init
/src/loki/pkg/storage/stores/tsdb/store.go:141
github.com/grafana/loki/pkg/storage/stores/tsdb.NewStore
/src/loki/pkg/storage/stores/tsdb/store.go:68
github.com/grafana/loki/pkg/storage.(*store).storeForPeriod
/src/loki/pkg/storage/store.go:274
github.com/grafana/loki/pkg/storage.(*store).init
/src/loki/pkg/storage/store.go:177
github.com/grafana/loki/pkg/storage.NewStore
/src/loki/pkg/storage/store.go:155
github.com/grafana/loki/pkg/loki.(*Loki).initStore
/src/loki/pkg/loki/modules.go:689
github.com/grafana/dskit/modules.(*Manager).initModule
/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:136
github.com/grafana/dskit/modules.(*Manager).InitModuleServices
/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108
github.com/grafana/loki/pkg/loki.(*Loki).Run
/src/loki/pkg/loki/loki.go:461
main.main
/src/loki/cmd/loki/main.go:110
runtime.main
/usr/local/go/src/runtime/proc.go:250
runtime.goexit
/usr/local/go/src/runtime/asm_amd64.s:1598
building TSDB from WALs
github.com/grafana/loki/pkg/storage/stores/tsdb.(*tsdbManager).BuildFromWALs
/src/loki/pkg/storage/stores/tsdb/manager.go:290
github.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).buildTSDBFromWALs
/src/loki/pkg/storage/stores/tsdb/head_manager.go:308
github.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).Start
/src/loki/pkg/storage/stores/tsdb/head_manager.go:274
github.com/grafana/loki/pkg/storage/stores/tsdb.(*store).init
/src/loki/pkg/storage/stores/tsdb/store.go:141
github.com/grafana/loki/pkg/storage/stores/tsdb.NewStore
/src/loki/pkg/storage/stores/tsdb/store.go:68
github.com/grafana/loki/pkg/storage.(*store).storeForPeriod
/src/loki/pkg/storage/store.go:274
github.com/grafana/loki/pkg/storage.(*store).init
/src/loki/pkg/storage/store.go:177
github.com/grafana/loki/pkg/storage.NewStore
/src/loki/pkg/storage/store.go:155
github.com/grafana/loki/pkg/loki.(*Loki).initStore
/src/loki/pkg/loki/modules.go:689
github.com/grafana/dskit/modules.(*Manager).initModule
/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:136
github.com/grafana/dskit/modules.(*Manager).InitModuleServices
/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108
github.com/grafana/loki/pkg/loki.(*Loki).Run
/src/loki/pkg/loki/loki.go:461
main.main
/src/loki/cmd/loki/main.go:110
runtime.main
/usr/local/go/src/runtime/proc.go:250
runtime.goexit
/usr/local/go/src/runtime/asm_amd64.s:1598
building tsdb from WALs
github.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).buildTSDBFromWALs
/src/loki/pkg/storage/stores/tsdb/head_manager.go:313
github.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).Start
/src/loki/pkg/storage/stores/tsdb/head_manager.go:274
github.com/grafana/loki/pkg/storage/stores/tsdb.(*store).init
/src/loki/pkg/storage/stores/tsdb/store.go:141
github.com/grafana/loki/pkg/storage/stores/tsdb.NewStore
/src/loki/pkg/storage/stores/tsdb/store.go:68
github.com/grafana/loki/pkg/storage.(*store).storeForPeriod
/src/loki/pkg/storage/store.go:274
github.com/grafana/loki/pkg/storage.(*store).init
/src/loki/pkg/storage/store.go:177
github.com/grafana/loki/pkg/storage.NewStore
/src/loki/pkg/storage/store.go:155
github.com/grafana/loki/pkg/loki.(*Loki).initStore
/src/loki/pkg/loki/modules.go:689
github.com/grafana/dskit/modules.(*Manager).initModule
/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:136
github.com/grafana/dskit/modules.(*Manager).InitModuleServices
/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108
github.com/grafana/loki/pkg/loki.(*Loki).Run
/src/loki/pkg/loki/loki.go:461
main.main
/src/loki/cmd/loki/main.go:110
runtime.main
/usr/local/go/src/runtime/proc.go:250
runtime.goexit
/usr/local/go/src/runtime/asm_amd64.s:1598
building tsdb from old WAL files
github.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).Start
/src/loki/pkg/storage/stores/tsdb/head_manager.go:275
github.com/grafana/loki/pkg/storage/stores/tsdb.(*store).init
/src/loki/pkg/storage/stores/tsdb/store.go:141
github.com/grafana/loki/pkg/storage/stores/tsdb.NewStore
/src/loki/pkg/storage/stores/tsdb/store.go:68
github.com/grafana/loki/pkg/storage.(*store).storeForPeriod
/src/loki/pkg/storage/store.go:274
github.com/grafana/loki/pkg/storage.(*store).init
/src/loki/pkg/storage/store.go:177
github.com/grafana/loki/pkg/storage.NewStore
/src/loki/pkg/storage/store.go:155
github.com/grafana/loki/pkg/loki.(*Loki).initStore
/src/loki/pkg/loki/modules.go:689
github.com/grafana/dskit/modules.(*Manager).initModule
/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:136
github.com/grafana/dskit/modules.(*Manager).InitModuleServices
/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108
github.com/grafana/loki/pkg/loki.(*Loki).Run
/src/loki/pkg/loki/loki.go:461
main.main
/src/loki/cmd/loki/main.go:110
runtime.main
/usr/local/go/src/runtime/proc.go:250
runtime.goexit
/usr/local/go/src/runtime/asm_amd64.s:1598
error initialising module: store
github.com/grafana/dskit/modules.(*Manager).initModule
/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:138
github.com/grafana/dskit/modules.(*Manager).InitModuleServices
/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108
github.com/grafana/loki/pkg/loki.(*Loki).Run
/src/loki/pkg/loki/loki.go:461
main.main
/src/loki/cmd/loki/main.go:110
runtime.main
/usr/local/go/src/runtime/proc.go:250
runtime.goexit
/usr/local/go/src/runtime/asm_amd64.s:1598
level=warn ts=2023-10-25T08:20:44.696738115Z caller=loki.go:288 msg="global timeout not configured, using default engine timeout (\"5m0s\"). This behavior will change in the next major to always use the default global timeout (\"5m\")."
level=info ts=2023-10-25T08:20:44.698449919Z caller=main.go:108 msg="Starting Loki" version="(version=2.9.1, branch=HEAD, revision=d9d5ed4a1)"
level=info ts=2023-10-25T08:20:44.698864944Z caller=server.go:322 http=[::]:3100 grpc=[::]:9095 msg="server listening on addresses"
level=info ts=2023-10-25T08:20:44.699784566Z caller=memberlist_client.go:434 msg="Using memberlist cluster label and node name" cluster_label= node=loki-loki-distributed-ingester-3-b152212e
ts=2023-10-25T08:20:44.699838717Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=warn ts=2023-10-25T08:20:44.699986658Z caller=experimental.go:20 msg="experimental feature in use" feature="In-memory (FIFO) cache - embedded-cache"
level=warn ts=2023-10-25T08:20:44.700060689Z caller=cache.go:127 msg="fifocache config is deprecated. use embedded-cache instead"
level=warn ts=2023-10-25T08:20:44.700073881Z caller=experimental.go:20 msg="experimental feature in use" feature="In-memory (FIFO) cache - chunksembedded-cache"
level=info ts=2023-10-25T08:20:44.700281598Z caller=shipper.go:165 index-store=tsdb-2020-05-15 msg="starting index shipper in WO mode"
level=debug ts=2023-10-25T08:20:44.700347875Z caller=tcp_transport.go:402 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.121.24.18 advertisePort=7946
level=info ts=2023-10-25T08:20:44.700444978Z caller=table_manager.go:136 index-store=tsdb-2020-05-15 msg="uploading tables"
level=info ts=2023-10-25T08:20:44.700523929Z caller=head_manager.go:299 index-store=tsdb-2020-05-15 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2023-10-25T08:20:44.700539418Z caller=manager.go:262 index-store=tsdb-2020-05-15 component=tsdb-manager msg="building WALs" n=0 ts=2023-10-25T08:20:44.700537905Z
level=debug ts=2023-10-25T08:20:44.700548124Z caller=manager.go:286 index-store=tsdb-2020-05-15 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2023-10-25T08:20:44.700970504Z caller=tcp_transport.go:402 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.121.24.18 advertisePort=7946
level=info ts=2023-10-25T08:20:44.701086312Z caller=memberlist_client.go:540 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2023-10-25T08:20:44.703742329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.121.16.84:7946"
ts=2023-10-25T08:20:44.706375546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.121.16.95:7946"
ts=2023-10-25T08:20:44.708960665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.121.16.11:7946"
ts=2023-10-25T08:20:44.711491039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.121.8.60:7946"
ts=2023-10-25T08:20:44.713512889Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.121.24.38:7946: dial tcp 10.121.24.38:7946: connect: connection refused"
ts=2023-10-25T08:20:44.714608079Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.121.8.102:7946: dial tcp 10.121.8.102:7946: connect: connection refused"
ts=2023-10-25T08:20:44.714894856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.121.24.78:7946"
ts=2023-10-25T08:20:44.716477297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.121.16.92:7946"
ts=2023-10-25T08:20:44.721547331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.121.8.154:7946"
ts=2023-10-25T08:20:44.723425734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.121.24.34:7946"
ts=2023-10-25T08:20:44.725168384Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.121.16.99:7946: dial tcp 10.121.16.99:7946: connect: connection refused"
ts=2023-10-25T08:20:44.726344948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.121.8.59:7946"
level=info ts=2023-10-25T08:20:44.728299778Z caller=manager.go:85 index-store=tsdb-2020-05-15 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=5 indices=238 failures=0
level=info ts=2023-10-25T08:20:44.728338232Z caller=head_manager.go:299 index-store=tsdb-2020-05-15 component=tsdb-head-manager msg="loaded wals by period" groups=1
level=debug ts=2023-10-25T08:20:44.728346614Z caller=manager.go:262 index-store=tsdb-2020-05-15 component=tsdb-manager msg="building WALs" n=1 ts=2023-10-25T08:20:44.7283453Z
level=debug ts=2023-10-25T08:20:44.728353141Z caller=manager.go:286 index-store=tsdb-2020-05-15 component=tsdb-manager msg="recovering tenant heads"
ts=2023-10-25T08:20:44.728470976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: 10.121.24.101:7946"
level=error ts=2023-10-25T08:20:44.729690428Z caller=log.go:230 msg="error running loki" err="corruption in segment /var/loki/index/wal/alibabacloud_2020-05-15/1697908069/00000000 at 20538: unexpected checksum 4aaaa698, expected 7ffa1b2b\nerror recovering from TSDB WAL\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.recoverHead\n\t/src/loki/pkg/storage/stores/tsdb/head_manager.go:594\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*tsdbManager).BuildFromWALs\n\t/src/loki/pkg/storage/stores/tsdb/manager.go:289\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).buildTSDBFromWALs\n\t/src/loki/pkg/storage/stores/tsdb/head_manager.go:308\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).Start\n\t/src/loki/pkg/storage/stores/tsdb/head_manager.go:274\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*store).init\n\t/src/loki/pkg/storage/stores/tsdb/store.go:141\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.NewStore\n\t/src/loki/pkg/storage/stores/tsdb/store.go:68\ngithub.com/grafana/loki/pkg/storage.(*store).storeForPeriod\n\t/src/loki/pkg/storage/store.go:274\ngithub.com/grafana/loki/pkg/storage.(*store).init\n\t/src/loki/pkg/storage/store.go:177\ngithub.com/grafana/loki/pkg/storage.NewStore\n\t/src/loki/pkg/storage/store.go:155\ngithub.com/grafana/loki/pkg/loki.(*Loki).initStore\n\t/src/loki/pkg/loki/modules.go:689\ngithub.com/grafana/dskit/modules.(*Manager).initModule\n\t/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:136\ngithub.com/grafana/dskit/modules.(*Manager).InitModuleServices\n\t/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108\ngithub.com/grafana/loki/pkg/loki.(*Loki).Run\n\t/src/loki/pkg/loki/loki.go:461\nmain.main\n\t/src/loki/cmd/loki/main.go:110\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1598\nbuilding TSDB from WALs\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*tsdbManager).BuildFromWALs\n\t/src/loki/pkg/storage/stores/tsdb/manager.go:290\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).buildTSDBFromWALs\n\t/src/loki/pkg/storage/stores/tsdb/head_manager.go:308\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).Start\n\t/src/loki/pkg/storage/stores/tsdb/head_manager.go:274\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*store).init\n\t/src/loki/pkg/storage/stores/tsdb/store.go:141\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.NewStore\n\t/src/loki/pkg/storage/stores/tsdb/store.go:68\ngithub.com/grafana/loki/pkg/storage.(*store).storeForPeriod\n\t/src/loki/pkg/storage/store.go:274\ngithub.com/grafana/loki/pkg/storage.(*store).init\n\t/src/loki/pkg/storage/store.go:177\ngithub.com/grafana/loki/pkg/storage.NewStore\n\t/src/loki/pkg/storage/store.go:155\ngithub.com/grafana/loki/pkg/loki.(*Loki).initStore\n\t/src/loki/pkg/loki/modules.go:689\ngithub.com/grafana/dskit/modules.(*Manager).initModule\n\t/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:136\ngithub.com/grafana/dskit/modules.(*Manager).InitModuleServices\n\t/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108\ngithub.com/grafana/loki/pkg/loki.(*Loki).Run\n\t/src/loki/pkg/loki/loki.go:461\nmain.main\n\t/src/loki/cmd/loki/main.go:110\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1598\nbuilding tsdb from WALs\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).buildTSDBFromWALs\n\t/src/loki/pkg/storage/stores/tsdb/head_manager.go:313\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).Start\n\t/src/loki/pkg/storage/stores/tsdb/head_manager.go:274\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*store).init\n\t/src/loki/pkg/storage/stores/tsdb/store.go:141\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.NewStore\n\t/src/loki/pkg/storage/stores/tsdb/store.go:68\ngithub.com/grafana/loki/pkg/storage.(*store).storeForPeriod\n\t/src/loki/pkg/storage/store.go:274\ngithub.com/grafana/loki/pkg/storage.(*store).init\n\t/src/loki/pkg/storage/store.go:177\ngithub.com/grafana/loki/pkg/storage.NewStore\n\t/src/loki/pkg/storage/store.go:155\ngithub.com/grafana/loki/pkg/loki.(*Loki).initStore\n\t/src/loki/pkg/loki/modules.go:689\ngithub.com/grafana/dskit/modules.(*Manager).initModule\n\t/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:136\ngithub.com/grafana/dskit/modules.(*Manager).InitModuleServices\n\t/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108\ngithub.com/grafana/loki/pkg/loki.(*Loki).Run\n\t/src/loki/pkg/loki/loki.go:461\nmain.main\n\t/src/loki/cmd/loki/main.go:110\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1598\nbuilding tsdb from old WAL files\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*HeadManager).Start\n\t/src/loki/pkg/storage/stores/tsdb/head_manager.go:275\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.(*store).init\n\t/src/loki/pkg/storage/stores/tsdb/store.go:141\ngithub.com/grafana/loki/pkg/storage/stores/tsdb.NewStore\n\t/src/loki/pkg/storage/stores/tsdb/store.go:68\ngithub.com/grafana/loki/pkg/storage.(*store).storeForPeriod\n\t/src/loki/pkg/storage/store.go:274\ngithub.com/grafana/loki/pkg/storage.(*store).init\n\t/src/loki/pkg/storage/store.go:177\ngithub.com/grafana/loki/pkg/storage.NewStore\n\t/src/loki/pkg/storage/store.go:155\ngithub.com/grafana/loki/pkg/loki.(*Loki).initStore\n\t/src/loki/pkg/loki/modules.go:689\ngithub.com/grafana/dskit/modules.(*Manager).initModule\n\t/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:136\ngithub.com/grafana/dskit/modules.(*Manager).InitModuleServices\n\t/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108\ngithub.com/grafana/loki/pkg/loki.(*Loki).Run\n\t/src/loki/pkg/loki/loki.go:461\nmain.main\n\t/src/loki/cmd/loki/main.go:110\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1598\nerror initialising module: store\ngithub.com/grafana/dskit/modules.(*Manager).initModule\n\t/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:138\ngithub.com/grafana/dskit/modules.(*Manager).InitModuleServices\n\t/src/loki/vendor/github.com/grafana/dskit/modules/modules.go:108\ngithub.com/grafana/loki/pkg/loki.(*Loki).Run\n\t/src/loki/pkg/loki/loki.go:461\nmain.main\n\t/src/loki/cmd/loki/main.go:110\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1598"