I upgraded Grafana from version 8.1.8 to 8.5.20 and after I did, I started seeing these errors in the logs about rolling up dashboards:
logger=analytics.summaries t=2023-02-09T21:41:06.67+0000 lvl=eror msg="error during daily rollup" err="context deadline exceeded"
logger=analytics.summaries t=2023-02-09T21:41:06.67+0000 lvl=eror msg="got error while rolling up dashboard" dashboard=8928
There isn’t any other information in the logs and I’m not sure why this is occurring. What does it mean to rollup a dashboard and how would I resolve this error?
1 Like
Hi @ngalagali,
Thanks for opening this post.
I will try to reproduce your case, can you please provide me with some of the data sources where you getting this message so that I can try to configure and reproduce it.
I have a same issues.
I use datasource for prometheus, loki and tempo.
this is my grafana server configs:
app_mode = production
instance_name = ${HOSTNAME}
force_migration = false
[paths]
data = /mon/data/grafana_server
temp_data_lifetime = 24h
logs = /mon/app/grafana_server/logs
plugins = /mon/app/grafana_server/provisioning/plugins
provisioning = /mon/app/grafana_server/provisioning
[server]
protocol = https
http_port = 3030
domain = 10.157.135.235
enforce_domain = false
root_url = %(protocol)s://%(domain)s:3000/
serve_from_sub_path = false
router_logging = false
static_root_path = public
enable_gzip = false
cert_file = /mon/app/grafana_server/certs/server.pem
cert_key = /mon/app/grafana_server/certs/server.key
read_timeout = 5m
[database]
type = postgres
host = 10.170.134.28:5432
name = pgdb_grafana_server
user = $__file{/mon/app/grafana_server/id}
password = $__file{/mon/app/grafana_server/pass}
ssl_mode = disable
max_idle_conn = 2
max_open_conn = 1000
conn_max_lifetime = 14400
[datasources]
datasource_limit = 5000
[remote_cache]
type = database
[dataproxy]
logging = false
timeout = 240
dialTimeout = 10
keep_alive_seconds = 240
tls_handshake_timeout_seconds = 10
expect_continue_timeout_seconds = 5
max_conns_per_host = 1000
max_idle_connections = 100
idle_conn_timeout_seconds = 300
send_user_header = false
response_limit = 0
row_limit = 1000000
[analytics]
reporting_enabled = false
check_for_updates = false
check_for_plugin_updates = false
[security]
disable_initial_admin_creation = false
disable_gravatar = false
cookie_secure = false
cookie_samesite = lax
allow_embedding = true
strict_transport_security = false
strict_transport_security_max_age_seconds = 86400
strict_transport_security_preload = false
strict_transport_security_subdomains = false
x_content_type_options = true
x_xss_protection = true
content_security_policy = false
content_security_policy_template = """script-src 'self' 'unsafe-eval' 'unsafe-inline' 'strict-dynamic' $NONCE;object-src 'none';font-src 'self';style-src 'self' 'unsafe-inline' blob:;img-src * data:;base-uri 'self';connect-src 'self' grafana.com ws://$ROOT_PATH wss://$ROOT_PATH;manifest-src 'self';media-src 'none';form-action 'self';"""
angular_support_enabled = true
[security.encryption]
data_keys_cache_ttl = 15m
data_keys_cache_cleanup_interval = 1m
[snapshots]
external_enabled = false
[dashboards]
versions_to_keep = 20
min_refresh_interval = 5s
default_home_dashboard_path =
[users]
allow_sign_up = false
allow_org_create = false
auto_assign_org = false
verify_email_enabled = false
login_hint = email or username
password_hint = password
default_theme = dark
home_page =
viewers_can_edit = true
editors_can_admin = false
user_invite_max_lifetime_duration = 24h
hidden_users =
[auth]
login_cookie_name = grafana_session
login_maximum_inactive_lifetime_duration = 7d
login_maximum_lifetime_duration = 30d
token_rotation_interval_minutes = 10
disable_login_form = false
disable_signout_menu = false
oauth_auto_login = false
api_key_max_seconds_to_live = -1
[auth.anonymous]
enabled = false
[auth.github]
enabled = false
[auth.gitlab]
enabled = false
[auth.google]
enabled = false
[auth.grafana_com]
enabled = false
[auth.azuread]
enabled = false
[auth.okta]
enabled = false
[auth.generic_oauth]
enabled = false
[auth.basic]
enabled = false
[auth.proxy]
enabled = false
[auth.jwt]
enabled = false
[auth.ldap]
enabled = true
config_file = /mon/app/grafana_server/conf/ldap.toml
allow_sign_up = true
[aws]
[azure]
[smtp]
enabled = true
host = mailhost.devcloud.mclocal.int:25
user =
password =
cert_file =
key_:file =
skip_verify = true
from_address = admin@grafana.localhost
from_name = Grafana
ehlo_identity = dashboard.example.com
startTLS_policy = NoStartTLS
[emails]
welcome_email_on_sign_up = false
[log]
mode = console file
level = info
[log.console]
level =
format = console
[log.file]
level = info
format = text
log_rotate = true
max_lines = 1000000
max_size_shift = 28
daily_rotate = true
max_days = 7
[log.syslog]
level =
format = text
network =
address =
facility =
tag =
[log.frontend]
enabled = false
[quota]
enabled = true
org_user = 10
org_dashboard = 100
org_data_source = 10
org_api_key = 10
org_alert_rule = 100
user_org = 10
global_user = -1
global_org = -1
global_dashboard = -1
global_api_key = -1
global_session = -1
global_alert_rule = -1
#################################### Unified Alerting ####################
[unified_alerting]
enabled = true
admin_config_poll_interval = 30s
alertmanager_config_poll_interval = 30s
ha_listen_address = "0.0.0.0:9094"
ha_advertise_address = "0.0.0.0:9094"
ha_peers = "10.157.135.235:9094,10.157.135.230:9094,10.168.136.165:9094,10.168.136.168:9094"
ha_peer_timeout = "15s"
ha_gossip_interval = "200ms"
ha_push_pull_interval = "60s"
execute_alerts = true
evaluation_timeout = 30s
max_attempts = 3
min_interval = 10s
#################################### Alerting ############################
[alerting]
enabled = false
[annotations]
cleanupjob_batchsize = 100
max_annotation_age = 1M
#max_annotations_to_keep = 3000
[annotations.dashboard]
max_age = 1M
#max_annotations_to_keep = 3000
[annotations.api]
max_age = 1M
#max_annotations_to_keep = 3000
[explore]
enabled = true
[help]
enabled = true
[profile]
enabled = true
[query_history]
enabled = false
[metrics]
enabled = true
interval_seconds = 30
disable_total_stats = false
[metrics.environment_info]
[metrics.graphite]
[grafana_com]
url = https://grafana.iemon.mastercard.int
[tracing.jaeger]
[tracing.opentelemetry.jaeger]
[external_image_storage]
[external_image_storage.s3]
[external_image_storage.webdav]
[external_image_storage.gcs]
[external_image_storage.azure_blob]
[external_image_storage.local]
[rendering]
[panels]
disable_sanitize_html = false
[plugins]
enable_alpha = true
app_tls_skip_verify_insecure = true
plugin_admin_enabled = true
plugin_admin_external_manage_enabled = true
plugin_catalog_url = https://grafana.com/grafana/plugins/
[live]
max_connections = 100
[plugin.grafana-image-renderer]
[enterprise]
[feature_toggles]
[date_formats]
full_date = YYYY-MM-DD HH:mm:ss
interval_second = HH:mm:ss
interval_minute = HH:mm
interval_hour = MM/DD HH:mm
interval_day = MM/DD
interval_month = YYYY-MM
interval_year = YYYY
default_timezone = browser
[expressions]
enabled = true
[geomap]
enable_custom_baselayers = true
[plugin.marcusolsson-csv-datasource]
allow_local_mode = true
I need your help.
I don’t know the impact. but the log is error level.
