Grafana send alarm email,not image

Grafana Alerting
1

png screenshot can be generated by sharing, but there is no image in the alarm email. and grafana log is norma

root@ubuntu-zabbix:~# grafana-cli plugins ls
installed plugins:
alexanderzobnin-zabbix-app @ 4.2.10
grafana-image-renderer @ 3.6.4

root@ubuntu-zabbix:~# ldd /var/lib/grafana/plugins/grafana-image-renderer/chrome-linux/chrome
linux-vdso.so.1 (0x00007ffd1646a000)
libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f550a981000)
libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f550a762000)
libgobject-2.0.so.0 => /usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0 (0x00007f550a50e000)
libglib-2.0.so.0 => /usr/lib/x86_64-linux-gnu/libglib-2.0.so.0 (0x00007f550a1f7000)
libnss3.so => /usr/lib/x86_64-linux-gnu/libnss3.so (0x00007f5509eb3000)
libnssutil3.so => /usr/lib/x86_64-linux-gnu/libnssutil3.so (0x00007f5509c84000)
libsmime3.so => /usr/lib/x86_64-linux-gnu/libsmime3.so (0x00007f5509a57000)
libnspr4.so => /usr/lib/x86_64-linux-gnu/libnspr4.so (0x00007f550981a000)
libatk-1.0.so.0 => /usr/lib/x86_64-linux-gnu/libatk-1.0.so.0 (0x00007f55095f4000)
libatk-bridge-2.0.so.0 => /usr/lib/x86_64-linux-gnu/libatk-bridge-2.0.so.0 (0x00007f55093c30 00)
libcups.so.2 => /usr/lib/x86_64-linux-gnu/libcups.so.2 (0x00007f5509137000)
libgio-2.0.so.0 => /usr/lib/x86_64-linux-gnu/libgio-2.0.so.0 (0x00007f5508d98000)
libdrm.so.2 => /usr/lib/x86_64-linux-gnu/libdrm.so.2 (0x00007f5508b87000)
libdbus-1.so.3 => /lib/x86_64-linux-gnu/libdbus-1.so.3 (0x00007f550893a000)
libexpat.so.1 => /lib/x86_64-linux-gnu/libexpat.so.1 (0x00007f5508708000)
libxcb.so.1 => /usr/lib/x86_64-linux-gnu/libxcb.so.1 (0x00007f55084e0000)
libxkbcommon.so.0 => /usr/lib/x86_64-linux-gnu/libxkbcommon.so.0 (0x00007f55082a1000)
libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f5507f03000)
libX11.so.6 => /usr/lib/x86_64-linux-gnu/libX11.so.6 (0x00007f5507bcb000)
libXcomposite.so.1 => /usr/lib/x86_64-linux-gnu/libXcomposite.so.1 (0x00007f55079c8000)
libXdamage.so.1 => /usr/lib/x86_64-linux-gnu/libXdamage.so.1 (0x00007f55077c5000)
libXext.so.6 => /usr/lib/x86_64-linux-gnu/libXext.so.6 (0x00007f55075b3000)
libXfixes.so.3 => /usr/lib/x86_64-linux-gnu/libXfixes.so.3 (0x00007f55073ad000)
libXrandr.so.2 => /usr/lib/x86_64-linux-gnu/libXrandr.so.2 (0x00007f55071a2000)
libgbm.so.1 => /usr/lib/x86_64-linux-gnu/libgbm.so.1 (0x00007f5506f93000)
libpango-1.0.so.0 => /usr/lib/x86_64-linux-gnu/libpango-1.0.so.0 (0x00007f5506d46000)
libcairo.so.2 => /usr/lib/x86_64-linux-gnu/libcairo.so.2 (0x00007f5506a29000)
libasound.so.2 => /usr/lib/x86_64-linux-gnu/libasound.so.2 (0x00007f5506722000)
libatspi.so.0 => /usr/lib/x86_64-linux-gnu/libatspi.so.0 (0x00007f55064f2000)
libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f55062da000)
libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f5505ee9000)
/lib64/ld-linux-x86-64.so.2 (0x00007f55184f1000)
libffi.so.6 => /usr/lib/x86_64-linux-gnu/libffi.so.6 (0x00007f5505ce1000)
libpcre.so.3 => /lib/x86_64-linux-gnu/libpcre.so.3 (0x00007f5505a70000)
libplc4.so => /usr/lib/x86_64-linux-gnu/libplc4.so (0x00007f550586b000)
libplds4.so => /usr/lib/x86_64-linux-gnu/libplds4.so (0x00007f5505667000)
librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007f550545f000)
libgssapi_krb5.so.2 => /usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2 (0x00007f5505214000)
libgnutls.so.30 => /usr/lib/x86_64-linux-gnu/libgnutls.so.30 (0x00007f5504eae000)
libavahi-common.so.3 => /usr/lib/x86_64-linux-gnu/libavahi-common.so.3 (0x00007f5504ca2000)
libavahi-client.so.3 => /usr/lib/x86_64-linux-gnu/libavahi-client.so.3 (0x00007f5504a91000)
libz.so.1 => /lib/x86_64-linux-gnu/libz.so.1 (0x00007f5504874000)
libgmodule-2.0.so.0 => /usr/lib/x86_64-linux-gnu/libgmodule-2.0.so.0 (0x00007f5504670000)
libselinux.so.1 => /lib/x86_64-linux-gnu/libselinux.so.1 (0x00007f5504448000)
libresolv.so.2 => /lib/x86_64-linux-gnu/libresolv.so.2 (0x00007f550422e000)
libmount.so.1 => /lib/x86_64-linux-gnu/libmount.so.1 (0x00007f5503fda000)
libsystemd.so.0 => /lib/x86_64-linux-gnu/libsystemd.so.0 (0x00007f5503d56000)
libXau.so.6 => /usr/lib/x86_64-linux-gnu/libXau.so.6 (0x00007f5503b52000)
libXdmcp.so.6 => /usr/lib/x86_64-linux-gnu/libXdmcp.so.6 (0x00007f550394c000)
libXrender.so.1 => /usr/lib/x86_64-linux-gnu/libXrender.so.1 (0x00007f5503742000)
libwayland-server.so.0 => /usr/lib/x86_64-linux-gnu/libwayland-server.so.0 (0x00007f550352f0 00)
libthai.so.0 => /usr/lib/x86_64-linux-gnu/libthai.so.0 (0x00007f5503326000)
libpixman-1.so.0 => /usr/lib/x86_64-linux-gnu/libpixman-1.so.0 (0x00007f5503081000)
libfontconfig.so.1 => /usr/lib/x86_64-linux-gnu/libfontconfig.so.1 (0x00007f5502e3c000)
libfreetype.so.6 => /usr/lib/x86_64-linux-gnu/libfreetype.so.6 (0x00007f5502b88000)
libpng16.so.16 => /usr/lib/x86_64-linux-gnu/libpng16.so.16 (0x00007f5502956000)
libxcb-shm.so.0 => /usr/lib/x86_64-linux-gnu/libxcb-shm.so.0 (0x00007f5502753000)
libxcb-render.so.0 => /usr/lib/x86_64-linux-gnu/libxcb-render.so.0 (0x00007f5502546000)
libkrb5.so.3 => /usr/lib/x86_64-linux-gnu/libkrb5.so.3 (0x00007f5502270000)
libk5crypto.so.3 => /usr/lib/x86_64-linux-gnu/libk5crypto.so.3 (0x00007f550203e000)
libcom_err.so.2 => /lib/x86_64-linux-gnu/libcom_err.so.2 (0x00007f5501e3a000)
libkrb5support.so.0 => /usr/lib/x86_64-linux-gnu/libkrb5support.so.0 (0x00007f5501c2f000)
libp11-kit.so.0 => /usr/lib/x86_64-linux-gnu/libp11-kit.so.0 (0x00007f5501900000)
libidn2.so.0 => /usr/lib/x86_64-linux-gnu/libidn2.so.0 (0x00007f55016e3000)
libunistring.so.2 => /usr/lib/x86_64-linux-gnu/libunistring.so.2 (0x00007f5501365000)
libtasn1.so.6 => /usr/lib/x86_64-linux-gnu/libtasn1.so.6 (0x00007f5501152000)
libnettle.so.6 => /usr/lib/x86_64-linux-gnu/libnettle.so.6 (0x00007f5500f1c000)
libhogweed.so.4 => /usr/lib/x86_64-linux-gnu/libhogweed.so.4 (0x00007f5500ce6000)
libgmp.so.10 => /usr/lib/x86_64-linux-gnu/libgmp.so.10 (0x00007f5500a65000)
libblkid.so.1 => /lib/x86_64-linux-gnu/libblkid.so.1 (0x00007f5500818000)
liblzma.so.5 => /lib/x86_64-linux-gnu/liblzma.so.5 (0x00007f55005f2000)
liblz4.so.1 => /usr/lib/x86_64-linux-gnu/liblz4.so.1 (0x00007f55003d6000)
libgcrypt.so.20 => /lib/x86_64-linux-gnu/libgcrypt.so.20 (0x00007f55000ba000)
libbsd.so.0 => /lib/x86_64-linux-gnu/libbsd.so.0 (0x00007f54ffea5000)
libdatrie.so.1 => /usr/lib/x86_64-linux-gnu/libdatrie.so.1 (0x00007f54ffc9e000)
libkeyutils.so.1 => /lib/x86_64-linux-gnu/libkeyutils.so.1 (0x00007f54ffa9a000)
libuuid.so.1 => /lib/x86_64-linux-gnu/libuuid.so.1 (0x00007f54ff893000)
libgpg-error.so.0 => /lib/x86_64-linux-gnu/libgpg-error.so.0 (0x00007f54ff67e000)

OS: ubuntu18.4

grafana: v9.2.4

image rendering plugin: 3.6.4

2

try a lot of things，Anyone help reply ？thank you

3

Hi! Have you read through the docs here Use images in notifications | Grafana documentation?

4

yes, i seem grafana and image renderer is Poor compatibility ，update to 9.4 ，email is norma but no image too

5

Hi! Have you followed the troubleshooting instructions? What do you see in logs?

6

i read all of troubleshooting instructions.

grafana.log is normal, and syslog too

i want to know why email can not include image

grafana9.2.4 update to latest 9.4 ,no image too

7

Can you show me the debug logs that contain the text ngalert.image?

8

Add an alarm, log as follows。No error，and i get email（no image）

logger=ngalert rule_uid=TQ3TWT-4z org_id=1 t=2023-03-13T10:02:18.67521636+08:00 level=info msg=“clearing the state of the rule because version has changed” version=1 new_version=2
logger=ngalert rule_uid=TQ3TWT-4z org_id=1 t=2023-03-13T10:02:18.702351928+08:00 level=info msg=“rules state was reset” deleted_states=1
logger=ngalert rule_uid=TQ3TWT-4z org_id=1 t=2023-03-13T10:02:26.670415685+08:00 level=info msg=“rules state was reset” deleted_states=0

9

Hi! I don’t see any level=debug logs in the output, just level=info. You’ll need to turn on debug logging. You can find the log_level in the ini file.

10

yes，i do，but it is not he text ngalert.image。
root@ubuntu-zabbix:~# grep -i ‘ngalert.image’ /var/log/grafana/grafana.log

I find out whether this log has anything to do with “no image”

logger=plugin.loader t=2023-03-13T17:30:15.499529866+08:00 level=debug msg=“Loading plugin” path=/var/lib/grafana/plugins/grafana-image-renderer/plugin.json

logger=plugin.loader t=2023-03-13T17:30:17.727086035+08:00 level=debug msg=“Plugin signature valid” id=grafana-image-renderer

logger=plugin.signature.validator t=2023-03-13T17:30:18.215815794+08:00 level=debug msg=“Plugin has valid signature” id=grafana-image-renderer

logger=plugin.loader t=2023-03-13T17:30:18.246240664+08:00 level=info msg=“Plugin registered” pluginID=grafana-image-renderer

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:18.439325322+08:00 level=debug msg=“starting plugin” path=/var/lib/grafana/plugins/grafana-image-renderer/plugin_start_linux_amd64 args=[/var/lib/grafana/plugins/grafana-image-renderer/plugin_start_linux_amd64]

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:18.440266688+08:00 level=debug msg=“plugin started” path=/var/lib/grafana/plugins/grafana-image-renderer/plugin_start_linux_amd64 pid=5722

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:18.440356903+08:00 level=debug msg=“waiting for RPC address” path=/var/lib/grafana/plugins/grafana-image-renderer/plugin_start_linux_amd64

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:19.240643336+08:00 level=debug msg=“Browser initialized” config=“map[args:[–no-sandbox --disable-gpu] chromeBin:/var/lib/grafana/plugins/grafana-image-renderer/chrome-linux/chrome clustering:map[maxConcurrency:5 mode:browser monitor:false timeout:30] deviceScaleFactor:1 dumpio:true emulateNetworkConditions:false headed:false height:500 ignoresHttpsErrors:false maxDeviceScaleFactor:4 maxHeight:3000 maxWidth:3000 mode:default pageZoomLevel:1 timingMetrics:false verboseLogging:true width:1000]”

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:19.340687965+08:00 level=debug msg=“[0313/173019.339025:ERROR:address_tracker_linux.cc(189)] Could not create NETLINK socket: Address family not supported by protocol (97)”

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:19.343403684+08:00 level=debug msg=“[0313/173019.342135:ERROR:udev_watcher.cc(52)] Failed to initialize a udev monitor.”

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:19.343899612+08:00 level=debug msg=

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:19.343945215+08:00 level=debug msg=“DevTools listening on ws://127.0.0.1:33417/devtools/browser/0968c5a7-61ed-4985-95ad-180b388137fe”

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:19.34397219+08:00 level=debug msg=“[0313/173019.343665:WARNING:bluez_dbus_manager.cc(247)] Floss manager not present, cannot set Floss enable/disable.”

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:19.38762403+08:00 level=debug msg=“[0313/173019.368129:WARNING:sandbox_linux.cc(376)] InitializeSandbox() called with multiple threads in process gpu-process.”

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:19.517526249+08:00 level=debug msg=“Using browser version” browserVersion=HeadlessChrome/106.0.5249.0

logger=plugin.grafana-image-renderer t=2023-03-13T17:30:19.521893953+08:00 level=debug msg=“using plugin” version=2

11

These look like start up logs for the image rendering plugin. You’ll want to find the logs from when an alert fired. If an alert hasn’t fired since Grafana was restarted with log_level=debug you can create an alert that is always firing to test. Please also make sure you are still on Grafana 9.4 as it contains the logging changes for debugging images.

12

now update grafana9.4.3 , this is log of ngalert.image

logger=ngalert.image rule_uid=SXGPf9aVk org_id=1 dashboard=Ygpgf9aVz panel=2 t=2023-03-18T16:24:50.080862659+08:00 level=debug msg=“Requesting screenshot”
logger=ngalert.image rule_uid=SXGPf9aVk org_id=1 dashboard=Ygpgf9aVz panel=2 t=2023-03-18T16:25:00.083816108+08:00 level=debug msg=“Requesting screenshot”
logger=ngalert.image rule_uid=SXGPf9aVk org_id=1 dashboard=Ygpgf9aVz panel=2 t=2023-03-18T16:25:10.076841231+08:00 level=debug msg=“Requesting screenshot”

13

Thanks! Looking at the logs I suspect that screenshots are not enabled. Can you share your Grafana configuration?

14

Manually click, you can take a screenshot, but there are only words without pictures in the email alarm

root@ubuntu-grafana:~# grafana-cli plugins ls
installed plugins:
alexanderzobnin-zabbix-app @ 4.2.10
grafana-image-renderer @ 3.6.4

root@ubuntu-grafana:~# apt list |grep grafana
grafana-enterprise/now 9.4.3 amd64 [installed,local]

[smtp]

enabled = true

host = smtp.***.com:465

user = @.com

password = ***

;cert_file =

;key_file =

skip_verify = true

from_address = @.com

from_name = Grafana107

ehlo_identity = dashboard.example.com

;startTLS_policy = NoStartTLS

[emails]

;welcome_email_on_sign_up = false

;templates_pattern = emails/.html, emails/.txt

;content_types = text/html

[alerting]
enabled = true

execute_alerts = true

;error_or_timeout = alerting

;nodata_or_nullvalues = no_data

;concurrent_render_limit = 5

;evaluation_timeout_seconds = 30

;notification_timeout_seconds = 30

;max_attempts = 3

;min_interval_seconds = 1

;max_annotation_age =

;max_annotations_to_keep =

[rendering]

;server_url =

;callback_url =

;renderer_token = -

;concurrent_render_request_limit = 30

;render_key_lifetime = 5m

[plugins]

;enable_alpha = false

;app_tls_skip_verify_insecure = false

allow_loading_unsigned_plugins = Grafana Image Renderer

;plugin_admin_enabled = false

;plugin_admin_external_manage_enabled = false

;plugin_catalog_url = Grafana Plugins - extend and customize your Grafana | Grafana Labs

;plugin_catalog_hidden_plugins =

[live]

;max_connections = 100

;allowed_origins =

;ha_engine =

;ha_engine_address = “127.0.0.1:6379”

[plugin.grafana-image-renderer]

;rendering_timezone =

;rendering_language =

;rendering_viewport_device_scale_factor =

;rendering_ignore_https_errors =

;rendering_verbose_logging =

;rendering_dumpio =

;rendering_args =

;rendering_chrome_bin =

;rendering_mode =

;rendering_clustering_mode =

;rendering_clustering_max_concurrency =

;rendering_clustering_timeout =

;rendering_viewport_max_width =

;rendering_viewport_max_height =

;rendering_viewport_max_device_scale_factor =

;grpc_host =

;grpc_port =

[snapshots]

;enabled = true

;external_enabled = true

;external_snapshot_url = https://snapshots.raintank.io

;external_snapshot_name = Publish to snapshots.raintank.io

;public_mode = false

;snapshot_remove_expired = true

15

OK I’m confused. You shared logs earlier from ngalert but the configuration file shared here tells me that you are not using Grafana Alerting, but the old Grafana 7 dashboard alerts?

[alerting]
enabled = true
execute_alerts = true
16

To eliminate distractions, as you suggested ，3 days ago I reinstalled a server with the latest grafana9.4.3 and renderer 3.6.4 , using the built-in grafana alarms. Browser click can be normal screenshot, but email alarm text without screenshot.

Don’t know what went wrong, and the log didn’t see anything unusual

17

But what version of alerting are you using? The new Grafana Alerting or the old dashboard alerts?

18

Grafana is up to date （9.4.3）, alerting is included and is up to date too

The alarm email received was also from grafana 9.4.3

alerting
alerting1418×711 67.3 KB

19

OK! Looking at your configuration file and the screenshot I suspect the issue is that Grafana has not been configured to take screenshots, you need to follow the instructions here Use images in notifications | Grafana documentation.

20

thanksbut i can not see the text unified_alerting.screenshots in the grafana.ini

root@ubuntu-grafana:~# grep -i ‘screenshots’ /etc/grafana/grafana.ini
root@ubuntu-grafana:~#

#################################### Unified Alerting ####################
892 [unified_alerting]
893 #Enable the Unified Alerting sub-system and interface. When enabled we’ll migrate all of your a lert rules and notification channels to the new system. New alert rules will be created and you r notification channels will be converted into an Alertmanager configuration. Previous data is preserved to enable backwards compatibility but new data is removed.```
894 ;enabled = true
895 # Comma-separated list of organization IDs for which to disable unified alerting. Only supporte d if unified alerting is enabled.
896 ;disabled_orgs =
897
898 # Specify the frequency of polling for admin config changes.
899 # The interval string is a possibly signed sequence of decimal numbers, followed by a unit suff ix (ms, s, m, h, d), e.g. 30s or 1m.
900 ;admin_config_poll_interval = 60s
901
902 # Specify the frequency of polling for Alertmanager config changes.
903 # The interval string is a possibly signed sequence of decimal numbers, followed by a unit suff ix (ms, s, m, h, d), e.g. 30s or 1m.
904 ;alertmanager_config_poll_interval = 60s

Listen address/hostname and port to receive unified alerting messages for other Grafana insta nces. The port is used for both TCP and UDP. It is assumed other Grafana instances are also run ning on the same port. The default value is 0.0.0.0:9094.

907 ;ha_listen_address = “0.0.0.0:9094”
908
909 # Listen address/hostname and port to receive unified alerting messages for other Grafana insta nces. The port is used for both TCP and UDP. It is assumed other Grafana instances are also run ning on the same port. The default value is 0.0.0.0:9094.
910 ;ha_advertise_address = “”
911
912 # Comma-separated list of initial instances (in a format of host:port) that will form the HA cl uster. Configuring this setting will enable High Availability mode for alerting.
913 ;ha_peers = “”
914
915 # Time to wait for an instance to send a notification via the Alertmanager. In HA, each Grafana instance will
916 # be assigned a position (e.g. 0, 1). We then multiply this position with the timeout to indica te how long should

each instance wait before sending the notification to take into account replication lag.

918 # The interval string is a possibly signed sequence of decimal numbers, followed by a unit suff ix (ms, s, m, h, d), e.g. 30s or 1m.
919 ;ha_peer_timeout = “15s”
920
921 # The interval between sending gossip messages. By lowering this value (more frequent) gossip m essages are propagated
922 # across cluster more quickly at the expense of increased bandwidth usage.
923 # The interval string is a possibly signed sequence of decimal numbers, followed by a unit suff ix (ms, s, m, h, d), e.g. 30s or 1m.
924 ;ha_gossip_interval = “200ms”
925
926 # The interval between gossip full state syncs. Setting this interval lower (more frequent) wil l increase convergence speeds
927 # across larger clusters at the expense of increased bandwidth usage.
928 # The interval string is a possibly signed sequence of decimal numbers, followed by a unit suff ix (ms, s, m, h, d), e.g. 30s or 1m.
929 ;ha_push_pull_interval = “60s”
930
931 # Enable or disable alerting rule execution. The alerting UI remains visible. This option has a legacy version in the [alerting] section that takes precedence.
932 ;execute_alerts = true

Alert evaluation timeout when fetching data from the datasource. This option has a legacy ver sion in the [alerting] section that takes precedence.

935 # The timeout string is a possibly signed sequence of decimal numbers, followed by a unit suffi x (ms, s, m, h, d), e.g. 30s or 1m.
936 ;evaluation_timeout = 30s
937
938 # Number of times we’ll attempt to evaluate an alert rule before giving up on that evaluation. This option has a legacy version in the [alerting] section that takes precedence.
939 ;max_attempts = 3
940
941 # Minimum interval to enforce between rule evaluations. Rules will be adjusted if they are less than this value or if they are not multiple of the scheduler interval (10s). Higher values ca n help with resource management as we’ll schedule fewer evaluations over time. This option has a legacy version in the [alerting] section that takes precedence.
942 # The interval string is a possibly signed sequence of decimal numbers, followed by a unit suff ix (ms, s, m, h, d), e.g. 30s or 1m.
943 ;min_interval = 10s
944
945 [unified_alerting.reserved_labels]
946 # Comma-separated list of reserved labels added by the Grafana Alerting engine that should be d isabled.
947 # For example: disabled_labels=grafana_folder
948 ;disabled_labels =
capture