/etc/netdata/health alarm notify.conf

cat /etc/netdata/health_alarm_notify.conf
# Configuration for alarm notifications
#
# This configuration is used by: alarm-notify.sh
# changes take effect immediately (the next alarm will use them).
#
# alarm-notify.sh can send:
# - e-mails (using the sendmail command),
# - push notifications to your mobile phone (pushover.net),
# - messages to your slack team (slack.com),
# - messages to your flock team (flock.com),
# - messages to your discord guild (discordapp.com),
# - messages to your telegram chat / group chat (telegram.org)
# - sms messages to your cell phone or any sms enabled device (twilio.com)
# - sms messages to your cell phone or any sms enabled device (messagebird.com)
# - notifications to users on pagerduty.com
#
# The 'to' line given at netdata alarms defines a *role*, so that many
# people can be notified for each role.
#
# This file is a BASH script itself.
#
#
#------------------------------------------------------------------------------
# proxy configuration
#
# If you need to send curl based notifications (pushover, pushbullet, slack,
# flock, discord, telegram) via a proxy, set these to your proxy address:
#export http_proxy="http://10.0.0.1:3128/"
#export https_proxy="http://10.0.0.1:3128/"


#------------------------------------------------------------------------------
# notifications images
#
# Images in notifications need to be downloaded from an Internet facing site.
# To allow notification providers fetch the icons/images, by default we set
# the URL of the global public netdata registry.
# If you have an Internet facing netdata (or you have copied the images/ folder
# of netdata to your web server), set its URL here, to fetch the notification
# images from it.
#images_base_url="http://my.public.netdata.server:19999"


#------------------------------------------------------------------------------
# external commands

# The full path to the sendmail command.
# If empty, the system $PATH will be searched for it.
# If not found, email notifications will be disabled (silently).
sendmail=""

# The full path of the curl command.
# If empty, the system $PATH will be searched for it.
# If not found, most notifications will be silently disabled.
curl=""


#------------------------------------------------------------------------------
# NOTE ABOUT RECIPIENTS
#
# When you define recipients (all types):
#
#  - emails addresses
#  - pushover user tokens
#  - telegram chat ids
#  - slack channels
#  - flock rooms
#  - discord channels
#  - hipchat rooms
#  - sms phone numbers
#  - pagerduty.com (pd) services
#
# You can append |critical to limit the notifications to be sent.
#
# In these examples, the first recipient receives all the alarms
# while the second one receives only the critical ones:
#
#  email      : "[email protected] [email protected]|critical"
#  pushover   : "2987343...9437837 8756278...2362736|critical"
#  telegram   : "111827421 112746832|critical"
#  slack      : "alarms disasters|critical"
#  flock      : "alarms disasters|critical"
#  discord    : "alarms disasters|critical"
#  twilio     : "+15555555555 +17777777777|critical"
#  messagebird: "+15555555555 +17777777777|critical"
#  kavenegar  : "09155555555 09177777777|critical"
#  pd         : "<pd_service_key_1> <pd_service_key_2>|critical"
#
# If a recipient is set to empty string, the default recipient of the given
# notification method (email, pushover, telegram, slack, etc) will be used.
# To disable a notification, use the recipient called: disabled
# This works for all notification methods (including the default recipients).


#------------------------------------------------------------------------------
# email global notification options

# multiple recipients can be given like this:
#              "[email protected] [email protected] ..."

# the email address sending email notifications
# the default is the system user netdata runs as (usually: netdata)
# The following formats are supported:
# EMAIL_SENDER="user@domain"
# EMAIL_SENDER="User Name <user@domain>"
# EMAIL_SENDER="'User Name' <user@domain>"
# EMAIL_SENDER="\"User Name\" <user@domain>"
EMAIL_SENDER=""

# enable/disable sending emails
SEND_EMAIL="YES"

# if a role recipient is not configured, an email will be send to:
DEFAULT_RECIPIENT_EMAIL="root"
# to receive only critical alarms, set it to "root|critical"

# Optionally specify the encoding to list in the Content-Type header.
# This doesn't change what encoding the e-mail is sent with, just what
# the headers say it was encoded as.
# This shouldn't need to be changed as it will almost always be
# autodetected from the environment.
#EMAIL_CHARSET="UTF-8"


#------------------------------------------------------------------------------
# pushover (pushover.net) global notification options

# multiple recipients can be given like this:
#                  "USERTOKEN1 USERTOKEN2 ..."

# enable/disable sending pushover notifications
SEND_PUSHOVER="YES"

# Login to pushover.net to get your pushover app token.
# You need only one for all your netdata servers (or you can have one for
# each of your netdata - your call).
# Without an app token, netdata cannot send pushover notifications.
PUSHOVER_APP_TOKEN=""

# if a role's recipients are not configured, a notification will be send to
# this pushover user token (empty = do not send a notification for unconfigured
# roles):
DEFAULT_RECIPIENT_PUSHOVER=""


#------------------------------------------------------------------------------
# pushbullet (pushbullet.com) push notification options

# multiple recipients can be given like this:
#                  "[email protected] [email protected]"

# enable/disable sending pushbullet notifications
SEND_PUSHBULLET="YES"

# Signup and Login to pushbullet.com
# To get your Access Token, go to https://www.pushbullet.com/#settings/account
# Create a new access token and paste it below.
# Then just set the recipients' emails.
# Please note that the if the email in the DEFAULT_RECIPIENT_PUSHBULLET does
# not have a pushbullet account, the pushbullet service will send an email
# to that address instead.

# Without an access token, netdata cannot send pushbullet notifications.
PUSHBULLET_ACCESS_TOKEN=""
DEFAULT_RECIPIENT_PUSHBULLET=""

# Device iden of the sending device. Optional.
PUSHBULLET_SOURCE_DEVICE=""


#------------------------------------------------------------------------------
# Twilio (twilio.com) SMS options

# multiple recipients can be given like this:
#                  "+15555555555 +17777777777"

# enable/disable sending twilio SMS
SEND_TWILIO="YES"

# Signup for free trial and select a SMS capable Twilio Number
# To get your Account SID and Token, go to https://www.twilio.com/console
# Place your sid, token and number below.
# Then just set the recipients' phone numbers.
# The trial account is only allowed to use the number specified when set up.

# Without an account sid and token, netdata cannot send Twilio text messages.
TWILIO_ACCOUNT_SID=""
TWILIO_ACCOUNT_TOKEN=""
TWILIO_NUMBER=""
DEFAULT_RECIPIENT_TWILIO=""


#------------------------------------------------------------------------------
# Messagebird (messagebird.com) SMS options

# multiple recipients can be given like this:
#                  "+15555555555 +17777777777"

# enable/disable sending messagebird SMS
SEND_MESSAGEBIRD="YES"

# to get an access key, create a free account at https://www.messagebird.com
# verify and activate the account (no CC info needed)
# login to your account and enter your phonenumber to get some free credits
# to get the API key, click on 'API' in the sidebar, then 'API Access (REST)'
# click 'Add access key' and fill in data (you want a live key to send SMS)

# Without an access key, netdata cannot send Messagebird text messages.
MESSAGEBIRD_ACCESS_KEY=""
MESSAGEBIRD_NUMBER=""
DEFAULT_RECIPIENT_MESSAGEBIRD=""


#------------------------------------------------------------------------------
# Kavenegar (Kavenegar.com) SMS options

# multiple recipients can be given like this:
#                  "09155555555 09177777777"

# enable/disable sending kavenegar SMS
SEND_KAVENEGAR="YES"

# to get an access key, after selecting and purchasing your desired service
# at http://kavenegar.com/pricing.html
# login to your account, go to your dashboard and my account are
# https://panel.kavenegar.com/Client/setting/account from API Key
# copy your api key. You can generate new API Key too.
# You can find and select kevenegar sender number from this place.

# Without an API key, netdata cannot send KAVENEGAR text messages.
KAVENEGAR_API_KEY=""
KAVENEGAR_SENDER=""
DEFAULT_RECIPIENT_KAVENEGAR=""


#------------------------------------------------------------------------------
# telegram (telegram.org) global notification options

# To get your chat ID send the command /my_id to telegram bot @get_id.
# Users also need to open a query with the bot (see below).

# note: multiple recipients can be given like this:
#                  "CHAT_ID_1 CHAT_ID_2 ..."

# enable/disable sending telegram messages
SEND_TELEGRAM="YES"

# Contact the bot @BotFather to create a new bot and receive a bot token.
# Without it, netdata cannot send telegram messages.
TELEGRAM_BOT_TOKEN=""

# If a role's recipients are not configured, a message will be send to
# this chat id (empty = do not send a notification for unconfigured roles):
DEFAULT_RECIPIENT_TELEGRAM=""


#------------------------------------------------------------------------------
# slack (slack.com) global notification options

# multiple recipients can be given like this:
#                  "CHANNEL1 CHANNEL2 ..."

# enable/disable sending slack notifications
SEND_SLACK="YES"

# Login to slack.com and create an incoming webhook. You need only one for all
# your netdata servers (or you can have one for each of your netdata).
# Without it, netdata cannot send slack notifications.
# Get yours from: https://api.slack.com/incoming-webhooks
SLACK_WEBHOOK_URL=""

# if a role's recipients are not configured, a notification will be send to
# this slack channel (empty = do not send a notification for unconfigured
# roles):
DEFAULT_RECIPIENT_SLACK=""


#------------------------------------------------------------------------------
# flock (flock.com) global notification options

# enable/disable sending flock notifications
SEND_FLOCK="YES"

# Login to flock.com and create an incoming webhook. You need only one for all
# your netdata servers (or you can have one for each of your netdata).
# Without it, netdata cannot send flock notifications.
FLOCK_WEBHOOK_URL=""

# if a role recipient is not configured, no notification will be sent
DEFAULT_RECIPIENT_FLOCK=""


#------------------------------------------------------------------------------
# discord (discordapp.com) global notification options

# multiple recipients can be given like this:
#                  "CHANNEL1 CHANNEL2 ..."

# enable/disable sending discord notifications
SEND_DISCORD="YES"

# Create a webhook by following the official documentation -
# https://support.discordapp.com/hc/en-us/articles/228383668-Intro-to-Webhooks
DISCORD_WEBHOOK_URL=""

# if a role's recipients are not configured, a notification will be send to
# this discord channel (empty = do not send a notification for unconfigured
# roles):
DEFAULT_RECIPIENT_DISCORD=""


#------------------------------------------------------------------------------
# hipchat global notification options

# multiple recipients can be given like this:
#                  "ROOM1 ROOM2 ..."

# enable/disable sending hipchat notifications
SEND_HIPCHAT="YES"

# define hipchat server
HIPCHAT_SERVER="api.hipchat.com"

# api.hipchat.com authorization token
# Without this, netdata cannot send hipchat notifications.
HIPCHAT_AUTH_TOKEN=""

# if a role's recipients are not configured, a notification will be send to
# this hipchat room (empty = do not send a notification for unconfigured
# roles):
DEFAULT_RECIPIENT_HIPCHAT=""


#------------------------------------------------------------------------------
# kafka notification options

# enable/disable sending kafka notifications
SEND_KAFKA="YES"

# The URL to POST kafka alarm data to. It should be the full URL.
KAFKA_URL=""

# The IP to be used in the kafka message as the sender.
KAFKA_SENDER_IP=""


#------------------------------------------------------------------------------
# pagerduty.com notification options
#
# pagerduty.com notifications require the pagerduty agent to be installed and
# a "Generic API" pagerduty service.
# https://www.pagerduty.com/docs/guides/agent-install-guide/

# multiple recipients can be given like this:
#              "<pd_service_key_1> <pd_service_key_2> ..."

# enable/disable sending pagerduty notifications
SEND_PD="YES"

# if a role's recipients are not configured, a notification will be sent to
# the "General API" pagerduty.com service that uses this service key.
# (empty = do not send a notification for unconfigured roles):
DEFAULT_RECIPIENT_PD=""


#------------------------------------------------------------------------------
# custom notifications
#

# enable/disable sending custom notifications
SEND_CUSTOM="YES"

# if a role's recipients are not configured, use the following.
# (empty = do not send a notification for unconfigured roles)
DEFAULT_RECIPIENT_CUSTOM=""

# The custom_sender() is a custom function to do whatever you need to do
custom_sender() {
    # variables you can use:
    # ${host}               the host generated this event
    # ${url_host}           same as ${host} but URL encoded
    # ${unique_id}          the unique id of this event
    # ${alarm_id}           the unique id of the alarm that generated this event
    # ${event_id}           the incremental id of the event, for this alarm id
    # ${when}               the timestamp this event occurred
    # ${name}               the name of the alarm, as given in netdata health.d entries
    # ${url_name}           same as ${name} but URL encoded
    # ${chart}              the name of the chart (type.id)
    # ${url_chart}          same as ${chart} but URL encoded
    # ${family}             the family of the chart
    # ${url_family}         same as ${family} but URL encoded
    # ${status}             the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
    # ${old_status}         the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
    # ${value}              the current value of the alarm
    # ${old_value}          the previous value of the alarm
    # ${src}                the line number and file the alarm has been configured
    # ${duration}           the duration in seconds of the previous alarm state
    # ${duration_txt}       same as ${duration} for humans
    # ${non_clear_duration} the total duration in seconds this is/was non-clear
    # ${non_clear_duration_txt} same as ${non_clear_duration} for humans
    # ${units}              the units of the value
    # ${info}               a short description of the alarm
    # ${value_string}       friendly value (with units)
    # ${old_value_string}   friendly old value (with units)
    # ${image}              the URL of an image to represent the status of the alarm
    # ${color}              a color in #AABBCC format for the alarm
    # ${goto_url}           the URL the user can click to see the netdata dashboard

    # these are more human friendly:
    # ${alarm}              like "name = value units"
    # ${status_message}     like "needs attention", "recovered", "is critical"
    # ${severity}           like "Escalated to CRITICAL", "Recovered from WARNING"
    # ${raised_for}         like "(alarm was raised for 10 minutes)"

    # example human readable SMS
    local msg="${host} ${status_message}: ${alarm} ${raised_for}"

    # limit it to 160 characters and encode it for use in a URL
    urlencode "${msg:0:160}" >/dev/null; msg="${REPLY}"

    # a space separated list of the recipients to send alarms to
    to="${1}"

    info "not sending custom notification to ${to}, for ${status} of '${host}.${chart}.${name}' - custom_sender() is not configured."
}


###############################################################################
# RECIPIENTS PER ROLE

# -----------------------------------------------------------------------------
# generic system alarms
# CPU, disks, network interfaces, entropy, etc

role_recipients_email[sysadmin]="${DEFAULT_RECIPIENT_EMAIL}"

role_recipients_pushover[sysadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"

role_recipients_pushbullet[sysadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"

role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"

role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}"

role_recipients_flock[sysadmin]="${DEFAULT_RECIPIENT_FLOCK}"

role_recipients_discord[sysadmin]="${DEFAULT_RECIPIENT_DISCORD}"

role_recipients_hipchat[sysadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"

role_recipients_twilio[sysadmin]="${DEFAULT_RECIPIENT_TWILIO}"

role_recipients_messagebird[sysadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

role_recipients_kavenegar[sysadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}"

role_recipients_pd[sysadmin]="${DEFAULT_RECIPIENT_PD}"

role_recipients_custom[sysadmin]="${DEFAULT_RECIPIENT_CUSTOM}"

# -----------------------------------------------------------------------------
# DNS related alarms

role_recipients_email[domainadmin]="${DEFAULT_RECIPIENT_EMAIL}"

role_recipients_pushover[domainadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"

role_recipients_pushbullet[domainadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"

role_recipients_telegram[domainadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"

role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}"

role_recipients_flock[domainadmin]="${DEFAULT_RECIPIENT_FLOCK}"

role_recipients_discord[domainadmin]="${DEFAULT_RECIPIENT_DISCORD}"

role_recipients_hipchat[domainadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"

role_recipients_twilio[domainadmin]="${DEFAULT_RECIPIENT_TWILIO}"

role_recipients_messagebird[domainadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

role_recipients_kavenegar[domainadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}"

role_recipients_pd[domainadmin]="${DEFAULT_RECIPIENT_PD}"

role_recipients_custom[domainadmin]="${DEFAULT_RECIPIENT_CUSTOM}"

# -----------------------------------------------------------------------------
# database servers alarms
# mysql, redis, memcached, postgres, etc

role_recipients_email[dba]="${DEFAULT_RECIPIENT_EMAIL}"

role_recipients_pushover[dba]="${DEFAULT_RECIPIENT_PUSHOVER}"

role_recipients_pushbullet[dba]="${DEFAULT_RECIPIENT_PUSHBULLET}"

role_recipients_telegram[dba]="${DEFAULT_RECIPIENT_TELEGRAM}"

role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}"

role_recipients_flock[dba]="${DEFAULT_RECIPIENT_FLOCK}"

role_recipients_discord[dba]="${DEFAULT_RECIPIENT_DISCORD}"

role_recipients_hipchat[dba]="${DEFAULT_RECIPIENT_HIPCHAT}"

role_recipients_twilio[dba]="${DEFAULT_RECIPIENT_TWILIO}"

role_recipients_messagebird[dba]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

role_recipients_kavenegar[dba]="${DEFAULT_RECIPIENT_KAVENEGAR}"

role_recipients_pd[dba]="${DEFAULT_RECIPIENT_PD}"

role_recipients_custom[dba]="${DEFAULT_RECIPIENT_CUSTOM}"

# -----------------------------------------------------------------------------
# web servers alarms
# apache, nginx, lighttpd, etc

role_recipients_email[webmaster]="${DEFAULT_RECIPIENT_EMAIL}"

role_recipients_pushover[webmaster]="${DEFAULT_RECIPIENT_PUSHOVER}"

role_recipients_pushbullet[webmaster]="${DEFAULT_RECIPIENT_PUSHBULLET}"

role_recipients_telegram[webmaster]="${DEFAULT_RECIPIENT_TELEGRAM}"

role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}"

role_recipients_flock[webmaster]="${DEFAULT_RECIPIENT_FLOCK}"

role_recipients_discord[webmaster]="${DEFAULT_RECIPIENT_DISCORD}"

role_recipients_hipchat[webmaster]="${DEFAULT_RECIPIENT_HIPCHAT}"

role_recipients_twilio[webmaster]="${DEFAULT_RECIPIENT_TWILIO}"

role_recipients_messagebird[webmaster]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

role_recipients_kavenegar[webmaster]="${DEFAULT_RECIPIENT_KAVENEGAR}"

role_recipients_pd[webmaster]="${DEFAULT_RECIPIENT_PD}"

role_recipients_custom[webmaster]="${DEFAULT_RECIPIENT_CUSTOM}"

# -----------------------------------------------------------------------------
# proxy servers alarms
# squid, etc

role_recipients_email[proxyadmin]="${DEFAULT_RECIPIENT_EMAIL}"

role_recipients_pushover[proxyadmin]="${DEFAULT_RECIPIENT_PUSHOVER}"

role_recipients_pushbullet[proxyadmin]="${DEFAULT_RECIPIENT_PUSHBULLET}"

role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"

role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}"

role_recipients_flock[proxyadmin]="${DEFAULT_RECIPIENT_FLOCK}"

role_recipients_discord[proxyadmin]="${DEFAULT_RECIPIENT_DISCORD}"

role_recipients_hipchat[proxyadmin]="${DEFAULT_RECIPIENT_HIPCHAT}"

role_recipients_twilio[proxyadmin]="${DEFAULT_RECIPIENT_TWILIO}"

role_recipients_messagebird[proxyadmin]="${DEFAULT_RECIPIENT_MESSAGEBIRD}"

role_recipients_kavenegar[proxyadmin]="${DEFAULT_RECIPIENT_KAVENEGAR}"

role_recipients_pd[proxyadmin]="${DEFAULT_RECIPIENT_PD}"

role_recipients_custom[proxyadmin]="${DEFAULT_RECIPIENT_CUSTOM}"

See also

• Netdata, /etc/netdata/netdata.conf