/etc/prometheus/alertmanager.yml

smtp, hipchat

Examples: https://github.com/prometheus/alertmanager/blob/main/doc/examples/simple.yml

cat /etc/prometheus/alertmanager.yml

# Sample configuration.
# See https://prometheus.io/docs/alerting/configuration/ for documentation.

global:
 # The smarthost and SMTP sender used for mail notifications.
 smtp_smarthost: 'localhost:25'
 smtp_from: '[email protected]'
 smtp_auth_username: 'alertmanager'
 smtp_auth_password: 'password'
 
 # The auth token for Hipchat.
 hipchat_auth_token: '1234556789'
 # Alternative host for Hipchat.
 hipchat_api_url: 'https://hipchat.foobar.org/'

# The directory from which notification templates are read.
templates:
 - '/etc/prometheus/alertmanager_templates/*.tmpl'

# The root route on which each incoming alert enters.
route:
 # The labels by which incoming alerts are grouped together. For example,
 # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
 # be batched into a single group.
 group_by: ['alertname', 'cluster', 'service']

 # When a new group of alerts is created by an incoming alert, wait at
 # least 'group_wait' to send the initial notification.
 # This way ensures that you get multiple alerts for the same group that start
 # firing shortly after another are batched together on the first
 # notification.
 group_wait: 30s

 # When the first notification was sent, wait 'group_interval' to send a batch
 # of new alerts that started firing for that group.
 group_interval: 5m

 # If an alert has successfully been sent, wait 'repeat_interval' to
 # resend them.
 repeat_interval: 3h

 # A default receiver
 receiver: team-X-mails

 # All the above attributes are inherited by all child routes and can
 # overwritten on each.

 # The child route trees.
 routes:
 # This routes performs a regular expression match on alert labels to
 # catch alerts that are related to a list of services.
 - match_re:
     service: ^(foo1|foo2|baz)$
   receiver: team-X-mails
   # The service has a sub-route for critical alerts, any alerts
   # that do not match, i.e. severity != critical, fall-back to the
   # parent node and are sent to 'team-X-mails'
   routes:
   - match:
       severity: critical
     receiver: team-X-pager
 - match:
     service: files
   receiver: team-Y-mails

   routes:
   - match:
       severity: critical
     receiver: team-Y-pager

 # This route handles all alerts coming from a database service. If there's
 # no team to handle it, it defaults to the DB team.
 - match:
     service: database
   receiver: team-DB-pager
   # Also group alerts by affected database.
   group_by: [alertname, cluster, database]
   routes:
   - match:
       owner: team-X
     receiver: team-X-pager
   - match:
       owner: team-Y
     receiver: team-Y-pager

# Inhibition rules allow to mute a set of alerts given that another alert is
# firing.
# We use this to mute any warning-level notifications if the same alert is
# already critical.
inhibit_rules:
- source_match:
   severity: 'critical'
 target_match:
   severity: 'warning'
 # Apply inhibition if the alertname is the same.
 equal: ['alertname', 'cluster', 'service']

receivers:
- name: 'team-X-mails'
 email_configs:
 - to: '[email protected]'

- name: 'team-X-pager'
 email_configs:
 - to: '[email protected]'
 pagerduty_configs:
 - service_key: <team-X-key>

- name: 'team-Y-mails'
 email_configs:
 - to: '[email protected]'

- name: 'team-Y-pager'
 pagerduty_configs:
 - service_key: <team-Y-key>

- name: 'team-DB-pager'
  pagerduty_configs:
  - service_key: <team-DB-key>
- name: 'team-X-hipchat'
  hipchat_configs:
  - auth_token: <auth_token>
    room_id: 85
    message_format: html
    notify: true

Related

• OpsGenie: https://prometheus.io/docs/alerting/latest/configuration/#opsgenie_config
• PagerDuty: https://prometheus.io/docs/alerting/latest/configuration/#pagerduty_config

See also

• Prometheus Alertmanager: amtool, alertmanager.yml, simple.yml, Karma, versions, Signl4.com, Zenduty
• Prometheus: Prometheus installation, 9090, storage, Prometheus exporters: Node exporter 9100, Blackbox exporter, Prometheus alertmanager (changelog), Prometheus versions, Prometheus Node exporter releases, promtool, amtool, tsdb, /etc/default/prometheus, /etc/prometheus/prometheus.yml, journalctl -u prometheus, /etc/default/prometheus-node-exporter, PromQL, alert:, Prometheus Operator, Thanos, VictoriaMetrics, PushGateway, Prometheus API, values.yml, Amazon Managed Service for Prometheus