Difference between revisions of "Datadog Airflow integration"

From wikieduonline
Jump to navigation Jump to search
 
(21 intermediate revisions by 4 users not shown)
Line 2: Line 2:
 
* [[Kubernetes]]: https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#datadog
 
* [[Kubernetes]]: https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#datadog
 
* Windows/Linux https://docs.datadoghq.com/integrations/airflow/
 
* Windows/Linux https://docs.datadoghq.com/integrations/airflow/
 +
* Official documentation with <code>[[dogstatsd_mapper_profile:]]</code> configuration for the agent: https://app.datadoghq.eu/integrations/airflow
 +
* https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html
  
 +
<code>[[statsd_datadog_enabled]]</code>
 +
<code>[[statsd_datadog_tags]]</code>
 +
 +
== Airflow monitoring ==
 +
{{airflow monitoring}}
 +
 +
 +
== Logs ==
 +
 +
[[triggerer]] | [2022-12-05 14:02:35,051] {base.py:317} INFO - [[Statsd]] buffering is disabled
 +
triggerer | [2022-12-05 14:02:35,052] {base.py:335} INFO - Statsd periodic buffer flush is disabled
 +
triggerer | [2022-12-05 14:02:35,053] {base.py:317} INFO - Statsd buffering is disabled
 +
triggerer | [2022-12-05 14:02:35,053] {base.py:335} INFO - Statsd periodic buffer flush is disabled
 +
 +
[[scheduler]] | [2022-12-05 14:02:35,223] {base.py:317} INFO - Statsd buffering is disabled
 +
scheduler | [2022-12-05 14:02:35,234] {base.py:335} INFO - Statsd periodic buffer flush is disabled
 +
scheduler | [2022-12-05 14:02:35,235] {base.py:317} INFO - Statsd buffering is disabled
 +
scheduler | [2022-12-05 14:02:35,235] {base.py:335} INFO - Statsd periodic buffer flush is disabled
 +
 +
[[webserver]] | [2022-12-05 14:02:36,135] {base.py:317} INFO - Statsd buffering is disabled
 +
webserver | [2022-12-05 14:02:36,135] {base.py:335} INFO - Statsd periodic buffer flush is disabled
 +
webserver | [2022-12-05 14:02:36,136] {base.py:317} INFO - Statsd buffering is disabled
 +
webserver | [2022-12-05 14:02:36,136] {base.py:335} INFO - Statsd periodic buffer flush is disabled
 +
 +
 +
 +
* https://docs.datadoghq.com/integrations/airflow/?tab=containerized
 +
INTEGRATION_NAME = {"airflow"}
 +
INIT_CONFIG = {}
 +
INSTANCE_CONFIG = {"url": http://%%host%%:8080"}
 +
 +
LOG_CONFIG = {"source": "airflow", "service": "<YOUR_APP_NAME>"}
 +
 +
== Errors ==
 +
* <code>[[ERROR airflow.stats Could not configure StatsClient: No module named 'datadog', using DummyStatsLogger instead]]</code>
 +
* <code>[[Error submitting packet]]: [Errno 111] Connection refused, dropping the packet and closing the socket, "level": "WARNING</code>
 +
 +
== Related ==
 +
* <code>[[airflow.cfg]]</code>
 +
* <code>[[airflow config list]] | grep [[statsd]]</code>
 +
* <code>[[8125]] port</code>
 +
* [[Airflow]] <code>[[base_url]]</code> default value <code>http://localhost:8080</code>
 +
* [[DogStatsD]]: <code>[[pip install]] 'apache-airflow[statsd]'</code>
 +
* <code>[[DD_DOGSTATSD_MAPPER_PROFILES]]</code>
  
 
== See also ==
 
== See also ==

Latest revision as of 14:11, 20 June 2024

statsd_datadog_enabled
statsd_datadog_tags

Airflow monitoring[edit]

[metrics]
statsd_on = True
statsd_host = localhost
statsd_port = 8125
statsd_prefix = airflow
#statsd_allow_list = scheduler,executor,dagrun
statsd_datadog_enabled = False


Logs[edit]

triggerer | [2022-12-05 14:02:35,051] {base.py:317} INFO - Statsd buffering is disabled
triggerer | [2022-12-05 14:02:35,052] {base.py:335} INFO - Statsd periodic buffer flush is disabled
triggerer | [2022-12-05 14:02:35,053] {base.py:317} INFO - Statsd buffering is disabled
triggerer | [2022-12-05 14:02:35,053] {base.py:335} INFO - Statsd periodic buffer flush is disabled

scheduler | [2022-12-05 14:02:35,223] {base.py:317} INFO - Statsd buffering is disabled
scheduler | [2022-12-05 14:02:35,234] {base.py:335} INFO - Statsd periodic buffer flush is disabled
scheduler | [2022-12-05 14:02:35,235] {base.py:317} INFO - Statsd buffering is disabled
scheduler | [2022-12-05 14:02:35,235] {base.py:335} INFO - Statsd periodic buffer flush is disabled

webserver | [2022-12-05 14:02:36,135] {base.py:317} INFO - Statsd buffering is disabled
webserver | [2022-12-05 14:02:36,135] {base.py:335} INFO - Statsd periodic buffer flush is disabled
webserver | [2022-12-05 14:02:36,136] {base.py:317} INFO - Statsd buffering is disabled
webserver | [2022-12-05 14:02:36,136] {base.py:335} INFO - Statsd periodic buffer flush is disabled



INTEGRATION_NAME = {"airflow"}
INIT_CONFIG = {}
INSTANCE_CONFIG = {"url": http://%%host%%:8080"}

LOG_CONFIG = {"source": "airflow", "service": "<YOUR_APP_NAME>"}

Errors[edit]

Related[edit]

See also[edit]

Retrieved from "https://www.wikieduonline.com/index.php?title=Datadog_Airflow_integration&oldid=329482"

Advertising: