diff --git a/config/_default/menus/main.en.yaml b/config/_default/menus/main.en.yaml index 90abae83032..c22de4dbc7e 100644 --- a/config/_default/menus/main.en.yaml +++ b/config/_default/menus/main.en.yaml @@ -6668,91 +6668,106 @@ menu: parent: observability_pipelines_processors identifier: observability_pipelines_processors_add_hostname weight: 302 + - name: Aggregate + url: observability_pipelines/processors/aggregate + parent: observability_pipelines_processors + identifier: observability_pipelines_processors_aggregate + weight: 303 - name: Custom Processor url: observability_pipelines/processors/custom_processor parent: observability_pipelines_processors identifier: observability_pipelines_processors_custom_processor - weight: 303 + weight: 304 - name: Deduplicate url: observability_pipelines/processors/dedupe parent: observability_pipelines_processors identifier: observability_pipelines_processors_dedupe - weight: 304 + weight: 305 - name: Edit fields url: observability_pipelines/processors/edit_fields parent: observability_pipelines_processors identifier: observability_pipelines_processors_edit_fields - weight: 305 + weight: 306 + - name: Edit Tags + url: observability_pipelines/processors/edit_tags + parent: observability_pipelines_processors + identifier: observability_pipelines_processors_edit_tags + weight: 307 - name: Enrichment Table url: observability_pipelines/processors/enrichment_table parent: observability_pipelines_processors identifier: observability_pipelines_processors_enrichment_table - weight: 306 + weight: 308 - name: Filter url: observability_pipelines/processors/filter parent: observability_pipelines_processors identifier: observability_pipelines_processors_filter - weight: 307 + weight: 309 - name: Generate Metrics url: observability_pipelines/processors/generate_metrics parent: observability_pipelines_processors identifier: observability_pipelines_processors_generate_metrics - weight: 308 + weight: 310 - name: Grok Parser url: observability_pipelines/processors/grok_parser parent: observability_pipelines_processors identifier: observability_pipelines_processors_grok_parser - weight: 309 + weight: 311 - name: Parse JSON url: observability_pipelines/processors/parse_json parent: observability_pipelines_processors identifier: observability_pipelines_processors_parse_json - weight: 310 + weight: 312 - name: Parse XML url: observability_pipelines/processors/parse_xml parent: observability_pipelines_processors identifier: observability_pipelines_processors_parse_xml - weight: 311 + weight: 313 - name: Quota url: observability_pipelines/processors/quota parent: observability_pipelines_processors identifier: observability_pipelines_processors_quota - weight: 312 + weight: 314 - name: Reduce url: observability_pipelines/processors/reduce parent: observability_pipelines_processors identifier: observability_pipelines_processors_reduce - weight: 313 + weight: 315 - name: Remap to OCSF url: observability_pipelines/processors/remap_ocsf parent: observability_pipelines_processors identifier: observability_pipelines_processors_remap_ocsf - weight: 314 + weight: 316 - name: Sample url: observability_pipelines/processors/sample parent: observability_pipelines_processors identifier: observability_pipelines_processors_sample - weight: 315 + weight: 317 - name: Sensitive Data Scanner url: observability_pipelines/processors/sensitive_data_scanner parent: observability_pipelines_processors identifier: observability_pipelines_processors_sensitive_data_scanner - weight: 316 + weight: 318 - name: Split Array url: observability_pipelines/processors/split_array parent: observability_pipelines_processors identifier: observability_pipelines_processors_split_array - weight: 317 + weight: 319 + - name: Tag Cardinality Control + url: observability_pipelines/processors/tag_cardinality_control + parent: observability_pipelines_processors + identifier: observability_pipelines_processors_tag_cardinality_control + weight: 320 - name: Tag Control url: observability_pipelines/processors/tag_control/logs/ parent: observability_pipelines_processors identifier: observability_pipelines_processors_tag_control - weight: 318 + weight: 321 - name: Throttle url: observability_pipelines/processors/throttle parent: observability_pipelines_processors identifier: observability_pipelines_processors_throttle - weight: 320 + weight: 322 - name: Destinations url: observability_pipelines/destinations/ parent: observability_pipelines diff --git a/content/en/observability_pipelines/processors/_index.md b/content/en/observability_pipelines/processors/_index.md index f2338156cc7..1defa27cfcf 100644 --- a/content/en/observability_pipelines/processors/_index.md +++ b/content/en/observability_pipelines/processors/_index.md @@ -72,11 +72,17 @@ These are the available processors: {{% /tab %}} {{% tab "Metrics" %}} -- [Filter][1] -- [Tag Control][2] - -[1]: /observability_pipelines/processors/filter/ -[2]: /observability_pipelines/processors/tag_control/ +- [Aggregate][1] +- [Edit Tags][2] +- [Filter][3] +- [Tag Cardinality Control][4] +- [Tag Control][5] + +[1]: /observability_pipelines/processors/aggregate/ +[2]: /observability_pipelines/processors/edit_tags/ +[3]: /observability_pipelines/processors/filter/ +[4]: /observability_pipelines/processors/tag_cardinality_control/ +[5]: /observability_pipelines/processors/tag_control/ {{% /tab %}} {{< /tabs >}} diff --git a/content/en/observability_pipelines/processors/aggregate.md b/content/en/observability_pipelines/processors/aggregate.md new file mode 100644 index 00000000000..574de671cba --- /dev/null +++ b/content/en/observability_pipelines/processors/aggregate.md @@ -0,0 +1,39 @@ +--- +title: Aggregate Processor +disable_toc: false +products: +- name: Metrics + icon: metrics + url: /observability_pipelines/configuration/?tab=metrics#pipeline-types +--- + +{{< product-availability >}} + +## Overview + +The Aggregate processor combines multiple metrics with the same tag values into a single sample based on the selected aggregation mode. Aggregating metrics can help reduce your metric volume and costs. + +## Setup + +To set up the Aggregate processor: + +1. Define a filter query. Only matching metrics are processed by the processor. All metrics, regardless of whether they match the filter query, are sent to the next step in the pipeline. See [Metrics Search Syntax][1] for information on creating queries. +1. In the **Mode** dropdown menu, select the aggregation function to use. See [Modes](#modes) for details. +1. In the **Interval** field, enter the time window in seconds for aggregating metrics. The maximum value is 60 seconds. + +## Modes + +The Aggregate processor can combine metrics into a single metric based on the following modes. Some modes are available only for specific [metric kinds][2], incremental or absolute. + +| Mode | Description | Incremental metrics | Absolute metrics | +| ------ | -------------------------------------------------------------------------------------- | :-----------------: | :--------------: | +| Auto | Default mode. Sums incremental metrics and uses the latest value for absolute metrics. | {{< X >}} | {{< X >}} | +| Sum | Sums the metric values. | {{< X >}} | | +| Count | Counts the number of times the metric is received. | {{< X >}} | {{< X >}} | +| Latest | Returns the latest metric value. | | {{< X >}} | +| Max | Returns the maximum metric value. | | {{< X >}} | +| Mean | Returns the mean metric value. | | {{< X >}} | +| Min | Returns the minimum metric value. | | {{< X >}} | + +[1]: /observability_pipelines/search_syntax/metrics/ +[2]: /observability_pipelines/configuration/?tab=metrics#metrics-data diff --git a/content/en/observability_pipelines/processors/edit_tags.md b/content/en/observability_pipelines/processors/edit_tags.md new file mode 100644 index 00000000000..ecd7d78ccc3 --- /dev/null +++ b/content/en/observability_pipelines/processors/edit_tags.md @@ -0,0 +1,48 @@ +--- +title: Edit Tags Processor +disable_toc: false +products: +- name: Metrics + icon: metrics + url: /observability_pipelines/configuration/?tab=metrics#pipeline-types +--- + +{{< product-availability >}} + +## Overview + +The Edit Tags processor can add or rename tags in your metrics. Use this processor to enrich your metrics with additional context and standardize naming across important attributes. + +The following tags can't be renamed because they provide specific platform functionality: + +- `host` +- `service` +- `ddsource` +- `function_arn` +- `datadog_` +- `_dd.*` + +## Setup + +### Add tag + +Use **Add tag** to append a new key-value tag to your metric. + +To set up the **Add tag** processor: + +1. Select **Add tag** in the **Action** dropdown menu. +1. Define a filter query. Only metrics that match the filter query are processed. All metrics, regardless of whether they match the filter query, are sent to the next step in the pipeline. See [Metrics Search Syntax][1] for information on creating queries. +1. Enter the tag key and value you want to add to metrics. **Note**: If the tag you want to add already exists, the Worker logs an error and the existing tag remains unchanged. + +### Rename tag + +Use **Rename tag** to rename a tag in your metric. + +To set up the **Rename tag** processor: + +1. Select **Rename tag** in the **Action** dropdown menu. +1. Define a filter query. Only metrics that match the filter query are processed. All metrics, regardless of whether they match the filter query, are sent to the next step in the pipeline. See [Metrics Search Syntax][1] for information on creating queries. +1. Enter the name of the tag key you want to rename in the **From** field. +1. In the **To** field, enter the tag key you want to rename the original tag to. **Note**: If the tag name in the **To** field already exists, the Worker logs an error and does not rename the tag key in the **From** field. + +[1]: /observability_pipelines/search_syntax/metrics/ diff --git a/content/en/observability_pipelines/processors/tag_cardinality_control.md b/content/en/observability_pipelines/processors/tag_cardinality_control.md new file mode 100644 index 00000000000..df3c97cd461 --- /dev/null +++ b/content/en/observability_pipelines/processors/tag_cardinality_control.md @@ -0,0 +1,55 @@ +--- +title: Tag Cardinality Control Processor +disable_toc: false +products: +- name: Metrics + icon: metrics + url: /observability_pipelines/configuration/?tab=metrics#pipeline-types +--- + +{{< product-availability >}} + +{{< callout url="#" + btn_hidden="true" header="Join the Preview!">}} +The Tag Cardinality Control processor is in Preview. Contact your account manager to request access. +{{< /callout >}} + +## Overview + +The Tag Cardinality Control processor limits the number of tag values for each metric. For example, a metric with unbounded tag keys, such as `userID`, can cause the metric's cardinality to spike and impact ingestion and indexing costs. To prevent these unexpected spikes, use the processor to set a cardinality limit for metrics that match the filter query, and either drop metrics received after the limit is reached or drop the tags for those metrics. + +Optionally, you can also configure [per-metric overrides](#optional-per-metric-override-settings) to set a limit for a specific metric or to exclude the metric from any cardinality limits. For each per-metric override, you can also set a custom limit for individual tags within the metric, or exclude the tag from the per-metric cardinality limit. + +## Setup + +To set up the Tag Cardinality Control processor: + +1. Define a filter query. Only matching metrics are processed by this processor. See [Metrics Search Syntax][1] for more information. +1. Enter a cardinality limit for the maximum number of distinct values per tag. This limit is applied to all metrics that match the filter query. +1. In the **When the limit is reached** dropdown menu, select whether to **Drop tag** or **Drop event** for metrics that have exceeded the cardinality limit. + +{{< img src="observability_pipelines/processors/tag_cardinality_control_settings.png" alt="The Tag Cardinality Control processor set to a cardinality limit of 200 and to drop tag when the limit is reached." style="width:40%;" >}} + +### Optional per-metric override settings + +If you want to set a cardinality limit for a specific metric: + +1. Click **Manage Overrides** and then **Add Metric Override** in the side panel. +1. Enter the name of the metric. +1. Select the override mode in the dropdown menu. + - **Custom limit**: Sets a cardinality limit for this metric. + - **Exclude from limit**: Excludes this metric from being counted toward the cardinality limit. This is useful when you are tracking an important metric and you don't want to drop any samples or tags due to a cardinality limit. +1. In the **When the limit is reached** dropdown menu, select whether to **Drop tag** or **Drop event** for metrics that have exceeded the cardinality limit. +1. If you want to add specific tag overrides for this metric: + 1. Click **Add Tag Override**. + 1. Enter the tag key on which to set a limit. + 1. Select the override mode in the dropdown menu: + - **Custom limit**: Sets a limit on the number of unique values per tag. For example, if the tag limit is set to `5`, the first five tag values received are used. + - **Note**: The tag values are only reset with a Worker restart or a pipeline configuration update, even if the Tag Cardinality Control processor isn't updated. + - **Exclude from limit**: Excludes metrics with the specified tag from being counted toward the cardinality limit. + 1. Enter the limit for the maximum number of tag key values. + 1. Click **Add Override**. + +{{< img src="observability_pipelines/processors/tag_cardinality_control_overrides.png" alt="The per-metric override panel with a custom limit set to 100 with per-tag overrides for the host tag excluded from the limit and the region tag limited to five." style="width:80%;" >}} + +[1]: /observability_pipelines/search_syntax/metrics/ diff --git a/static/images/observability_pipelines/processors/tag_cardinality_control_overrides.png b/static/images/observability_pipelines/processors/tag_cardinality_control_overrides.png new file mode 100644 index 00000000000..61d870fafc1 Binary files /dev/null and b/static/images/observability_pipelines/processors/tag_cardinality_control_overrides.png differ diff --git a/static/images/observability_pipelines/processors/tag_cardinality_control_settings.png b/static/images/observability_pipelines/processors/tag_cardinality_control_settings.png new file mode 100644 index 00000000000..954267cc98e Binary files /dev/null and b/static/images/observability_pipelines/processors/tag_cardinality_control_settings.png differ