From aca7bf82dee2e231ad563fec75319df05e6c8951 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Wed, 26 May 2021 00:09:44 +0200 Subject: [PATCH 1/7] wip Signed-off-by: Cyril Tovena --- docs/sources/operations/storage/retention.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/sources/operations/storage/retention.md b/docs/sources/operations/storage/retention.md index 24617d091ea4..cba6d96865dc 100644 --- a/docs/sources/operations/storage/retention.md +++ b/docs/sources/operations/storage/retention.md @@ -3,7 +3,16 @@ title: Retention --- # Loki Storage Retention -Retention in Loki is achieved through the [Table Manager](../table-manager/). +Retention in Loki is achieved either through the [Table Manager](../table-manager/) or the [Compactor](./#compactor). + +Retention through the table manager is achieved by relying on the object store TTL feature and will work for both boltdb-shipper store and chunk store. However retention through the [Compactor](./#compactor) + + +Going forward the compactor retention will become the default and long term supported one. + +## Compactor +## Table Manager + In order to enable the retention support, the Table Manager needs to be configured to enable deletions and a retention period. Please refer to the [`table_manager_config`](../../../configuration#table_manager_config) From 36bdc1ed272c717ba74f8998b57235e6b710c643 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Fri, 28 May 2021 17:11:04 +0200 Subject: [PATCH 2/7] Finish docs. Signed-off-by: Cyril Tovena --- docs/sources/configuration/_index.md | 31 +++- docs/sources/operations/storage/retention.md | 147 ++++++++++++++++++- 2 files changed, 173 insertions(+), 5 deletions(-) diff --git a/docs/sources/configuration/_index.md b/docs/sources/configuration/_index.md index bdaa8937da13..9639c813ec9b 100644 --- a/docs/sources/configuration/_index.md +++ b/docs/sources/configuration/_index.md @@ -1631,8 +1631,17 @@ compacts index shards to more performant forms. # Prefix should never start with a separator but should always end with it. [shared_store_key_prefix: | default = "index/"] -# Interval at which to re-run the compaction operation. -[compaction_interval: | default = 2h] +# Interval at which to re-run the compaction operation (or retention if enabled). +[compaction_interval: | default = 10m] + +# (Experimental) Activate custom (per-stream,per-tenant) retention. +[retention_enabled: | default = false] + +# Delay after which chunks will be fully deleted during retention. +[retention_delete_delay: | default = 2h] + +# The total amount of worker to use to delete chunks. +[retention_delete_worker_count: | default = 150] ``` ## limits_config @@ -1753,6 +1762,24 @@ logs in Loki. # CLI flag: -ruler.max-rule-groups-per-tenant [ruler_max_rule_groups_per_tenant: | default = 0] +# Retention to apply for the store, if the retention is enable on the compactor side. +# CLI flag: -store.retention +[retention_period: | default = 744h] + +# Per-stream retention to apply, if the retention is enable on the compactor side. +# Example: +# retention_stream: +# - selector: '{namespace="dev"}' +# priority: 1 +# period: 24h +# - selector: '{container="nginx"}' +# priority: 1 +# period: 744h +# Selector is a Prometheus labels matchers that will apply the `period` retention only if +# the stream is matching. In case multiple stream are matching, the highest priority will be picked. +# If no rule is matched the `retention_period` is used. +[retention_stream: | default = none] + # Feature renamed to 'runtime configuration', flag deprecated in favor of -runtime-config.file (runtime_config.file in YAML). # CLI flag: -limits.per-user-override-config [per_tenant_override_config: ] diff --git a/docs/sources/operations/storage/retention.md b/docs/sources/operations/storage/retention.md index cba6d96865dc..3c62009929ea 100644 --- a/docs/sources/operations/storage/retention.md +++ b/docs/sources/operations/storage/retention.md @@ -3,14 +3,155 @@ title: Retention --- # Loki Storage Retention -Retention in Loki is achieved either through the [Table Manager](../table-manager/) or the [Compactor](./#compactor). +Retention in Loki is achieved either through the [Table Manager](#table-manager) or the [Compactor](#Compactor). -Retention through the table manager is achieved by relying on the object store TTL feature and will work for both boltdb-shipper store and chunk store. However retention through the [Compactor](./#compactor) +## Which one should you use ? +Retention through the [Table Manager](../table-manager/) is achieved by relying on the object store TTL feature and will work for both [boltdb-shipper](../boltdb-shipper) store and chunk/index store. However retention through the [Compactor](../boltdb-shipper#compactor) is only supported when using [boltdb-shipper](../boltdb-shipper) store. -Going forward the compactor retention will become the default and long term supported one. +Going forward the [Compactor](#Compactor) retention will become the default and long term supported one. While this retention is still **experimental** it supports more granular retention (per tenant and per stream) use cases. ## Compactor + +As explained in the [Compactor](../boltdb-shipper#compactor) section, the compactor can deduplicate index entries but can also apply granular retention when activated. +When applying retention with the Compactor, you can totally remove the [Table Manager](../table-manager/) from your Loki workload as it become totally unnecessary. + +> The compactor should always be run as a singleton. (single instance) + +### How it works ? + +Once activated, the Compactor will run a in loop that will apply compaction and retention at every `compaction_interval`. If it runs behind, the will executed as soon as possible. + +The execution is as follow: + +- First the compactor will download all available tables (previously uploaded by ingesters) for each day. +- For each day and table it will: + - First compact the table into a single index file. + - Then traverse the whole index to mark chunks that need to be removed based on the tenant configuration. + - Marked chunks are directly removed from the index and their reference is saved into a file on disk. + - Finally re-upload the new modified index files. + +Retention is applying directly on the index, however chunks are not deleted directly. They actually will be deleted by the compactor asynchronously in another loop (sweeper). + +Marked chunks will only be deleted after `retention_delete_delay` configured is expired because: + +1. boltdb-shipper indexes are refreshed from the shared store on components using it (querier and ruler) at a specific interval. This means deleting chunks instantly could lead to components still having reference to old chunks and so they could fails to execute queries. Having a delay allows for components to refresh their store and so remove gracefully their reference of those chunks. + +2. It gives you a short period to cancel chunks deletion in case of mistakes. + +Marker files (containing chunks to deletes), should be store on a persistent disk, since this is the sole reference to them anymore. + +### Configuration + +The following compactor config shows how to activate retention with the compactor. + +```yaml +compactor: + working_directory: /data/retention + shared_store: gcs + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 +schema_config: + configs: + - from: "2020-07-31" + index: + period: 24h + prefix: loki_index_ + object_store: gcs + schema: v11 + store: boltdb-shipper +storage_config: + boltdb_shipper: + active_index_directory: /data/index + cache_location: /data/boltdb-cache + shared_store: gcs + gcs: + bucket_name: loki +``` + +> Note that retention is only available if the index period is 24h. + +As you can see the retention needs `schema_config` and `storage_config` to access the storage. +To activate retention you'll need to set `retention_enabled` to true otherwise the compactor will only compact tables. + +The `working_directory` is the directory where marked chunks and temporary tables will be saved. +The `compaction_interval` dictates how often retention and/or compaction is applied, if it falls behind it will execute as soon as possible. +`retention_delete_delay` is the delay after which the compactor will actually delete marked chunks. + +And finally `retention_delete_worker_count` setup the amount of goroutine to use to delete chunks. + +> Compaction and retention is idempotent, in case of restart the compactor will restart where he left off. + +#### Configuring retention period + +So far we've only talked about how to setup the compactor to apply retention, in this section we will explain how to configure the actual retention period to apply. + +Retention period is configure via the [`limits_config`](./../../../configuration/#limits_config) configuration section. It supports two type of retention `retention_stream` which is applied to all chunks matching the selector + +The example below can be added to your Loki configuration to configure global retention. + +```yaml +... +limits_config: + retention_period: 744h + retention_stream: + - selector: '{namespace="dev"}' + priority: 1 + period: 24h + per_tenant_override_config: /etc/overrides.yaml +... +``` + +Per tenant retention can be defined using the `/etc/overrides.yaml` files. For example: + +```yaml +overrides: + "29": + retention_period: 168h + retention_stream: + - selector: '{namespace="prod"}' + priority: 2 + period: 336h + - selector: '{container="loki"}' + priority: 1 + period: 72h + "30": + retention_stream: + - selector: '{container="nginx"}' + priority: 1 + period: 24h +``` + +The most specific matching rule is selected. The rule selection is as follow: + +- If a per-tenant `retention_stream` matches the current stream, the highest priority is picked. +- Otherwise if a global `retention_stream` matches the current stream, the highest priority is picked. +- Otherwise if a per-tenant `retention_period` is specified it will be applied. +- The the global `retention_period` will be selected if nothing else matched. +- And finally if no global `retention_period` is set, the default `30days (744h)` retention is selected. + +Stream matching is using the same syntax as Prometheus label matching: + +- `=`: Select labels that are exactly equal to the provided string. +- `!=`: Select labels that are not equal to the provided string. +- `=~`: Select labels that regex-match the provided string. +- `!~`: Select labels that do not regex-match the provided string. + +The two configurations will, for example, set those rules: + +- All tenants except `29` and `30` will have the retention for streams that are in the `dev` namespace to `24h` hours, and other streams will have a retention of `744h`. + - For tenant `29`: + - All streams except, the container `loki` or the `namespace` prod, will have retention of 1 week (`168h`). + - All streams in `prod` will have a retention of `336h` (2 weeks), even the if the container label is also `loki` since the priority of the `prod` rule is higher. + - Steams that have the label container `loki` but are not in the namespace `prod` will have 72h retention. + - For tenant `30`: + - All streams, except those having the label container `nginx` will have a retention of `744h` (the global retention,since it doesn't overrides a new one). + - Streams that have the label `nginx` will have a retention of `24h`. + +> The minimum retention period is 24h + ## Table Manager In order to enable the retention support, the Table Manager needs to be From aac61d1c4a71a4fafd502078a5941bdfba2869d6 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 3 Jun 2021 11:24:35 +0200 Subject: [PATCH 3/7] Addresses Karen's feedback. Signed-off-by: Cyril Tovena --- docs/sources/operations/storage/retention.md | 101 ++++++++++--------- 1 file changed, 53 insertions(+), 48 deletions(-) diff --git a/docs/sources/operations/storage/retention.md b/docs/sources/operations/storage/retention.md index 3c62009929ea..2e638f2345dd 100644 --- a/docs/sources/operations/storage/retention.md +++ b/docs/sources/operations/storage/retention.md @@ -5,33 +5,31 @@ title: Retention Retention in Loki is achieved either through the [Table Manager](#table-manager) or the [Compactor](#Compactor). -## Which one should you use ? +Which one should you use ? -Retention through the [Table Manager](../table-manager/) is achieved by relying on the object store TTL feature and will work for both [boltdb-shipper](../boltdb-shipper) store and chunk/index store. However retention through the [Compactor](../boltdb-shipper#compactor) is only supported when using [boltdb-shipper](../boltdb-shipper) store. +Retention through the [Table Manager](../table-manager/) is achieved by relying on the object store TTL feature, and will work for both [boltdb-shipper](../boltdb-shipper) store and chunk/index store. However retention through the [Compactor](../boltdb-shipper#compactor) is supported only with the [boltdb-shipper](../boltdb-shipper) store. -Going forward the [Compactor](#Compactor) retention will become the default and long term supported one. While this retention is still **experimental** it supports more granular retention (per tenant and per stream) use cases. +The [Compactor](#Compactor) retention will become the default and have long term support. While this retention is still **experimental**, it supports more granular retention policies on per tenant and per stream use cases. ## Compactor -As explained in the [Compactor](../boltdb-shipper#compactor) section, the compactor can deduplicate index entries but can also apply granular retention when activated. -When applying retention with the Compactor, you can totally remove the [Table Manager](../table-manager/) from your Loki workload as it become totally unnecessary. +The [Compactor](../boltdb-shipper#compactor) can deduplicate index entries. It can also apply granular retention. When applying retention with the Compactor, the [Table Manager](../table-manager/) is unnecessary. -> The compactor should always be run as a singleton. (single instance) +> Run the compactor as a singleton (a single instance). -### How it works ? +Compaction and retention are idempotent. If the compactor restarts, it will continue from where it left off. -Once activated, the Compactor will run a in loop that will apply compaction and retention at every `compaction_interval`. If it runs behind, the will executed as soon as possible. +The Compactor loops to apply compaction and retention at every `compaction_interval`, or as soon as possible if running behind. -The execution is as follow: +The compactor's algorithm to update the index: -- First the compactor will download all available tables (previously uploaded by ingesters) for each day. -- For each day and table it will: - - First compact the table into a single index file. - - Then traverse the whole index to mark chunks that need to be removed based on the tenant configuration. - - Marked chunks are directly removed from the index and their reference is saved into a file on disk. - - Finally re-upload the new modified index files. +- For each table within each day: + - Compact the table into a single index file. + - Traverse the entire index. Use the tenant configuration to identify and mark chunks that need to be removed. + - Remove marked chunks from the index and save their reference in a file on disk. + - Upload the new modified index files. -Retention is applying directly on the index, however chunks are not deleted directly. They actually will be deleted by the compactor asynchronously in another loop (sweeper). +The retention algorithm is applied to the index. Chunks are not deleted while applying the retention algorithm. The chunks will be deleted by the compactor asynchronously when swept. Marked chunks will only be deleted after `retention_delete_delay` configured is expired because: @@ -41,9 +39,9 @@ Marked chunks will only be deleted after `retention_delete_delay` configured is Marker files (containing chunks to deletes), should be store on a persistent disk, since this is the sole reference to them anymore. -### Configuration +### Retention Configuration -The following compactor config shows how to activate retention with the compactor. +This compactor configuration example activates retention. ```yaml compactor: @@ -73,24 +71,32 @@ storage_config: > Note that retention is only available if the index period is 24h. -As you can see the retention needs `schema_config` and `storage_config` to access the storage. -To activate retention you'll need to set `retention_enabled` to true otherwise the compactor will only compact tables. +Set `retention_enabled` to true. Without this, the Compactor will only compact tables. -The `working_directory` is the directory where marked chunks and temporary tables will be saved. -The `compaction_interval` dictates how often retention and/or compaction is applied, if it falls behind it will execute as soon as possible. -`retention_delete_delay` is the delay after which the compactor will actually delete marked chunks. +Define `schema_config` and `storage_config` to access the storage. -And finally `retention_delete_worker_count` setup the amount of goroutine to use to delete chunks. +The index period must be 24h. -> Compaction and retention is idempotent, in case of restart the compactor will restart where he left off. +`working_directory` is the directory where marked chunks and temporary tables will be saved. -#### Configuring retention period +`compaction_interval` dictates how often compaction and/or retention is applied. If the Compactor falls behind, compaction and/or retention occur as soon as possible. -So far we've only talked about how to setup the compactor to apply retention, in this section we will explain how to configure the actual retention period to apply. +`retention_delete_delay` is the delay after which the compactor will delete marked chunks. -Retention period is configure via the [`limits_config`](./../../../configuration/#limits_config) configuration section. It supports two type of retention `retention_stream` which is applied to all chunks matching the selector +`retention_delete_worker_count` specifies the maximum quantity of goroutine workers instantiated to delete chunks. -The example below can be added to your Loki configuration to configure global retention. +#### Configuring the retention period + +Retention period is configured within the [`limits_config`](./../../../configuration/#limits_config) configuration section. + +There are two ways of setting retention policies: + +- `retention_period` which is applied globally. +- `retention_stream` which is only applied to chunks matching the selector + +> The minimum retention period is 24h. + +This example configures global retention: ```yaml ... @@ -124,33 +130,32 @@ overrides: period: 24h ``` -The most specific matching rule is selected. The rule selection is as follow: +A rule to apply is selected by choosing the first in this list that matches: -- If a per-tenant `retention_stream` matches the current stream, the highest priority is picked. -- Otherwise if a global `retention_stream` matches the current stream, the highest priority is picked. -- Otherwise if a per-tenant `retention_period` is specified it will be applied. -- The the global `retention_period` will be selected if nothing else matched. -- And finally if no global `retention_period` is set, the default `30days (744h)` retention is selected. +1. If a per-tenant `retention_stream` matches the current stream, the highest priority is picked. +2. If a global `retention_stream` matches the current stream, the highest priority is picked. +3. If a per-tenant `retention_period` is specified, it will be applied. +4. The global `retention_period` will be selected if nothing else matched. +5. If no global `retention_period` is specified, the default value of `744h` (30days) retention is used. -Stream matching is using the same syntax as Prometheus label matching: +Stream matching is uses the same syntax as Prometheus label matching: - `=`: Select labels that are exactly equal to the provided string. - `!=`: Select labels that are not equal to the provided string. - `=~`: Select labels that regex-match the provided string. - `!~`: Select labels that do not regex-match the provided string. -The two configurations will, for example, set those rules: - -- All tenants except `29` and `30` will have the retention for streams that are in the `dev` namespace to `24h` hours, and other streams will have a retention of `744h`. - - For tenant `29`: - - All streams except, the container `loki` or the `namespace` prod, will have retention of 1 week (`168h`). - - All streams in `prod` will have a retention of `336h` (2 weeks), even the if the container label is also `loki` since the priority of the `prod` rule is higher. - - Steams that have the label container `loki` but are not in the namespace `prod` will have 72h retention. - - For tenant `30`: - - All streams, except those having the label container `nginx` will have a retention of `744h` (the global retention,since it doesn't overrides a new one). - - Streams that have the label `nginx` will have a retention of `24h`. - -> The minimum retention period is 24h +The example configurations will, set these rules: + +- All tenants except `29` and `30` in the `dev` namespace will have a retention period of `24h` hours. +- All tenants except `29` and `30` that are not in the `dev` namespace will have the retention period of `744h`. +- For tenant `29`: + - All streams except those in the container `loki` or in the namespace `prod` will have retention period of `168h` (1 week). + - All streams in the `prod` namespace will have a retention period of `336h` (2 weeks), even if the container label is `loki`, since the priority of the `prod` rule is higher. + - Streams that have the container label `loki` but are not in the namespace `prod` will have a `72h` retention period. +- For tenant `30`: + - All streams except those having the container label `nginx` will have the global retention period of `744h`, since there is no override specified. + - Streams that have the label `nginx` will have a retention period of `24h`. ## Table Manager From 0c0407705fafbb80e73c7457a5458105e7b41958 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 7 Jun 2021 09:02:57 +0200 Subject: [PATCH 4/7] Update docs/sources/operations/storage/retention.md Co-authored-by: Karen Miller <84039272+KMiller-Grafana@users.noreply.github.com> --- docs/sources/operations/storage/retention.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sources/operations/storage/retention.md b/docs/sources/operations/storage/retention.md index 2e638f2345dd..0880647fabd8 100644 --- a/docs/sources/operations/storage/retention.md +++ b/docs/sources/operations/storage/retention.md @@ -5,7 +5,6 @@ title: Retention Retention in Loki is achieved either through the [Table Manager](#table-manager) or the [Compactor](#Compactor). -Which one should you use ? Retention through the [Table Manager](../table-manager/) is achieved by relying on the object store TTL feature, and will work for both [boltdb-shipper](../boltdb-shipper) store and chunk/index store. However retention through the [Compactor](../boltdb-shipper#compactor) is supported only with the [boltdb-shipper](../boltdb-shipper) store. From 6d75a96b1f4201b5aa3fe22d1025bfa42994eb64 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 7 Jun 2021 09:03:28 +0200 Subject: [PATCH 5/7] Update docs/sources/operations/storage/retention.md Co-authored-by: Karen Miller <84039272+KMiller-Grafana@users.noreply.github.com> --- docs/sources/operations/storage/retention.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/operations/storage/retention.md b/docs/sources/operations/storage/retention.md index 0880647fabd8..4371a97b9299 100644 --- a/docs/sources/operations/storage/retention.md +++ b/docs/sources/operations/storage/retention.md @@ -144,7 +144,7 @@ Stream matching is uses the same syntax as Prometheus label matching: - `=~`: Select labels that regex-match the provided string. - `!~`: Select labels that do not regex-match the provided string. -The example configurations will, set these rules: +The example configurations will set these rules: - All tenants except `29` and `30` in the `dev` namespace will have a retention period of `24h` hours. - All tenants except `29` and `30` that are not in the `dev` namespace will have the retention period of `744h`. From 33255cbb9985b0d0e32fc9250f26959f9eee4cc3 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 7 Jun 2021 09:03:37 +0200 Subject: [PATCH 6/7] Update docs/sources/operations/storage/retention.md Co-authored-by: Karen Miller <84039272+KMiller-Grafana@users.noreply.github.com> --- docs/sources/operations/storage/retention.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/operations/storage/retention.md b/docs/sources/operations/storage/retention.md index 4371a97b9299..33179e532206 100644 --- a/docs/sources/operations/storage/retention.md +++ b/docs/sources/operations/storage/retention.md @@ -137,7 +137,7 @@ A rule to apply is selected by choosing the first in this list that matches: 4. The global `retention_period` will be selected if nothing else matched. 5. If no global `retention_period` is specified, the default value of `744h` (30days) retention is used. -Stream matching is uses the same syntax as Prometheus label matching: +Stream matching uses the same syntax as Prometheus label matching: - `=`: Select labels that are exactly equal to the provided string. - `!=`: Select labels that are not equal to the provided string. From b4420384cb8ed1f50644a4893871d118ddc21cda Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Mon, 7 Jun 2021 09:19:37 +0200 Subject: [PATCH 7/7] Update docs/sources/operations/storage/retention.md Co-authored-by: Karen Miller <84039272+KMiller-Grafana@users.noreply.github.com> --- docs/sources/operations/storage/retention.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/sources/operations/storage/retention.md b/docs/sources/operations/storage/retention.md index 33179e532206..75170ab1bfed 100644 --- a/docs/sources/operations/storage/retention.md +++ b/docs/sources/operations/storage/retention.md @@ -36,7 +36,13 @@ Marked chunks will only be deleted after `retention_delete_delay` configured is 2. It gives you a short period to cancel chunks deletion in case of mistakes. -Marker files (containing chunks to deletes), should be store on a persistent disk, since this is the sole reference to them anymore. +Marked chunks will only be deleted after `retention_delete_delay` configured is expired because + +- boltdb-shipper indexes are refreshed from the shared store on components using them (querier and ruler) at a specific interval. Deleting chunks instantly could lead to components still having a reference to old chunks. This could, in turn, cause query execution failure. Having a delay allows components to refresh their store and gracefully remove their reference to those chunks. + +- It provides a short window of time in which to cancel chunk deletion in the case of a configuration mistake. + +Marker files (containing chunks to delete) should be stored on a persistent disk, since the disk will be the sole reference to them. ### Retention Configuration