From ebb39db864f56986cb8385166d220980fd8be684 Mon Sep 17 00:00:00 2001 From: Artem Timchenko <77285301+timchenko-a@users.noreply.github.com> Date: Thu, 5 May 2022 14:22:00 +0300 Subject: [PATCH] lambda-promtail: Add multi-tenancy support (#6102) * lambda-promtail: adding multi-tenancy support * update changelog and fix import order * fix import order * fix import order --- CHANGELOG.md | 1 + docs/sources/clients/lambda-promtail/_index.md | 6 ++++++ tools/lambda-promtail/README.md | 4 ++-- tools/lambda-promtail/lambda-promtail/main.go | 14 ++++++++------ tools/lambda-promtail/lambda-promtail/promtail.go | 4 ++++ tools/lambda-promtail/main.tf | 1 + tools/lambda-promtail/template.yaml | 5 +++++ tools/lambda-promtail/variables.tf | 6 ++++++ 8 files changed, 33 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbbb274439f9..53d8e56b44fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Main +* [6102](https://github.com/grafana/loki/pull/6102) **timchenko-a**: Add multi-tenancy support to lambda-promtail * [5971](https://github.com/grafana/loki/pull/5971) **kavirajk**: Record statistics about metadata queries such as labels and series queries in `metrics.go` as well * [5790](https://github.com/grafana/loki/pull/5790) **chaudum**: Add UDP support for Promtail's syslog target. * [5984](https://github.com/grafana/loki/pull/5984) **dannykopping** and **salvacorts**: Querier: prevent unnecessary calls to ingesters. diff --git a/docs/sources/clients/lambda-promtail/_index.md b/docs/sources/clients/lambda-promtail/_index.md index 01e10254f3d1..e17a7167bc26 100644 --- a/docs/sources/clients/lambda-promtail/_index.md +++ b/docs/sources/clients/lambda-promtail/_index.md @@ -22,6 +22,8 @@ There's also a flag to keep the log stream label when propagating the logs from Additionally, an environment variable can be configured to add extra lables to the logs streamed by lambda-protmail. These extra labels will take the form `__extra_=` +Optional environment variable can be configured to add tenant id to the logs streamed by lambda-protmail. + In an effort to make deployment of lambda-promtail as simple as possible, we've created a [public ECR repo](https://gallery.ecr.aws/grafana/lambda-promtail) to publish our builds of lambda-promtail. Users are still able to clone this repo, make their own modifications to the Go code, and upload their own image to their own ECR repo if they wish. ### Examples @@ -42,6 +44,8 @@ To keep the log group label add `-var "keep_stream=true"`. To add extra labels add `-var 'extra_labels="name1,value1,name2,value2"'` +To add tenant id add `-var "tenant_id=value"` + Note that the creation of subscription filter on Cloudwatch in the provided Terraform file only accepts an array of log group names, it does **not** accept strings for regex filtering on the logs contents via the subscription filters. We suggest extending the Terraform file to do so, or having lambda-promtail write to Promtail and using [pipeline stages](https://grafana.com/docs/loki/latest/clients/promtail/stages/drop/). CloudFormation: @@ -63,6 +67,8 @@ To keep the log group label add `ParameterKey=KeepStream,ParameterValue=true`. To add extra labels, include `ParameterKey=ExtraLabels,ParameterValue="name1,value1,name2,value2"` +To add tenant id add `ParameterKey=TenantID,ParameterValue=value`. + To modify an already created CloudFormation stack you need to use [update-stack](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/update-stack.html). ## Uses diff --git a/tools/lambda-promtail/README.md b/tools/lambda-promtail/README.md index f2188a476bde..b828657683a9 100644 --- a/tools/lambda-promtail/README.md +++ b/tools/lambda-promtail/README.md @@ -50,13 +50,13 @@ Also, if your deployment requires a [VPC configuration](https://registry.terrafo Then use Terraform to deploy: ```bash -terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' +terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=" ``` or CloudFormation: ```bash -aws cloudformation create-stack --stack-name lambda-promtail-stack --template-body file://template.yaml --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM --region us-east-2 --parameters ParameterKey=WriteAddress,ParameterValue=https://your-loki-url/loki/api/v1/push ParameterKey=Username,ParameterValue= ParameterKey=Password,ParameterValue= ParameterKey=LambdaPromtailImage,ParameterValue=: ParameterKey=ExtraLabels,ParameterValue="name1,value1,name2,value2" +aws cloudformation create-stack --stack-name lambda-promtail-stack --template-body file://template.yaml --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM --region us-east-2 --parameters ParameterKey=WriteAddress,ParameterValue=https://your-loki-url/loki/api/v1/push ParameterKey=Username,ParameterValue= ParameterKey=Password,ParameterValue= ParameterKey=LambdaPromtailImage,ParameterValue=: ParameterKey=ExtraLabels,ParameterValue="name1,value1,name2,value2" ParameterKey=TenantID,ParameterValue= ``` # Appendix diff --git a/tools/lambda-promtail/lambda-promtail/main.go b/tools/lambda-promtail/lambda-promtail/main.go index 3b8a9857c861..9aca2126f9d9 100644 --- a/tools/lambda-promtail/lambda-promtail/main.go +++ b/tools/lambda-promtail/lambda-promtail/main.go @@ -26,12 +26,12 @@ const ( ) var ( - writeAddress *url.URL - username, password, extraLabelsRaw string - keepStream bool - batchSize int - s3Clients map[string]*s3.Client - extraLabels model.LabelSet + writeAddress *url.URL + username, password, extraLabelsRaw, tenantID string + keepStream bool + batchSize int + s3Clients map[string]*s3.Client + extraLabels model.LabelSet ) func setupArguments() { @@ -61,6 +61,8 @@ func setupArguments() { panic("both username and password must be set if either one is set") } + tenantID = os.Getenv("TENANT_ID") + keep := os.Getenv("KEEP_STREAM") // Anything other than case-insensitive 'true' is treated as 'false'. if strings.EqualFold(keep, "true") { diff --git a/tools/lambda-promtail/lambda-promtail/promtail.go b/tools/lambda-promtail/lambda-promtail/promtail.go index bd3cf3f3d05f..64d1348b76d6 100644 --- a/tools/lambda-promtail/lambda-promtail/promtail.go +++ b/tools/lambda-promtail/lambda-promtail/promtail.go @@ -169,6 +169,10 @@ func send(ctx context.Context, buf []byte) (int, error) { req.Header.Set("Content-Type", contentType) req.Header.Set("User-Agent", userAgent) + if tenantID != "" { + req.Header.Set("X-Scope-OrgID", tenantID) + } + if username != "" && password != "" { req.SetBasicAuth(username, password) } diff --git a/tools/lambda-promtail/main.tf b/tools/lambda-promtail/main.tf index b4c2f580cbcd..ba71bb198a26 100644 --- a/tools/lambda-promtail/main.tf +++ b/tools/lambda-promtail/main.tf @@ -86,6 +86,7 @@ resource "aws_lambda_function" "lambda_promtail" { KEEP_STREAM = var.keep_stream BATCH_SIZE = var.batch_size EXTRA_LABELS = var.extra_labels + TENANT_ID = var.tenant_id } } diff --git a/tools/lambda-promtail/template.yaml b/tools/lambda-promtail/template.yaml index 21e60e8215cb..5e15b48d5e94 100644 --- a/tools/lambda-promtail/template.yaml +++ b/tools/lambda-promtail/template.yaml @@ -34,6 +34,10 @@ Parameters: Description: Comma separated list of extra labels, in the format 'name1,value1,name2,value2,...,nameN,valueN' to add to entries forwarded by lambda-promtail. Type: String Default: "" + TenantID: + Description: Tenant ID to be added when writing logs from lambda-promtail. + Type: String + Default: "" Resources: LambdaPromtailRole: @@ -80,6 +84,7 @@ Resources: PASSWORD: !Ref Password KEEP_STREAM: !Ref KeepStream EXTRA_LABELS: !Ref ExtraLabels + TENANT_ID: !Ref TenantID LambdaPromtailVersion: Type: AWS::Lambda::Version Properties: diff --git a/tools/lambda-promtail/variables.tf b/tools/lambda-promtail/variables.tf index f94c6d2ce9df..1a7d6dcdd02f 100644 --- a/tools/lambda-promtail/variables.tf +++ b/tools/lambda-promtail/variables.tf @@ -35,6 +35,12 @@ variable "password" { default = "" } +variable "tenant_id" { + type = string + description = "Tenant ID to be added when writing logs from lambda-promtail." + default = "" +} + variable "keep_stream" { type = string description = "Determines whether to keep the CloudWatch Log Stream value as a Loki label when writing logs from lambda-promtail."