lambda-promtail: Add multi-tenancy support (#6102)

* lambda-promtail: adding multi-tenancy support

* update changelog and fix import order

* fix import order

* fix import order

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+* [6102](https://github.com/grafana/loki/pull/6102) **timchenko-a**: Add multi-tenancy support to lambda-promtail
 * [5971](https://github.com/grafana/loki/pull/5971) **kavirajk**: Record statistics about metadata queries such as labels and series queries in `metrics.go` as well
 * [5790](https://github.com/grafana/loki/pull/5790) **chaudum**: Add UDP support for Promtail's syslog target.
 * [5984](https://github.com/grafana/loki/pull/5984) **dannykopping** and **salvacorts**: Querier: prevent unnecessary calls to ingesters.

docs/sources/clients/lambda-promtail/_index.md

@@ -22,6 +22,8 @@ There's also a flag to keep the log stream label when propagating the logs from
 Additionally, an environment variable can be configured to add extra lables to the logs streamed by lambda-protmail.
 These extra labels will take the form `__extra_<name>=<value>`
 
+Optional environment variable can be configured to add tenant id to the logs streamed by lambda-protmail.
+
 In an effort to make deployment of lambda-promtail as simple as possible, we've created a [public ECR repo](https://gallery.ecr.aws/grafana/lambda-promtail) to publish our builds of lambda-promtail. Users are still able to clone this repo, make their own modifications to the Go code, and upload their own image to their own ECR repo if they wish.
 
 ### Examples
@@ -42,6 +44,8 @@ To keep the log group label add `-var "keep_stream=true"`.
 
 To add extra labels add `-var 'extra_labels="name1,value1,name2,value2"'`
 
+To add tenant id add `-var "tenant_id=value"`
+
 Note that the creation of subscription filter on Cloudwatch in the provided Terraform file only accepts an array of log group names, it does **not** accept strings for regex filtering on the logs contents via the subscription filters. We suggest extending the Terraform file to do so, or having lambda-promtail write to Promtail and using [pipeline stages](https://grafana.com/docs/loki/latest/clients/promtail/stages/drop/).
 
 CloudFormation:
@@ -63,6 +67,8 @@ To keep the log group label add `ParameterKey=KeepStream,ParameterValue=true`.
 
 To add extra labels, include `ParameterKey=ExtraLabels,ParameterValue="name1,value1,name2,value2"`
 
+To add tenant id add `ParameterKey=TenantID,ParameterValue=value`.
+
 To modify an already created CloudFormation stack you need to use [update-stack](https://docs.aws.amazon.com/cli/latest/reference/cloudformation/update-stack.html).
 
 ## Uses

tools/lambda-promtail/README.md

@@ -50,13 +50,13 @@ Also, if your deployment requires a [VPC configuration](https://registry.terrafo
 Then use Terraform to deploy:
 
 ```bash
-terraform apply -var "<ecr-repo>:<tag>" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"'
+terraform apply -var "<ecr-repo>:<tag>" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=<value>"
 ```
 
 or CloudFormation:
 
 ```bash
-aws cloudformation create-stack --stack-name lambda-promtail-stack --template-body file://template.yaml --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM --region us-east-2 --parameters ParameterKey=WriteAddress,ParameterValue=https://your-loki-url/loki/api/v1/push ParameterKey=Username,ParameterValue=<basic-auth-username> ParameterKey=Password,ParameterValue=<basic-auth-pw> ParameterKey=LambdaPromtailImage,ParameterValue=<ecr-repo>:<tag> ParameterKey=ExtraLabels,ParameterValue="name1,value1,name2,value2"
+aws cloudformation create-stack --stack-name lambda-promtail-stack --template-body file://template.yaml --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM --region us-east-2 --parameters ParameterKey=WriteAddress,ParameterValue=https://your-loki-url/loki/api/v1/push ParameterKey=Username,ParameterValue=<basic-auth-username> ParameterKey=Password,ParameterValue=<basic-auth-pw> ParameterKey=LambdaPromtailImage,ParameterValue=<ecr-repo>:<tag> ParameterKey=ExtraLabels,ParameterValue="name1,value1,name2,value2" ParameterKey=TenantID,ParameterValue=<value>
 ```
 
 # Appendix

tools/lambda-promtail/lambda-promtail/main.go

@@ -26,12 +26,12 @@ const (
 )
 
 var (
-	writeAddress                       *url.URL
-	username, password, extraLabelsRaw string
-	keepStream                         bool
-	batchSize                          int
-	s3Clients                          map[string]*s3.Client
-	extraLabels                        model.LabelSet
+	writeAddress                                 *url.URL
+	username, password, extraLabelsRaw, tenantID string
+	keepStream                                   bool
+	batchSize                                    int
+	s3Clients                                    map[string]*s3.Client
+	extraLabels                                  model.LabelSet
 )
 
 func setupArguments() {
@@ -61,6 +61,8 @@ func setupArguments() {
 		panic("both username and password must be set if either one is set")
 	}
 
+	tenantID = os.Getenv("TENANT_ID")
+
 	keep := os.Getenv("KEEP_STREAM")
 	// Anything other than case-insensitive 'true' is treated as 'false'.
 	if strings.EqualFold(keep, "true") {

tools/lambda-promtail/lambda-promtail/promtail.go

@@ -169,6 +169,10 @@ func send(ctx context.Context, buf []byte) (int, error) {
 	req.Header.Set("Content-Type", contentType)
 	req.Header.Set("User-Agent", userAgent)
 
+	if tenantID != "" {
+		req.Header.Set("X-Scope-OrgID", tenantID)
+	}
+
 	if username != "" && password != "" {
 		req.SetBasicAuth(username, password)
 	}

tools/lambda-promtail/main.tf

@@ -86,6 +86,7 @@ resource "aws_lambda_function" "lambda_promtail" {
       KEEP_STREAM   = var.keep_stream
       BATCH_SIZE    = var.batch_size
       EXTRA_LABELS  = var.extra_labels
+      TENANT_ID     = var.tenant_id
     }
   }
 

tools/lambda-promtail/template.yaml

@@ -34,6 +34,10 @@ Parameters:
     Description: Comma separated list of extra labels, in the format 'name1,value1,name2,value2,...,nameN,valueN' to add to entries forwarded by lambda-promtail.
     Type: String
     Default: ""
+  TenantID:
+    Description: Tenant ID to be added when writing logs from lambda-promtail.
+    Type: String
+    Default: ""
 
 Resources:
   LambdaPromtailRole:
@@ -80,6 +84,7 @@ Resources:
           PASSWORD: !Ref Password
           KEEP_STREAM: !Ref KeepStream
           EXTRA_LABELS: !Ref ExtraLabels
+          TENANT_ID: !Ref TenantID
   LambdaPromtailVersion:
     Type: AWS::Lambda::Version
     Properties:

tools/lambda-promtail/variables.tf

@@ -35,6 +35,12 @@ variable "password" {
   default     = ""
 }
 
+variable "tenant_id" {
+  type        = string
+  description = "Tenant ID to be added when writing logs from lambda-promtail."
+  default     = ""
+}
+
 variable "keep_stream" {
   type        = string
   description = "Determines whether to keep the CloudWatch Log Stream value as a Loki label when writing logs from lambda-promtail."