impr/clients: Handle TLS config and MTLS for logcli and promtail (#540)

* impr/clients: Handle TLS config and MTLS for logcli and promtail

* fix/tls: Please gofmt...

* impr/clients: use prometheus HTTPClientConfig for logcli and promtail

* fix/promtail: Set proper Client config name

* impr/promtail: Use prometheus HTTPClientConfig configuration

* adapt with master

* address review

* fix conflicts

* address requested changes

* remove file

Signed-off-by: Goutham Veeramachaneni <gouthamve@gmail.com>

cmd/logcli/client.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/gorilla/websocket"
+	"github.com/prometheus/common/config"
 
 	"github.com/grafana/loki/pkg/logproto"
 )
@@ -60,9 +61,25 @@ func doRequest(path string, out interface{}) error {
 	if err != nil {
 		return err
 	}
+
 	req.SetBasicAuth(*username, *password)
 
-	resp, err := http.DefaultClient.Do(req)
+	clientConfig := config.HTTPClientConfig{
+		TLSConfig: config.TLSConfig{
+			CAFile:             *tlsCACertPath,
+			CertFile:           *tlsClientCertPath,
+			KeyFile:            *tlsClientCertKeyPath,
+			ServerName:         url,
+			InsecureSkipVerify: *tlsSkipVerify,
+		},
+	}
+
+	client, err := config.NewClientFromConfig(clientConfig, "logcli")
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}
@@ -86,6 +103,18 @@ func liveTailQueryConn() (*websocket.Conn, error) {
 }
 
 func wsConnect(path string) (*websocket.Conn, error) {
+
+	tlsConfig, err := config.NewTLSConfig(&config.TLSConfig{
+		CAFile:             *tlsCACertPath,
+		CertFile:           *tlsClientCertPath,
+		KeyFile:            *tlsClientCertKeyPath,
+		ServerName:         *addr,
+		InsecureSkipVerify: *tlsSkipVerify,
+	})
+	if err != nil {
+		return nil, err
+	}
+
 	url := *addr + path
 	if strings.HasPrefix(url, "https") {
 		url = strings.Replace(url, "https", "wss", 1)
@@ -95,7 +124,12 @@ func wsConnect(path string) (*websocket.Conn, error) {
 	log.Println(url)
 
 	h := http.Header{"Authorization": {"Basic " + base64.StdEncoding.EncodeToString([]byte(*username+":"+*password))}}
-	c, resp, err := websocket.DefaultDialer.Dial(url, h)
+
+	ws := websocket.Dialer{
+		TLSClientConfig: tlsConfig,
+	}
+
+	c, resp, err := ws.Dial(url, h)
 
 	if err != nil {
 		if resp == nil {

cmd/logcli/main.go

@@ -14,6 +14,11 @@ var (
 	username = app.Flag("username", "Username for HTTP basic auth.").Default("").Envar("GRAFANA_USERNAME").String()
 	password = app.Flag("password", "Password for HTTP basic auth.").Default("").Envar("GRAFANA_PASSWORD").String()
 
+	tlsCACertPath        = app.Flag("ca-cert", "Path to the server Certificate Authority.").Default("").Envar("LOKI_CA_CERT_PATH").String()
+	tlsSkipVerify        = app.Flag("tls-skip-verify", "Server certificate TLS skip verify.").Default("false").Bool()
+	tlsClientCertPath    = app.Flag("cert", "Path to the client certificate.").Default("").Envar("LOKI_CLIENT_CERT_PATH").String()
+	tlsClientCertKeyPath = app.Flag("key", "Path to the client certificate key.").Default("").Envar("LOKI_CLIENT_KEY_PATH").String()
+
 	queryCmd        = app.Command("query", "Run a LogQL query.")
 	queryStr        = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String()
 	regexpStr       = queryCmd.Arg("regex", "").String()

docs/logcli.md

@@ -44,26 +44,31 @@ Common labels: {job="cortex-ops/consul", namespace="cortex-ops"}
 
 ### Configuration
 
-
 Configuration values are considered in the following order (lowest to highest):
+
 - environment value
 - command line
 
 The URLs of the requests are printed to help with integration work.
 
 ### Details
 
-```
+```console
 $ logcli help
 usage: logcli [<flags>] <command> [<args> ...]
 
 A command-line for loki.
 
 Flags:
-  --help         Show context-sensitive help (also try --help-long and --help-man).
-  --addr=""      Server address, need to specify.
-  --username=""  Username for HTTP basic auth.
-  --password=""  Password for HTTP basic auth.
+  --help             Show context-sensitive help (also try --help-long and --help-man).
+  --addr="https://logs-us-west1.grafana.net"
+                     Server address.
+  --username=""      Username for HTTP basic auth.
+  --password=""      Password for HTTP basic auth.
+  --ca-cert=""       Path to the server Certificate Authority.
+  --tls-skip-verify  Server certificate TLS skip verify.
+  --cert=""          Path to the client certificate.
+  --key=""           Path to the client certificate key.
 
 Commands:
   help [<command>...]
@@ -72,7 +77,7 @@ Commands:
   query [<flags>] <query> [<regex>]
     Run a LogQL query.
 
-  labels <label>
+  labels [<label>]
     Find values for a given label.
 
 $ logcli help query

docs/operations.md

@@ -6,6 +6,7 @@ This page lists operational aspects of running Loki in alphabetical order:
 
 Loki does not have an authentication layer.
 You are expected to run an authenticating reverse proxy in front of your services, such as an Nginx with basic auth or an OAuth2 proxy.
+See [client options](promtail-setup.md#custom-client-options) for more details about supported authentication methods.
 
 ### Multi-tenancy
 
@@ -129,7 +130,6 @@ storage_config:
       dynamodb: dynamodb://region
 ```
 
-
 #### S3
 
 Loki is using S3 as object storage. It stores log within directories based on
@@ -158,4 +158,3 @@ create the table manually you cannot easily erase old data and your index just g
 If you set your DynamoDB table manually, ensure you set the primary index key to `h`
 (string) and use `r` (binary) as the sort key. Also set the "period" attribute in the yaml to zero.
 Make sure adjust your throughput base on your usage.
-

docs/promtail-setup.md

@@ -2,7 +2,7 @@
 
 ## Daemonset method
 
-Daemonset will deploy promtail on every node within the kubernetes cluster.
+Daemonset will deploy `promtail` on every node within the Kubernetes cluster.
 
 Daemonset deployment is great to collect all of the container logs within the
 cluster. It is great solution for single tenant.  All of the logs will send to a
@@ -11,6 +11,7 @@ single Loki server.
 Check the `production` folder for examples of a daemonset deployment for kubernetes using both helm and ksonnet.
 
 ### Example
+
 ```yaml
 ---Daemonset.yaml
 apiVersion: extensions/v1beta1
@@ -87,22 +88,23 @@ roleRef:
 
 ## Sidecar Method
 
-Sidecar method will deploy promtail as a container within a pod that
+Sidecar method will deploy `promtail` as a container within a pod that
 developer/devops create.
 
-This method will deploy promtail as a sidecar container within a pod.
+This method will deploy `promtail` as a sidecar container within a pod.
 In a multi-tenant environment, this enables teams to aggregate logs
 for specific pods and deployments for example for all pods in a namespace.
 
 ### Example
+
 ```yaml
 ---Deployment.yaml
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
   name: my_test_app
   ...
-spec: 
+spec:
   ...
   template:
     spec:
@@ -133,10 +135,11 @@ spec:
 Sometime application create customized log files.  To collect those logs, you
 would need to have a customized `__path__` in your scrape_config.
 
-Right now, the best way to watch and tail custom log path is define log filepath
+Right now, the best way to watch and tail custom log path is define log file path
 as a label for the pod.
 
 #### Example
+
 ```yaml
 ---Deployment.yaml
 apiVersion: extensions/v1beta1
@@ -164,3 +167,87 @@ scrape_configs:
         replacement: /your_log_file_dir/$1.log
       ...
 ```
+
+### Custom Client options
+
+`promtail` client configuration uses the [Prometheus http client](https://godoc.org/github.com/prometheus/common/config) implementation.
+Therefore you can configure the following authentication parameters in the `client` or `clients` section.
+
+```yaml
+---promtail_config.yaml
+...
+
+# Simple client
+client:
+  [ <client_option> ]
+
+# Multiple clients
+clients:
+  [ - <client_option> ]
+...
+```
+
+>Note: Passing the `-client.url` from command line is only valid if you set the `client` section.
+
+#### `<client_option>`
+
+```yaml
+  # Sets the `url` of loki api push endpoint
+  url: http[s]://<host>:<port>/api/prom/push
+
+  # Sets the `Authorization` header on every promtail request with the
+  # configured username and password.
+  # password and password_file are mutually exclusive.
+  basic_auth:
+    username: <string>
+    password: <secret>
+    password_file: <string>
+
+  # Sets the `Authorization` header on every promtail request with
+  # the configured bearer token. It is mutually exclusive with `bearer_token_file`.
+  bearer_token: <secret>
+
+  # Sets the `Authorization` header on every promtail request with the bearer token
+  # read from the configured file. It is mutually exclusive with `bearer_token`.
+  bearer_token_file: /path/to/bearer/token/file
+
+  # Configures the promtail request's TLS settings.
+  tls_config:
+    # CA certificate to validate API server certificate with.
+    # If not provided Trusted CA from sytem will be used.
+    ca_file: <filename>
+
+    # Certificate and key files for client cert authentication to the server.
+    cert_file: <filename>
+    key_file: <filename>
+
+    # ServerName extension to indicate the name of the server.
+    # https://tools.ietf.org/html/rfc4366#section-3.1
+    server_name: <string>
+
+    # Disable validation of the server certificate.
+    insecure_skip_verify: <boolean>
+
+  # Optional proxy URL.
+  proxy_url: <string>
+
+  # Maximum wait period before sending batch
+  batchwait: 1s
+
+  # Maximum batch size to accrue before sending, unit is byte
+  batchsize: 102400
+
+  # Maximum time to wait for server to respond to a request
+  timeout: 10s
+
+  backoff_config:
+    # Initial backoff time between retries
+    minbackoff: 100ms
+    # Maximum backoff time between retries
+    maxbackoff: 5s
+    # Maximum number of retires when sending batches, 0 means infinite retries
+    maxretries: 5
+
+  # The labels to add to any time series or alerts when communicating with loki
+  external_labels: {}
+```

pkg/promtail/client/client.go

@@ -18,9 +18,11 @@ import (
 	"github.com/go-kit/kit/log/level"
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
+
 	"github.com/grafana/loki/pkg/helpers"
 	"github.com/grafana/loki/pkg/logproto"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 )
 
@@ -62,6 +64,7 @@ type Client interface {
 type client struct {
 	logger  log.Logger
 	cfg     Config
+	client  *http.Client
 	quit    chan struct{}
 	entries chan entry
 	wg      sync.WaitGroup
@@ -75,7 +78,7 @@ type entry struct {
 }
 
 // New makes a new Client.
-func New(cfg Config, logger log.Logger) Client {
+func New(cfg Config, logger log.Logger) (Client, error) {
 	c := &client{
 		logger:  log.With(logger, "component", "client", "host", cfg.URL.Host),
 		cfg:     cfg,
@@ -84,9 +87,22 @@ func New(cfg Config, logger log.Logger) Client {
 
 		externalLabels: cfg.ExternalLabels.LabelSet,
 	}
+
+	err := cfg.Client.Validate()
+	if err != nil {
+		return nil, err
+	}
+
+	c.client, err = config.NewClientFromConfig(cfg.Client, "promtail")
+	if err != nil {
+		return nil, err
+	}
+
+	c.client.Timeout = cfg.Timeout
+
 	c.wg.Add(1)
 	go c.run()
-	return c
+	return c, nil
 }
 
 func (c *client) run() {
@@ -194,7 +210,7 @@ func (c *client) send(ctx context.Context, buf []byte) (int, error) {
 	req = req.WithContext(ctx)
 	req.Header.Set("Content-Type", contentType)
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := c.client.Do(req)
 	if err != nil {
 		return -1, err
 	}

pkg/promtail/client/config.go

@@ -7,6 +7,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 	lokiflag "github.com/grafana/loki/pkg/util/flagext"
+	"github.com/prometheus/common/config"
 )
 
 // Config describes configuration for a HTTP pusher client.
@@ -15,6 +16,8 @@ type Config struct {
 	BatchWait time.Duration
 	BatchSize int
 
+	Client config.HTTPClientConfig `yaml:",inline"`
+
 	BackoffConfig util.BackoffConfig `yaml:"backoff_config"`
 	// The labels to add to any time series or alerts when communicating with loki
 	ExternalLabels lokiflag.LabelSet `yaml:"external_labels,omitempty"`

pkg/promtail/client/multi.go

@@ -17,9 +17,14 @@ func NewMulti(logger log.Logger, cfgs ...Config) (Client, error) {
 	if len(cfgs) == 0 {
 		return nil, errors.New("at least one client config should be provided")
 	}
-	var clients []Client
+
+	clients := make([]Client, 0, len(cfgs))
 	for _, cfg := range cfgs {
-		clients = append(clients, New(cfg, logger))
+		client, err := New(cfg, logger)
+		if err != nil {
+			return nil, err
+		}
+		clients = append(clients, client)
 	}
 	return MultiClient(clients), nil
 }