Skip to content

Commit

Permalink
impr/clients: Handle TLS config and MTLS for logcli and promtail (#540)
Browse files Browse the repository at this point in the history
* impr/clients: Handle TLS config and MTLS for logcli and promtail

* fix/tls: Please gofmt...

* impr/clients: use prometheus HTTPClientConfig for logcli and promtail

* fix/promtail: Set proper Client config name

* impr/promtail: Use prometheus HTTPClientConfig configuration

* adapt with master

* address review

* fix conflicts

* address requested changes

* remove file
  • Loading branch information
CyrilPeponnet authored and cyriltovena committed May 15, 2019
1 parent 0885f64 commit bbaf1cd
Show file tree
Hide file tree
Showing 9 changed files with 175 additions and 21 deletions.
38 changes: 36 additions & 2 deletions cmd/logcli/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"time"

"github.com/gorilla/websocket"
"github.com/prometheus/common/config"

"github.com/grafana/loki/pkg/logproto"
)
Expand Down Expand Up @@ -60,9 +61,25 @@ func doRequest(path string, out interface{}) error {
if err != nil {
return err
}

req.SetBasicAuth(*username, *password)

resp, err := http.DefaultClient.Do(req)
clientConfig := config.HTTPClientConfig{
TLSConfig: config.TLSConfig{
CAFile: *tlsCACertPath,
CertFile: *tlsClientCertPath,
KeyFile: *tlsClientCertKeyPath,
ServerName: url,
InsecureSkipVerify: *tlsSkipVerify,
},
}

client, err := config.NewClientFromConfig(clientConfig, "logcli")
if err != nil {
return err
}

resp, err := client.Do(req)
if err != nil {
return err
}
Expand All @@ -86,6 +103,18 @@ func liveTailQueryConn() (*websocket.Conn, error) {
}

func wsConnect(path string) (*websocket.Conn, error) {

tlsConfig, err := config.NewTLSConfig(&config.TLSConfig{
CAFile: *tlsCACertPath,
CertFile: *tlsClientCertPath,
KeyFile: *tlsClientCertKeyPath,
ServerName: *addr,
InsecureSkipVerify: *tlsSkipVerify,
})
if err != nil {
return nil, err
}

url := *addr + path
if strings.HasPrefix(url, "https") {
url = strings.Replace(url, "https", "wss", 1)
Expand All @@ -95,7 +124,12 @@ func wsConnect(path string) (*websocket.Conn, error) {
log.Println(url)

h := http.Header{"Authorization": {"Basic " + base64.StdEncoding.EncodeToString([]byte(*username+":"+*password))}}
c, resp, err := websocket.DefaultDialer.Dial(url, h)

ws := websocket.Dialer{
TLSClientConfig: tlsConfig,
}

c, resp, err := ws.Dial(url, h)

if err != nil {
if resp == nil {
Expand Down
5 changes: 5 additions & 0 deletions cmd/logcli/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ var (
username = app.Flag("username", "Username for HTTP basic auth.").Default("").Envar("GRAFANA_USERNAME").String()
password = app.Flag("password", "Password for HTTP basic auth.").Default("").Envar("GRAFANA_PASSWORD").String()

tlsCACertPath = app.Flag("ca-cert", "Path to the server Certificate Authority.").Default("").Envar("LOKI_CA_CERT_PATH").String()
tlsSkipVerify = app.Flag("tls-skip-verify", "Server certificate TLS skip verify.").Default("false").Bool()
tlsClientCertPath = app.Flag("cert", "Path to the client certificate.").Default("").Envar("LOKI_CLIENT_CERT_PATH").String()
tlsClientCertKeyPath = app.Flag("key", "Path to the client certificate key.").Default("").Envar("LOKI_CLIENT_KEY_PATH").String()

queryCmd = app.Command("query", "Run a LogQL query.")
queryStr = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String()
regexpStr = queryCmd.Arg("regex", "").String()
Expand Down
19 changes: 12 additions & 7 deletions docs/logcli.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,26 +44,31 @@ Common labels: {job="cortex-ops/consul", namespace="cortex-ops"}

### Configuration


Configuration values are considered in the following order (lowest to highest):

- environment value
- command line

The URLs of the requests are printed to help with integration work.

### Details

```
```console
$ logcli help
usage: logcli [<flags>] <command> [<args> ...]

A command-line for loki.

Flags:
--help Show context-sensitive help (also try --help-long and --help-man).
--addr="" Server address, need to specify.
--username="" Username for HTTP basic auth.
--password="" Password for HTTP basic auth.
--help Show context-sensitive help (also try --help-long and --help-man).
--addr="https://logs-us-west1.grafana.net"
Server address.
--username="" Username for HTTP basic auth.
--password="" Password for HTTP basic auth.
--ca-cert="" Path to the server Certificate Authority.
--tls-skip-verify Server certificate TLS skip verify.
--cert="" Path to the client certificate.
--key="" Path to the client certificate key.

Commands:
help [<command>...]
Expand All @@ -72,7 +77,7 @@ Commands:
query [<flags>] <query> [<regex>]
Run a LogQL query.

labels <label>
labels [<label>]
Find values for a given label.

$ logcli help query
Expand Down
3 changes: 1 addition & 2 deletions docs/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ This page lists operational aspects of running Loki in alphabetical order:

Loki does not have an authentication layer.
You are expected to run an authenticating reverse proxy in front of your services, such as an Nginx with basic auth or an OAuth2 proxy.
See [client options](promtail-setup.md#custom-client-options) for more details about supported authentication methods.

### Multi-tenancy

Expand Down Expand Up @@ -129,7 +130,6 @@ storage_config:
dynamodb: dynamodb://region
```
#### S3
Loki is using S3 as object storage. It stores log within directories based on
Expand Down Expand Up @@ -158,4 +158,3 @@ create the table manually you cannot easily erase old data and your index just g
If you set your DynamoDB table manually, ensure you set the primary index key to `h`
(string) and use `r` (binary) as the sort key. Also set the "period" attribute in the yaml to zero.
Make sure adjust your throughput base on your usage.

97 changes: 92 additions & 5 deletions docs/promtail-setup.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

## Daemonset method

Daemonset will deploy promtail on every node within the kubernetes cluster.
Daemonset will deploy `promtail` on every node within the Kubernetes cluster.

Daemonset deployment is great to collect all of the container logs within the
cluster. It is great solution for single tenant. All of the logs will send to a
Expand All @@ -11,6 +11,7 @@ single Loki server.
Check the `production` folder for examples of a daemonset deployment for kubernetes using both helm and ksonnet.

### Example

```yaml
---Daemonset.yaml
apiVersion: extensions/v1beta1
Expand Down Expand Up @@ -87,22 +88,23 @@ roleRef:
## Sidecar Method
Sidecar method will deploy promtail as a container within a pod that
Sidecar method will deploy `promtail` as a container within a pod that
developer/devops create.

This method will deploy promtail as a sidecar container within a pod.
This method will deploy `promtail` as a sidecar container within a pod.
In a multi-tenant environment, this enables teams to aggregate logs
for specific pods and deployments for example for all pods in a namespace.

### Example

```yaml
---Deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: my_test_app
...
spec:
spec:
...
template:
spec:
Expand Down Expand Up @@ -133,10 +135,11 @@ spec:
Sometime application create customized log files. To collect those logs, you
would need to have a customized `__path__` in your scrape_config.

Right now, the best way to watch and tail custom log path is define log filepath
Right now, the best way to watch and tail custom log path is define log file path
as a label for the pod.

#### Example

```yaml
---Deployment.yaml
apiVersion: extensions/v1beta1
Expand Down Expand Up @@ -164,3 +167,87 @@ scrape_configs:
replacement: /your_log_file_dir/$1.log
...
```

### Custom Client options

`promtail` client configuration uses the [Prometheus http client](https://godoc.org/github.com/prometheus/common/config) implementation.
Therefore you can configure the following authentication parameters in the `client` or `clients` section.

```yaml
---promtail_config.yaml
...
# Simple client
client:
[ <client_option> ]
# Multiple clients
clients:
[ - <client_option> ]
...
```

>Note: Passing the `-client.url` from command line is only valid if you set the `client` section.

#### `<client_option>`

```yaml
# Sets the `url` of loki api push endpoint
url: http[s]://<host>:<port>/api/prom/push

# Sets the `Authorization` header on every promtail request with the
# configured username and password.
# password and password_file are mutually exclusive.
basic_auth:
username: <string>
password: <secret>
password_file: <string>

# Sets the `Authorization` header on every promtail request with
# the configured bearer token. It is mutually exclusive with `bearer_token_file`.
bearer_token: <secret>

# Sets the `Authorization` header on every promtail request with the bearer token
# read from the configured file. It is mutually exclusive with `bearer_token`.
bearer_token_file: /path/to/bearer/token/file

# Configures the promtail request's TLS settings.
tls_config:
# CA certificate to validate API server certificate with.
# If not provided Trusted CA from sytem will be used.
ca_file: <filename>

# Certificate and key files for client cert authentication to the server.
cert_file: <filename>
key_file: <filename>

# ServerName extension to indicate the name of the server.
# https://tools.ietf.org/html/rfc4366#section-3.1
server_name: <string>

# Disable validation of the server certificate.
insecure_skip_verify: <boolean>

# Optional proxy URL.
proxy_url: <string>

# Maximum wait period before sending batch
batchwait: 1s

# Maximum batch size to accrue before sending, unit is byte
batchsize: 102400

# Maximum time to wait for server to respond to a request
timeout: 10s

backoff_config:
# Initial backoff time between retries
minbackoff: 100ms
# Maximum backoff time between retries
maxbackoff: 5s
# Maximum number of retires when sending batches, 0 means infinite retries
maxretries: 5

# The labels to add to any time series or alerts when communicating with loki
external_labels: {}
```
Binary file added main
Binary file not shown.
22 changes: 19 additions & 3 deletions pkg/promtail/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ import (
"github.com/go-kit/kit/log/level"
"github.com/gogo/protobuf/proto"
"github.com/golang/snappy"

"github.com/grafana/loki/pkg/helpers"
"github.com/grafana/loki/pkg/logproto"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
)

Expand Down Expand Up @@ -62,6 +64,7 @@ type Client interface {
type client struct {
logger log.Logger
cfg Config
client *http.Client
quit chan struct{}
entries chan entry
wg sync.WaitGroup
Expand All @@ -75,7 +78,7 @@ type entry struct {
}

// New makes a new Client.
func New(cfg Config, logger log.Logger) Client {
func New(cfg Config, logger log.Logger) (Client, error) {
c := &client{
logger: log.With(logger, "component", "client", "host", cfg.URL.Host),
cfg: cfg,
Expand All @@ -84,9 +87,22 @@ func New(cfg Config, logger log.Logger) Client {

externalLabels: cfg.ExternalLabels.LabelSet,
}

err := cfg.Client.Validate()
if err != nil {
return nil, err
}

c.client, err = config.NewClientFromConfig(cfg.Client, "promtail")
if err != nil {
return nil, err
}

c.client.Timeout = cfg.Timeout

c.wg.Add(1)
go c.run()
return c
return c, nil
}

func (c *client) run() {
Expand Down Expand Up @@ -194,7 +210,7 @@ func (c *client) send(ctx context.Context, buf []byte) (int, error) {
req = req.WithContext(ctx)
req.Header.Set("Content-Type", contentType)

resp, err := http.DefaultClient.Do(req)
resp, err := c.client.Do(req)
if err != nil {
return -1, err
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/promtail/client/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/cortexproject/cortex/pkg/util"
"github.com/cortexproject/cortex/pkg/util/flagext"
lokiflag "github.com/grafana/loki/pkg/util/flagext"
"github.com/prometheus/common/config"
)

// Config describes configuration for a HTTP pusher client.
Expand All @@ -15,6 +16,8 @@ type Config struct {
BatchWait time.Duration
BatchSize int

Client config.HTTPClientConfig `yaml:",inline"`

BackoffConfig util.BackoffConfig `yaml:"backoff_config"`
// The labels to add to any time series or alerts when communicating with loki
ExternalLabels lokiflag.LabelSet `yaml:"external_labels,omitempty"`
Expand Down
9 changes: 7 additions & 2 deletions pkg/promtail/client/multi.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,14 @@ func NewMulti(logger log.Logger, cfgs ...Config) (Client, error) {
if len(cfgs) == 0 {
return nil, errors.New("at least one client config should be provided")
}
var clients []Client

clients := make([]Client, 0, len(cfgs))
for _, cfg := range cfgs {
clients = append(clients, New(cfg, logger))
client, err := New(cfg, logger)
if err != nil {
return nil, err
}
clients = append(clients, client)
}
return MultiClient(clients), nil
}
Expand Down

0 comments on commit bbaf1cd

Please sign in to comment.