From cadf8f31db634d0dd3dc6f22256353610faf0196 Mon Sep 17 00:00:00 2001 From: "m.conraux" Date: Fri, 7 Sep 2018 14:53:56 +0200 Subject: [PATCH 01/11] initial commit for prometheus and sys/metrics support --- command/server.go | 40 +- command/server/config.go | 6 + http/handler.go | 1 + http/sys_metrics.go | 42 ++ vault/request_handling.go | 10 +- .../prometheus/promhttp/delegator.go | 199 ++++++++ .../prometheus/promhttp/delegator_1_8.go | 181 +++++++ .../prometheus/promhttp/delegator_pre_1_8.go | 44 ++ .../client_golang/prometheus/promhttp/http.go | 204 ++++++++ .../prometheus/promhttp/instrument_client.go | 98 ++++ .../promhttp/instrument_client_1_8.go | 144 ++++++ .../prometheus/promhttp/instrument_server.go | 440 ++++++++++++++++++ 12 files changed, 1393 insertions(+), 16 deletions(-) create mode 100644 http/sys_metrics.go create mode 100644 vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator.go create mode 100644 vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_1_8.go create mode 100644 vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_pre_1_8.go create mode 100644 vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go create mode 100644 vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client.go create mode 100644 vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client_1_8.go create mode 100644 vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go diff --git a/command/server.go b/command/server.go index f07b6d129e09..525fd6981ba4 100644 --- a/command/server.go +++ b/command/server.go @@ -23,6 +23,7 @@ import ( metrics "github.com/armon/go-metrics" "github.com/armon/go-metrics/circonus" "github.com/armon/go-metrics/datadog" + "github.com/armon/go-metrics/prometheus" "github.com/hashicorp/errwrap" log "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" @@ -466,7 +467,8 @@ func (c *ServerCommand) Run(args []string) int { "in a Docker container, provide the IPC_LOCK cap to the container.")) } - if err := c.setupTelemetry(config); err != nil { + InMemMetrics, err := c.setupTelemetry(config) + if err != nil { c.UI.Error(fmt.Sprintf("Error initializing telemetry: %s", err)) return 1 } @@ -1076,10 +1078,12 @@ CLUSTER_SYNTHESIS_COMPLETE: // Initialize the HTTP servers for _, ln := range lns { handler := vaulthttp.Handler(&vault.HandlerProperties{ - Core: core, - MaxRequestSize: ln.maxRequestSize, - MaxRequestDuration: ln.maxRequestDuration, - DisablePrintableCheck: config.DisablePrintableCheck, + Core: core, + MaxRequestSize: ln.maxRequestSize, + MaxRequestDuration: ln.maxRequestDuration, + DisablePrintableCheck: config.DisablePrintableCheck, + TelemetryMemSink: InMemMetrics, + TelemetryPrometheusRetention: config.Telemetry.PrometheusRetentionTime, }) // We perform validation on the config earlier, we can just cast here @@ -1666,8 +1670,8 @@ func (c *ServerCommand) detectRedirect(detect physical.RedirectDetect, return url.String(), nil } -// setupTelemetry is used to setup the telemetry sub-systems -func (c *ServerCommand) setupTelemetry(config *server.Config) error { +// setupTelemetry is used to setup the telemetry sub-systems and returns the in-memory sink to be used in http configuration +func (c *ServerCommand) setupTelemetry(config *server.Config) (*metrics.InmemSink, error) { /* Setup telemetry Aggregate on 10 second intervals for 1 minute. Expose the metrics over stderr when there is a SIGUSR1 received. @@ -1690,7 +1694,7 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) error { if telConfig.StatsiteAddr != "" { sink, err := metrics.NewStatsiteSink(telConfig.StatsiteAddr) if err != nil { - return err + return nil, err } fanout = append(fanout, sink) } @@ -1699,7 +1703,7 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) error { if telConfig.StatsdAddr != "" { sink, err := metrics.NewStatsdSink(telConfig.StatsdAddr) if err != nil { - return err + return nil, err } fanout = append(fanout, sink) } @@ -1735,12 +1739,24 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) error { sink, err := circonus.NewCirconusSink(cfg) if err != nil { - return err + return nil, err } sink.Start() fanout = append(fanout, sink) } + // Configure the Prometheus sink + if telConfig.PrometheusRetentionTime.Nanoseconds() > 0 { + prometheusOpts := prometheus.PrometheusOpts{ + Expiration: telConfig.PrometheusRetentionTime, + } + sink, err := prometheus.NewPrometheusSinkFrom(prometheusOpts) + if err != nil { + return nil, err + } + fanout = append(fanout, sink) + } + if telConfig.DogStatsDAddr != "" { var tags []string @@ -1750,7 +1766,7 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) error { sink, err := datadog.NewDogStatsdSink(telConfig.DogStatsDAddr, metricsConf.HostName) if err != nil { - return errwrap.Wrapf("failed to start DogStatsD sink: {{err}}", err) + return nil, errwrap.Wrapf("failed to start DogStatsD sink: {{err}}", err) } sink.SetTags(tags) fanout = append(fanout, sink) @@ -1764,7 +1780,7 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) error { metricsConf.EnableHostname = false metrics.NewGlobal(metricsConf, inm) } - return nil + return inm, nil } func (c *ServerCommand) Reload(lock *sync.RWMutex, reloadFuncs *map[string][]reload.ReloadFunc, configPath []string) error { diff --git a/command/server/config.go b/command/server/config.go index 1b0b7fb3a173..025ef04f800e 100644 --- a/command/server/config.go +++ b/command/server/config.go @@ -233,6 +233,12 @@ type Telemetry struct { // DogStatsdTags are the global tags that should be sent with each packet to dogstatsd // It is a list of strings, where each string looks like "my_tag_name:my_tag_value" DogStatsDTags []string `hcl:"dogstatsd_tags"` + + // Prometheus: + // PrometheusRetentionTime is the retention time for prometheus metrics if greater than 0. + // A value of 0 disable Prometheus support. + // Default: 0 + PrometheusRetentionTime time.Duration `hcl:"prometheus_retention_time"` } func (s *Telemetry) GoString() string { diff --git a/http/handler.go b/http/handler.go index c4ccc40c0da8..37fa0528cd73 100644 --- a/http/handler.go +++ b/http/handler.go @@ -107,6 +107,7 @@ func Handler(props *vault.HandlerProperties) http.Handler { mux.Handle("/v1/sys/unseal", handleSysUnseal(core)) mux.Handle("/v1/sys/leader", handleSysLeader(core)) mux.Handle("/v1/sys/health", handleSysHealth(core)) + mux.Handle("/v1/sys/metrics", handleSysMetrics(props.TelemetryMemSink, props.TelemetryPrometheusRetention)) mux.Handle("/v1/sys/generate-root/attempt", handleRequestForwarding(core, handleSysGenerateRootAttempt(core, vault.GenerateStandardRootTokenStrategy))) mux.Handle("/v1/sys/generate-root/update", handleRequestForwarding(core, handleSysGenerateRootUpdate(core, vault.GenerateStandardRootTokenStrategy))) mux.Handle("/v1/sys/rekey/init", handleRequestForwarding(core, handleSysRekeyInit(core, false))) diff --git a/http/sys_metrics.go b/http/sys_metrics.go new file mode 100644 index 000000000000..fb7df6179d01 --- /dev/null +++ b/http/sys_metrics.go @@ -0,0 +1,42 @@ +package http + +import ( + "encoding/json" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + metrics "github.com/armon/go-metrics" +) + +func handleSysMetrics(inm *metrics.InmemSink, prometheusRetention time.Duration) http.Handler { + return http.HandlerFunc( + func(res http.ResponseWriter, req *http.Request) { + if format := req.URL.Query().Get("format"); format == "prometheus" { + handlerOptions := promhttp.HandlerOpts{ + ErrorHandling: promhttp.ContinueOnError, + DisableCompression: true, + } + + handler := promhttp.HandlerFor(prometheus.DefaultGatherer, handlerOptions) + handler.ServeHTTP(res, req) + return + } + summary, err := inm.DisplayMetrics(res, req) + if err != nil { + res.WriteHeader(500) + res.Write([]byte(err.Error())) + } else { + content, err := json.Marshal(summary) + if err != nil { + res.WriteHeader(500) + res.Write([]byte("can't marshal internal metrics into json")) + } else { + res.WriteHeader(200) + res.Write(content) + } + } + }) +} diff --git a/vault/request_handling.go b/vault/request_handling.go index b8471f8d2ca0..ba483dd41532 100644 --- a/vault/request_handling.go +++ b/vault/request_handling.go @@ -39,10 +39,12 @@ var ( // HandlerProperties is used to seed configuration into a vaulthttp.Handler. // It's in this package to avoid a circular dependency type HandlerProperties struct { - Core *Core - MaxRequestSize int64 - MaxRequestDuration time.Duration - DisablePrintableCheck bool + Core *Core + MaxRequestSize int64 + MaxRequestDuration time.Duration + DisablePrintableCheck bool + TelemetryMemSink *metrics.InmemSink + TelemetryPrometheusRetention time.Duration } // fetchEntityAndDerivedPolicies returns the entity object for the given entity diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator.go new file mode 100644 index 000000000000..5ee095b09e96 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator.go @@ -0,0 +1,199 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promhttp + +import ( + "bufio" + "io" + "net" + "net/http" +) + +const ( + closeNotifier = 1 << iota + flusher + hijacker + readerFrom + pusher +) + +type delegator interface { + http.ResponseWriter + + Status() int + Written() int64 +} + +type responseWriterDelegator struct { + http.ResponseWriter + + handler, method string + status int + written int64 + wroteHeader bool + observeWriteHeader func(int) +} + +func (r *responseWriterDelegator) Status() int { + return r.status +} + +func (r *responseWriterDelegator) Written() int64 { + return r.written +} + +func (r *responseWriterDelegator) WriteHeader(code int) { + r.status = code + r.wroteHeader = true + r.ResponseWriter.WriteHeader(code) + if r.observeWriteHeader != nil { + r.observeWriteHeader(code) + } +} + +func (r *responseWriterDelegator) Write(b []byte) (int, error) { + if !r.wroteHeader { + r.WriteHeader(http.StatusOK) + } + n, err := r.ResponseWriter.Write(b) + r.written += int64(n) + return n, err +} + +type closeNotifierDelegator struct{ *responseWriterDelegator } +type flusherDelegator struct{ *responseWriterDelegator } +type hijackerDelegator struct{ *responseWriterDelegator } +type readerFromDelegator struct{ *responseWriterDelegator } + +func (d *closeNotifierDelegator) CloseNotify() <-chan bool { + return d.ResponseWriter.(http.CloseNotifier).CloseNotify() +} +func (d *flusherDelegator) Flush() { + d.ResponseWriter.(http.Flusher).Flush() +} +func (d *hijackerDelegator) Hijack() (net.Conn, *bufio.ReadWriter, error) { + return d.ResponseWriter.(http.Hijacker).Hijack() +} +func (d *readerFromDelegator) ReadFrom(re io.Reader) (int64, error) { + if !d.wroteHeader { + d.WriteHeader(http.StatusOK) + } + n, err := d.ResponseWriter.(io.ReaderFrom).ReadFrom(re) + d.written += n + return n, err +} + +var pickDelegator = make([]func(*responseWriterDelegator) delegator, 32) + +func init() { + // TODO(beorn7): Code generation would help here. + pickDelegator[0] = func(d *responseWriterDelegator) delegator { // 0 + return d + } + pickDelegator[closeNotifier] = func(d *responseWriterDelegator) delegator { // 1 + return closeNotifierDelegator{d} + } + pickDelegator[flusher] = func(d *responseWriterDelegator) delegator { // 2 + return flusherDelegator{d} + } + pickDelegator[flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 3 + return struct { + *responseWriterDelegator + http.Flusher + http.CloseNotifier + }{d, &flusherDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[hijacker] = func(d *responseWriterDelegator) delegator { // 4 + return hijackerDelegator{d} + } + pickDelegator[hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 5 + return struct { + *responseWriterDelegator + http.Hijacker + http.CloseNotifier + }{d, &hijackerDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 6 + return struct { + *responseWriterDelegator + http.Hijacker + http.Flusher + }{d, &hijackerDelegator{d}, &flusherDelegator{d}} + } + pickDelegator[hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 7 + return struct { + *responseWriterDelegator + http.Hijacker + http.Flusher + http.CloseNotifier + }{d, &hijackerDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[readerFrom] = func(d *responseWriterDelegator) delegator { // 8 + return readerFromDelegator{d} + } + pickDelegator[readerFrom+closeNotifier] = func(d *responseWriterDelegator) delegator { // 9 + return struct { + *responseWriterDelegator + io.ReaderFrom + http.CloseNotifier + }{d, &readerFromDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[readerFrom+flusher] = func(d *responseWriterDelegator) delegator { // 10 + return struct { + *responseWriterDelegator + io.ReaderFrom + http.Flusher + }{d, &readerFromDelegator{d}, &flusherDelegator{d}} + } + pickDelegator[readerFrom+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 11 + return struct { + *responseWriterDelegator + io.ReaderFrom + http.Flusher + http.CloseNotifier + }{d, &readerFromDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[readerFrom+hijacker] = func(d *responseWriterDelegator) delegator { // 12 + return struct { + *responseWriterDelegator + io.ReaderFrom + http.Hijacker + }{d, &readerFromDelegator{d}, &hijackerDelegator{d}} + } + pickDelegator[readerFrom+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 13 + return struct { + *responseWriterDelegator + io.ReaderFrom + http.Hijacker + http.CloseNotifier + }{d, &readerFromDelegator{d}, &hijackerDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[readerFrom+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 14 + return struct { + *responseWriterDelegator + io.ReaderFrom + http.Hijacker + http.Flusher + }{d, &readerFromDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}} + } + pickDelegator[readerFrom+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 15 + return struct { + *responseWriterDelegator + io.ReaderFrom + http.Hijacker + http.Flusher + http.CloseNotifier + }{d, &readerFromDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} + } +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_1_8.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_1_8.go new file mode 100644 index 000000000000..f4d386f7a393 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_1_8.go @@ -0,0 +1,181 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.8 + +package promhttp + +import ( + "io" + "net/http" +) + +type pusherDelegator struct{ *responseWriterDelegator } + +func (d *pusherDelegator) Push(target string, opts *http.PushOptions) error { + return d.ResponseWriter.(http.Pusher).Push(target, opts) +} + +func init() { + pickDelegator[pusher] = func(d *responseWriterDelegator) delegator { // 16 + return pusherDelegator{d} + } + pickDelegator[pusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 17 + return struct { + *responseWriterDelegator + http.Pusher + http.CloseNotifier + }{d, &pusherDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[pusher+flusher] = func(d *responseWriterDelegator) delegator { // 18 + return struct { + *responseWriterDelegator + http.Pusher + http.Flusher + }{d, &pusherDelegator{d}, &flusherDelegator{d}} + } + pickDelegator[pusher+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 19 + return struct { + *responseWriterDelegator + http.Pusher + http.Flusher + http.CloseNotifier + }{d, &pusherDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[pusher+hijacker] = func(d *responseWriterDelegator) delegator { // 20 + return struct { + *responseWriterDelegator + http.Pusher + http.Hijacker + }{d, &pusherDelegator{d}, &hijackerDelegator{d}} + } + pickDelegator[pusher+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 21 + return struct { + *responseWriterDelegator + http.Pusher + http.Hijacker + http.CloseNotifier + }{d, &pusherDelegator{d}, &hijackerDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[pusher+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 22 + return struct { + *responseWriterDelegator + http.Pusher + http.Hijacker + http.Flusher + }{d, &pusherDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}} + } + pickDelegator[pusher+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { //23 + return struct { + *responseWriterDelegator + http.Pusher + http.Hijacker + http.Flusher + http.CloseNotifier + }{d, &pusherDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[pusher+readerFrom] = func(d *responseWriterDelegator) delegator { // 24 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + }{d, &pusherDelegator{d}, &readerFromDelegator{d}} + } + pickDelegator[pusher+readerFrom+closeNotifier] = func(d *responseWriterDelegator) delegator { // 25 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.CloseNotifier + }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[pusher+readerFrom+flusher] = func(d *responseWriterDelegator) delegator { // 26 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Flusher + }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &flusherDelegator{d}} + } + pickDelegator[pusher+readerFrom+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 27 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Flusher + http.CloseNotifier + }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[pusher+readerFrom+hijacker] = func(d *responseWriterDelegator) delegator { // 28 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Hijacker + }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &hijackerDelegator{d}} + } + pickDelegator[pusher+readerFrom+hijacker+closeNotifier] = func(d *responseWriterDelegator) delegator { // 29 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Hijacker + http.CloseNotifier + }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &hijackerDelegator{d}, &closeNotifierDelegator{d}} + } + pickDelegator[pusher+readerFrom+hijacker+flusher] = func(d *responseWriterDelegator) delegator { // 30 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Hijacker + http.Flusher + }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}} + } + pickDelegator[pusher+readerFrom+hijacker+flusher+closeNotifier] = func(d *responseWriterDelegator) delegator { // 31 + return struct { + *responseWriterDelegator + http.Pusher + io.ReaderFrom + http.Hijacker + http.Flusher + http.CloseNotifier + }{d, &pusherDelegator{d}, &readerFromDelegator{d}, &hijackerDelegator{d}, &flusherDelegator{d}, &closeNotifierDelegator{d}} + } +} + +func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator { + d := &responseWriterDelegator{ + ResponseWriter: w, + observeWriteHeader: observeWriteHeaderFunc, + } + + id := 0 + if _, ok := w.(http.CloseNotifier); ok { + id += closeNotifier + } + if _, ok := w.(http.Flusher); ok { + id += flusher + } + if _, ok := w.(http.Hijacker); ok { + id += hijacker + } + if _, ok := w.(io.ReaderFrom); ok { + id += readerFrom + } + if _, ok := w.(http.Pusher); ok { + id += pusher + } + + return pickDelegator[id](d) +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_pre_1_8.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_pre_1_8.go new file mode 100644 index 000000000000..8bb9b8b68f8b --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_pre_1_8.go @@ -0,0 +1,44 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !go1.8 + +package promhttp + +import ( + "io" + "net/http" +) + +func newDelegator(w http.ResponseWriter, observeWriteHeaderFunc func(int)) delegator { + d := &responseWriterDelegator{ + ResponseWriter: w, + observeWriteHeader: observeWriteHeaderFunc, + } + + id := 0 + if _, ok := w.(http.CloseNotifier); ok { + id += closeNotifier + } + if _, ok := w.(http.Flusher); ok { + id += flusher + } + if _, ok := w.(http.Hijacker); ok { + id += hijacker + } + if _, ok := w.(io.ReaderFrom); ok { + id += readerFrom + } + + return pickDelegator[id](d) +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go new file mode 100644 index 000000000000..2d67f2496293 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go @@ -0,0 +1,204 @@ +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package promhttp provides tooling around HTTP servers and clients. +// +// First, the package allows the creation of http.Handler instances to expose +// Prometheus metrics via HTTP. promhttp.Handler acts on the +// prometheus.DefaultGatherer. With HandlerFor, you can create a handler for a +// custom registry or anything that implements the Gatherer interface. It also +// allows the creation of handlers that act differently on errors or allow to +// log errors. +// +// Second, the package provides tooling to instrument instances of http.Handler +// via middleware. Middleware wrappers follow the naming scheme +// InstrumentHandlerX, where X describes the intended use of the middleware. +// See each function's doc comment for specific details. +// +// Finally, the package allows for an http.RoundTripper to be instrumented via +// middleware. Middleware wrappers follow the naming scheme +// InstrumentRoundTripperX, where X describes the intended use of the +// middleware. See each function's doc comment for specific details. +package promhttp + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "net/http" + "strings" + "sync" + + "github.com/prometheus/common/expfmt" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + contentTypeHeader = "Content-Type" + contentLengthHeader = "Content-Length" + contentEncodingHeader = "Content-Encoding" + acceptEncodingHeader = "Accept-Encoding" +) + +var bufPool sync.Pool + +func getBuf() *bytes.Buffer { + buf := bufPool.Get() + if buf == nil { + return &bytes.Buffer{} + } + return buf.(*bytes.Buffer) +} + +func giveBuf(buf *bytes.Buffer) { + buf.Reset() + bufPool.Put(buf) +} + +// Handler returns an HTTP handler for the prometheus.DefaultGatherer. The +// Handler uses the default HandlerOpts, i.e. report the first error as an HTTP +// error, no error logging, and compression if requested by the client. +// +// If you want to create a Handler for the DefaultGatherer with different +// HandlerOpts, create it with HandlerFor with prometheus.DefaultGatherer and +// your desired HandlerOpts. +func Handler() http.Handler { + return HandlerFor(prometheus.DefaultGatherer, HandlerOpts{}) +} + +// HandlerFor returns an http.Handler for the provided Gatherer. The behavior +// of the Handler is defined by the provided HandlerOpts. +func HandlerFor(reg prometheus.Gatherer, opts HandlerOpts) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + mfs, err := reg.Gather() + if err != nil { + if opts.ErrorLog != nil { + opts.ErrorLog.Println("error gathering metrics:", err) + } + switch opts.ErrorHandling { + case PanicOnError: + panic(err) + case ContinueOnError: + if len(mfs) == 0 { + http.Error(w, "No metrics gathered, last error:\n\n"+err.Error(), http.StatusInternalServerError) + return + } + case HTTPErrorOnError: + http.Error(w, "An error has occurred during metrics gathering:\n\n"+err.Error(), http.StatusInternalServerError) + return + } + } + + contentType := expfmt.Negotiate(req.Header) + buf := getBuf() + defer giveBuf(buf) + writer, encoding := decorateWriter(req, buf, opts.DisableCompression) + enc := expfmt.NewEncoder(writer, contentType) + var lastErr error + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + lastErr = err + if opts.ErrorLog != nil { + opts.ErrorLog.Println("error encoding metric family:", err) + } + switch opts.ErrorHandling { + case PanicOnError: + panic(err) + case ContinueOnError: + // Handled later. + case HTTPErrorOnError: + http.Error(w, "An error has occurred during metrics encoding:\n\n"+err.Error(), http.StatusInternalServerError) + return + } + } + } + if closer, ok := writer.(io.Closer); ok { + closer.Close() + } + if lastErr != nil && buf.Len() == 0 { + http.Error(w, "No metrics encoded, last error:\n\n"+lastErr.Error(), http.StatusInternalServerError) + return + } + header := w.Header() + header.Set(contentTypeHeader, string(contentType)) + header.Set(contentLengthHeader, fmt.Sprint(buf.Len())) + if encoding != "" { + header.Set(contentEncodingHeader, encoding) + } + w.Write(buf.Bytes()) + // TODO(beorn7): Consider streaming serving of metrics. + }) +} + +// HandlerErrorHandling defines how a Handler serving metrics will handle +// errors. +type HandlerErrorHandling int + +// These constants cause handlers serving metrics to behave as described if +// errors are encountered. +const ( + // Serve an HTTP status code 500 upon the first error + // encountered. Report the error message in the body. + HTTPErrorOnError HandlerErrorHandling = iota + // Ignore errors and try to serve as many metrics as possible. However, + // if no metrics can be served, serve an HTTP status code 500 and the + // last error message in the body. Only use this in deliberate "best + // effort" metrics collection scenarios. It is recommended to at least + // log errors (by providing an ErrorLog in HandlerOpts) to not mask + // errors completely. + ContinueOnError + // Panic upon the first error encountered (useful for "crash only" apps). + PanicOnError +) + +// Logger is the minimal interface HandlerOpts needs for logging. Note that +// log.Logger from the standard library implements this interface, and it is +// easy to implement by custom loggers, if they don't do so already anyway. +type Logger interface { + Println(v ...interface{}) +} + +// HandlerOpts specifies options how to serve metrics via an http.Handler. The +// zero value of HandlerOpts is a reasonable default. +type HandlerOpts struct { + // ErrorLog specifies an optional logger for errors collecting and + // serving metrics. If nil, errors are not logged at all. + ErrorLog Logger + // ErrorHandling defines how errors are handled. Note that errors are + // logged regardless of the configured ErrorHandling provided ErrorLog + // is not nil. + ErrorHandling HandlerErrorHandling + // If DisableCompression is true, the handler will never compress the + // response, even if requested by the client. + DisableCompression bool +} + +// decorateWriter wraps a writer to handle gzip compression if requested. It +// returns the decorated writer and the appropriate "Content-Encoding" header +// (which is empty if no compression is enabled). +func decorateWriter(request *http.Request, writer io.Writer, compressionDisabled bool) (io.Writer, string) { + if compressionDisabled { + return writer, "" + } + header := request.Header.Get(acceptEncodingHeader) + parts := strings.Split(header, ",") + for _, part := range parts { + part := strings.TrimSpace(part) + if part == "gzip" || strings.HasPrefix(part, "gzip;") { + return gzip.NewWriter(writer), "gzip" + } + } + return writer, "" +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client.go new file mode 100644 index 000000000000..65f942544549 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client.go @@ -0,0 +1,98 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promhttp + +import ( + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// The RoundTripperFunc type is an adapter to allow the use of ordinary +// functions as RoundTrippers. If f is a function with the appropriate +// signature, RountTripperFunc(f) is a RoundTripper that calls f. +type RoundTripperFunc func(req *http.Request) (*http.Response, error) + +// RoundTrip implements the RoundTripper interface. +func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return rt(r) +} + +// InstrumentRoundTripperInFlight is a middleware that wraps the provided +// http.RoundTripper. It sets the provided prometheus.Gauge to the number of +// requests currently handled by the wrapped http.RoundTripper. +// +// See the example for ExampleInstrumentRoundTripperDuration for example usage. +func InstrumentRoundTripperInFlight(gauge prometheus.Gauge, next http.RoundTripper) RoundTripperFunc { + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + gauge.Inc() + defer gauge.Dec() + return next.RoundTrip(r) + }) +} + +// InstrumentRoundTripperCounter is a middleware that wraps the provided +// http.RoundTripper to observe the request result with the provided CounterVec. +// The CounterVec must have zero, one, or two labels. The only allowed label +// names are "code" and "method". The function panics if any other instance +// labels are provided. Partitioning of the CounterVec happens by HTTP status +// code and/or HTTP method if the respective instance label names are present +// in the CounterVec. For unpartitioned counting, use a CounterVec with +// zero labels. +// +// If the wrapped RoundTripper panics or returns a non-nil error, the Counter +// is not incremented. +// +// See the example for ExampleInstrumentRoundTripperDuration for example usage. +func InstrumentRoundTripperCounter(counter *prometheus.CounterVec, next http.RoundTripper) RoundTripperFunc { + code, method := checkLabels(counter) + + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + resp, err := next.RoundTrip(r) + if err == nil { + counter.With(labels(code, method, r.Method, resp.StatusCode)).Inc() + } + return resp, err + }) +} + +// InstrumentRoundTripperDuration is a middleware that wraps the provided +// http.RoundTripper to observe the request duration with the provided ObserverVec. +// The ObserverVec must have zero, one, or two labels. The only allowed label +// names are "code" and "method". The function panics if any other instance +// labels are provided. The Observe method of the Observer in the ObserverVec +// is called with the request duration in seconds. Partitioning happens by HTTP +// status code and/or HTTP method if the respective instance label names are +// present in the ObserverVec. For unpartitioned observations, use an +// ObserverVec with zero labels. Note that partitioning of Histograms is +// expensive and should be used judiciously. +// +// If the wrapped RoundTripper panics or returns a non-nil error, no values are +// reported. +// +// Note that this method is only guaranteed to never observe negative durations +// if used with Go1.9+. +func InstrumentRoundTripperDuration(obs prometheus.ObserverVec, next http.RoundTripper) RoundTripperFunc { + code, method := checkLabels(obs) + + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + start := time.Now() + resp, err := next.RoundTrip(r) + if err == nil { + obs.With(labels(code, method, r.Method, resp.StatusCode)).Observe(time.Since(start).Seconds()) + } + return resp, err + }) +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client_1_8.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client_1_8.go new file mode 100644 index 000000000000..0bd80c35521d --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client_1_8.go @@ -0,0 +1,144 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.8 + +package promhttp + +import ( + "context" + "crypto/tls" + "net/http" + "net/http/httptrace" + "time" +) + +// InstrumentTrace is used to offer flexibility in instrumenting the available +// httptrace.ClientTrace hook functions. Each function is passed a float64 +// representing the time in seconds since the start of the http request. A user +// may choose to use separately buckets Histograms, or implement custom +// instance labels on a per function basis. +type InstrumentTrace struct { + GotConn func(float64) + PutIdleConn func(float64) + GotFirstResponseByte func(float64) + Got100Continue func(float64) + DNSStart func(float64) + DNSDone func(float64) + ConnectStart func(float64) + ConnectDone func(float64) + TLSHandshakeStart func(float64) + TLSHandshakeDone func(float64) + WroteHeaders func(float64) + Wait100Continue func(float64) + WroteRequest func(float64) +} + +// InstrumentRoundTripperTrace is a middleware that wraps the provided +// RoundTripper and reports times to hook functions provided in the +// InstrumentTrace struct. Hook functions that are not present in the provided +// InstrumentTrace struct are ignored. Times reported to the hook functions are +// time since the start of the request. Only with Go1.9+, those times are +// guaranteed to never be negative. (Earlier Go versions are not using a +// monotonic clock.) Note that partitioning of Histograms is expensive and +// should be used judiciously. +// +// For hook functions that receive an error as an argument, no observations are +// made in the event of a non-nil error value. +// +// See the example for ExampleInstrumentRoundTripperDuration for example usage. +func InstrumentRoundTripperTrace(it *InstrumentTrace, next http.RoundTripper) RoundTripperFunc { + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + start := time.Now() + + trace := &httptrace.ClientTrace{ + GotConn: func(_ httptrace.GotConnInfo) { + if it.GotConn != nil { + it.GotConn(time.Since(start).Seconds()) + } + }, + PutIdleConn: func(err error) { + if err != nil { + return + } + if it.PutIdleConn != nil { + it.PutIdleConn(time.Since(start).Seconds()) + } + }, + DNSStart: func(_ httptrace.DNSStartInfo) { + if it.DNSStart != nil { + it.DNSStart(time.Since(start).Seconds()) + } + }, + DNSDone: func(_ httptrace.DNSDoneInfo) { + if it.DNSStart != nil { + it.DNSStart(time.Since(start).Seconds()) + } + }, + ConnectStart: func(_, _ string) { + if it.ConnectStart != nil { + it.ConnectStart(time.Since(start).Seconds()) + } + }, + ConnectDone: func(_, _ string, err error) { + if err != nil { + return + } + if it.ConnectDone != nil { + it.ConnectDone(time.Since(start).Seconds()) + } + }, + GotFirstResponseByte: func() { + if it.GotFirstResponseByte != nil { + it.GotFirstResponseByte(time.Since(start).Seconds()) + } + }, + Got100Continue: func() { + if it.Got100Continue != nil { + it.Got100Continue(time.Since(start).Seconds()) + } + }, + TLSHandshakeStart: func() { + if it.TLSHandshakeStart != nil { + it.TLSHandshakeStart(time.Since(start).Seconds()) + } + }, + TLSHandshakeDone: func(_ tls.ConnectionState, err error) { + if err != nil { + return + } + if it.TLSHandshakeDone != nil { + it.TLSHandshakeDone(time.Since(start).Seconds()) + } + }, + WroteHeaders: func() { + if it.WroteHeaders != nil { + it.WroteHeaders(time.Since(start).Seconds()) + } + }, + Wait100Continue: func() { + if it.Wait100Continue != nil { + it.Wait100Continue(time.Since(start).Seconds()) + } + }, + WroteRequest: func(_ httptrace.WroteRequestInfo) { + if it.WroteRequest != nil { + it.WroteRequest(time.Since(start).Seconds()) + } + }, + } + r = r.WithContext(httptrace.WithClientTrace(context.Background(), trace)) + + return next.RoundTrip(r) + }) +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go new file mode 100644 index 000000000000..3d145adbf096 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go @@ -0,0 +1,440 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promhttp + +import ( + "net/http" + "strconv" + "strings" + "time" + + dto "github.com/prometheus/client_model/go" + + "github.com/prometheus/client_golang/prometheus" +) + +// magicString is used for the hacky label test in checkLabels. Remove once fixed. +const magicString = "zZgWfBxLqvG8kc8IMv3POi2Bb0tZI3vAnBx+gBaFi9FyPzB/CzKUer1yufDa" + +// InstrumentHandlerInFlight is a middleware that wraps the provided +// http.Handler. It sets the provided prometheus.Gauge to the number of +// requests currently handled by the wrapped http.Handler. +// +// See the example for InstrumentHandlerDuration for example usage. +func InstrumentHandlerInFlight(g prometheus.Gauge, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + g.Inc() + defer g.Dec() + next.ServeHTTP(w, r) + }) +} + +// InstrumentHandlerDuration is a middleware that wraps the provided +// http.Handler to observe the request duration with the provided ObserverVec. +// The ObserverVec must have zero, one, or two labels. The only allowed label +// names are "code" and "method". The function panics if any other instance +// labels are provided. The Observe method of the Observer in the ObserverVec +// is called with the request duration in seconds. Partitioning happens by HTTP +// status code and/or HTTP method if the respective instance label names are +// present in the ObserverVec. For unpartitioned observations, use an +// ObserverVec with zero labels. Note that partitioning of Histograms is +// expensive and should be used judiciously. +// +// If the wrapped Handler does not set a status code, a status code of 200 is assumed. +// +// If the wrapped Handler panics, no values are reported. +// +// Note that this method is only guaranteed to never observe negative durations +// if used with Go1.9+. +func InstrumentHandlerDuration(obs prometheus.ObserverVec, next http.Handler) http.HandlerFunc { + code, method := checkLabels(obs) + + if code { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + now := time.Now() + d := newDelegator(w, nil) + next.ServeHTTP(d, r) + + obs.With(labels(code, method, r.Method, d.Status())).Observe(time.Since(now).Seconds()) + }) + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + now := time.Now() + next.ServeHTTP(w, r) + obs.With(labels(code, method, r.Method, 0)).Observe(time.Since(now).Seconds()) + }) +} + +// InstrumentHandlerCounter is a middleware that wraps the provided +// http.Handler to observe the request result with the provided CounterVec. +// The CounterVec must have zero, one, or two labels. The only allowed label +// names are "code" and "method". The function panics if any other instance +// labels are provided. Partitioning of the CounterVec happens by HTTP status +// code and/or HTTP method if the respective instance label names are present +// in the CounterVec. For unpartitioned counting, use a CounterVec with +// zero labels. +// +// If the wrapped Handler does not set a status code, a status code of 200 is assumed. +// +// If the wrapped Handler panics, the Counter is not incremented. +// +// See the example for InstrumentHandlerDuration for example usage. +func InstrumentHandlerCounter(counter *prometheus.CounterVec, next http.Handler) http.HandlerFunc { + code, method := checkLabels(counter) + + if code { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + d := newDelegator(w, nil) + next.ServeHTTP(d, r) + counter.With(labels(code, method, r.Method, d.Status())).Inc() + }) + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + counter.With(labels(code, method, r.Method, 0)).Inc() + }) +} + +// InstrumentHandlerTimeToWriteHeader is a middleware that wraps the provided +// http.Handler to observe with the provided ObserverVec the request duration +// until the response headers are written. The ObserverVec must have zero, one, +// or two labels. The only allowed label names are "code" and "method". The +// function panics if any other instance labels are provided. The Observe +// method of the Observer in the ObserverVec is called with the request +// duration in seconds. Partitioning happens by HTTP status code and/or HTTP +// method if the respective instance label names are present in the +// ObserverVec. For unpartitioned observations, use an ObserverVec with zero +// labels. Note that partitioning of Histograms is expensive and should be used +// judiciously. +// +// If the wrapped Handler panics before calling WriteHeader, no value is +// reported. +// +// Note that this method is only guaranteed to never observe negative durations +// if used with Go1.9+. +// +// See the example for InstrumentHandlerDuration for example usage. +func InstrumentHandlerTimeToWriteHeader(obs prometheus.ObserverVec, next http.Handler) http.HandlerFunc { + code, method := checkLabels(obs) + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + now := time.Now() + d := newDelegator(w, func(status int) { + obs.With(labels(code, method, r.Method, status)).Observe(time.Since(now).Seconds()) + }) + next.ServeHTTP(d, r) + }) +} + +// InstrumentHandlerRequestSize is a middleware that wraps the provided +// http.Handler to observe the request size with the provided ObserverVec. +// The ObserverVec must have zero, one, or two labels. The only allowed label +// names are "code" and "method". The function panics if any other instance +// labels are provided. The Observe method of the Observer in the ObserverVec +// is called with the request size in bytes. Partitioning happens by HTTP +// status code and/or HTTP method if the respective instance label names are +// present in the ObserverVec. For unpartitioned observations, use an +// ObserverVec with zero labels. Note that partitioning of Histograms is +// expensive and should be used judiciously. +// +// If the wrapped Handler does not set a status code, a status code of 200 is assumed. +// +// If the wrapped Handler panics, no values are reported. +// +// See the example for InstrumentHandlerDuration for example usage. +func InstrumentHandlerRequestSize(obs prometheus.ObserverVec, next http.Handler) http.HandlerFunc { + code, method := checkLabels(obs) + + if code { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + d := newDelegator(w, nil) + next.ServeHTTP(d, r) + size := computeApproximateRequestSize(r) + obs.With(labels(code, method, r.Method, d.Status())).Observe(float64(size)) + }) + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + size := computeApproximateRequestSize(r) + obs.With(labels(code, method, r.Method, 0)).Observe(float64(size)) + }) +} + +// InstrumentHandlerResponseSize is a middleware that wraps the provided +// http.Handler to observe the response size with the provided ObserverVec. +// The ObserverVec must have zero, one, or two labels. The only allowed label +// names are "code" and "method". The function panics if any other instance +// labels are provided. The Observe method of the Observer in the ObserverVec +// is called with the response size in bytes. Partitioning happens by HTTP +// status code and/or HTTP method if the respective instance label names are +// present in the ObserverVec. For unpartitioned observations, use an +// ObserverVec with zero labels. Note that partitioning of Histograms is +// expensive and should be used judiciously. +// +// If the wrapped Handler does not set a status code, a status code of 200 is assumed. +// +// If the wrapped Handler panics, no values are reported. +// +// See the example for InstrumentHandlerDuration for example usage. +func InstrumentHandlerResponseSize(obs prometheus.ObserverVec, next http.Handler) http.Handler { + code, method := checkLabels(obs) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + d := newDelegator(w, nil) + next.ServeHTTP(d, r) + obs.With(labels(code, method, r.Method, d.Status())).Observe(float64(d.Written())) + }) +} + +func checkLabels(c prometheus.Collector) (code bool, method bool) { + // TODO(beorn7): Remove this hacky way to check for instance labels + // once Descriptors can have their dimensionality queried. + var ( + desc *prometheus.Desc + pm dto.Metric + ) + + descc := make(chan *prometheus.Desc, 1) + c.Describe(descc) + + select { + case desc = <-descc: + default: + panic("no description provided by collector") + } + select { + case <-descc: + panic("more than one description provided by collector") + default: + } + + close(descc) + + if _, err := prometheus.NewConstMetric(desc, prometheus.UntypedValue, 0); err == nil { + return + } + if m, err := prometheus.NewConstMetric(desc, prometheus.UntypedValue, 0, magicString); err == nil { + if err := m.Write(&pm); err != nil { + panic("error checking metric for labels") + } + for _, label := range pm.Label { + name, value := label.GetName(), label.GetValue() + if value != magicString { + continue + } + switch name { + case "code": + code = true + case "method": + method = true + default: + panic("metric partitioned with non-supported labels") + } + return + } + panic("previously set label not found – this must never happen") + } + if m, err := prometheus.NewConstMetric(desc, prometheus.UntypedValue, 0, magicString, magicString); err == nil { + if err := m.Write(&pm); err != nil { + panic("error checking metric for labels") + } + for _, label := range pm.Label { + name, value := label.GetName(), label.GetValue() + if value != magicString { + continue + } + if name == "code" || name == "method" { + continue + } + panic("metric partitioned with non-supported labels") + } + code = true + method = true + return + } + panic("metric partitioned with non-supported labels") +} + +// emptyLabels is a one-time allocation for non-partitioned metrics to avoid +// unnecessary allocations on each request. +var emptyLabels = prometheus.Labels{} + +func labels(code, method bool, reqMethod string, status int) prometheus.Labels { + if !(code || method) { + return emptyLabels + } + labels := prometheus.Labels{} + + if code { + labels["code"] = sanitizeCode(status) + } + if method { + labels["method"] = sanitizeMethod(reqMethod) + } + + return labels +} + +func computeApproximateRequestSize(r *http.Request) int { + s := 0 + if r.URL != nil { + s += len(r.URL.String()) + } + + s += len(r.Method) + s += len(r.Proto) + for name, values := range r.Header { + s += len(name) + for _, value := range values { + s += len(value) + } + } + s += len(r.Host) + + // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL. + + if r.ContentLength != -1 { + s += int(r.ContentLength) + } + return s +} + +func sanitizeMethod(m string) string { + switch m { + case "GET", "get": + return "get" + case "PUT", "put": + return "put" + case "HEAD", "head": + return "head" + case "POST", "post": + return "post" + case "DELETE", "delete": + return "delete" + case "CONNECT", "connect": + return "connect" + case "OPTIONS", "options": + return "options" + case "NOTIFY", "notify": + return "notify" + default: + return strings.ToLower(m) + } +} + +// If the wrapped http.Handler has not set a status code, i.e. the value is +// currently 0, santizeCode will return 200, for consistency with behavior in +// the stdlib. +func sanitizeCode(s int) string { + switch s { + case 100: + return "100" + case 101: + return "101" + + case 200, 0: + return "200" + case 201: + return "201" + case 202: + return "202" + case 203: + return "203" + case 204: + return "204" + case 205: + return "205" + case 206: + return "206" + + case 300: + return "300" + case 301: + return "301" + case 302: + return "302" + case 304: + return "304" + case 305: + return "305" + case 307: + return "307" + + case 400: + return "400" + case 401: + return "401" + case 402: + return "402" + case 403: + return "403" + case 404: + return "404" + case 405: + return "405" + case 406: + return "406" + case 407: + return "407" + case 408: + return "408" + case 409: + return "409" + case 410: + return "410" + case 411: + return "411" + case 412: + return "412" + case 413: + return "413" + case 414: + return "414" + case 415: + return "415" + case 416: + return "416" + case 417: + return "417" + case 418: + return "418" + + case 500: + return "500" + case 501: + return "501" + case 502: + return "502" + case 503: + return "503" + case 504: + return "504" + case 505: + return "505" + + case 428: + return "428" + case 429: + return "429" + case 431: + return "431" + case 511: + return "511" + + default: + return strconv.Itoa(s) + } +} From 0b78c525df0e710a2086944f4e73653a2ac55325 Mon Sep 17 00:00:00 2001 From: "m.conraux" Date: Fri, 7 Sep 2018 17:44:12 +0200 Subject: [PATCH 02/11] Throw an error if prometheusRetentionTime is 0,add prometheus in devmode --- command/server/config.go | 5 ++++- http/sys_metrics.go | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/command/server/config.go b/command/server/config.go index 025ef04f800e..1f21b55f7eae 100644 --- a/command/server/config.go +++ b/command/server/config.go @@ -98,7 +98,10 @@ func DevConfig(ha, transactional bool) *Config { EnableUI: true, - Telemetry: &Telemetry{}, + Telemetry: &Telemetry{ + PrometheusRetentionTime: 24 * time.Hour, + DisableHostname: true, + }, } switch { diff --git a/http/sys_metrics.go b/http/sys_metrics.go index fb7df6179d01..29bb7b6a3d67 100644 --- a/http/sys_metrics.go +++ b/http/sys_metrics.go @@ -15,6 +15,12 @@ func handleSysMetrics(inm *metrics.InmemSink, prometheusRetention time.Duration) return http.HandlerFunc( func(res http.ResponseWriter, req *http.Request) { if format := req.URL.Query().Get("format"); format == "prometheus" { + + if prometheusRetention.Nanoseconds() == 0 { + res.WriteHeader(500) + res.Write([]byte("prometheus support is not enabled")) + } + handlerOptions := promhttp.HandlerOpts{ ErrorHandling: promhttp.ContinueOnError, DisableCompression: true, From 431a73bcc23bec78d9a6b25ea93d2922f8690eda Mon Sep 17 00:00:00 2001 From: "m.conraux" Date: Mon, 10 Sep 2018 14:38:40 +0200 Subject: [PATCH 03/11] return when format=prometheus is used and prom is disable --- http/sys_metrics.go | 1 + 1 file changed, 1 insertion(+) diff --git a/http/sys_metrics.go b/http/sys_metrics.go index 29bb7b6a3d67..e813b3ef5ebf 100644 --- a/http/sys_metrics.go +++ b/http/sys_metrics.go @@ -19,6 +19,7 @@ func handleSysMetrics(inm *metrics.InmemSink, prometheusRetention time.Duration) if prometheusRetention.Nanoseconds() == 0 { res.WriteHeader(500) res.Write([]byte("prometheus support is not enabled")) + return } handlerOptions := promhttp.HandlerOpts{ From c9e51ad906ae871772e2a68e13772fac230464fb Mon Sep 17 00:00:00 2001 From: "m.conraux" Date: Mon, 10 Sep 2018 14:39:14 +0200 Subject: [PATCH 04/11] parse prometheus_retention_time from string instead of int --- command/server/config.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/command/server/config.go b/command/server/config.go index 1f21b55f7eae..f0c44daa630b 100644 --- a/command/server/config.go +++ b/command/server/config.go @@ -241,7 +241,8 @@ type Telemetry struct { // PrometheusRetentionTime is the retention time for prometheus metrics if greater than 0. // A value of 0 disable Prometheus support. // Default: 0 - PrometheusRetentionTime time.Duration `hcl:"prometheus_retention_time"` + PrometheusRetentionTime time.Duration `hcl:-` + PrometheusRetentionTimeRaw interface{} `hcl:"prometheus_retention_time"` } func (s *Telemetry) GoString() string { @@ -825,5 +826,13 @@ func parseTelemetry(result *Config, list *ast.ObjectList) error { if err := hcl.DecodeObject(&result.Telemetry, item.Val); err != nil { return multierror.Prefix(err, "telemetry:") } + + if result.Telemetry.PrometheusRetentionTimeRaw != nil { + var err error + if result.Telemetry.PrometheusRetentionTime, err = parseutil.ParseDurationSecond(result.Telemetry.PrometheusRetentionTimeRaw); err != nil { + return err + } + } + return nil } From bdea686abebd2d9bcc901779ac1fbdf72b9159ee Mon Sep 17 00:00:00 2001 From: Martin Conraux Date: Tue, 18 Sep 2018 11:10:07 +0200 Subject: [PATCH 05/11] Initialize config.Telemetry if nil --- command/server.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/command/server.go b/command/server.go index 525fd6981ba4..70feacf39a8a 100644 --- a/command/server.go +++ b/command/server.go @@ -1681,10 +1681,9 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) (*metrics.InmemSin var telConfig *server.Telemetry if config.Telemetry == nil { - telConfig = &server.Telemetry{} - } else { - telConfig = config.Telemetry + config.Telemetry = &server.Telemetry{} } + telConfig = config.Telemetry metricsConf := metrics.DefaultConfig("vault") metricsConf.EnableHostname = !telConfig.DisableHostname From e592cff7bb4692e5964e5f50c0c29d685159c48f Mon Sep 17 00:00:00 2001 From: Martin Conraux Date: Tue, 18 Dec 2018 18:19:47 +0100 Subject: [PATCH 06/11] address PR issues --- command/server.go | 22 ++++++------ http/handler.go | 1 - http/sys_metrics.go | 49 -------------------------- vault/core.go | 10 ++++++ vault/logical_system.go | 74 +++++++++++++++++++++++++++++++++++++++ vault/mount.go | 3 ++ vault/request_handling.go | 10 +++--- 7 files changed, 103 insertions(+), 66 deletions(-) delete mode 100644 http/sys_metrics.go diff --git a/command/server.go b/command/server.go index 70feacf39a8a..f41995505995 100644 --- a/command/server.go +++ b/command/server.go @@ -467,11 +467,12 @@ func (c *ServerCommand) Run(args []string) int { "in a Docker container, provide the IPC_LOCK cap to the container.")) } - InMemMetrics, err := c.setupTelemetry(config) + inMemMetrics, err := c.setupTelemetry(config) if err != nil { c.UI.Error(fmt.Sprintf("Error initializing telemetry: %s", err)) return 1 } + prometheusEnabled := config.Telemetry != nil && config.Telemetry.PrometheusRetentionTime > 0 // Initialize the backend factory, exists := c.PhysicalBackends[config.Storage.Type] @@ -563,6 +564,8 @@ func (c *ServerCommand) Run(args []string) int { AllLoggers: allLoggers, BuiltinRegistry: builtinplugins.Registry, DisableKeyEncodingChecks: config.DisablePrintableCheck, + InMemSink: inMemMetrics, + PrometheusEnabled: prometheusEnabled, } if c.flagDev { coreConfig.DevToken = c.flagDevRootTokenID @@ -1078,12 +1081,10 @@ CLUSTER_SYNTHESIS_COMPLETE: // Initialize the HTTP servers for _, ln := range lns { handler := vaulthttp.Handler(&vault.HandlerProperties{ - Core: core, - MaxRequestSize: ln.maxRequestSize, - MaxRequestDuration: ln.maxRequestDuration, - DisablePrintableCheck: config.DisablePrintableCheck, - TelemetryMemSink: InMemMetrics, - TelemetryPrometheusRetention: config.Telemetry.PrometheusRetentionTime, + Core: core, + MaxRequestSize: ln.maxRequestSize, + MaxRequestDuration: ln.maxRequestDuration, + DisablePrintableCheck: config.DisablePrintableCheck, }) // We perform validation on the config earlier, we can just cast here @@ -1680,10 +1681,11 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) (*metrics.InmemSin metrics.DefaultInmemSignal(inm) var telConfig *server.Telemetry - if config.Telemetry == nil { - config.Telemetry = &server.Telemetry{} + if config.Telemetry != nil { + telConfig = config.Telemetry + } else { + telConfig = &server.Telemetry{} } - telConfig = config.Telemetry metricsConf := metrics.DefaultConfig("vault") metricsConf.EnableHostname = !telConfig.DisableHostname diff --git a/http/handler.go b/http/handler.go index 37fa0528cd73..c4ccc40c0da8 100644 --- a/http/handler.go +++ b/http/handler.go @@ -107,7 +107,6 @@ func Handler(props *vault.HandlerProperties) http.Handler { mux.Handle("/v1/sys/unseal", handleSysUnseal(core)) mux.Handle("/v1/sys/leader", handleSysLeader(core)) mux.Handle("/v1/sys/health", handleSysHealth(core)) - mux.Handle("/v1/sys/metrics", handleSysMetrics(props.TelemetryMemSink, props.TelemetryPrometheusRetention)) mux.Handle("/v1/sys/generate-root/attempt", handleRequestForwarding(core, handleSysGenerateRootAttempt(core, vault.GenerateStandardRootTokenStrategy))) mux.Handle("/v1/sys/generate-root/update", handleRequestForwarding(core, handleSysGenerateRootUpdate(core, vault.GenerateStandardRootTokenStrategy))) mux.Handle("/v1/sys/rekey/init", handleRequestForwarding(core, handleSysRekeyInit(core, false))) diff --git a/http/sys_metrics.go b/http/sys_metrics.go deleted file mode 100644 index e813b3ef5ebf..000000000000 --- a/http/sys_metrics.go +++ /dev/null @@ -1,49 +0,0 @@ -package http - -import ( - "encoding/json" - "net/http" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - - metrics "github.com/armon/go-metrics" -) - -func handleSysMetrics(inm *metrics.InmemSink, prometheusRetention time.Duration) http.Handler { - return http.HandlerFunc( - func(res http.ResponseWriter, req *http.Request) { - if format := req.URL.Query().Get("format"); format == "prometheus" { - - if prometheusRetention.Nanoseconds() == 0 { - res.WriteHeader(500) - res.Write([]byte("prometheus support is not enabled")) - return - } - - handlerOptions := promhttp.HandlerOpts{ - ErrorHandling: promhttp.ContinueOnError, - DisableCompression: true, - } - - handler := promhttp.HandlerFor(prometheus.DefaultGatherer, handlerOptions) - handler.ServeHTTP(res, req) - return - } - summary, err := inm.DisplayMetrics(res, req) - if err != nil { - res.WriteHeader(500) - res.Write([]byte(err.Error())) - } else { - content, err := json.Marshal(summary) - if err != nil { - res.WriteHeader(500) - res.Write([]byte("can't marshal internal metrics into json")) - } else { - res.WriteHeader(200) - res.Write(content) - } - } - }) -} diff --git a/vault/core.go b/vault/core.go index 5c8a4db0e98e..2b9b65e5b786 100644 --- a/vault/core.go +++ b/vault/core.go @@ -402,6 +402,10 @@ type Core struct { // Stores loggers so we can reset the level allLoggers []log.Logger allLoggersLock sync.RWMutex + + // Telemetry objects + inMemSink *metrics.InmemSink + prometheusEnabled bool } // CoreConfig is used to parameterize a core @@ -470,6 +474,10 @@ type CoreConfig struct { DisableKeyEncodingChecks bool AllLoggers []log.Logger + + // Telemetry objects + InMemSink *metrics.InmemSink + PrometheusEnabled bool } func (c *CoreConfig) Clone() *CoreConfig { @@ -576,6 +584,8 @@ func NewCore(conf *CoreConfig) (*Core, error) { activeContextCancelFunc: new(atomic.Value), allLoggers: conf.AllLoggers, builtinRegistry: conf.BuiltinRegistry, + inMemSink: conf.InMemSink, + prometheusEnabled: conf.PrometheusEnabled, } atomic.StoreUint32(c.sealed, 1) diff --git a/vault/logical_system.go b/vault/logical_system.go index 76be568473a1..e8c48feeee69 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -1,6 +1,7 @@ package vault import ( + "bytes" "context" "crypto/sha256" "crypto/sha512" @@ -18,6 +19,8 @@ import ( "sync" "time" + "github.com/prometheus/common/expfmt" + "github.com/hashicorp/errwrap" log "github.com/hashicorp/go-hclog" memdb "github.com/hashicorp/go-memdb" @@ -33,6 +36,7 @@ import ( "github.com/hashicorp/vault/logical" "github.com/hashicorp/vault/logical/framework" "github.com/mitchellh/mapstructure" + "github.com/prometheus/client_golang/prometheus" ) var ( @@ -125,6 +129,20 @@ func NewSystemBackend(core *Core, logger log.Logger) *SystemBackend { LocalStorage: []string{ expirationSubPath, }, + &framework.Path{ + Pattern: "metrics", + Fields: map[string]*framework.FieldSchema{ + "format": &framework.FieldSchema{ + Type: framework.TypeString, + Description: "Format to export metrics into. Currently accept only \"prometheus\"", + }, + }, + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.ReadOperation: b.handleMetrics, + }, + HelpSynopsis: strings.TrimSpace(sysHelp["metrics"][0]), + HelpDescription: strings.TrimSpace(sysHelp["metrics"][1]), + }, }, } @@ -2473,6 +2491,58 @@ func (b *SystemBackend) responseWrappingUnwrap(ctx context.Context, te *logical. return response, nil } +func (b *SystemBackend) handleMetrics(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + + format := data.Get("format").(string) + + acceptHeaders := req.Headers["Accept"] + if format == "prometheus" || (len(acceptHeaders) > 0 && strings.HasPrefix(acceptHeaders[0], "application/openmetrics-text")) { + if !b.Core.prometheusEnabled { + err := "prometheus support is not enabled" + return nil, fmt.Errorf(err) + } + + metricsFamilies, err := prometheus.DefaultGatherer.Gather() + if err != nil && len(metricsFamilies) == 0 { + return nil, fmt.Errorf("no prometheus metrics could be decoded: %s", err) + } + + // Initialize a byte buffer. + buf := &bytes.Buffer{} + defer buf.Reset() + + e := expfmt.NewEncoder(buf, expfmt.FmtText) + for _, mf := range metricsFamilies { + err := e.Encode(mf) + if err != nil { + return nil, fmt.Errorf("error during the encoding of metrics: %s", err) + } + } + return &logical.Response{ + Data: map[string]interface{}{ + logical.HTTPContentType: "text/plain", + logical.HTTPRawBody: buf.Bytes(), + logical.HTTPStatusCode: 200, + }, + }, nil + } + summary, err := b.Core.inMemSink.DisplayMetrics(nil, nil) + if err != nil { + return nil, fmt.Errorf("error while fetching the in-memory metrics: %s", err) + } + content, err := json.Marshal(summary) + if err != nil { + return nil, fmt.Errorf("error while marshalling the in-memory metrics: %s", err) + } + return &logical.Response{ + Data: map[string]interface{}{ + logical.HTTPContentType: "application/json", + logical.HTTPRawBody: content, + logical.HTTPStatusCode: 200, + }, + }, nil +} + func (b *SystemBackend) handleWrappingLookup(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { // This ordering of lookups has been validated already in the wrapping // validation func, we're just doing this for a safety check @@ -3853,4 +3923,8 @@ This path responds to the following HTTP methods. "Information about a token's resultant ACL. Internal API; its location, inputs, and outputs may change.", "", }, + "metrics": { + "Export the metrics aggregated for telemetry purpose.", + "", + }, } diff --git a/vault/mount.go b/vault/mount.go index 355b150543b3..06262ec773b4 100644 --- a/vault/mount.go +++ b/vault/mount.go @@ -1267,6 +1267,9 @@ func (c *Core) requiredMountTable() *MountTable { UUID: sysUUID, Accessor: sysAccessor, BackendAwareUUID: sysBackendUUID, + Config: MountConfig{ + PassthroughRequestHeaders: []string{"Accept"}, + }, } identityUUID, err := uuid.GenerateUUID() diff --git a/vault/request_handling.go b/vault/request_handling.go index ba483dd41532..b8471f8d2ca0 100644 --- a/vault/request_handling.go +++ b/vault/request_handling.go @@ -39,12 +39,10 @@ var ( // HandlerProperties is used to seed configuration into a vaulthttp.Handler. // It's in this package to avoid a circular dependency type HandlerProperties struct { - Core *Core - MaxRequestSize int64 - MaxRequestDuration time.Duration - DisablePrintableCheck bool - TelemetryMemSink *metrics.InmemSink - TelemetryPrometheusRetention time.Duration + Core *Core + MaxRequestSize int64 + MaxRequestDuration time.Duration + DisablePrintableCheck bool } // fetchEntityAndDerivedPolicies returns the entity object for the given entity From 27cf6a6ae19d18e8ea0b5cdc2c5259e5fcd1d7ba Mon Sep 17 00:00:00 2001 From: Martin Conraux Date: Mon, 21 Jan 2019 18:24:22 +0100 Subject: [PATCH 07/11] add sys/metrics framework.Path in a factory --- vault/logical_system.go | 15 +-------------- vault/logical_system_paths.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/vault/logical_system.go b/vault/logical_system.go index e8c48feeee69..eb91fe4cf1bb 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -129,20 +129,6 @@ func NewSystemBackend(core *Core, logger log.Logger) *SystemBackend { LocalStorage: []string{ expirationSubPath, }, - &framework.Path{ - Pattern: "metrics", - Fields: map[string]*framework.FieldSchema{ - "format": &framework.FieldSchema{ - Type: framework.TypeString, - Description: "Format to export metrics into. Currently accept only \"prometheus\"", - }, - }, - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.handleMetrics, - }, - HelpSynopsis: strings.TrimSpace(sysHelp["metrics"][0]), - HelpDescription: strings.TrimSpace(sysHelp["metrics"][1]), - }, }, } @@ -163,6 +149,7 @@ func NewSystemBackend(core *Core, logger log.Logger) *SystemBackend { b.Backend.Paths = append(b.Backend.Paths, b.capabilitiesPaths()...) b.Backend.Paths = append(b.Backend.Paths, b.internalPaths()...) b.Backend.Paths = append(b.Backend.Paths, b.remountPath()) + b.Backend.Paths = append(b.Backend.Paths, b.metricsPath()) if core.rawEnabled { b.Backend.Paths = append(b.Backend.Paths, &framework.Path{ diff --git a/vault/logical_system_paths.go b/vault/logical_system_paths.go index bcd0a6ef2445..54b5eb499b9d 100644 --- a/vault/logical_system_paths.go +++ b/vault/logical_system_paths.go @@ -1100,6 +1100,24 @@ func (b *SystemBackend) remountPath() *framework.Path { } } +func (b *SystemBackend) metricsPath() *framework.Path { + return &framework.Path{ + Pattern: "metrics", + Fields: map[string]*framework.FieldSchema{ + "format": &framework.FieldSchema{ + Type: framework.TypeString, + Description: "Format to export metrics into. Currently accept only \"prometheus\"", + }, + }, + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.ReadOperation: b.handleMetrics, + }, + HelpSynopsis: strings.TrimSpace(sysHelp["metrics"][0]), + HelpDescription: strings.TrimSpace(sysHelp["metrics"][1]), + } + +} + func (b *SystemBackend) authPaths() []*framework.Path { return []*framework.Path{ { From fa05d1ef84d12c25cefc1c698ce699b9de8679fb Mon Sep 17 00:00:00 2001 From: Martin Conraux Date: Fri, 1 Feb 2019 10:24:23 +0100 Subject: [PATCH 08/11] Apply requiredMountTable entries's MountConfig to existing core table --- vault/mount.go | 1 + 1 file changed, 1 insertion(+) diff --git a/vault/mount.go b/vault/mount.go index 06262ec773b4..e8db560defff 100644 --- a/vault/mount.go +++ b/vault/mount.go @@ -843,6 +843,7 @@ func (c *Core) loadMounts(ctx context.Context) error { for _, coreMount := range c.mounts.Entries { if coreMount.Type == requiredMount.Type { foundRequired = true + coreMount.Config = requiredMount.Config break } } From 743ef908dfcf9fa59f587e168c940c7a607dda0b Mon Sep 17 00:00:00 2001 From: Martin Conraux Date: Fri, 1 Feb 2019 17:13:45 +0100 Subject: [PATCH 09/11] address pr comments --- command/server.go | 5 +++++ vault/logical_system.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/command/server.go b/command/server.go index f41995505995..3a98b25218f5 100644 --- a/command/server.go +++ b/command/server.go @@ -1755,6 +1755,11 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) (*metrics.InmemSin if err != nil { return nil, err } + // Hostname enabled will create poor quality metrics name + if !telConfig.DisableHostname { + c.UI.Warn("telemetry.disable_hostname has been set to false. Recommended setting is true for Prometheus to avoid poorly named metrics.") + + } fanout = append(fanout, sink) } diff --git a/vault/logical_system.go b/vault/logical_system.go index eb91fe4cf1bb..52e9f650d658 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -2507,7 +2507,7 @@ func (b *SystemBackend) handleMetrics(ctx context.Context, req *logical.Request, } return &logical.Response{ Data: map[string]interface{}{ - logical.HTTPContentType: "text/plain", + logical.HTTPContentType: expfmt.FmtText, logical.HTTPRawBody: buf.Bytes(), logical.HTTPStatusCode: 200, }, From cc4512456f38fd9344b22a90ab7ec9d123122991 Mon Sep 17 00:00:00 2001 From: Martin Conraux Date: Mon, 4 Feb 2019 16:20:13 +0100 Subject: [PATCH 10/11] enable prometheus sink by default --- command/server.go | 47 ++++++++++++++++++++-------------------- command/server/config.go | 11 +++++++--- vault/core.go | 3 --- vault/logical_system.go | 4 ---- 4 files changed, 32 insertions(+), 33 deletions(-) diff --git a/command/server.go b/command/server.go index 3a98b25218f5..6405f31b1a7a 100644 --- a/command/server.go +++ b/command/server.go @@ -472,7 +472,6 @@ func (c *ServerCommand) Run(args []string) int { c.UI.Error(fmt.Sprintf("Error initializing telemetry: %s", err)) return 1 } - prometheusEnabled := config.Telemetry != nil && config.Telemetry.PrometheusRetentionTime > 0 // Initialize the backend factory, exists := c.PhysicalBackends[config.Storage.Type] @@ -565,7 +564,6 @@ func (c *ServerCommand) Run(args []string) int { BuiltinRegistry: builtinplugins.Registry, DisableKeyEncodingChecks: config.DisablePrintableCheck, InMemSink: inMemMetrics, - PrometheusEnabled: prometheusEnabled, } if c.flagDev { coreConfig.DevToken = c.flagDevRootTokenID @@ -1692,6 +1690,23 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) (*metrics.InmemSin // Configure the statsite sink var fanout metrics.FanoutSink + + // Configure the Prometheus sink + if telConfig.PrometheusRetentionTime == 0 { + return nil, fmt.Errorf("telemetry.prometheus_retention_time must be > 0") + } + + prometheusOpts := prometheus.PrometheusOpts{ + Expiration: telConfig.PrometheusRetentionTime, + } + + sink, err := prometheus.NewPrometheusSinkFrom(prometheusOpts) + if err != nil { + return nil, err + } + + fanout = append(fanout, sink) + if telConfig.StatsiteAddr != "" { sink, err := metrics.NewStatsiteSink(telConfig.StatsiteAddr) if err != nil { @@ -1746,23 +1761,6 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) (*metrics.InmemSin fanout = append(fanout, sink) } - // Configure the Prometheus sink - if telConfig.PrometheusRetentionTime.Nanoseconds() > 0 { - prometheusOpts := prometheus.PrometheusOpts{ - Expiration: telConfig.PrometheusRetentionTime, - } - sink, err := prometheus.NewPrometheusSinkFrom(prometheusOpts) - if err != nil { - return nil, err - } - // Hostname enabled will create poor quality metrics name - if !telConfig.DisableHostname { - c.UI.Warn("telemetry.disable_hostname has been set to false. Recommended setting is true for Prometheus to avoid poorly named metrics.") - - } - fanout = append(fanout, sink) - } - if telConfig.DogStatsDAddr != "" { var tags []string @@ -1779,13 +1777,16 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) (*metrics.InmemSin } // Initialize the global sink - if len(fanout) > 0 { - fanout = append(fanout, inm) - metrics.NewGlobal(metricsConf, fanout) + if len(fanout) > 1 { + // Hostname enabled will create poor quality metrics name for prometheus + if !telConfig.DisableHostname { + c.UI.Warn("telemetry.disable_hostname has been set to false. Recommended setting is true for Prometheus to avoid poorly named metrics.") + } } else { metricsConf.EnableHostname = false - metrics.NewGlobal(metricsConf, inm) } + fanout = append(fanout, inm) + metrics.NewGlobal(metricsConf, fanout) return inm, nil } diff --git a/command/server/config.go b/command/server/config.go index f0c44daa630b..c71de454ba38 100644 --- a/command/server/config.go +++ b/command/server/config.go @@ -19,6 +19,10 @@ import ( "github.com/hashicorp/vault/helper/parseutil" ) +const ( + prometheusDefaultRetentionTime = 24 * time.Hour +) + // Config is the configuration for the vault server. type Config struct { Listeners []*Listener `hcl:"-"` @@ -99,7 +103,7 @@ func DevConfig(ha, transactional bool) *Config { EnableUI: true, Telemetry: &Telemetry{ - PrometheusRetentionTime: 24 * time.Hour, + PrometheusRetentionTime: prometheusDefaultRetentionTime, DisableHostname: true, }, } @@ -239,8 +243,7 @@ type Telemetry struct { // Prometheus: // PrometheusRetentionTime is the retention time for prometheus metrics if greater than 0. - // A value of 0 disable Prometheus support. - // Default: 0 + // Default: 24h PrometheusRetentionTime time.Duration `hcl:-` PrometheusRetentionTimeRaw interface{} `hcl:"prometheus_retention_time"` } @@ -832,6 +835,8 @@ func parseTelemetry(result *Config, list *ast.ObjectList) error { if result.Telemetry.PrometheusRetentionTime, err = parseutil.ParseDurationSecond(result.Telemetry.PrometheusRetentionTimeRaw); err != nil { return err } + } else { + result.Telemetry.PrometheusRetentionTime = prometheusDefaultRetentionTime } return nil diff --git a/vault/core.go b/vault/core.go index 2b9b65e5b786..6275d8776cae 100644 --- a/vault/core.go +++ b/vault/core.go @@ -405,7 +405,6 @@ type Core struct { // Telemetry objects inMemSink *metrics.InmemSink - prometheusEnabled bool } // CoreConfig is used to parameterize a core @@ -477,7 +476,6 @@ type CoreConfig struct { // Telemetry objects InMemSink *metrics.InmemSink - PrometheusEnabled bool } func (c *CoreConfig) Clone() *CoreConfig { @@ -585,7 +583,6 @@ func NewCore(conf *CoreConfig) (*Core, error) { allLoggers: conf.AllLoggers, builtinRegistry: conf.BuiltinRegistry, inMemSink: conf.InMemSink, - prometheusEnabled: conf.PrometheusEnabled, } atomic.StoreUint32(c.sealed, 1) diff --git a/vault/logical_system.go b/vault/logical_system.go index 52e9f650d658..9c8c12418a57 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -2484,10 +2484,6 @@ func (b *SystemBackend) handleMetrics(ctx context.Context, req *logical.Request, acceptHeaders := req.Headers["Accept"] if format == "prometheus" || (len(acceptHeaders) > 0 && strings.HasPrefix(acceptHeaders[0], "application/openmetrics-text")) { - if !b.Core.prometheusEnabled { - err := "prometheus support is not enabled" - return nil, fmt.Errorf(err) - } metricsFamilies, err := prometheus.DefaultGatherer.Gather() if err != nil && len(metricsFamilies) == 0 { From cdb2cc793d64903ea6379782152f61edf47b25d7 Mon Sep 17 00:00:00 2001 From: Martin Conraux Date: Tue, 12 Feb 2019 10:54:19 +0100 Subject: [PATCH 11/11] Move Metric-related code in a separate metricsutil helper --- command/server.go | 38 ++++++----- helper/metricsutil/metricsutil.go | 104 ++++++++++++++++++++++++++++++ vault/core.go | 7 +- vault/logical_system.go | 51 ++------------- 4 files changed, 135 insertions(+), 65 deletions(-) create mode 100644 helper/metricsutil/metricsutil.go diff --git a/command/server.go b/command/server.go index 6405f31b1a7a..9a426e087e02 100644 --- a/command/server.go +++ b/command/server.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "github.com/hashicorp/vault/helper/metricsutil" "io" "io/ioutil" "net" @@ -467,7 +468,7 @@ func (c *ServerCommand) Run(args []string) int { "in a Docker container, provide the IPC_LOCK cap to the container.")) } - inMemMetrics, err := c.setupTelemetry(config) + metricsHelper, err := c.setupTelemetry(config) if err != nil { c.UI.Error(fmt.Sprintf("Error initializing telemetry: %s", err)) return 1 @@ -563,7 +564,7 @@ func (c *ServerCommand) Run(args []string) int { AllLoggers: allLoggers, BuiltinRegistry: builtinplugins.Registry, DisableKeyEncodingChecks: config.DisablePrintableCheck, - InMemSink: inMemMetrics, + MetricsHelper: metricsHelper, } if c.flagDev { coreConfig.DevToken = c.flagDevRootTokenID @@ -1670,7 +1671,7 @@ func (c *ServerCommand) detectRedirect(detect physical.RedirectDetect, } // setupTelemetry is used to setup the telemetry sub-systems and returns the in-memory sink to be used in http configuration -func (c *ServerCommand) setupTelemetry(config *server.Config) (*metrics.InmemSink, error) { +func (c *ServerCommand) setupTelemetry(config *server.Config) (*metricsutil.MetricsHelper, error) { /* Setup telemetry Aggregate on 10 second intervals for 1 minute. Expose the metrics over stderr when there is a SIGUSR1 received. @@ -1690,22 +1691,24 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) (*metrics.InmemSin // Configure the statsite sink var fanout metrics.FanoutSink + var prometheusEnabled bool // Configure the Prometheus sink - if telConfig.PrometheusRetentionTime == 0 { - return nil, fmt.Errorf("telemetry.prometheus_retention_time must be > 0") - } + if telConfig.PrometheusRetentionTime != 0 { + prometheusEnabled = true + prometheusOpts := prometheus.PrometheusOpts{ + Expiration: telConfig.PrometheusRetentionTime, + } - prometheusOpts := prometheus.PrometheusOpts{ - Expiration: telConfig.PrometheusRetentionTime, + sink, err := prometheus.NewPrometheusSinkFrom(prometheusOpts) + if err != nil { + return nil, err + } + fanout = append(fanout, sink) } - sink, err := prometheus.NewPrometheusSinkFrom(prometheusOpts) - if err != nil { - return nil, err - } + metricHelper := metricsutil.NewMetricsHelper(inm, prometheusEnabled) - fanout = append(fanout, sink) if telConfig.StatsiteAddr != "" { sink, err := metrics.NewStatsiteSink(telConfig.StatsiteAddr) @@ -1786,8 +1789,13 @@ func (c *ServerCommand) setupTelemetry(config *server.Config) (*metrics.InmemSin metricsConf.EnableHostname = false } fanout = append(fanout, inm) - metrics.NewGlobal(metricsConf, fanout) - return inm, nil + _, err := metrics.NewGlobal(metricsConf, fanout) + + if err != nil { + return nil, err + } + + return metricHelper, nil } func (c *ServerCommand) Reload(lock *sync.RWMutex, reloadFuncs *map[string][]reload.ReloadFunc, configPath []string) error { diff --git a/helper/metricsutil/metricsutil.go b/helper/metricsutil/metricsutil.go new file mode 100644 index 000000000000..3083d2dc10ca --- /dev/null +++ b/helper/metricsutil/metricsutil.go @@ -0,0 +1,104 @@ +package metricsutil + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/armon/go-metrics" + "github.com/hashicorp/vault/logical" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" + "strings" +) + +const ( + OpenMetricsMIMEType = "application/openmetrics-text" +) + +const ( + PrometheusMetricFormat = "prometheus" +) + +type MetricsHelper struct { + inMemSink *metrics.InmemSink + PrometheusEnabled bool +} + +func NewMetricsHelper(inMem *metrics.InmemSink, enablePrometheus bool) *MetricsHelper{ + return &MetricsHelper{inMem, enablePrometheus} +} + +func FormatFromRequest(req *logical.Request) (string) { + acceptHeaders := req.Headers["Accept"] + if len(acceptHeaders) > 0 { + acceptHeader := acceptHeaders[0] + if strings.HasPrefix(acceptHeader, OpenMetricsMIMEType) { + return "prometheus" + } + } + return "" +} + +func (m *MetricsHelper) ResponseForFormat(format string) (*logical.Response, error) { + switch format { + case PrometheusMetricFormat: + return m.PrometheusResponse() + case "": + return m.GenericResponse() + default: + return nil, fmt.Errorf("metric response format \"%s\" unknown", format) + } +} + +func (m *MetricsHelper) PrometheusResponse() (*logical.Response, error) { + if !m.PrometheusEnabled { + return &logical.Response{ + Data: map[string]interface{}{ + logical.HTTPContentType: "text/plain", + logical.HTTPRawBody: "prometheus is not enabled", + logical.HTTPStatusCode: 400, + }, + }, nil + } + metricsFamilies, err := prometheus.DefaultGatherer.Gather() + if err != nil && len(metricsFamilies) == 0 { + return nil, fmt.Errorf("no prometheus metrics could be decoded: %s", err) + } + + // Initialize a byte buffer. + buf := &bytes.Buffer{} + defer buf.Reset() + + e := expfmt.NewEncoder(buf, expfmt.FmtText) + for _, mf := range metricsFamilies { + err := e.Encode(mf) + if err != nil { + return nil, fmt.Errorf("error during the encoding of metrics: %s", err) + } + } + return &logical.Response{ + Data: map[string]interface{}{ + logical.HTTPContentType: string(expfmt.FmtText), + logical.HTTPRawBody: buf.Bytes(), + logical.HTTPStatusCode: 200, + }, + }, nil +} + +func (m *MetricsHelper) GenericResponse() (*logical.Response, error) { + summary, err := m.inMemSink.DisplayMetrics(nil,nil) + if err != nil { + return nil, fmt.Errorf("error while fetching the in-memory metrics: %s", err) + } + content, err := json.Marshal(summary) + if err != nil { + return nil, fmt.Errorf("error while marshalling the in-memory metrics: %s", err) + } + return &logical.Response{ + Data: map[string]interface{}{ + logical.HTTPContentType: "application/json", + logical.HTTPRawBody: content, + logical.HTTPStatusCode: 200, + }, + }, nil +} diff --git a/vault/core.go b/vault/core.go index 6275d8776cae..de50ab7f9949 100644 --- a/vault/core.go +++ b/vault/core.go @@ -7,6 +7,7 @@ import ( "crypto/x509" "errors" "fmt" + "github.com/hashicorp/vault/helper/metricsutil" "net" "net/http" "net/url" @@ -404,7 +405,7 @@ type Core struct { allLoggersLock sync.RWMutex // Telemetry objects - inMemSink *metrics.InmemSink + metricsHelper *metricsutil.MetricsHelper } // CoreConfig is used to parameterize a core @@ -475,7 +476,7 @@ type CoreConfig struct { AllLoggers []log.Logger // Telemetry objects - InMemSink *metrics.InmemSink + MetricsHelper *metricsutil.MetricsHelper } func (c *CoreConfig) Clone() *CoreConfig { @@ -582,7 +583,7 @@ func NewCore(conf *CoreConfig) (*Core, error) { activeContextCancelFunc: new(atomic.Value), allLoggers: conf.AllLoggers, builtinRegistry: conf.BuiltinRegistry, - inMemSink: conf.InMemSink, + metricsHelper: conf.MetricsHelper, } atomic.StoreUint32(c.sealed, 1) diff --git a/vault/logical_system.go b/vault/logical_system.go index 9c8c12418a57..71494347cd36 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -1,7 +1,6 @@ package vault import ( - "bytes" "context" "crypto/sha256" "crypto/sha512" @@ -19,8 +18,6 @@ import ( "sync" "time" - "github.com/prometheus/common/expfmt" - "github.com/hashicorp/errwrap" log "github.com/hashicorp/go-hclog" memdb "github.com/hashicorp/go-memdb" @@ -29,6 +26,7 @@ import ( "github.com/hashicorp/vault/helper/consts" "github.com/hashicorp/vault/helper/identity" "github.com/hashicorp/vault/helper/jsonutil" + "github.com/hashicorp/vault/helper/metricsutil" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/helper/parseutil" "github.com/hashicorp/vault/helper/strutil" @@ -36,7 +34,6 @@ import ( "github.com/hashicorp/vault/logical" "github.com/hashicorp/vault/logical/framework" "github.com/mitchellh/mapstructure" - "github.com/prometheus/client_golang/prometheus" ) var ( @@ -2479,51 +2476,11 @@ func (b *SystemBackend) responseWrappingUnwrap(ctx context.Context, te *logical. } func (b *SystemBackend) handleMetrics(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { - format := data.Get("format").(string) - - acceptHeaders := req.Headers["Accept"] - if format == "prometheus" || (len(acceptHeaders) > 0 && strings.HasPrefix(acceptHeaders[0], "application/openmetrics-text")) { - - metricsFamilies, err := prometheus.DefaultGatherer.Gather() - if err != nil && len(metricsFamilies) == 0 { - return nil, fmt.Errorf("no prometheus metrics could be decoded: %s", err) - } - - // Initialize a byte buffer. - buf := &bytes.Buffer{} - defer buf.Reset() - - e := expfmt.NewEncoder(buf, expfmt.FmtText) - for _, mf := range metricsFamilies { - err := e.Encode(mf) - if err != nil { - return nil, fmt.Errorf("error during the encoding of metrics: %s", err) - } - } - return &logical.Response{ - Data: map[string]interface{}{ - logical.HTTPContentType: expfmt.FmtText, - logical.HTTPRawBody: buf.Bytes(), - logical.HTTPStatusCode: 200, - }, - }, nil - } - summary, err := b.Core.inMemSink.DisplayMetrics(nil, nil) - if err != nil { - return nil, fmt.Errorf("error while fetching the in-memory metrics: %s", err) + if format == "" { + format = metricsutil.FormatFromRequest(req) } - content, err := json.Marshal(summary) - if err != nil { - return nil, fmt.Errorf("error while marshalling the in-memory metrics: %s", err) - } - return &logical.Response{ - Data: map[string]interface{}{ - logical.HTTPContentType: "application/json", - logical.HTTPRawBody: content, - logical.HTTPStatusCode: 200, - }, - }, nil + return b.Core.metricsHelper.ResponseForFormat(format) } func (b *SystemBackend) handleWrappingLookup(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {